SQLite and JSON toggle support added
Co-authored-by: Copilot <copilot@github.com>
This commit is contained in:
Jason
@@ -0,0 +1,278 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Migration script to convert JSON data to SQLite database.
|
||||
Run this script to migrate data from JSON files to the new SQLite database.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
# Add the app directory to the path
|
||||
basedir = os.path.abspath(os.path.dirname(__file__))
|
||||
sys.path.insert(0, basedir)
|
||||
|
||||
from app import app
|
||||
from models import db, User, Product, Location, ProductAttribute, Question, Accessory
|
||||
|
||||
def load_json_file(filename):
|
||||
"""Load a JSON file from the data directory"""
|
||||
filepath = os.path.join(basedir, 'data', filename)
|
||||
if not os.path.exists(filepath):
|
||||
print(f"Warning: {filename} not found, skipping...")
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
print(f"Error loading {filename}: {e}")
|
||||
return None
|
||||
|
||||
def migrate_users():
|
||||
"""Migrate users from JSON to database"""
|
||||
print("Migrating users...")
|
||||
users_data = load_json_file('users.json')
|
||||
if not users_data:
|
||||
return
|
||||
|
||||
count = 0
|
||||
for user_data in users_data:
|
||||
# Check if user already exists
|
||||
existing = User.query.filter_by(username=user_data['username']).first()
|
||||
if existing:
|
||||
print(f" User '{user_data['username']}' already exists, skipping...")
|
||||
continue
|
||||
|
||||
user = User(
|
||||
username=user_data['username'],
|
||||
password=user_data['password'],
|
||||
default_location=user_data.get('defaultLocation'),
|
||||
active=user_data.get('active', True),
|
||||
super_admin=user_data.get('superAdmin', False)
|
||||
)
|
||||
user.permissions = user_data.get('permissions', {})
|
||||
user.locationSettings = user_data.get('locationSettings', {})
|
||||
|
||||
db.session.add(user)
|
||||
count += 1
|
||||
|
||||
db.session.commit()
|
||||
print(f" ✓ Migrated {count} users")
|
||||
|
||||
def migrate_locations():
|
||||
"""Migrate locations from product_attributes.json"""
|
||||
print("Migrating locations...")
|
||||
attributes_data = load_json_file('product_attributes.json')
|
||||
if not attributes_data or 'locations' not in attributes_data:
|
||||
return
|
||||
|
||||
count = 0
|
||||
for loc_data in attributes_data['locations']:
|
||||
# Check if location already exists
|
||||
existing = Location.query.filter_by(code=loc_data['code']).first()
|
||||
if existing:
|
||||
print(f" Location '{loc_data['code']}' already exists, skipping...")
|
||||
continue
|
||||
|
||||
location = Location(
|
||||
code=loc_data['code'],
|
||||
name=loc_data['name'],
|
||||
active=True
|
||||
)
|
||||
db.session.add(location)
|
||||
count += 1
|
||||
|
||||
db.session.commit()
|
||||
print(f" ✓ Migrated {count} locations")
|
||||
|
||||
def migrate_product_attributes():
|
||||
"""Migrate product attributes from JSON"""
|
||||
print("Migrating product attributes...")
|
||||
attributes_data = load_json_file('product_attributes.json')
|
||||
if not attributes_data:
|
||||
return
|
||||
|
||||
# Check if attributes already exist
|
||||
existing = ProductAttribute.query.first()
|
||||
if existing:
|
||||
print(" Product attributes already exist, updating...")
|
||||
attr = existing
|
||||
else:
|
||||
attr = ProductAttribute()
|
||||
db.session.add(attr)
|
||||
|
||||
# Set the attributes
|
||||
attr.statuses = attributes_data.get('statuses', [])
|
||||
attr.productTypes = attributes_data.get('productTypes', [])
|
||||
attr.codeModifiers = attributes_data.get('codeModifiers', [])
|
||||
|
||||
db.session.commit()
|
||||
print(f" ✓ Product attributes migrated")
|
||||
|
||||
def migrate_products():
|
||||
"""Migrate products from JSON to database"""
|
||||
print("Migrating products...")
|
||||
products_data = load_json_file('products.json')
|
||||
if not products_data:
|
||||
return
|
||||
|
||||
count = 0
|
||||
skipped = 0
|
||||
for prod_data in products_data:
|
||||
# Check if product already exists
|
||||
existing = Product.query.filter_by(product_code=prod_data['productCode']).first()
|
||||
if existing:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
product = Product(
|
||||
product_code=prod_data['productCode'],
|
||||
category=prod_data.get('category'),
|
||||
description=prod_data.get('description'),
|
||||
discontinued=prod_data.get('discontinued', False),
|
||||
location=prod_data.get('location'),
|
||||
base_type=prod_data.get('baseType', ''),
|
||||
is_accessory=prod_data.get('isAccessory', False)
|
||||
)
|
||||
|
||||
# Set JSON properties
|
||||
product.subType = prod_data.get('subType', {})
|
||||
product.materials = prod_data.get('materials', [])
|
||||
product.colors = prod_data.get('colors', [])
|
||||
product.compatibleAccessories = prod_data.get('compatibleAccessories', [])
|
||||
product.imageConfig = prod_data.get('imageConfig', {})
|
||||
|
||||
db.session.add(product)
|
||||
count += 1
|
||||
|
||||
db.session.commit()
|
||||
print(f" ✓ Migrated {count} products (skipped {skipped} existing)")
|
||||
|
||||
def migrate_questions():
|
||||
"""Migrate questions from JSON to database"""
|
||||
print("Migrating questions...")
|
||||
questions_data = load_json_file('questions.json')
|
||||
if not questions_data:
|
||||
return
|
||||
|
||||
# Check if questions already exist
|
||||
existing_count = Question.query.count()
|
||||
if existing_count > 0:
|
||||
print(f" Questions already exist ({existing_count} records), skipping...")
|
||||
return
|
||||
|
||||
count = 0
|
||||
for idx, question_data in enumerate(questions_data):
|
||||
question = Question(
|
||||
question_data_json=json.dumps(question_data),
|
||||
order=idx,
|
||||
active=True
|
||||
)
|
||||
db.session.add(question)
|
||||
count += 1
|
||||
|
||||
db.session.commit()
|
||||
print(f" ✓ Migrated {count} questions")
|
||||
|
||||
def migrate_accessories():
|
||||
"""Migrate accessories from JSON to database"""
|
||||
print("Migrating accessories...")
|
||||
accessories_data = load_json_file('accessories.json')
|
||||
if not accessories_data:
|
||||
return
|
||||
|
||||
# Check if accessories already exist
|
||||
existing_count = Accessory.query.count()
|
||||
if existing_count > 0:
|
||||
print(f" Accessories already exist ({existing_count} records), skipping...")
|
||||
return
|
||||
|
||||
count = 0
|
||||
for category, items in accessories_data.items():
|
||||
for item_data in items:
|
||||
accessory = Accessory(
|
||||
accessory_data_json=json.dumps(item_data),
|
||||
category=category,
|
||||
active=True
|
||||
)
|
||||
db.session.add(accessory)
|
||||
count += 1
|
||||
|
||||
db.session.commit()
|
||||
print(f" ✓ Migrated {count} accessories")
|
||||
|
||||
def backup_json_files():
|
||||
"""Create backup of JSON files before migration"""
|
||||
print("Creating backup of JSON files...")
|
||||
backup_dir = os.path.join(basedir, 'data', 'json_backup')
|
||||
os.makedirs(backup_dir, exist_ok=True)
|
||||
|
||||
json_files = ['users.json', 'products.json', 'product_attributes.json',
|
||||
'questions.json', 'accessories.json', 'navigation.json',
|
||||
'filter_config.json', 'product_bitwise.json']
|
||||
|
||||
count = 0
|
||||
for filename in json_files:
|
||||
src = os.path.join(basedir, 'data', filename)
|
||||
if os.path.exists(src):
|
||||
dst = os.path.join(backup_dir, f"{filename}.backup_{datetime.now().strftime('%Y%m%d_%H%M%S')}")
|
||||
import shutil
|
||||
shutil.copy2(src, dst)
|
||||
count += 1
|
||||
|
||||
print(f" ✓ Backed up {count} files to {backup_dir}")
|
||||
|
||||
def main():
|
||||
"""Main migration function"""
|
||||
print("=" * 60)
|
||||
print("JSON to SQLite Migration Script")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
# Create application context
|
||||
with app.app_context():
|
||||
# Create all tables
|
||||
print("Creating database tables...")
|
||||
db.create_all()
|
||||
print(" ✓ Tables created")
|
||||
print()
|
||||
|
||||
# Create backup
|
||||
backup_json_files()
|
||||
print()
|
||||
|
||||
# Run migrations
|
||||
try:
|
||||
migrate_users()
|
||||
migrate_locations()
|
||||
migrate_product_attributes()
|
||||
migrate_products()
|
||||
migrate_questions()
|
||||
migrate_accessories()
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("✅ Migration completed successfully!")
|
||||
print("=" * 60)
|
||||
print()
|
||||
print("Database location:", os.path.join(basedir, 'data', 'products.db'))
|
||||
print()
|
||||
print("Next steps:")
|
||||
print("1. Test the application to ensure everything works")
|
||||
print("2. Update blueprints to use database instead of JSON")
|
||||
print("3. Keep JSON files as backup or remove them")
|
||||
|
||||
except Exception as e:
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("❌ Migration failed!")
|
||||
print("=" * 60)
|
||||
print(f"Error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
db.session.rollback()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user