Initial
This commit is contained in:
Jason
@@ -0,0 +1,295 @@
|
||||
"""
|
||||
Generate Bitwise Helper File for Product Classification
|
||||
Creates a helper file with bitwise flags based on product attributes (columns G-S)
|
||||
"""
|
||||
|
||||
import csv
|
||||
import json
|
||||
from collections import defaultdict
|
||||
|
||||
# Bit position definitions
|
||||
BIT_DEFINITIONS = {
|
||||
# Base Type (Bits 0-1)
|
||||
'base_door': 0, # 2^0 = 1
|
||||
'base_window': 1, # 2^1 = 2
|
||||
|
||||
# Product Flags (Bits 2-3)
|
||||
'is_accessory': 2, # 2^2 = 4
|
||||
'specific_item': 3, # 2^3 = 8
|
||||
|
||||
# Materials (Bits 4-5)
|
||||
'material_aluminum': 4, # 2^4 = 16
|
||||
'material_vinyl': 5, # 2^5 = 32
|
||||
|
||||
# Colors (Bits 6-11)
|
||||
'color_black': 6, # 2^6 = 64
|
||||
'color_white': 7, # 2^7 = 128
|
||||
'color_bronze': 8, # 2^8 = 256
|
||||
'color_tan': 9, # 2^9 = 512
|
||||
'color_mill': 10, # 2^10 = 1024
|
||||
'color_sandstone': 11, # 2^11 = 2048
|
||||
}
|
||||
|
||||
# Sub-type bits will be assigned dynamically (starting at bit 12)
|
||||
SUBTYPE_BIT_START = 12
|
||||
|
||||
def parse_bool(value):
|
||||
"""Parse TRUE/FALSE string to boolean"""
|
||||
if isinstance(value, str):
|
||||
return value.strip().upper() == 'TRUE'
|
||||
return bool(value)
|
||||
|
||||
def calculate_bit_value(row, subtype_map):
|
||||
"""Calculate the bitwise value for a product row"""
|
||||
bit_value = 0
|
||||
explanation = []
|
||||
|
||||
# Base Type (Column G - index 6)
|
||||
base_type = row[6].strip() if len(row) > 6 and row[6] else ""
|
||||
if base_type.lower() == 'door':
|
||||
bit_value |= (1 << BIT_DEFINITIONS['base_door'])
|
||||
explanation.append('Door')
|
||||
elif base_type.lower() == 'window':
|
||||
bit_value |= (1 << BIT_DEFINITIONS['base_window'])
|
||||
explanation.append('Window')
|
||||
|
||||
# Accessory Flag (Column J - index 9)
|
||||
if len(row) > 9 and parse_bool(row[9]):
|
||||
bit_value |= (1 << BIT_DEFINITIONS['is_accessory'])
|
||||
explanation.append('Accessory')
|
||||
|
||||
# Specific Item Link (Column K - index 10)
|
||||
if len(row) > 10 and parse_bool(row[10]):
|
||||
bit_value |= (1 << BIT_DEFINITIONS['specific_item'])
|
||||
explanation.append('SpecificItem')
|
||||
|
||||
# Materials
|
||||
if len(row) > 11 and parse_bool(row[11]): # Aluminum (Column L)
|
||||
bit_value |= (1 << BIT_DEFINITIONS['material_aluminum'])
|
||||
explanation.append('Aluminum')
|
||||
|
||||
if len(row) > 12 and parse_bool(row[12]): # Vinyl (Column M)
|
||||
bit_value |= (1 << BIT_DEFINITIONS['material_vinyl'])
|
||||
explanation.append('Vinyl')
|
||||
|
||||
# Colors
|
||||
color_columns = [
|
||||
(13, 'color_black', 'Black'),
|
||||
(14, 'color_white', 'White'),
|
||||
(15, 'color_bronze', 'Bronze'),
|
||||
(16, 'color_tan', 'Tan'),
|
||||
(17, 'color_mill', 'Mill'),
|
||||
(18, 'color_sandstone', 'Sandstone'),
|
||||
]
|
||||
|
||||
for col_idx, bit_key, color_name in color_columns:
|
||||
if len(row) > col_idx and parse_bool(row[col_idx]):
|
||||
bit_value |= (1 << BIT_DEFINITIONS[bit_key])
|
||||
explanation.append(color_name)
|
||||
|
||||
# Sub-types (Columns H and I - indices 7 and 8)
|
||||
door_subtype = row[7].strip() if len(row) > 7 and row[7] else ""
|
||||
window_subtype = row[8].strip() if len(row) > 8 and row[8] else ""
|
||||
|
||||
if door_subtype and door_subtype in subtype_map:
|
||||
bit_pos = subtype_map[door_subtype]
|
||||
bit_value |= (1 << bit_pos)
|
||||
explanation.append(f'Subtype:{door_subtype}')
|
||||
|
||||
if window_subtype and window_subtype in subtype_map:
|
||||
bit_pos = subtype_map[window_subtype]
|
||||
bit_value |= (1 << bit_pos)
|
||||
explanation.append(f'Subtype:{window_subtype}')
|
||||
|
||||
return bit_value, explanation
|
||||
|
||||
def collect_subtypes(csv_path):
|
||||
"""Collect all unique sub-types from the CSV"""
|
||||
subtypes = set()
|
||||
|
||||
with open(csv_path, 'r', encoding='utf-8') as f:
|
||||
reader = csv.reader(f)
|
||||
next(reader) # Skip first header row
|
||||
next(reader) # Skip second header row
|
||||
|
||||
for row in reader:
|
||||
if len(row) < 8:
|
||||
continue
|
||||
|
||||
# Skip discontinued items
|
||||
if len(row) > 0 and parse_bool(row[0]):
|
||||
continue
|
||||
|
||||
# Collect door subtype (column H - index 7)
|
||||
if len(row) > 7 and row[7].strip():
|
||||
subtypes.add(row[7].strip())
|
||||
|
||||
# Collect window subtype (column I - index 8)
|
||||
if len(row) > 8 and row[8].strip():
|
||||
subtypes.add(row[8].strip())
|
||||
|
||||
return sorted(subtypes)
|
||||
|
||||
def generate_helper_files(csv_path):
|
||||
"""Generate helper CSV and JSON files with bitwise values"""
|
||||
|
||||
print("Analyzing product data...")
|
||||
|
||||
# First pass: collect all unique sub-types
|
||||
subtypes = collect_subtypes(csv_path)
|
||||
|
||||
# Create subtype bit mapping
|
||||
subtype_map = {}
|
||||
for idx, subtype in enumerate(subtypes):
|
||||
subtype_map[subtype] = SUBTYPE_BIT_START + idx
|
||||
|
||||
print(f"Found {len(subtypes)} unique sub-types")
|
||||
|
||||
# Second pass: calculate bit values
|
||||
results = []
|
||||
|
||||
with open(csv_path, 'r', encoding='utf-8') as f:
|
||||
reader = csv.reader(f)
|
||||
next(reader) # Skip first header row
|
||||
next(reader) # Skip second header row
|
||||
|
||||
for row in reader:
|
||||
if len(row) < 3:
|
||||
continue
|
||||
|
||||
prod_code = row[2].strip() # Column C - PROD_CODE
|
||||
discontinued = parse_bool(row[0]) # Column A - DISCONT
|
||||
|
||||
bit_value, explanation = calculate_bit_value(row, subtype_map)
|
||||
|
||||
results.append({
|
||||
'PROD_CODE': prod_code,
|
||||
'BIT_VALUE': bit_value,
|
||||
'BIT_HEX': f"0x{bit_value:X}",
|
||||
'BIT_BINARY': f"{bit_value:b}",
|
||||
'DISCONTINUED': discontinued,
|
||||
'FLAGS': ' | '.join(explanation) if explanation else 'None'
|
||||
})
|
||||
|
||||
# Filter out discontinued products for output files
|
||||
active_results = [r for r in results if not r['DISCONTINUED']]
|
||||
|
||||
print(f"Total products processed: {len(results)}")
|
||||
print(f"Active products: {len(active_results)}")
|
||||
print(f"Discontinued products (filtered out): {len(results) - len(active_results)}")
|
||||
|
||||
# Write CSV helper file (only active products)
|
||||
csv_output = 'data/product_bitwise.csv'
|
||||
with open(csv_output, 'w', newline='', encoding='utf-8') as f:
|
||||
writer = csv.DictWriter(f, fieldnames=['PROD_CODE', 'BIT_VALUE', 'BIT_HEX', 'BIT_BINARY', 'DISCONTINUED', 'FLAGS'])
|
||||
writer.writeheader()
|
||||
writer.writerows(active_results)
|
||||
|
||||
print(f"✓ Created {csv_output}")
|
||||
|
||||
# Write JSON helper file (only active products)
|
||||
json_output = 'data/product_bitwise.json'
|
||||
with open(json_output, 'w', encoding='utf-8') as f:
|
||||
json.dump(active_results, f, indent=2)
|
||||
|
||||
print(f"✓ Created {json_output}")
|
||||
|
||||
# Create bit legend/documentation
|
||||
legend = {
|
||||
'description': 'Bitwise flag system for product classification',
|
||||
'bit_definitions': {
|
||||
'base_attributes': {
|
||||
'bit_0 (1)': 'Base Type = Door',
|
||||
'bit_1 (2)': 'Base Type = Window',
|
||||
'bit_2 (4)': 'Is Accessory',
|
||||
'bit_3 (8)': 'Specific Item Link',
|
||||
},
|
||||
'materials': {
|
||||
'bit_4 (16)': 'Material = Aluminum',
|
||||
'bit_5 (32)': 'Material = Vinyl',
|
||||
},
|
||||
'colors': {
|
||||
'bit_6 (64)': 'Color = Black',
|
||||
'bit_7 (128)': 'Color = White',
|
||||
'bit_8 (256)': 'Color = Bronze',
|
||||
'bit_9 (512)': 'Color = Tan',
|
||||
'bit_10 (1024)': 'Color = Mill',
|
||||
'bit_11 (2048)': 'Color = Sandstone',
|
||||
},
|
||||
'subtypes': {}
|
||||
},
|
||||
'usage_examples': {
|
||||
'check_if_door': 'bit_value & 1',
|
||||
'check_if_window': 'bit_value & 2',
|
||||
'check_if_aluminum': 'bit_value & 16',
|
||||
'check_if_white': 'bit_value & 128',
|
||||
'check_multiple': '(bit_value & 1) and (bit_value & 16) # Door AND Aluminum',
|
||||
}
|
||||
}
|
||||
|
||||
# Add subtype mappings to legend
|
||||
for subtype, bit_pos in sorted(subtype_map.items(), key=lambda x: x[1]):
|
||||
bit_value = 1 << bit_pos
|
||||
legend['bit_definitions']['subtypes'][f'bit_{bit_pos} ({bit_value})'] = f'Subtype = {subtype}'
|
||||
legend['usage_examples'][f'check_subtype_{subtype.replace(" ", "_").lower()}'] = f'bit_value & {bit_value}'
|
||||
|
||||
# Write legend file
|
||||
legend_output = 'data/bitwise_legend.json'
|
||||
with open(legend_output, 'w', encoding='utf-8') as f:
|
||||
json.dump(legend, f, indent=2)
|
||||
|
||||
print(f"✓ Created {legend_output}")
|
||||
|
||||
# Generate statistics
|
||||
stats = {
|
||||
'total_products': len([r for r in results if not r['DISCONTINUED']]),
|
||||
'total_discontinued': len([r for r in results if r['DISCONTINUED']]),
|
||||
'total_accessories': len([r for r in results if not r['DISCONTINUED'] and (r['BIT_VALUE'] & 4)]),
|
||||
'doors': len([r for r in results if not r['DISCONTINUED'] and (r['BIT_VALUE'] & 1)]),
|
||||
'windows': len([r for r in results if not r['DISCONTINUED'] and (r['BIT_VALUE'] & 2)]),
|
||||
'aluminum_products': len([r for r in results if not r['DISCONTINUED'] and (r['BIT_VALUE'] & 16)]),
|
||||
'vinyl_products': len([r for r in results if not r['DISCONTINUED'] and (r['BIT_VALUE'] & 32)]),
|
||||
'multi_material': len([r for r in results if not r['DISCONTINUED'] and (r['BIT_VALUE'] & 16) and (r['BIT_VALUE'] & 32)]),
|
||||
}
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("STATISTICS")
|
||||
print("="*60)
|
||||
print(f"Total Active Products: {stats['total_products']}")
|
||||
print(f"Total Discontinued: {stats['total_discontinued']}")
|
||||
print(f"Accessories: {stats['total_accessories']}")
|
||||
print(f"Doors: {stats['doors']}")
|
||||
print(f"Windows: {stats['windows']}")
|
||||
print(f"Aluminum Products: {stats['aluminum_products']}")
|
||||
print(f"Vinyl Products: {stats['vinyl_products']}")
|
||||
print(f"Multi-Material Products: {stats['multi_material']}")
|
||||
print(f"Unique Sub-types: {len(subtypes)}")
|
||||
print("="*60)
|
||||
|
||||
# Show example products
|
||||
print("\nEXAMPLE PRODUCTS:")
|
||||
print("-"*60)
|
||||
for i, result in enumerate(active_results[:5]):
|
||||
print(f"{result['PROD_CODE']:12} | {result['BIT_VALUE']:6} | {result['BIT_HEX']:8} | {result['FLAGS']}")
|
||||
print("-"*60)
|
||||
|
||||
return active_results, legend, subtype_map
|
||||
|
||||
if __name__ == '__main__':
|
||||
csv_path = 'data/products.csv'
|
||||
|
||||
try:
|
||||
results, legend, subtype_map = generate_helper_files(csv_path)
|
||||
print("\n✓ All helper files generated successfully!")
|
||||
print("\nGenerated files:")
|
||||
print(" - data/product_bitwise.csv (CSV format with bit values)")
|
||||
print(" - data/product_bitwise.json (JSON format with bit values)")
|
||||
print(" - data/bitwise_legend.json (Bit mapping documentation)")
|
||||
|
||||
except FileNotFoundError:
|
||||
print(f"Error: Could not find {csv_path}")
|
||||
print("Make sure products.csv exists in the data/ folder")
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
Reference in New Issue
Block a user