demo seed change 2
This commit is contained in:
Urtzi Alfaro
283
scripts/migrate_json_to_base_ts.py
Normal file
283
scripts/migrate_json_to_base_ts.py
Normal file
@@ -0,0 +1,283 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Migrate all demo JSON files from offset_days/ISO timestamps to BASE_TS markers.
|
||||
This script performs a one-time migration to align with the new architecture.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict
|
||||
|
||||
# Base reference date used in current JSON files
|
||||
BASE_REFERENCE_ISO = "2025-01-15T06:00:00Z"
|
||||
BASE_REFERENCE = datetime.fromisoformat(BASE_REFERENCE_ISO.replace('Z', '+00:00'))
|
||||
|
||||
# Date fields to transform by entity type
|
||||
DATE_FIELDS_MAP = {
|
||||
'purchase_orders': [
|
||||
'order_date', 'required_delivery_date', 'estimated_delivery_date',
|
||||
'expected_delivery_date', 'sent_to_supplier_at', 'supplier_confirmation_date',
|
||||
'created_at', 'updated_at'
|
||||
],
|
||||
'batches': [
|
||||
'planned_start_time', 'planned_end_time', 'actual_start_time',
|
||||
'actual_end_time', 'completed_at', 'created_at', 'updated_at'
|
||||
],
|
||||
'equipment': [
|
||||
'install_date', 'last_maintenance_date', 'next_maintenance_date',
|
||||
'created_at', 'updated_at'
|
||||
],
|
||||
'ingredients': ['created_at', 'updated_at'],
|
||||
'stock_batches': [
|
||||
'received_date', 'expiration_date', 'best_before_date',
|
||||
'created_at', 'updated_at'
|
||||
],
|
||||
'customers': ['last_order_date', 'created_at', 'updated_at'],
|
||||
'orders': [
|
||||
'order_date', 'delivery_date', 'promised_date',
|
||||
'completed_at', 'created_at', 'updated_at'
|
||||
],
|
||||
'completed_orders': [
|
||||
'order_date', 'delivery_date', 'promised_date',
|
||||
'completed_at', 'created_at', 'updated_at'
|
||||
],
|
||||
'forecasts': ['forecast_date', 'created_at', 'updated_at'],
|
||||
'prediction_batches': ['prediction_date', 'created_at', 'updated_at'],
|
||||
'sales_data': ['created_at', 'updated_at'],
|
||||
'quality_controls': ['created_at', 'updated_at'],
|
||||
'quality_alerts': ['created_at', 'updated_at'],
|
||||
'customer_orders': [
|
||||
'order_date', 'delivery_date', 'promised_date',
|
||||
'completed_at', 'created_at', 'updated_at'
|
||||
],
|
||||
'order_items': ['created_at', 'updated_at'],
|
||||
'procurement_requirements': ['created_at', 'updated_at'],
|
||||
'replenishment_plans': ['created_at', 'updated_at'],
|
||||
'production_schedules': ['schedule_date', 'created_at', 'updated_at'],
|
||||
'users': ['created_at', 'updated_at'],
|
||||
'stock': ['expiration_date', 'received_date', 'created_at', 'updated_at'],
|
||||
'recipes': ['created_at', 'updated_at'],
|
||||
'recipe_ingredients': ['created_at', 'updated_at'],
|
||||
'suppliers': ['created_at', 'updated_at'],
|
||||
'production_batches': ['start_time', 'end_time', 'created_at', 'updated_at'],
|
||||
'purchase_order_items': ['created_at', 'updated_at'],
|
||||
# Enterprise children files
|
||||
'local_inventory': ['expiration_date', 'received_date', 'created_at', 'updated_at'],
|
||||
'local_sales': ['created_at', 'updated_at'],
|
||||
'local_orders': ['order_date', 'delivery_date', 'created_at', 'updated_at'],
|
||||
'local_production_batches': [
|
||||
'planned_start_time', 'planned_end_time', 'actual_start_time',
|
||||
'actual_end_time', 'created_at', 'updated_at'
|
||||
],
|
||||
'local_forecasts': ['forecast_date', 'created_at', 'updated_at']
|
||||
}
|
||||
|
||||
|
||||
def calculate_offset_from_base(iso_timestamp: str) -> str:
|
||||
"""
|
||||
Calculate BASE_TS offset from an ISO timestamp.
|
||||
|
||||
Args:
|
||||
iso_timestamp: ISO 8601 timestamp string
|
||||
|
||||
Returns:
|
||||
BASE_TS marker string (e.g., "BASE_TS + 2d 3h")
|
||||
"""
|
||||
try:
|
||||
target_time = datetime.fromisoformat(iso_timestamp.replace('Z', '+00:00'))
|
||||
except (ValueError, AttributeError):
|
||||
return None
|
||||
|
||||
# Calculate offset from BASE_REFERENCE
|
||||
offset = target_time - BASE_REFERENCE
|
||||
total_seconds = int(offset.total_seconds())
|
||||
|
||||
if total_seconds == 0:
|
||||
return "BASE_TS"
|
||||
|
||||
# Convert to days, hours, minutes
|
||||
days = offset.days
|
||||
remaining_seconds = total_seconds - (days * 86400)
|
||||
hours = remaining_seconds // 3600
|
||||
minutes = (remaining_seconds % 3600) // 60
|
||||
|
||||
# Build BASE_TS expression
|
||||
parts = []
|
||||
if days != 0:
|
||||
parts.append(f"{abs(days)}d")
|
||||
if hours != 0:
|
||||
parts.append(f"{abs(hours)}h")
|
||||
if minutes != 0:
|
||||
parts.append(f"{abs(minutes)}m")
|
||||
|
||||
if not parts:
|
||||
return "BASE_TS"
|
||||
|
||||
operator = "+" if total_seconds > 0 else "-"
|
||||
return f"BASE_TS {operator} {' '.join(parts)}"
|
||||
|
||||
|
||||
def migrate_date_field(value: Any, field_name: str) -> Any:
|
||||
"""
|
||||
Migrate a single date field to BASE_TS format.
|
||||
|
||||
Args:
|
||||
value: Field value (can be ISO string, offset_days dict, or None)
|
||||
field_name: Name of the field being migrated
|
||||
|
||||
Returns:
|
||||
BASE_TS marker string or original value (if already BASE_TS or None)
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
# Already a BASE_TS marker - keep as-is
|
||||
if isinstance(value, str) and value.startswith("BASE_TS"):
|
||||
return value
|
||||
|
||||
# Handle ISO timestamp strings
|
||||
if isinstance(value, str) and ('T' in value or 'Z' in value):
|
||||
return calculate_offset_from_base(value)
|
||||
|
||||
# Handle offset_days dictionary format (from inventory stock)
|
||||
if isinstance(value, dict) and 'offset_days' in value:
|
||||
days = value.get('offset_days', 0)
|
||||
hour = value.get('hour', 0)
|
||||
minute = value.get('minute', 0)
|
||||
|
||||
parts = []
|
||||
if days != 0:
|
||||
parts.append(f"{abs(days)}d")
|
||||
if hour != 0:
|
||||
parts.append(f"{abs(hour)}h")
|
||||
if minute != 0:
|
||||
parts.append(f"{abs(minute)}m")
|
||||
|
||||
if not parts:
|
||||
return "BASE_TS"
|
||||
|
||||
operator = "+" if days >= 0 else "-"
|
||||
return f"BASE_TS {operator} {' '.join(parts)}"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def migrate_entity(entity: Dict[str, Any], date_fields: list) -> Dict[str, Any]:
|
||||
"""
|
||||
Migrate all date fields in an entity to BASE_TS format.
|
||||
|
||||
Also removes *_offset_days fields as they're now redundant.
|
||||
|
||||
Args:
|
||||
entity: Entity dictionary
|
||||
date_fields: List of date field names to migrate
|
||||
|
||||
Returns:
|
||||
Migrated entity dictionary
|
||||
"""
|
||||
migrated = entity.copy()
|
||||
|
||||
# Remove offset_days fields and migrate their values
|
||||
offset_fields_to_remove = []
|
||||
for key in list(migrated.keys()):
|
||||
if key.endswith('_offset_days'):
|
||||
# Extract base field name
|
||||
base_field = key.replace('_offset_days', '')
|
||||
|
||||
# Calculate BASE_TS marker
|
||||
offset_days = migrated[key]
|
||||
if offset_days == 0:
|
||||
migrated[base_field] = "BASE_TS"
|
||||
else:
|
||||
operator = "+" if offset_days > 0 else "-"
|
||||
migrated[base_field] = f"BASE_TS {operator} {abs(offset_days)}d"
|
||||
|
||||
offset_fields_to_remove.append(key)
|
||||
|
||||
# Remove offset_days fields
|
||||
for key in offset_fields_to_remove:
|
||||
del migrated[key]
|
||||
|
||||
# Migrate ISO timestamp fields
|
||||
for field in date_fields:
|
||||
if field in migrated:
|
||||
migrated[field] = migrate_date_field(migrated[field], field)
|
||||
|
||||
return migrated
|
||||
|
||||
|
||||
def migrate_json_file(file_path: Path) -> bool:
|
||||
"""
|
||||
Migrate a single JSON file to BASE_TS format.
|
||||
|
||||
Args:
|
||||
file_path: Path to JSON file
|
||||
|
||||
Returns:
|
||||
True if file was modified, False otherwise
|
||||
"""
|
||||
print(f"\n📄 Processing: {file_path.relative_to(file_path.parents[3])}")
|
||||
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
except Exception as e:
|
||||
print(f" ❌ Failed to load: {e}")
|
||||
return False
|
||||
|
||||
modified = False
|
||||
|
||||
# Migrate each entity type
|
||||
for entity_type, date_fields in DATE_FIELDS_MAP.items():
|
||||
if entity_type in data:
|
||||
original_count = len(data[entity_type])
|
||||
data[entity_type] = [
|
||||
migrate_entity(entity, date_fields)
|
||||
for entity in data[entity_type]
|
||||
]
|
||||
if original_count > 0:
|
||||
print(f" ✅ Migrated {original_count} {entity_type}")
|
||||
modified = True
|
||||
|
||||
if modified:
|
||||
# Write back with pretty formatting
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
print(f" 💾 File updated successfully")
|
||||
|
||||
return modified
|
||||
|
||||
|
||||
def main():
|
||||
"""Main migration function"""
|
||||
# Find all JSON files in demo fixtures
|
||||
root_dir = Path(__file__).parent.parent
|
||||
fixtures_dir = root_dir / "shared" / "demo" / "fixtures"
|
||||
|
||||
if not fixtures_dir.exists():
|
||||
print(f"❌ Fixtures directory not found: {fixtures_dir}")
|
||||
return 1
|
||||
|
||||
# Find all JSON files
|
||||
json_files = list(fixtures_dir.rglob("*.json"))
|
||||
|
||||
if not json_files:
|
||||
print(f"❌ No JSON files found in {fixtures_dir}")
|
||||
return 1
|
||||
|
||||
print(f"🔍 Found {len(json_files)} JSON files to migrate")
|
||||
|
||||
# Migrate each file
|
||||
total_modified = 0
|
||||
for json_file in sorted(json_files):
|
||||
if migrate_json_file(json_file):
|
||||
total_modified += 1
|
||||
|
||||
print(f"\n✅ Migration complete: {total_modified}/{len(json_files)} files modified")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
242
scripts/validate_demo_dates.py
Normal file
242
scripts/validate_demo_dates.py
Normal file
@@ -0,0 +1,242 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Validate demo JSON files to ensure all dates use BASE_TS markers.
|
||||
This script enforces the new architecture requirement that all temporal
|
||||
data in demo fixtures must use BASE_TS markers for deterministic sessions.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Tuple, Set
|
||||
|
||||
# Date/time fields that should use BASE_TS markers or be null
|
||||
DATE_TIME_FIELDS = {
|
||||
# Common fields
|
||||
'created_at', 'updated_at',
|
||||
|
||||
# Procurement
|
||||
'order_date', 'required_delivery_date', 'estimated_delivery_date',
|
||||
'expected_delivery_date', 'sent_to_supplier_at', 'supplier_confirmation_date',
|
||||
'approval_deadline',
|
||||
|
||||
# Production
|
||||
'planned_start_time', 'planned_end_time', 'actual_start_time',
|
||||
'actual_end_time', 'completed_at', 'install_date', 'last_maintenance_date',
|
||||
'next_maintenance_date',
|
||||
|
||||
# Inventory
|
||||
'received_date', 'expiration_date', 'best_before_date',
|
||||
'original_expiration_date', 'transformation_date', 'final_expiration_date',
|
||||
|
||||
# Orders
|
||||
'order_date', 'delivery_date', 'promised_date', 'last_order_date',
|
||||
|
||||
# Forecasting
|
||||
'forecast_date', 'prediction_date',
|
||||
|
||||
# Schedules
|
||||
'schedule_date', 'shift_start', 'shift_end', 'finalized_at',
|
||||
|
||||
# Quality
|
||||
'check_time',
|
||||
|
||||
# Generic
|
||||
'date', 'start_time', 'end_time'
|
||||
}
|
||||
|
||||
|
||||
class ValidationError:
|
||||
"""Represents a validation error"""
|
||||
|
||||
def __init__(self, file_path: Path, entity_type: str, entity_index: int,
|
||||
field_name: str, value: any, message: str):
|
||||
self.file_path = file_path
|
||||
self.entity_type = entity_type
|
||||
self.entity_index = entity_index
|
||||
self.field_name = field_name
|
||||
self.value = value
|
||||
self.message = message
|
||||
|
||||
def __str__(self):
|
||||
return (
|
||||
f"❌ {self.file_path.name} » {self.entity_type}[{self.entity_index}] » "
|
||||
f"{self.field_name}: {self.message}\n"
|
||||
f" Value: {self.value}"
|
||||
)
|
||||
|
||||
|
||||
def validate_date_value(value: any, field_name: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Validate a single date field value.
|
||||
|
||||
Returns:
|
||||
(is_valid, error_message)
|
||||
"""
|
||||
# Null values are allowed
|
||||
if value is None:
|
||||
return True, ""
|
||||
|
||||
# BASE_TS markers are the expected format
|
||||
if isinstance(value, str) and value.startswith("BASE_TS"):
|
||||
# Validate BASE_TS marker format
|
||||
if value == "BASE_TS":
|
||||
return True, ""
|
||||
|
||||
# Should be "BASE_TS + ..." or "BASE_TS - ..."
|
||||
parts = value.split()
|
||||
if len(parts) < 3:
|
||||
return False, f"Invalid BASE_TS marker format (expected 'BASE_TS +/- <offset>')"
|
||||
|
||||
if parts[1] not in ['+', '-']:
|
||||
return False, f"Invalid BASE_TS operator (expected + or -)"
|
||||
|
||||
# Extract offset parts (starting from index 2)
|
||||
offset_parts = ' '.join(parts[2:])
|
||||
|
||||
# Validate offset components (must contain d, h, or m)
|
||||
if not any(c in offset_parts for c in ['d', 'h', 'm']):
|
||||
return False, f"BASE_TS offset must contain at least one of: d (days), h (hours), m (minutes)"
|
||||
|
||||
return True, ""
|
||||
|
||||
# ISO 8601 timestamps are NOT allowed (should use BASE_TS)
|
||||
if isinstance(value, str) and ('T' in value or 'Z' in value):
|
||||
return False, "Found ISO 8601 timestamp - should use BASE_TS marker instead"
|
||||
|
||||
# offset_days dictionaries are NOT allowed (legacy format)
|
||||
if isinstance(value, dict) and 'offset_days' in value:
|
||||
return False, "Found offset_days dictionary - should use BASE_TS marker instead"
|
||||
|
||||
# Unknown format
|
||||
return False, f"Unknown date format (type: {type(value).__name__})"
|
||||
|
||||
|
||||
def validate_entity(entity: Dict, entity_type: str, entity_index: int,
|
||||
file_path: Path) -> List[ValidationError]:
|
||||
"""
|
||||
Validate all date fields in a single entity.
|
||||
|
||||
Returns:
|
||||
List of validation errors
|
||||
"""
|
||||
errors = []
|
||||
|
||||
# Check for legacy offset_days fields
|
||||
for key in entity.keys():
|
||||
if key.endswith('_offset_days'):
|
||||
base_field = key.replace('_offset_days', '')
|
||||
errors.append(ValidationError(
|
||||
file_path, entity_type, entity_index, key,
|
||||
entity[key],
|
||||
f"Legacy offset_days field found - migrate to BASE_TS marker in '{base_field}' field"
|
||||
))
|
||||
|
||||
# Validate date/time fields
|
||||
for field_name, value in entity.items():
|
||||
if field_name in DATE_TIME_FIELDS:
|
||||
is_valid, error_msg = validate_date_value(value, field_name)
|
||||
if not is_valid:
|
||||
errors.append(ValidationError(
|
||||
file_path, entity_type, entity_index, field_name,
|
||||
value, error_msg
|
||||
))
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def validate_json_file(file_path: Path) -> List[ValidationError]:
|
||||
"""
|
||||
Validate all entities in a JSON file.
|
||||
|
||||
Returns:
|
||||
List of validation errors
|
||||
"""
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
except json.JSONDecodeError as e:
|
||||
return [ValidationError(
|
||||
file_path, "FILE", 0, "JSON",
|
||||
None, f"Invalid JSON: {e}"
|
||||
)]
|
||||
except Exception as e:
|
||||
return [ValidationError(
|
||||
file_path, "FILE", 0, "READ",
|
||||
None, f"Failed to read file: {e}"
|
||||
)]
|
||||
|
||||
errors = []
|
||||
|
||||
# Validate each entity type
|
||||
for entity_type, entities in data.items():
|
||||
if isinstance(entities, list):
|
||||
for i, entity in enumerate(entities):
|
||||
if isinstance(entity, dict):
|
||||
errors.extend(
|
||||
validate_entity(entity, entity_type, i, file_path)
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def main():
|
||||
"""Main validation function"""
|
||||
# Find all JSON files in demo fixtures
|
||||
root_dir = Path(__file__).parent.parent
|
||||
fixtures_dir = root_dir / "shared" / "demo" / "fixtures"
|
||||
|
||||
if not fixtures_dir.exists():
|
||||
print(f"❌ Fixtures directory not found: {fixtures_dir}")
|
||||
return 1
|
||||
|
||||
# Find all JSON files
|
||||
json_files = sorted(fixtures_dir.rglob("*.json"))
|
||||
|
||||
if not json_files:
|
||||
print(f"❌ No JSON files found in {fixtures_dir}")
|
||||
return 1
|
||||
|
||||
print(f"🔍 Validating {len(json_files)} JSON files...\n")
|
||||
|
||||
# Validate each file
|
||||
all_errors = []
|
||||
files_with_errors = 0
|
||||
|
||||
for json_file in json_files:
|
||||
errors = validate_json_file(json_file)
|
||||
|
||||
if errors:
|
||||
files_with_errors += 1
|
||||
all_errors.extend(errors)
|
||||
|
||||
# Print file header
|
||||
relative_path = json_file.relative_to(fixtures_dir)
|
||||
print(f"\n📄 {relative_path}")
|
||||
print(f" Found {len(errors)} error(s):")
|
||||
|
||||
# Print each error
|
||||
for error in errors:
|
||||
print(f" {error}")
|
||||
|
||||
# Print summary
|
||||
print("\n" + "=" * 80)
|
||||
if all_errors:
|
||||
print(f"\n❌ VALIDATION FAILED")
|
||||
print(f" Total errors: {len(all_errors)}")
|
||||
print(f" Files with errors: {files_with_errors}/{len(json_files)}")
|
||||
print(f"\n💡 Fix these errors by:")
|
||||
print(f" 1. Replacing ISO timestamps with BASE_TS markers")
|
||||
print(f" 2. Removing *_offset_days fields")
|
||||
print(f" 3. Using format: 'BASE_TS +/- <offset>' where offset uses d/h/m")
|
||||
print(f" Examples: 'BASE_TS', 'BASE_TS + 2d', 'BASE_TS - 4h', 'BASE_TS + 1h30m'")
|
||||
return 1
|
||||
else:
|
||||
print(f"\n✅ ALL VALIDATIONS PASSED")
|
||||
print(f" Files validated: {len(json_files)}")
|
||||
print(f" All date fields use BASE_TS markers correctly")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user