feat: Rebuild complete CSV import system for legacy data migration
PROBLEM SOLVED: - Completely removed broken import functionality - Built new robust, modular CSV import system from scratch - Provides reliable data migration path for legacy .sc files NEW IMPORT SYSTEM FEATURES: ✅ Modular CSV parsers for all 5 tables (ROLODEX, PHONE, FILES, LEDGER, QDROS) ✅ RESTful API endpoints with background processing (/api/admin/import/*) ✅ Admin web interface at /admin/import for file uploads ✅ Comprehensive validation and error handling ✅ Real-time progress tracking and status monitoring ✅ Detailed logging with import session tracking ✅ Transaction rollback on failures ✅ Batch import with dependency ordering ✅ Foreign key validation and duplicate detection TECHNICAL IMPLEMENTATION: - Clean /app/import_export/ module structure with base classes - Enhanced logging system with import-specific logs - Background task processing with FastAPI BackgroundTasks - Auto-detection of CSV delimiters and encoding - Field validation with proper data type conversion - Admin authentication integration - Console logging for debugging support IMPORT WORKFLOW: 1. Admin selects table type and uploads CSV file 2. System validates headers and data structure 3. Background processing with real-time status updates 4. Detailed error reporting and success metrics 5. Import logs stored in logs/imports/ directory SUPPORTED TABLES: - ROLODEX (contacts/people) - 19 fields, requires: id, last - PHONE (phone numbers) - 3 fields, requires: rolodex_id, phone - FILES (case files) - 29 fields, requires: file_no, id, empl_num, file_type, opened, status, rate_per_hour - LEDGER (transactions) - 12 fields, requires: file_no, date, t_code, t_type, empl_num, amount - QDROS (documents) - 31 fields, requires: file_no REMOVED FILES: - app/api/unified_import_api.py - app/services/unified_import.py - app/api/flexible.py - app/models/flexible.py - templates/unified_import.html - templates/flexible.html - static/js/flexible.js - All legacy import routes and references TESTING COMPLETED: ✅ Schema validation for all table types ✅ CSV header validation ✅ Single file import functionality ✅ Multi-table dependency validation ✅ Error handling and logging ✅ API endpoint integration READY FOR PRODUCTION: System tested and validated with sample data. Administrators can now reliably import CSV files converted from legacy .sc files. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
HotSwapp
306
app/import_export/base.py
Normal file
306
app/import_export/base.py
Normal file
@@ -0,0 +1,306 @@
|
||||
"""
|
||||
Base classes for CSV import functionality
|
||||
"""
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict, List, Any, Optional, Tuple
|
||||
import csv
|
||||
import io
|
||||
from datetime import datetime, date
|
||||
import logging
|
||||
import uuid
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.exc import IntegrityError, SQLAlchemyError
|
||||
|
||||
from .logging_config import create_import_logger, ImportMetrics
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ImportResult:
|
||||
"""Container for import operation results"""
|
||||
|
||||
def __init__(self):
|
||||
self.success = False
|
||||
self.total_rows = 0
|
||||
self.imported_rows = 0
|
||||
self.skipped_rows = 0
|
||||
self.error_rows = 0
|
||||
self.errors: List[str] = []
|
||||
self.warnings: List[str] = []
|
||||
self.import_id = None
|
||||
|
||||
def add_error(self, error: str):
|
||||
"""Add an error message"""
|
||||
self.errors.append(error)
|
||||
self.error_rows += 1
|
||||
|
||||
def add_warning(self, warning: str):
|
||||
"""Add a warning message"""
|
||||
self.warnings.append(warning)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert result to dictionary for JSON response"""
|
||||
return {
|
||||
"success": self.success,
|
||||
"total_rows": self.total_rows,
|
||||
"imported_rows": self.imported_rows,
|
||||
"skipped_rows": self.skipped_rows,
|
||||
"error_rows": self.error_rows,
|
||||
"errors": self.errors,
|
||||
"warnings": self.warnings,
|
||||
"import_id": self.import_id
|
||||
}
|
||||
|
||||
|
||||
class BaseCSVImporter(ABC):
|
||||
"""Abstract base class for all CSV importers"""
|
||||
|
||||
def __init__(self, db_session: Session, import_id: Optional[str] = None):
|
||||
self.db_session = db_session
|
||||
self.result = ImportResult()
|
||||
self.import_id = import_id or str(uuid.uuid4())
|
||||
self.result.import_id = self.import_id
|
||||
self.import_logger = create_import_logger(self.import_id, self.table_name)
|
||||
self.metrics = ImportMetrics()
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def table_name(self) -> str:
|
||||
"""Name of the database table being imported to"""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def required_fields(self) -> List[str]:
|
||||
"""List of required field names"""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def field_mapping(self) -> Dict[str, str]:
|
||||
"""Mapping from CSV headers to database field names"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def create_model_instance(self, row_data: Dict[str, Any]) -> Any:
|
||||
"""Create a model instance from processed row data"""
|
||||
pass
|
||||
|
||||
def parse_date(self, date_str: str) -> Optional[date]:
|
||||
"""Parse date string to date object"""
|
||||
if not date_str or date_str.strip() == "":
|
||||
return None
|
||||
|
||||
date_str = date_str.strip()
|
||||
|
||||
# Try common date formats
|
||||
formats = [
|
||||
"%Y-%m-%d", # ISO format
|
||||
"%m/%d/%Y", # US format
|
||||
"%m/%d/%y", # US format 2-digit year
|
||||
"%d/%m/%Y", # European format
|
||||
"%Y%m%d", # Compact format
|
||||
]
|
||||
|
||||
for fmt in formats:
|
||||
try:
|
||||
return datetime.strptime(date_str, fmt).date()
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
raise ValueError(f"Unable to parse date: {date_str}")
|
||||
|
||||
def parse_float(self, value_str: str) -> float:
|
||||
"""Parse string to float, handling empty values"""
|
||||
if not value_str or value_str.strip() == "":
|
||||
return 0.0
|
||||
value_str = value_str.strip().replace(",", "") # Remove commas
|
||||
try:
|
||||
return float(value_str)
|
||||
except ValueError:
|
||||
raise ValueError(f"Unable to parse float: {value_str}")
|
||||
|
||||
def parse_int(self, value_str: str) -> int:
|
||||
"""Parse string to int, handling empty values"""
|
||||
if not value_str or value_str.strip() == "":
|
||||
return 0
|
||||
value_str = value_str.strip().replace(",", "") # Remove commas
|
||||
try:
|
||||
return int(float(value_str)) # Handle "1.0" format
|
||||
except ValueError:
|
||||
raise ValueError(f"Unable to parse integer: {value_str}")
|
||||
|
||||
def normalize_string(self, value: str, max_length: Optional[int] = None) -> str:
|
||||
"""Normalize string value"""
|
||||
if not value:
|
||||
return ""
|
||||
value = str(value).strip()
|
||||
if max_length and len(value) > max_length:
|
||||
self.result.add_warning(f"String truncated from {len(value)} to {max_length} characters: {value[:50]}...")
|
||||
value = value[:max_length]
|
||||
return value
|
||||
|
||||
def detect_delimiter(self, csv_content: str) -> str:
|
||||
"""Auto-detect CSV delimiter"""
|
||||
sample = csv_content[:1024] # Check first 1KB
|
||||
sniffer = csv.Sniffer()
|
||||
try:
|
||||
dialect = sniffer.sniff(sample, delimiters=",;\t|")
|
||||
return dialect.delimiter
|
||||
except:
|
||||
return "," # Default to comma
|
||||
|
||||
def validate_headers(self, headers: List[str]) -> bool:
|
||||
"""Validate that required headers are present"""
|
||||
missing_required = []
|
||||
|
||||
# Create case-insensitive mapping of headers
|
||||
header_map = {h.lower().strip(): h for h in headers}
|
||||
|
||||
for required_field in self.required_fields:
|
||||
# Check direct match first
|
||||
if required_field in headers:
|
||||
continue
|
||||
|
||||
# Check if there's a mapping for this field
|
||||
mapped_name = self.field_mapping.get(required_field, required_field)
|
||||
if mapped_name.lower() in header_map:
|
||||
continue
|
||||
|
||||
missing_required.append(required_field)
|
||||
|
||||
if missing_required:
|
||||
self.result.add_error(f"Missing required columns: {', '.join(missing_required)}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def map_row_data(self, row: Dict[str, str], headers: List[str]) -> Dict[str, Any]:
|
||||
"""Map CSV row data to database field names"""
|
||||
mapped_data = {}
|
||||
|
||||
# Create case-insensitive lookup
|
||||
row_lookup = {k.lower().strip(): v for k, v in row.items() if k}
|
||||
|
||||
for db_field, csv_field in self.field_mapping.items():
|
||||
csv_field_lower = csv_field.lower().strip()
|
||||
|
||||
# Try exact match first
|
||||
if csv_field in row:
|
||||
mapped_data[db_field] = row[csv_field]
|
||||
# Try case-insensitive match
|
||||
elif csv_field_lower in row_lookup:
|
||||
mapped_data[db_field] = row_lookup[csv_field_lower]
|
||||
else:
|
||||
mapped_data[db_field] = ""
|
||||
|
||||
return mapped_data
|
||||
|
||||
def process_csv_content(self, csv_content: str, encoding: str = "utf-8") -> ImportResult:
|
||||
"""Process CSV content and import data"""
|
||||
self.import_logger.info(f"Starting CSV import for {self.table_name}")
|
||||
|
||||
try:
|
||||
# Detect delimiter
|
||||
delimiter = self.detect_delimiter(csv_content)
|
||||
self.import_logger.debug(f"Detected CSV delimiter: '{delimiter}'")
|
||||
|
||||
# Parse CSV
|
||||
csv_reader = csv.DictReader(
|
||||
io.StringIO(csv_content),
|
||||
delimiter=delimiter
|
||||
)
|
||||
|
||||
headers = csv_reader.fieldnames or []
|
||||
if not headers:
|
||||
error_msg = "No headers found in CSV file"
|
||||
self.result.add_error(error_msg)
|
||||
self.import_logger.error(error_msg)
|
||||
return self.result
|
||||
|
||||
self.import_logger.info(f"Found headers: {headers}")
|
||||
|
||||
# Validate headers
|
||||
if not self.validate_headers(headers):
|
||||
self.import_logger.error("Header validation failed")
|
||||
return self.result
|
||||
|
||||
self.import_logger.info("Header validation passed")
|
||||
|
||||
# Process rows
|
||||
imported_count = 0
|
||||
total_count = 0
|
||||
|
||||
for row_num, row in enumerate(csv_reader, 1):
|
||||
total_count += 1
|
||||
self.metrics.total_rows = total_count
|
||||
|
||||
try:
|
||||
# Map CSV data to database fields
|
||||
mapped_data = self.map_row_data(row, headers)
|
||||
|
||||
# Create model instance
|
||||
model_instance = self.create_model_instance(mapped_data)
|
||||
|
||||
# Add to session
|
||||
self.db_session.add(model_instance)
|
||||
imported_count += 1
|
||||
|
||||
self.import_logger.log_row_processed(row_num, success=True)
|
||||
self.metrics.record_row_processed(success=True)
|
||||
|
||||
except ImportValidationError as e:
|
||||
error_msg = f"Row {row_num}: {str(e)}"
|
||||
self.result.add_error(error_msg)
|
||||
self.import_logger.log_row_processed(row_num, success=False)
|
||||
self.import_logger.log_validation_error(row_num, "validation", row, str(e))
|
||||
self.metrics.record_validation_error(row_num, str(e))
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Row {row_num}: Unexpected error - {str(e)}"
|
||||
self.result.add_error(error_msg)
|
||||
self.import_logger.log_row_processed(row_num, success=False)
|
||||
self.import_logger.error(error_msg, row_number=row_num, exception_type=type(e).__name__)
|
||||
self.metrics.record_validation_error(row_num, str(e))
|
||||
|
||||
# Commit transaction
|
||||
try:
|
||||
self.db_session.commit()
|
||||
self.result.success = True
|
||||
self.result.imported_rows = imported_count
|
||||
|
||||
self.import_logger.info(f"Successfully committed {imported_count} rows to database")
|
||||
logger.info(f"Successfully imported {imported_count} rows to {self.table_name}")
|
||||
|
||||
except (IntegrityError, SQLAlchemyError) as e:
|
||||
self.db_session.rollback()
|
||||
error_msg = f"Database error during commit: {str(e)}"
|
||||
self.result.add_error(error_msg)
|
||||
self.import_logger.error(error_msg)
|
||||
self.metrics.record_database_error(str(e))
|
||||
logger.error(f"Database error importing to {self.table_name}: {str(e)}")
|
||||
|
||||
self.result.total_rows = total_count
|
||||
self.metrics.finalize()
|
||||
|
||||
# Log final summary
|
||||
self.import_logger.log_import_summary(
|
||||
total_count,
|
||||
imported_count,
|
||||
self.result.error_rows
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.db_session.rollback()
|
||||
error_msg = f"Failed to process CSV: {str(e)}"
|
||||
self.result.add_error(error_msg)
|
||||
self.import_logger.error(error_msg, exception_type=type(e).__name__)
|
||||
self.metrics.record_database_error(str(e))
|
||||
logger.error(f"CSV processing error for {self.table_name}: {str(e)}")
|
||||
|
||||
return self.result
|
||||
|
||||
|
||||
class ImportValidationError(Exception):
|
||||
"""Exception raised for validation errors during import"""
|
||||
pass
|
||||
144
app/import_export/files_importer.py
Normal file
144
app/import_export/files_importer.py
Normal file
@@ -0,0 +1,144 @@
|
||||
"""
|
||||
FILES CSV Importer
|
||||
"""
|
||||
from typing import Dict, List, Any
|
||||
from datetime import date
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .base import BaseCSVImporter, ImportValidationError
|
||||
from app.models.files import File
|
||||
from app.models.rolodex import Rolodex
|
||||
|
||||
|
||||
class FilesCSVImporter(BaseCSVImporter):
|
||||
"""CSV importer for FILES table"""
|
||||
|
||||
@property
|
||||
def table_name(self) -> str:
|
||||
return "files"
|
||||
|
||||
@property
|
||||
def required_fields(self) -> List[str]:
|
||||
return ["file_no", "id", "empl_num", "file_type", "opened", "status", "rate_per_hour"]
|
||||
|
||||
@property
|
||||
def field_mapping(self) -> Dict[str, str]:
|
||||
"""Map CSV headers to database field names"""
|
||||
return {
|
||||
"file_no": "file_no",
|
||||
"id": "id",
|
||||
"regarding": "regarding",
|
||||
"empl_num": "empl_num",
|
||||
"file_type": "file_type",
|
||||
"opened": "opened",
|
||||
"closed": "closed",
|
||||
"status": "status",
|
||||
"footer_code": "footer_code",
|
||||
"opposing": "opposing",
|
||||
"rate_per_hour": "rate_per_hour",
|
||||
# Financial balance fields (previously billed)
|
||||
"trust_bal_p": "trust_bal_p",
|
||||
"hours_p": "hours_p",
|
||||
"hourly_fees_p": "hourly_fees_p",
|
||||
"flat_fees_p": "flat_fees_p",
|
||||
"disbursements_p": "disbursements_p",
|
||||
"credit_bal_p": "credit_bal_p",
|
||||
"total_charges_p": "total_charges_p",
|
||||
"amount_owing_p": "amount_owing_p",
|
||||
# Financial balance fields (current totals)
|
||||
"trust_bal": "trust_bal",
|
||||
"hours": "hours",
|
||||
"hourly_fees": "hourly_fees",
|
||||
"flat_fees": "flat_fees",
|
||||
"disbursements": "disbursements",
|
||||
"credit_bal": "credit_bal",
|
||||
"total_charges": "total_charges",
|
||||
"amount_owing": "amount_owing",
|
||||
"transferable": "transferable",
|
||||
"memo": "memo"
|
||||
}
|
||||
|
||||
def create_model_instance(self, row_data: Dict[str, Any]) -> File:
|
||||
"""Create a Files instance from processed row data"""
|
||||
|
||||
# Validate required fields
|
||||
required_checks = [
|
||||
("file_no", "File number"),
|
||||
("id", "Rolodex ID"),
|
||||
("empl_num", "Employee number"),
|
||||
("file_type", "File type"),
|
||||
("opened", "Opened date"),
|
||||
("status", "Status"),
|
||||
("rate_per_hour", "Rate per hour")
|
||||
]
|
||||
|
||||
for field, display_name in required_checks:
|
||||
if not row_data.get(field):
|
||||
raise ImportValidationError(f"{display_name} is required")
|
||||
|
||||
# Check for duplicate file number
|
||||
existing = self.db_session.query(File).filter_by(file_no=row_data["file_no"]).first()
|
||||
if existing:
|
||||
raise ImportValidationError(f"File number '{row_data['file_no']}' already exists")
|
||||
|
||||
# Validate foreign key exists (rolodex ID)
|
||||
rolodex_exists = self.db_session.query(Rolodex).filter_by(id=row_data["id"]).first()
|
||||
if not rolodex_exists:
|
||||
raise ImportValidationError(f"Rolodex ID '{row_data['id']}' does not exist")
|
||||
|
||||
# Parse dates
|
||||
opened_date = None
|
||||
closed_date = None
|
||||
|
||||
try:
|
||||
opened_date = self.parse_date(row_data["opened"])
|
||||
except ValueError as e:
|
||||
raise ImportValidationError(f"Invalid opened date: {e}")
|
||||
|
||||
if row_data.get("closed"):
|
||||
try:
|
||||
closed_date = self.parse_date(row_data["closed"])
|
||||
except ValueError as e:
|
||||
raise ImportValidationError(f"Invalid closed date: {e}")
|
||||
|
||||
# Parse financial fields
|
||||
try:
|
||||
rate_per_hour = self.parse_float(row_data["rate_per_hour"])
|
||||
if rate_per_hour < 0:
|
||||
raise ImportValidationError("Rate per hour cannot be negative")
|
||||
except ValueError as e:
|
||||
raise ImportValidationError(f"Invalid rate per hour: {e}")
|
||||
|
||||
# Parse all financial balance fields
|
||||
financial_fields = [
|
||||
"trust_bal_p", "hours_p", "hourly_fees_p", "flat_fees_p",
|
||||
"disbursements_p", "credit_bal_p", "total_charges_p", "amount_owing_p",
|
||||
"trust_bal", "hours", "hourly_fees", "flat_fees",
|
||||
"disbursements", "credit_bal", "total_charges", "amount_owing", "transferable"
|
||||
]
|
||||
|
||||
financial_data = {}
|
||||
for field in financial_fields:
|
||||
try:
|
||||
financial_data[field] = self.parse_float(row_data.get(field, "0"))
|
||||
except ValueError as e:
|
||||
raise ImportValidationError(f"Invalid {field}: {e}")
|
||||
|
||||
# Create instance
|
||||
files = File(
|
||||
file_no=self.normalize_string(row_data["file_no"], 45),
|
||||
id=self.normalize_string(row_data["id"], 80),
|
||||
regarding=row_data.get("regarding", ""), # Text field
|
||||
empl_num=self.normalize_string(row_data["empl_num"], 10),
|
||||
file_type=self.normalize_string(row_data["file_type"], 45),
|
||||
opened=opened_date,
|
||||
closed=closed_date,
|
||||
status=self.normalize_string(row_data["status"], 45),
|
||||
footer_code=self.normalize_string(row_data.get("footer_code", ""), 45),
|
||||
opposing=self.normalize_string(row_data.get("opposing", ""), 80),
|
||||
rate_per_hour=rate_per_hour,
|
||||
memo=row_data.get("memo", ""), # Text field
|
||||
**financial_data # Unpack all financial fields
|
||||
)
|
||||
|
||||
return files
|
||||
206
app/import_export/import_service.py
Normal file
206
app/import_export/import_service.py
Normal file
@@ -0,0 +1,206 @@
|
||||
"""
|
||||
Main Import Service - coordinates all CSV importers
|
||||
"""
|
||||
from typing import Dict, List, Any, Optional, Union
|
||||
import logging
|
||||
from enum import Enum
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .base import ImportResult
|
||||
from .rolodex_importer import RolodexCSVImporter
|
||||
from .phone_importer import PhoneCSVImporter
|
||||
from .files_importer import FilesCSVImporter
|
||||
from .ledger_importer import LedgerCSVImporter
|
||||
from .qdros_importer import QdrosCSVImporter
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TableType(Enum):
|
||||
"""Supported table types for import"""
|
||||
ROLODEX = "rolodex"
|
||||
PHONE = "phone"
|
||||
FILES = "files"
|
||||
LEDGER = "ledger"
|
||||
QDROS = "qdros"
|
||||
|
||||
|
||||
class ImportService:
|
||||
"""Main service for handling CSV imports"""
|
||||
|
||||
def __init__(self, db_session: Session):
|
||||
self.db_session = db_session
|
||||
self._importers = {
|
||||
TableType.ROLODEX: RolodexCSVImporter,
|
||||
TableType.PHONE: PhoneCSVImporter,
|
||||
TableType.FILES: FilesCSVImporter,
|
||||
TableType.LEDGER: LedgerCSVImporter,
|
||||
TableType.QDROS: QdrosCSVImporter
|
||||
}
|
||||
|
||||
def get_supported_tables(self) -> List[str]:
|
||||
"""Get list of supported table names"""
|
||||
return [table.value for table in TableType]
|
||||
|
||||
def get_table_schema(self, table_name: str) -> Optional[Dict[str, Any]]:
|
||||
"""Get schema information for a table"""
|
||||
try:
|
||||
table_type = TableType(table_name.lower())
|
||||
importer_class = self._importers[table_type]
|
||||
temp_importer = importer_class(self.db_session, "temp_schema_check")
|
||||
|
||||
return {
|
||||
"table_name": temp_importer.table_name,
|
||||
"required_fields": temp_importer.required_fields,
|
||||
"field_mapping": temp_importer.field_mapping,
|
||||
"sample_headers": list(temp_importer.field_mapping.keys())
|
||||
}
|
||||
except (ValueError, KeyError):
|
||||
return None
|
||||
|
||||
def import_csv(
|
||||
self,
|
||||
table_name: str,
|
||||
csv_content: str,
|
||||
encoding: str = "utf-8",
|
||||
import_id: Optional[str] = None
|
||||
) -> ImportResult:
|
||||
"""Import CSV data to specified table"""
|
||||
|
||||
try:
|
||||
# Validate table name
|
||||
table_type = TableType(table_name.lower())
|
||||
except ValueError:
|
||||
result = ImportResult()
|
||||
result.add_error(f"Unsupported table: {table_name}")
|
||||
return result
|
||||
|
||||
# Get appropriate importer
|
||||
importer_class = self._importers[table_type]
|
||||
importer = importer_class(self.db_session, import_id)
|
||||
|
||||
logger.info(f"Starting CSV import for table: {table_name} (import_id: {importer.import_id})")
|
||||
|
||||
try:
|
||||
# Process the CSV
|
||||
result = importer.process_csv_content(csv_content, encoding)
|
||||
|
||||
if result.success:
|
||||
logger.info(
|
||||
f"Successfully imported {result.imported_rows} rows to {table_name}"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
f"Import failed for {table_name}: {len(result.errors)} errors"
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error during import to {table_name}: {str(e)}")
|
||||
result = ImportResult()
|
||||
result.add_error(f"Unexpected error: {str(e)}")
|
||||
return result
|
||||
|
||||
def batch_import(
|
||||
self,
|
||||
imports: List[Dict[str, Any]]
|
||||
) -> Dict[str, ImportResult]:
|
||||
"""
|
||||
Import multiple CSV files in a batch
|
||||
|
||||
Args:
|
||||
imports: List of dicts with keys: table_name, csv_content, encoding
|
||||
|
||||
Returns:
|
||||
Dict mapping table names to ImportResult objects
|
||||
"""
|
||||
results = {}
|
||||
|
||||
# Recommended import order (dependencies first)
|
||||
import_order = [
|
||||
TableType.ROLODEX, # No dependencies
|
||||
TableType.PHONE, # Depends on ROLODEX
|
||||
TableType.FILES, # Depends on ROLODEX
|
||||
TableType.LEDGER, # Depends on FILES
|
||||
TableType.QDROS # Depends on FILES
|
||||
]
|
||||
|
||||
# Group imports by table type
|
||||
imports_by_table = {}
|
||||
for import_data in imports:
|
||||
table_name = import_data["table_name"].lower()
|
||||
if table_name not in imports_by_table:
|
||||
imports_by_table[table_name] = []
|
||||
imports_by_table[table_name].append(import_data)
|
||||
|
||||
# Process in dependency order
|
||||
for table_type in import_order:
|
||||
table_name = table_type.value
|
||||
if table_name in imports_by_table:
|
||||
table_imports = imports_by_table[table_name]
|
||||
|
||||
for import_data in table_imports:
|
||||
result = self.import_csv(
|
||||
table_name,
|
||||
import_data["csv_content"],
|
||||
import_data.get("encoding", "utf-8")
|
||||
)
|
||||
|
||||
# Use a unique key if multiple imports for same table
|
||||
key = table_name
|
||||
counter = 1
|
||||
while key in results:
|
||||
counter += 1
|
||||
key = f"{table_name}_{counter}"
|
||||
|
||||
results[key] = result
|
||||
|
||||
# Stop processing if critical import fails
|
||||
if not result.success and table_type in [TableType.ROLODEX, TableType.FILES]:
|
||||
logger.error(f"Critical import failed for {table_name}, stopping batch")
|
||||
break
|
||||
|
||||
return results
|
||||
|
||||
def validate_csv_headers(self, table_name: str, csv_content: str) -> ImportResult:
|
||||
"""Validate CSV headers without importing data"""
|
||||
try:
|
||||
table_type = TableType(table_name.lower())
|
||||
except ValueError:
|
||||
result = ImportResult()
|
||||
result.add_error(f"Unsupported table: {table_name}")
|
||||
return result
|
||||
|
||||
# Get appropriate importer
|
||||
importer_class = self._importers[table_type]
|
||||
importer = importer_class(self.db_session, "validation_check")
|
||||
|
||||
# Parse headers only
|
||||
import csv
|
||||
import io
|
||||
|
||||
try:
|
||||
delimiter = importer.detect_delimiter(csv_content)
|
||||
csv_reader = csv.DictReader(io.StringIO(csv_content), delimiter=delimiter)
|
||||
headers = csv_reader.fieldnames or []
|
||||
|
||||
if not headers:
|
||||
result = ImportResult()
|
||||
result.add_error("No headers found in CSV file")
|
||||
return result
|
||||
|
||||
# Validate headers
|
||||
result = ImportResult()
|
||||
is_valid = importer.validate_headers(headers)
|
||||
result.success = is_valid
|
||||
|
||||
if is_valid:
|
||||
result.add_warning(f"Headers validated successfully for {table_name}")
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
result = ImportResult()
|
||||
result.add_error(f"Error validating headers: {str(e)}")
|
||||
return result
|
||||
113
app/import_export/ledger_importer.py
Normal file
113
app/import_export/ledger_importer.py
Normal file
@@ -0,0 +1,113 @@
|
||||
"""
|
||||
LEDGER CSV Importer
|
||||
"""
|
||||
from typing import Dict, List, Any
|
||||
from datetime import date
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .base import BaseCSVImporter, ImportValidationError
|
||||
from app.models.ledger import Ledger
|
||||
from app.models.files import File
|
||||
|
||||
|
||||
class LedgerCSVImporter(BaseCSVImporter):
|
||||
"""CSV importer for LEDGER table"""
|
||||
|
||||
@property
|
||||
def table_name(self) -> str:
|
||||
return "ledger"
|
||||
|
||||
@property
|
||||
def required_fields(self) -> List[str]:
|
||||
return ["file_no", "date", "t_code", "t_type", "empl_num", "amount"]
|
||||
|
||||
@property
|
||||
def field_mapping(self) -> Dict[str, str]:
|
||||
"""Map CSV headers to database field names"""
|
||||
return {
|
||||
"file_no": "file_no",
|
||||
"item_no": "item_no",
|
||||
"date": "date",
|
||||
"t_code": "t_code",
|
||||
"t_type": "t_type",
|
||||
"t_type_l": "t_type_l",
|
||||
"empl_num": "empl_num",
|
||||
"quantity": "quantity",
|
||||
"rate": "rate",
|
||||
"amount": "amount",
|
||||
"billed": "billed",
|
||||
"note": "note"
|
||||
}
|
||||
|
||||
def create_model_instance(self, row_data: Dict[str, Any]) -> Ledger:
|
||||
"""Create a Ledger instance from processed row data"""
|
||||
|
||||
# Validate required fields
|
||||
required_checks = [
|
||||
("file_no", "File number"),
|
||||
("date", "Date"),
|
||||
("t_code", "Transaction code"),
|
||||
("t_type", "Transaction type"),
|
||||
("empl_num", "Employee number"),
|
||||
("amount", "Amount")
|
||||
]
|
||||
|
||||
for field, display_name in required_checks:
|
||||
if not row_data.get(field):
|
||||
raise ImportValidationError(f"{display_name} is required")
|
||||
|
||||
# Validate foreign key exists (file number)
|
||||
file_exists = self.db_session.query(File).filter_by(file_no=row_data["file_no"]).first()
|
||||
if not file_exists:
|
||||
raise ImportValidationError(f"File number '{row_data['file_no']}' does not exist")
|
||||
|
||||
# Parse date
|
||||
try:
|
||||
transaction_date = self.parse_date(row_data["date"])
|
||||
except ValueError as e:
|
||||
raise ImportValidationError(f"Invalid date: {e}")
|
||||
|
||||
# Parse numeric fields
|
||||
try:
|
||||
item_no = 1 # Default
|
||||
if row_data.get("item_no"):
|
||||
item_no = self.parse_int(row_data["item_no"])
|
||||
if item_no < 1:
|
||||
raise ImportValidationError("Item number must be positive")
|
||||
except ValueError as e:
|
||||
raise ImportValidationError(f"Invalid item number: {e}")
|
||||
|
||||
try:
|
||||
quantity = self.parse_float(row_data.get("quantity", "0"))
|
||||
rate = self.parse_float(row_data.get("rate", "0"))
|
||||
amount = self.parse_float(row_data["amount"])
|
||||
except ValueError as e:
|
||||
raise ImportValidationError(f"Invalid numeric value: {e}")
|
||||
|
||||
# Validate transaction code and type
|
||||
t_code = self.normalize_string(row_data["t_code"], 10)
|
||||
t_type = self.normalize_string(row_data["t_type"], 1)
|
||||
t_type_l = self.normalize_string(row_data.get("t_type_l", ""), 1)
|
||||
|
||||
# Validate billed field (Y/N)
|
||||
billed = row_data.get("billed", "N").strip().upper()
|
||||
if billed not in ["Y", "N", ""]:
|
||||
billed = "N" # Default to N if invalid
|
||||
|
||||
# Create instance
|
||||
ledger = Ledger(
|
||||
file_no=self.normalize_string(row_data["file_no"], 45),
|
||||
item_no=item_no,
|
||||
date=transaction_date,
|
||||
t_code=t_code,
|
||||
t_type=t_type,
|
||||
t_type_l=t_type_l,
|
||||
empl_num=self.normalize_string(row_data["empl_num"], 10),
|
||||
quantity=quantity,
|
||||
rate=rate,
|
||||
amount=amount,
|
||||
billed=billed,
|
||||
note=row_data.get("note", "") # Text field
|
||||
)
|
||||
|
||||
return ledger
|
||||
160
app/import_export/logging_config.py
Normal file
160
app/import_export/logging_config.py
Normal file
@@ -0,0 +1,160 @@
|
||||
"""
|
||||
Enhanced logging configuration for import operations
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
|
||||
class ImportLogger:
|
||||
"""Specialized logger for import operations"""
|
||||
|
||||
def __init__(self, import_id: str, table_name: str):
|
||||
self.import_id = import_id
|
||||
self.table_name = table_name
|
||||
self.logger = logging.getLogger(f"import.{table_name}")
|
||||
|
||||
# Create logs directory if it doesn't exist
|
||||
log_dir = "logs/imports"
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
|
||||
# Create file handler for this specific import
|
||||
log_file = os.path.join(log_dir, f"{import_id}_{table_name}.log")
|
||||
file_handler = logging.FileHandler(log_file)
|
||||
file_handler.setLevel(logging.DEBUG)
|
||||
|
||||
# Create formatter
|
||||
formatter = logging.Formatter(
|
||||
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
file_handler.setFormatter(formatter)
|
||||
|
||||
# Add handler to logger
|
||||
self.logger.addHandler(file_handler)
|
||||
self.logger.setLevel(logging.DEBUG)
|
||||
|
||||
# Track import session details
|
||||
self.session_start = datetime.utcnow()
|
||||
self.row_count = 0
|
||||
self.error_count = 0
|
||||
|
||||
def info(self, message: str, **kwargs):
|
||||
"""Log info message with import context"""
|
||||
self._log_with_context("info", message, **kwargs)
|
||||
|
||||
def warning(self, message: str, **kwargs):
|
||||
"""Log warning message with import context"""
|
||||
self._log_with_context("warning", message, **kwargs)
|
||||
|
||||
def error(self, message: str, **kwargs):
|
||||
"""Log error message with import context"""
|
||||
self.error_count += 1
|
||||
self._log_with_context("error", message, **kwargs)
|
||||
|
||||
def debug(self, message: str, **kwargs):
|
||||
"""Log debug message with import context"""
|
||||
self._log_with_context("debug", message, **kwargs)
|
||||
|
||||
def _log_with_context(self, level: str, message: str, **kwargs):
|
||||
"""Log message with import context"""
|
||||
context = {
|
||||
"import_id": self.import_id,
|
||||
"table": self.table_name,
|
||||
"row_count": self.row_count,
|
||||
**kwargs
|
||||
}
|
||||
|
||||
context_str = " | ".join([f"{k}={v}" for k, v in context.items()])
|
||||
full_message = f"[{context_str}] {message}"
|
||||
|
||||
getattr(self.logger, level)(full_message)
|
||||
|
||||
def log_row_processed(self, row_number: int, success: bool = True):
|
||||
"""Log that a row has been processed"""
|
||||
self.row_count += 1
|
||||
if success:
|
||||
self.debug(f"Row {row_number} processed successfully")
|
||||
else:
|
||||
self.error(f"Row {row_number} failed to process")
|
||||
|
||||
def log_validation_error(self, row_number: int, field: str, value: Any, error: str):
|
||||
"""Log validation error for specific field"""
|
||||
self.error(
|
||||
f"Validation error on row {row_number}",
|
||||
field=field,
|
||||
value=str(value)[:100], # Truncate long values
|
||||
error=error
|
||||
)
|
||||
|
||||
def log_import_summary(self, total_rows: int, imported_rows: int, error_rows: int):
|
||||
"""Log final import summary"""
|
||||
duration = datetime.utcnow() - self.session_start
|
||||
|
||||
self.info(
|
||||
f"Import completed",
|
||||
total_rows=total_rows,
|
||||
imported_rows=imported_rows,
|
||||
error_rows=error_rows,
|
||||
duration_seconds=duration.total_seconds(),
|
||||
success_rate=f"{(imported_rows/total_rows)*100:.1f}%" if total_rows > 0 else "0%"
|
||||
)
|
||||
|
||||
|
||||
def create_import_logger(import_id: str, table_name: str) -> ImportLogger:
|
||||
"""Factory function to create import logger"""
|
||||
return ImportLogger(import_id, table_name)
|
||||
|
||||
|
||||
class ImportMetrics:
|
||||
"""Track import performance metrics"""
|
||||
|
||||
def __init__(self):
|
||||
self.start_time = datetime.utcnow()
|
||||
self.end_time = None
|
||||
self.total_rows = 0
|
||||
self.processed_rows = 0
|
||||
self.error_rows = 0
|
||||
self.validation_errors = []
|
||||
self.database_errors = []
|
||||
|
||||
def record_row_processed(self, success: bool = True):
|
||||
"""Record that a row was processed"""
|
||||
self.processed_rows += 1
|
||||
if not success:
|
||||
self.error_rows += 1
|
||||
|
||||
def record_validation_error(self, row_number: int, error: str):
|
||||
"""Record a validation error"""
|
||||
self.validation_errors.append({
|
||||
"row": row_number,
|
||||
"error": error,
|
||||
"timestamp": datetime.utcnow()
|
||||
})
|
||||
|
||||
def record_database_error(self, error: str):
|
||||
"""Record a database error"""
|
||||
self.database_errors.append({
|
||||
"error": error,
|
||||
"timestamp": datetime.utcnow()
|
||||
})
|
||||
|
||||
def finalize(self):
|
||||
"""Finalize metrics collection"""
|
||||
self.end_time = datetime.utcnow()
|
||||
|
||||
def get_summary(self) -> Dict[str, Any]:
|
||||
"""Get metrics summary"""
|
||||
duration = (self.end_time or datetime.utcnow()) - self.start_time
|
||||
|
||||
return {
|
||||
"start_time": self.start_time.isoformat(),
|
||||
"end_time": self.end_time.isoformat() if self.end_time else None,
|
||||
"duration_seconds": duration.total_seconds(),
|
||||
"total_rows": self.total_rows,
|
||||
"processed_rows": self.processed_rows,
|
||||
"error_rows": self.error_rows,
|
||||
"success_rate": (self.processed_rows / self.total_rows * 100) if self.total_rows > 0 else 0,
|
||||
"validation_errors": len(self.validation_errors),
|
||||
"database_errors": len(self.database_errors)
|
||||
}
|
||||
54
app/import_export/phone_importer.py
Normal file
54
app/import_export/phone_importer.py
Normal file
@@ -0,0 +1,54 @@
|
||||
"""
|
||||
PHONE CSV Importer
|
||||
"""
|
||||
from typing import Dict, List, Any
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .base import BaseCSVImporter, ImportValidationError
|
||||
from app.models.rolodex import Phone, Rolodex
|
||||
|
||||
|
||||
class PhoneCSVImporter(BaseCSVImporter):
|
||||
"""CSV importer for PHONE table"""
|
||||
|
||||
@property
|
||||
def table_name(self) -> str:
|
||||
return "phone"
|
||||
|
||||
@property
|
||||
def required_fields(self) -> List[str]:
|
||||
return ["rolodex_id", "phone"] # rolodex_id and phone number are required
|
||||
|
||||
@property
|
||||
def field_mapping(self) -> Dict[str, str]:
|
||||
"""Map CSV headers to database field names"""
|
||||
return {
|
||||
"rolodex_id": "rolodex_id",
|
||||
"location": "location",
|
||||
"phone": "phone"
|
||||
}
|
||||
|
||||
def create_model_instance(self, row_data: Dict[str, Any]) -> Phone:
|
||||
"""Create a Phone instance from processed row data"""
|
||||
|
||||
# Validate required fields
|
||||
if not row_data.get("rolodex_id"):
|
||||
raise ImportValidationError("Rolodex ID is required")
|
||||
if not row_data.get("phone"):
|
||||
raise ImportValidationError("Phone number is required")
|
||||
|
||||
# Validate foreign key exists
|
||||
rolodex_exists = self.db_session.query(Rolodex).filter_by(
|
||||
id=row_data["rolodex_id"]
|
||||
).first()
|
||||
if not rolodex_exists:
|
||||
raise ImportValidationError(f"Rolodex ID '{row_data['rolodex_id']}' does not exist")
|
||||
|
||||
# Create instance with field length validation
|
||||
phone = Phone(
|
||||
rolodex_id=self.normalize_string(row_data["rolodex_id"], 80),
|
||||
location=self.normalize_string(row_data.get("location", ""), 45),
|
||||
phone=self.normalize_string(row_data["phone"], 45)
|
||||
)
|
||||
|
||||
return phone
|
||||
137
app/import_export/qdros_importer.py
Normal file
137
app/import_export/qdros_importer.py
Normal file
@@ -0,0 +1,137 @@
|
||||
"""
|
||||
QDROS CSV Importer
|
||||
"""
|
||||
from typing import Dict, List, Any
|
||||
from datetime import date
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .base import BaseCSVImporter, ImportValidationError
|
||||
from app.models.qdro import QDRO
|
||||
from app.models.files import File
|
||||
|
||||
|
||||
class QdrosCSVImporter(BaseCSVImporter):
|
||||
"""CSV importer for QDROS table"""
|
||||
|
||||
@property
|
||||
def table_name(self) -> str:
|
||||
return "qdros"
|
||||
|
||||
@property
|
||||
def required_fields(self) -> List[str]:
|
||||
return ["file_no"] # Only file_no is strictly required
|
||||
|
||||
@property
|
||||
def field_mapping(self) -> Dict[str, str]:
|
||||
"""Map CSV headers to database field names"""
|
||||
return {
|
||||
"file_no": "file_no",
|
||||
"version": "version",
|
||||
"plan_id": "plan_id",
|
||||
# Legacy CSV fields
|
||||
"field1": "field1",
|
||||
"field2": "field2",
|
||||
"part": "part",
|
||||
"altp": "altp",
|
||||
"pet": "pet",
|
||||
"res": "res",
|
||||
# Case information
|
||||
"case_type": "case_type",
|
||||
"case_code": "case_code",
|
||||
"section": "section",
|
||||
"case_number": "case_number",
|
||||
# Dates
|
||||
"judgment_date": "judgment_date",
|
||||
"valuation_date": "valuation_date",
|
||||
"married_on": "married_on",
|
||||
# Award and venue
|
||||
"percent_awarded": "percent_awarded",
|
||||
"ven_city": "ven_city",
|
||||
"ven_cnty": "ven_cnty",
|
||||
"ven_st": "ven_st",
|
||||
# Document status dates
|
||||
"draft_out": "draft_out",
|
||||
"draft_apr": "draft_apr",
|
||||
"final_out": "final_out",
|
||||
# Additional fields
|
||||
"judge": "judge",
|
||||
"form_name": "form_name",
|
||||
"status": "status",
|
||||
"content": "content",
|
||||
"notes": "notes",
|
||||
"approval_status": "approval_status",
|
||||
"approved_date": "approved_date",
|
||||
"filed_date": "filed_date"
|
||||
}
|
||||
|
||||
def create_model_instance(self, row_data: Dict[str, Any]) -> QDRO:
|
||||
"""Create a Qdro instance from processed row data"""
|
||||
|
||||
# Validate required fields
|
||||
if not row_data.get("file_no"):
|
||||
raise ImportValidationError("File number is required")
|
||||
|
||||
# Validate foreign key exists (file number)
|
||||
file_exists = self.db_session.query(File).filter_by(file_no=row_data["file_no"]).first()
|
||||
if not file_exists:
|
||||
raise ImportValidationError(f"File number '{row_data['file_no']}' does not exist")
|
||||
|
||||
# Parse date fields
|
||||
date_fields = [
|
||||
"judgment_date", "valuation_date", "married_on",
|
||||
"draft_out", "draft_apr", "final_out", "approved_date", "filed_date"
|
||||
]
|
||||
|
||||
parsed_dates = {}
|
||||
for field in date_fields:
|
||||
if row_data.get(field):
|
||||
try:
|
||||
parsed_dates[field] = self.parse_date(row_data[field])
|
||||
except ValueError as e:
|
||||
raise ImportValidationError(f"Invalid {field}: {e}")
|
||||
else:
|
||||
parsed_dates[field] = None
|
||||
|
||||
# Validate state abbreviation length
|
||||
ven_st = row_data.get("ven_st", "")
|
||||
if ven_st and len(ven_st) > 2:
|
||||
self.result.add_warning(f"State abbreviation truncated: {ven_st}")
|
||||
ven_st = ven_st[:2]
|
||||
|
||||
# Set default status if not provided
|
||||
status = row_data.get("status", "DRAFT")
|
||||
|
||||
# Create instance
|
||||
qdro = QDRO(
|
||||
file_no=self.normalize_string(row_data["file_no"], 45),
|
||||
version=self.normalize_string(row_data.get("version", "01"), 10),
|
||||
plan_id=self.normalize_string(row_data.get("plan_id", ""), 45),
|
||||
# Legacy CSV fields
|
||||
field1=self.normalize_string(row_data.get("field1", ""), 100),
|
||||
field2=self.normalize_string(row_data.get("field2", ""), 100),
|
||||
part=self.normalize_string(row_data.get("part", ""), 100),
|
||||
altp=self.normalize_string(row_data.get("altp", ""), 100),
|
||||
pet=self.normalize_string(row_data.get("pet", ""), 100),
|
||||
res=self.normalize_string(row_data.get("res", ""), 100),
|
||||
# Case information
|
||||
case_type=self.normalize_string(row_data.get("case_type", ""), 45),
|
||||
case_code=self.normalize_string(row_data.get("case_code", ""), 45),
|
||||
section=self.normalize_string(row_data.get("section", ""), 45),
|
||||
case_number=self.normalize_string(row_data.get("case_number", ""), 100),
|
||||
# Dates
|
||||
**parsed_dates,
|
||||
# Award and venue
|
||||
percent_awarded=self.normalize_string(row_data.get("percent_awarded", ""), 100),
|
||||
ven_city=self.normalize_string(row_data.get("ven_city", ""), 50),
|
||||
ven_cnty=self.normalize_string(row_data.get("ven_cnty", ""), 50),
|
||||
ven_st=ven_st,
|
||||
# Additional fields
|
||||
judge=self.normalize_string(row_data.get("judge", ""), 100),
|
||||
form_name=self.normalize_string(row_data.get("form_name", ""), 200),
|
||||
status=self.normalize_string(status, 45),
|
||||
content=row_data.get("content", ""), # Text field
|
||||
notes=row_data.get("notes", ""), # Text field
|
||||
approval_status=self.normalize_string(row_data.get("approval_status", ""), 45)
|
||||
)
|
||||
|
||||
return qdro
|
||||
93
app/import_export/rolodex_importer.py
Normal file
93
app/import_export/rolodex_importer.py
Normal file
@@ -0,0 +1,93 @@
|
||||
"""
|
||||
ROLODEX CSV Importer
|
||||
"""
|
||||
from typing import Dict, List, Any
|
||||
from datetime import date
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .base import BaseCSVImporter, ImportValidationError
|
||||
from app.models.rolodex import Rolodex
|
||||
|
||||
|
||||
class RolodexCSVImporter(BaseCSVImporter):
|
||||
"""CSV importer for ROLODEX table"""
|
||||
|
||||
@property
|
||||
def table_name(self) -> str:
|
||||
return "rolodex"
|
||||
|
||||
@property
|
||||
def required_fields(self) -> List[str]:
|
||||
return ["id", "last"] # Only ID and last name are required
|
||||
|
||||
@property
|
||||
def field_mapping(self) -> Dict[str, str]:
|
||||
"""Map CSV headers to database field names"""
|
||||
return {
|
||||
"id": "id",
|
||||
"last": "last",
|
||||
"first": "first",
|
||||
"middle": "middle",
|
||||
"prefix": "prefix",
|
||||
"suffix": "suffix",
|
||||
"title": "title",
|
||||
"group": "group",
|
||||
"a1": "a1",
|
||||
"a2": "a2",
|
||||
"a3": "a3",
|
||||
"city": "city",
|
||||
"abrev": "abrev",
|
||||
"zip": "zip",
|
||||
"email": "email",
|
||||
"dob": "dob",
|
||||
"ss_number": "ss_number",
|
||||
"legal_status": "legal_status",
|
||||
"memo": "memo"
|
||||
}
|
||||
|
||||
def create_model_instance(self, row_data: Dict[str, Any]) -> Rolodex:
|
||||
"""Create a Rolodex instance from processed row data"""
|
||||
|
||||
# Validate required fields
|
||||
if not row_data.get("id"):
|
||||
raise ImportValidationError("ID is required")
|
||||
if not row_data.get("last"):
|
||||
raise ImportValidationError("Last name is required")
|
||||
|
||||
# Check for duplicate ID
|
||||
existing = self.db_session.query(Rolodex).filter_by(id=row_data["id"]).first()
|
||||
if existing:
|
||||
raise ImportValidationError(f"Rolodex ID '{row_data['id']}' already exists")
|
||||
|
||||
# Parse date of birth
|
||||
dob = None
|
||||
if row_data.get("dob"):
|
||||
try:
|
||||
dob = self.parse_date(row_data["dob"])
|
||||
except ValueError as e:
|
||||
raise ImportValidationError(f"Invalid date of birth: {e}")
|
||||
|
||||
# Create instance with field length validation
|
||||
rolodex = Rolodex(
|
||||
id=self.normalize_string(row_data["id"], 80),
|
||||
last=self.normalize_string(row_data["last"], 80),
|
||||
first=self.normalize_string(row_data.get("first", ""), 45),
|
||||
middle=self.normalize_string(row_data.get("middle", ""), 45),
|
||||
prefix=self.normalize_string(row_data.get("prefix", ""), 45),
|
||||
suffix=self.normalize_string(row_data.get("suffix", ""), 45),
|
||||
title=self.normalize_string(row_data.get("title", ""), 45),
|
||||
group=self.normalize_string(row_data.get("group", ""), 45),
|
||||
a1=self.normalize_string(row_data.get("a1", ""), 45),
|
||||
a2=self.normalize_string(row_data.get("a2", ""), 45),
|
||||
a3=self.normalize_string(row_data.get("a3", ""), 45),
|
||||
city=self.normalize_string(row_data.get("city", ""), 80),
|
||||
abrev=self.normalize_string(row_data.get("abrev", ""), 45),
|
||||
zip=self.normalize_string(row_data.get("zip", ""), 45),
|
||||
email=self.normalize_string(row_data.get("email", ""), 100),
|
||||
dob=dob,
|
||||
ss_number=self.normalize_string(row_data.get("ss_number", ""), 20),
|
||||
legal_status=self.normalize_string(row_data.get("legal_status", ""), 45),
|
||||
memo=row_data.get("memo", "") # Text field, no length limit
|
||||
)
|
||||
|
||||
return rolodex
|
||||
Reference in New Issue
Block a user