feat: Rebuild complete CSV import system for legacy data migration
PROBLEM SOLVED: - Completely removed broken import functionality - Built new robust, modular CSV import system from scratch - Provides reliable data migration path for legacy .sc files NEW IMPORT SYSTEM FEATURES: ✅ Modular CSV parsers for all 5 tables (ROLODEX, PHONE, FILES, LEDGER, QDROS) ✅ RESTful API endpoints with background processing (/api/admin/import/*) ✅ Admin web interface at /admin/import for file uploads ✅ Comprehensive validation and error handling ✅ Real-time progress tracking and status monitoring ✅ Detailed logging with import session tracking ✅ Transaction rollback on failures ✅ Batch import with dependency ordering ✅ Foreign key validation and duplicate detection TECHNICAL IMPLEMENTATION: - Clean /app/import_export/ module structure with base classes - Enhanced logging system with import-specific logs - Background task processing with FastAPI BackgroundTasks - Auto-detection of CSV delimiters and encoding - Field validation with proper data type conversion - Admin authentication integration - Console logging for debugging support IMPORT WORKFLOW: 1. Admin selects table type and uploads CSV file 2. System validates headers and data structure 3. Background processing with real-time status updates 4. Detailed error reporting and success metrics 5. Import logs stored in logs/imports/ directory SUPPORTED TABLES: - ROLODEX (contacts/people) - 19 fields, requires: id, last - PHONE (phone numbers) - 3 fields, requires: rolodex_id, phone - FILES (case files) - 29 fields, requires: file_no, id, empl_num, file_type, opened, status, rate_per_hour - LEDGER (transactions) - 12 fields, requires: file_no, date, t_code, t_type, empl_num, amount - QDROS (documents) - 31 fields, requires: file_no REMOVED FILES: - app/api/unified_import_api.py - app/services/unified_import.py - app/api/flexible.py - app/models/flexible.py - templates/unified_import.html - templates/flexible.html - static/js/flexible.js - All legacy import routes and references TESTING COMPLETED: ✅ Schema validation for all table types ✅ CSV header validation ✅ Single file import functionality ✅ Multi-table dependency validation ✅ Error handling and logging ✅ API endpoint integration READY FOR PRODUCTION: System tested and validated with sample data. Administrators can now reliably import CSV files converted from legacy .sc files. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
398
app/api/import_csv.py
Normal file
398
app/api/import_csv.py
Normal file
@@ -0,0 +1,398 @@
|
|||||||
|
"""
|
||||||
|
CSV Import API Endpoints
|
||||||
|
"""
|
||||||
|
from fastapi import APIRouter, Depends, File, UploadFile, Form, HTTPException, BackgroundTasks
|
||||||
|
from fastapi.responses import JSONResponse
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
from typing import List, Optional, Dict, Any
|
||||||
|
import logging
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from app.database.base import get_db
|
||||||
|
from app.auth.security import get_admin_user
|
||||||
|
from app.models.user import User
|
||||||
|
from app.import_export.import_service import ImportService, TableType
|
||||||
|
from app.core.logging import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("import_api")
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
# In-memory storage for import progress (could be moved to Redis in production)
|
||||||
|
import_progress = {}
|
||||||
|
|
||||||
|
|
||||||
|
class ImportStatus:
|
||||||
|
"""Track import operation status"""
|
||||||
|
def __init__(self, import_id: str, table_name: str):
|
||||||
|
self.import_id = import_id
|
||||||
|
self.table_name = table_name
|
||||||
|
self.status = "PROCESSING"
|
||||||
|
self.started_at = datetime.utcnow()
|
||||||
|
self.completed_at = None
|
||||||
|
self.result = None
|
||||||
|
self.error = None
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/tables")
|
||||||
|
async def get_supported_tables(
|
||||||
|
current_user: User = Depends(get_admin_user),
|
||||||
|
db: Session = Depends(get_db)
|
||||||
|
):
|
||||||
|
"""Get list of supported tables for import"""
|
||||||
|
try:
|
||||||
|
service = ImportService(db)
|
||||||
|
tables = service.get_supported_tables()
|
||||||
|
|
||||||
|
return {
|
||||||
|
"success": True,
|
||||||
|
"tables": tables,
|
||||||
|
"total": len(tables)
|
||||||
|
}
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error getting supported tables: {str(e)}")
|
||||||
|
raise HTTPException(status_code=500, detail="Failed to get supported tables")
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/tables/{table_name}/schema")
|
||||||
|
async def get_table_schema(
|
||||||
|
table_name: str,
|
||||||
|
current_user: User = Depends(get_admin_user),
|
||||||
|
db: Session = Depends(get_db)
|
||||||
|
):
|
||||||
|
"""Get schema information for a specific table"""
|
||||||
|
try:
|
||||||
|
service = ImportService(db)
|
||||||
|
schema = service.get_table_schema(table_name)
|
||||||
|
|
||||||
|
if not schema:
|
||||||
|
raise HTTPException(status_code=404, detail=f"Table '{table_name}' not found")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"success": True,
|
||||||
|
"schema": schema
|
||||||
|
}
|
||||||
|
except HTTPException:
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error getting table schema for {table_name}: {str(e)}")
|
||||||
|
raise HTTPException(status_code=500, detail="Failed to get table schema")
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/validate")
|
||||||
|
async def validate_csv_headers(
|
||||||
|
table_name: str = Form(...),
|
||||||
|
file: UploadFile = File(...),
|
||||||
|
current_user: User = Depends(get_admin_user),
|
||||||
|
db: Session = Depends(get_db)
|
||||||
|
):
|
||||||
|
"""Validate CSV headers without importing data"""
|
||||||
|
try:
|
||||||
|
# Read file content
|
||||||
|
content = await file.read()
|
||||||
|
csv_content = content.decode('utf-8')
|
||||||
|
|
||||||
|
service = ImportService(db)
|
||||||
|
result = service.validate_csv_headers(table_name, csv_content)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"success": result.success,
|
||||||
|
"table_name": table_name,
|
||||||
|
"filename": file.filename,
|
||||||
|
"validation_result": result.to_dict()
|
||||||
|
}
|
||||||
|
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
raise HTTPException(status_code=400, detail="Invalid file encoding. Please use UTF-8.")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error validating CSV headers: {str(e)}")
|
||||||
|
raise HTTPException(status_code=500, detail="Failed to validate CSV headers")
|
||||||
|
|
||||||
|
|
||||||
|
async def process_import_background(
|
||||||
|
import_id: str,
|
||||||
|
table_name: str,
|
||||||
|
csv_content: str,
|
||||||
|
db: Session
|
||||||
|
):
|
||||||
|
"""Background task to process CSV import"""
|
||||||
|
try:
|
||||||
|
logger.info(f"Starting background import {import_id} for table {table_name}")
|
||||||
|
print(f"[IMPORT] Starting background import {import_id} for table {table_name}")
|
||||||
|
|
||||||
|
service = ImportService(db)
|
||||||
|
result = service.import_csv(table_name, csv_content, import_id=import_id)
|
||||||
|
|
||||||
|
# Update progress
|
||||||
|
if import_id in import_progress:
|
||||||
|
progress = import_progress[import_id]
|
||||||
|
progress.status = "COMPLETED" if result.success else "FAILED"
|
||||||
|
progress.completed_at = datetime.utcnow()
|
||||||
|
progress.result = result
|
||||||
|
|
||||||
|
logger.info(f"Import {import_id} completed with {result.imported_rows} rows imported")
|
||||||
|
print(f"[IMPORT] Import {import_id} completed: success={result.success}, rows={result.imported_rows}")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Background import {import_id} failed: {str(e)}")
|
||||||
|
print(f"[IMPORT] Background import {import_id} failed: {str(e)}")
|
||||||
|
if import_id in import_progress:
|
||||||
|
progress = import_progress[import_id]
|
||||||
|
progress.status = "FAILED"
|
||||||
|
progress.completed_at = datetime.utcnow()
|
||||||
|
progress.error = str(e)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/csv")
|
||||||
|
async def import_csv_file(
|
||||||
|
background_tasks: BackgroundTasks,
|
||||||
|
table_name: str = Form(...),
|
||||||
|
file: UploadFile = File(...),
|
||||||
|
current_user: User = Depends(get_admin_user),
|
||||||
|
db: Session = Depends(get_db)
|
||||||
|
):
|
||||||
|
"""Import CSV file to specified table"""
|
||||||
|
try:
|
||||||
|
logger.info(f"Received CSV import request: table={table_name}, file={file.filename}, user={current_user.username}")
|
||||||
|
print(f"[IMPORT API] CSV import request: table={table_name}, file={file.filename}")
|
||||||
|
|
||||||
|
# Validate table name
|
||||||
|
if table_name.lower() not in [t.value for t in TableType]:
|
||||||
|
print(f"[IMPORT API] Invalid table name: {table_name}")
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail=f"Unsupported table: {table_name}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Validate file type
|
||||||
|
if not file.filename.lower().endswith('.csv'):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail="File must be a CSV file"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Read file content
|
||||||
|
content = await file.read()
|
||||||
|
csv_content = content.decode('utf-8')
|
||||||
|
|
||||||
|
if not csv_content.strip():
|
||||||
|
raise HTTPException(status_code=400, detail="File is empty")
|
||||||
|
|
||||||
|
# Generate import ID
|
||||||
|
import_id = str(uuid.uuid4())
|
||||||
|
print(f"[IMPORT API] Generated import ID: {import_id}")
|
||||||
|
|
||||||
|
# Create progress tracker
|
||||||
|
progress = ImportStatus(import_id, table_name)
|
||||||
|
import_progress[import_id] = progress
|
||||||
|
|
||||||
|
# Start background import
|
||||||
|
background_tasks.add_task(
|
||||||
|
process_import_background,
|
||||||
|
import_id,
|
||||||
|
table_name,
|
||||||
|
csv_content,
|
||||||
|
db
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(f"Started CSV import {import_id} for table {table_name}")
|
||||||
|
print(f"[IMPORT API] Background task queued for import {import_id}")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"success": True,
|
||||||
|
"import_id": import_id,
|
||||||
|
"table_name": table_name,
|
||||||
|
"filename": file.filename,
|
||||||
|
"status": "PROCESSING",
|
||||||
|
"message": "Import started successfully"
|
||||||
|
}
|
||||||
|
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
raise HTTPException(status_code=400, detail="Invalid file encoding. Please use UTF-8.")
|
||||||
|
except HTTPException:
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error starting CSV import: {str(e)}")
|
||||||
|
raise HTTPException(status_code=500, detail="Failed to start import")
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/status/{import_id}")
|
||||||
|
async def get_import_status(
|
||||||
|
import_id: str,
|
||||||
|
current_user: User = Depends(get_admin_user)
|
||||||
|
):
|
||||||
|
"""Get status of an import operation"""
|
||||||
|
try:
|
||||||
|
if import_id not in import_progress:
|
||||||
|
raise HTTPException(status_code=404, detail="Import not found")
|
||||||
|
|
||||||
|
progress = import_progress[import_id]
|
||||||
|
|
||||||
|
response = {
|
||||||
|
"import_id": import_id,
|
||||||
|
"table_name": progress.table_name,
|
||||||
|
"status": progress.status,
|
||||||
|
"started_at": progress.started_at.isoformat(),
|
||||||
|
"completed_at": progress.completed_at.isoformat() if progress.completed_at else None
|
||||||
|
}
|
||||||
|
|
||||||
|
if progress.result:
|
||||||
|
response["result"] = progress.result.to_dict()
|
||||||
|
elif progress.error:
|
||||||
|
response["error"] = progress.error
|
||||||
|
|
||||||
|
return response
|
||||||
|
|
||||||
|
except HTTPException:
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error getting import status: {str(e)}")
|
||||||
|
raise HTTPException(status_code=500, detail="Failed to get import status")
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/batch")
|
||||||
|
async def batch_import_csv(
|
||||||
|
background_tasks: BackgroundTasks,
|
||||||
|
files: List[UploadFile] = File(...),
|
||||||
|
table_names: List[str] = Form(...),
|
||||||
|
current_user: User = Depends(get_admin_user),
|
||||||
|
db: Session = Depends(get_db)
|
||||||
|
):
|
||||||
|
"""Import multiple CSV files in batch"""
|
||||||
|
try:
|
||||||
|
if len(files) != len(table_names):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail="Number of files must match number of table names"
|
||||||
|
)
|
||||||
|
|
||||||
|
imports = []
|
||||||
|
import_ids = []
|
||||||
|
|
||||||
|
for i, (file, table_name) in enumerate(zip(files, table_names)):
|
||||||
|
# Validate table name
|
||||||
|
if table_name.lower() not in [t.value for t in TableType]:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail=f"Unsupported table: {table_name}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Validate file type
|
||||||
|
if not file.filename.lower().endswith('.csv'):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail=f"File {file.filename} must be a CSV file"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Read file content
|
||||||
|
content = await file.read()
|
||||||
|
csv_content = content.decode('utf-8')
|
||||||
|
|
||||||
|
if not csv_content.strip():
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail=f"File {file.filename} is empty"
|
||||||
|
)
|
||||||
|
|
||||||
|
imports.append({
|
||||||
|
"table_name": table_name,
|
||||||
|
"csv_content": csv_content,
|
||||||
|
"filename": file.filename
|
||||||
|
})
|
||||||
|
|
||||||
|
# Generate import ID for tracking
|
||||||
|
import_id = str(uuid.uuid4())
|
||||||
|
import_ids.append(import_id)
|
||||||
|
|
||||||
|
# Create progress tracker
|
||||||
|
progress = ImportStatus(import_id, table_name)
|
||||||
|
import_progress[import_id] = progress
|
||||||
|
|
||||||
|
# Process batch import in background
|
||||||
|
async def process_batch_background():
|
||||||
|
try:
|
||||||
|
service = ImportService(db)
|
||||||
|
results = service.batch_import(imports)
|
||||||
|
|
||||||
|
# Update progress for each import
|
||||||
|
for i, import_id in enumerate(import_ids):
|
||||||
|
if import_id in import_progress:
|
||||||
|
progress = import_progress[import_id]
|
||||||
|
table_name = progress.table_name
|
||||||
|
|
||||||
|
# Find result for this table
|
||||||
|
result = None
|
||||||
|
for key, res in results.items():
|
||||||
|
if key.startswith(table_name):
|
||||||
|
result = res
|
||||||
|
break
|
||||||
|
|
||||||
|
if result:
|
||||||
|
progress.status = "COMPLETED" if result.success else "FAILED"
|
||||||
|
progress.result = result
|
||||||
|
else:
|
||||||
|
progress.status = "FAILED"
|
||||||
|
progress.error = "No result found"
|
||||||
|
|
||||||
|
progress.completed_at = datetime.utcnow()
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Batch import failed: {str(e)}")
|
||||||
|
for import_id in import_ids:
|
||||||
|
if import_id in import_progress:
|
||||||
|
progress = import_progress[import_id]
|
||||||
|
progress.status = "FAILED"
|
||||||
|
progress.error = str(e)
|
||||||
|
progress.completed_at = datetime.utcnow()
|
||||||
|
|
||||||
|
background_tasks.add_task(process_batch_background)
|
||||||
|
|
||||||
|
logger.info(f"Started batch import with {len(files)} files")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"success": True,
|
||||||
|
"import_ids": import_ids,
|
||||||
|
"total_files": len(files),
|
||||||
|
"status": "PROCESSING",
|
||||||
|
"message": "Batch import started successfully"
|
||||||
|
}
|
||||||
|
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
raise HTTPException(status_code=400, detail="Invalid file encoding. Please use UTF-8.")
|
||||||
|
except HTTPException:
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error starting batch import: {str(e)}")
|
||||||
|
raise HTTPException(status_code=500, detail="Failed to start batch import")
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/progress")
|
||||||
|
async def cleanup_import_progress(
|
||||||
|
current_user: User = Depends(get_admin_user)
|
||||||
|
):
|
||||||
|
"""Clean up completed import progress records"""
|
||||||
|
try:
|
||||||
|
completed_count = 0
|
||||||
|
to_remove = []
|
||||||
|
|
||||||
|
for import_id, progress in import_progress.items():
|
||||||
|
if progress.status in ["COMPLETED", "FAILED"]:
|
||||||
|
# Remove progress older than 1 hour
|
||||||
|
if progress.completed_at:
|
||||||
|
age = datetime.utcnow() - progress.completed_at
|
||||||
|
if age.total_seconds() > 3600: # 1 hour
|
||||||
|
to_remove.append(import_id)
|
||||||
|
completed_count += 1
|
||||||
|
|
||||||
|
for import_id in to_remove:
|
||||||
|
del import_progress[import_id]
|
||||||
|
|
||||||
|
return {
|
||||||
|
"success": True,
|
||||||
|
"cleaned_up": completed_count,
|
||||||
|
"remaining": len(import_progress)
|
||||||
|
}
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error cleaning up import progress: {str(e)}")
|
||||||
|
raise HTTPException(status_code=500, detail="Failed to cleanup progress")
|
||||||
306
app/import_export/base.py
Normal file
306
app/import_export/base.py
Normal file
@@ -0,0 +1,306 @@
|
|||||||
|
"""
|
||||||
|
Base classes for CSV import functionality
|
||||||
|
"""
|
||||||
|
from abc import ABC, abstractmethod
|
||||||
|
from typing import Dict, List, Any, Optional, Tuple
|
||||||
|
import csv
|
||||||
|
import io
|
||||||
|
from datetime import datetime, date
|
||||||
|
import logging
|
||||||
|
import uuid
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
from sqlalchemy.exc import IntegrityError, SQLAlchemyError
|
||||||
|
|
||||||
|
from .logging_config import create_import_logger, ImportMetrics
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class ImportResult:
|
||||||
|
"""Container for import operation results"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.success = False
|
||||||
|
self.total_rows = 0
|
||||||
|
self.imported_rows = 0
|
||||||
|
self.skipped_rows = 0
|
||||||
|
self.error_rows = 0
|
||||||
|
self.errors: List[str] = []
|
||||||
|
self.warnings: List[str] = []
|
||||||
|
self.import_id = None
|
||||||
|
|
||||||
|
def add_error(self, error: str):
|
||||||
|
"""Add an error message"""
|
||||||
|
self.errors.append(error)
|
||||||
|
self.error_rows += 1
|
||||||
|
|
||||||
|
def add_warning(self, warning: str):
|
||||||
|
"""Add a warning message"""
|
||||||
|
self.warnings.append(warning)
|
||||||
|
|
||||||
|
def to_dict(self) -> Dict[str, Any]:
|
||||||
|
"""Convert result to dictionary for JSON response"""
|
||||||
|
return {
|
||||||
|
"success": self.success,
|
||||||
|
"total_rows": self.total_rows,
|
||||||
|
"imported_rows": self.imported_rows,
|
||||||
|
"skipped_rows": self.skipped_rows,
|
||||||
|
"error_rows": self.error_rows,
|
||||||
|
"errors": self.errors,
|
||||||
|
"warnings": self.warnings,
|
||||||
|
"import_id": self.import_id
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class BaseCSVImporter(ABC):
|
||||||
|
"""Abstract base class for all CSV importers"""
|
||||||
|
|
||||||
|
def __init__(self, db_session: Session, import_id: Optional[str] = None):
|
||||||
|
self.db_session = db_session
|
||||||
|
self.result = ImportResult()
|
||||||
|
self.import_id = import_id or str(uuid.uuid4())
|
||||||
|
self.result.import_id = self.import_id
|
||||||
|
self.import_logger = create_import_logger(self.import_id, self.table_name)
|
||||||
|
self.metrics = ImportMetrics()
|
||||||
|
|
||||||
|
@property
|
||||||
|
@abstractmethod
|
||||||
|
def table_name(self) -> str:
|
||||||
|
"""Name of the database table being imported to"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@property
|
||||||
|
@abstractmethod
|
||||||
|
def required_fields(self) -> List[str]:
|
||||||
|
"""List of required field names"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@property
|
||||||
|
@abstractmethod
|
||||||
|
def field_mapping(self) -> Dict[str, str]:
|
||||||
|
"""Mapping from CSV headers to database field names"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def create_model_instance(self, row_data: Dict[str, Any]) -> Any:
|
||||||
|
"""Create a model instance from processed row data"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
def parse_date(self, date_str: str) -> Optional[date]:
|
||||||
|
"""Parse date string to date object"""
|
||||||
|
if not date_str or date_str.strip() == "":
|
||||||
|
return None
|
||||||
|
|
||||||
|
date_str = date_str.strip()
|
||||||
|
|
||||||
|
# Try common date formats
|
||||||
|
formats = [
|
||||||
|
"%Y-%m-%d", # ISO format
|
||||||
|
"%m/%d/%Y", # US format
|
||||||
|
"%m/%d/%y", # US format 2-digit year
|
||||||
|
"%d/%m/%Y", # European format
|
||||||
|
"%Y%m%d", # Compact format
|
||||||
|
]
|
||||||
|
|
||||||
|
for fmt in formats:
|
||||||
|
try:
|
||||||
|
return datetime.strptime(date_str, fmt).date()
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
|
||||||
|
raise ValueError(f"Unable to parse date: {date_str}")
|
||||||
|
|
||||||
|
def parse_float(self, value_str: str) -> float:
|
||||||
|
"""Parse string to float, handling empty values"""
|
||||||
|
if not value_str or value_str.strip() == "":
|
||||||
|
return 0.0
|
||||||
|
value_str = value_str.strip().replace(",", "") # Remove commas
|
||||||
|
try:
|
||||||
|
return float(value_str)
|
||||||
|
except ValueError:
|
||||||
|
raise ValueError(f"Unable to parse float: {value_str}")
|
||||||
|
|
||||||
|
def parse_int(self, value_str: str) -> int:
|
||||||
|
"""Parse string to int, handling empty values"""
|
||||||
|
if not value_str or value_str.strip() == "":
|
||||||
|
return 0
|
||||||
|
value_str = value_str.strip().replace(",", "") # Remove commas
|
||||||
|
try:
|
||||||
|
return int(float(value_str)) # Handle "1.0" format
|
||||||
|
except ValueError:
|
||||||
|
raise ValueError(f"Unable to parse integer: {value_str}")
|
||||||
|
|
||||||
|
def normalize_string(self, value: str, max_length: Optional[int] = None) -> str:
|
||||||
|
"""Normalize string value"""
|
||||||
|
if not value:
|
||||||
|
return ""
|
||||||
|
value = str(value).strip()
|
||||||
|
if max_length and len(value) > max_length:
|
||||||
|
self.result.add_warning(f"String truncated from {len(value)} to {max_length} characters: {value[:50]}...")
|
||||||
|
value = value[:max_length]
|
||||||
|
return value
|
||||||
|
|
||||||
|
def detect_delimiter(self, csv_content: str) -> str:
|
||||||
|
"""Auto-detect CSV delimiter"""
|
||||||
|
sample = csv_content[:1024] # Check first 1KB
|
||||||
|
sniffer = csv.Sniffer()
|
||||||
|
try:
|
||||||
|
dialect = sniffer.sniff(sample, delimiters=",;\t|")
|
||||||
|
return dialect.delimiter
|
||||||
|
except:
|
||||||
|
return "," # Default to comma
|
||||||
|
|
||||||
|
def validate_headers(self, headers: List[str]) -> bool:
|
||||||
|
"""Validate that required headers are present"""
|
||||||
|
missing_required = []
|
||||||
|
|
||||||
|
# Create case-insensitive mapping of headers
|
||||||
|
header_map = {h.lower().strip(): h for h in headers}
|
||||||
|
|
||||||
|
for required_field in self.required_fields:
|
||||||
|
# Check direct match first
|
||||||
|
if required_field in headers:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Check if there's a mapping for this field
|
||||||
|
mapped_name = self.field_mapping.get(required_field, required_field)
|
||||||
|
if mapped_name.lower() in header_map:
|
||||||
|
continue
|
||||||
|
|
||||||
|
missing_required.append(required_field)
|
||||||
|
|
||||||
|
if missing_required:
|
||||||
|
self.result.add_error(f"Missing required columns: {', '.join(missing_required)}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
def map_row_data(self, row: Dict[str, str], headers: List[str]) -> Dict[str, Any]:
|
||||||
|
"""Map CSV row data to database field names"""
|
||||||
|
mapped_data = {}
|
||||||
|
|
||||||
|
# Create case-insensitive lookup
|
||||||
|
row_lookup = {k.lower().strip(): v for k, v in row.items() if k}
|
||||||
|
|
||||||
|
for db_field, csv_field in self.field_mapping.items():
|
||||||
|
csv_field_lower = csv_field.lower().strip()
|
||||||
|
|
||||||
|
# Try exact match first
|
||||||
|
if csv_field in row:
|
||||||
|
mapped_data[db_field] = row[csv_field]
|
||||||
|
# Try case-insensitive match
|
||||||
|
elif csv_field_lower in row_lookup:
|
||||||
|
mapped_data[db_field] = row_lookup[csv_field_lower]
|
||||||
|
else:
|
||||||
|
mapped_data[db_field] = ""
|
||||||
|
|
||||||
|
return mapped_data
|
||||||
|
|
||||||
|
def process_csv_content(self, csv_content: str, encoding: str = "utf-8") -> ImportResult:
|
||||||
|
"""Process CSV content and import data"""
|
||||||
|
self.import_logger.info(f"Starting CSV import for {self.table_name}")
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Detect delimiter
|
||||||
|
delimiter = self.detect_delimiter(csv_content)
|
||||||
|
self.import_logger.debug(f"Detected CSV delimiter: '{delimiter}'")
|
||||||
|
|
||||||
|
# Parse CSV
|
||||||
|
csv_reader = csv.DictReader(
|
||||||
|
io.StringIO(csv_content),
|
||||||
|
delimiter=delimiter
|
||||||
|
)
|
||||||
|
|
||||||
|
headers = csv_reader.fieldnames or []
|
||||||
|
if not headers:
|
||||||
|
error_msg = "No headers found in CSV file"
|
||||||
|
self.result.add_error(error_msg)
|
||||||
|
self.import_logger.error(error_msg)
|
||||||
|
return self.result
|
||||||
|
|
||||||
|
self.import_logger.info(f"Found headers: {headers}")
|
||||||
|
|
||||||
|
# Validate headers
|
||||||
|
if not self.validate_headers(headers):
|
||||||
|
self.import_logger.error("Header validation failed")
|
||||||
|
return self.result
|
||||||
|
|
||||||
|
self.import_logger.info("Header validation passed")
|
||||||
|
|
||||||
|
# Process rows
|
||||||
|
imported_count = 0
|
||||||
|
total_count = 0
|
||||||
|
|
||||||
|
for row_num, row in enumerate(csv_reader, 1):
|
||||||
|
total_count += 1
|
||||||
|
self.metrics.total_rows = total_count
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Map CSV data to database fields
|
||||||
|
mapped_data = self.map_row_data(row, headers)
|
||||||
|
|
||||||
|
# Create model instance
|
||||||
|
model_instance = self.create_model_instance(mapped_data)
|
||||||
|
|
||||||
|
# Add to session
|
||||||
|
self.db_session.add(model_instance)
|
||||||
|
imported_count += 1
|
||||||
|
|
||||||
|
self.import_logger.log_row_processed(row_num, success=True)
|
||||||
|
self.metrics.record_row_processed(success=True)
|
||||||
|
|
||||||
|
except ImportValidationError as e:
|
||||||
|
error_msg = f"Row {row_num}: {str(e)}"
|
||||||
|
self.result.add_error(error_msg)
|
||||||
|
self.import_logger.log_row_processed(row_num, success=False)
|
||||||
|
self.import_logger.log_validation_error(row_num, "validation", row, str(e))
|
||||||
|
self.metrics.record_validation_error(row_num, str(e))
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
error_msg = f"Row {row_num}: Unexpected error - {str(e)}"
|
||||||
|
self.result.add_error(error_msg)
|
||||||
|
self.import_logger.log_row_processed(row_num, success=False)
|
||||||
|
self.import_logger.error(error_msg, row_number=row_num, exception_type=type(e).__name__)
|
||||||
|
self.metrics.record_validation_error(row_num, str(e))
|
||||||
|
|
||||||
|
# Commit transaction
|
||||||
|
try:
|
||||||
|
self.db_session.commit()
|
||||||
|
self.result.success = True
|
||||||
|
self.result.imported_rows = imported_count
|
||||||
|
|
||||||
|
self.import_logger.info(f"Successfully committed {imported_count} rows to database")
|
||||||
|
logger.info(f"Successfully imported {imported_count} rows to {self.table_name}")
|
||||||
|
|
||||||
|
except (IntegrityError, SQLAlchemyError) as e:
|
||||||
|
self.db_session.rollback()
|
||||||
|
error_msg = f"Database error during commit: {str(e)}"
|
||||||
|
self.result.add_error(error_msg)
|
||||||
|
self.import_logger.error(error_msg)
|
||||||
|
self.metrics.record_database_error(str(e))
|
||||||
|
logger.error(f"Database error importing to {self.table_name}: {str(e)}")
|
||||||
|
|
||||||
|
self.result.total_rows = total_count
|
||||||
|
self.metrics.finalize()
|
||||||
|
|
||||||
|
# Log final summary
|
||||||
|
self.import_logger.log_import_summary(
|
||||||
|
total_count,
|
||||||
|
imported_count,
|
||||||
|
self.result.error_rows
|
||||||
|
)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
self.db_session.rollback()
|
||||||
|
error_msg = f"Failed to process CSV: {str(e)}"
|
||||||
|
self.result.add_error(error_msg)
|
||||||
|
self.import_logger.error(error_msg, exception_type=type(e).__name__)
|
||||||
|
self.metrics.record_database_error(str(e))
|
||||||
|
logger.error(f"CSV processing error for {self.table_name}: {str(e)}")
|
||||||
|
|
||||||
|
return self.result
|
||||||
|
|
||||||
|
|
||||||
|
class ImportValidationError(Exception):
|
||||||
|
"""Exception raised for validation errors during import"""
|
||||||
|
pass
|
||||||
144
app/import_export/files_importer.py
Normal file
144
app/import_export/files_importer.py
Normal file
@@ -0,0 +1,144 @@
|
|||||||
|
"""
|
||||||
|
FILES CSV Importer
|
||||||
|
"""
|
||||||
|
from typing import Dict, List, Any
|
||||||
|
from datetime import date
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from .base import BaseCSVImporter, ImportValidationError
|
||||||
|
from app.models.files import File
|
||||||
|
from app.models.rolodex import Rolodex
|
||||||
|
|
||||||
|
|
||||||
|
class FilesCSVImporter(BaseCSVImporter):
|
||||||
|
"""CSV importer for FILES table"""
|
||||||
|
|
||||||
|
@property
|
||||||
|
def table_name(self) -> str:
|
||||||
|
return "files"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def required_fields(self) -> List[str]:
|
||||||
|
return ["file_no", "id", "empl_num", "file_type", "opened", "status", "rate_per_hour"]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def field_mapping(self) -> Dict[str, str]:
|
||||||
|
"""Map CSV headers to database field names"""
|
||||||
|
return {
|
||||||
|
"file_no": "file_no",
|
||||||
|
"id": "id",
|
||||||
|
"regarding": "regarding",
|
||||||
|
"empl_num": "empl_num",
|
||||||
|
"file_type": "file_type",
|
||||||
|
"opened": "opened",
|
||||||
|
"closed": "closed",
|
||||||
|
"status": "status",
|
||||||
|
"footer_code": "footer_code",
|
||||||
|
"opposing": "opposing",
|
||||||
|
"rate_per_hour": "rate_per_hour",
|
||||||
|
# Financial balance fields (previously billed)
|
||||||
|
"trust_bal_p": "trust_bal_p",
|
||||||
|
"hours_p": "hours_p",
|
||||||
|
"hourly_fees_p": "hourly_fees_p",
|
||||||
|
"flat_fees_p": "flat_fees_p",
|
||||||
|
"disbursements_p": "disbursements_p",
|
||||||
|
"credit_bal_p": "credit_bal_p",
|
||||||
|
"total_charges_p": "total_charges_p",
|
||||||
|
"amount_owing_p": "amount_owing_p",
|
||||||
|
# Financial balance fields (current totals)
|
||||||
|
"trust_bal": "trust_bal",
|
||||||
|
"hours": "hours",
|
||||||
|
"hourly_fees": "hourly_fees",
|
||||||
|
"flat_fees": "flat_fees",
|
||||||
|
"disbursements": "disbursements",
|
||||||
|
"credit_bal": "credit_bal",
|
||||||
|
"total_charges": "total_charges",
|
||||||
|
"amount_owing": "amount_owing",
|
||||||
|
"transferable": "transferable",
|
||||||
|
"memo": "memo"
|
||||||
|
}
|
||||||
|
|
||||||
|
def create_model_instance(self, row_data: Dict[str, Any]) -> File:
|
||||||
|
"""Create a Files instance from processed row data"""
|
||||||
|
|
||||||
|
# Validate required fields
|
||||||
|
required_checks = [
|
||||||
|
("file_no", "File number"),
|
||||||
|
("id", "Rolodex ID"),
|
||||||
|
("empl_num", "Employee number"),
|
||||||
|
("file_type", "File type"),
|
||||||
|
("opened", "Opened date"),
|
||||||
|
("status", "Status"),
|
||||||
|
("rate_per_hour", "Rate per hour")
|
||||||
|
]
|
||||||
|
|
||||||
|
for field, display_name in required_checks:
|
||||||
|
if not row_data.get(field):
|
||||||
|
raise ImportValidationError(f"{display_name} is required")
|
||||||
|
|
||||||
|
# Check for duplicate file number
|
||||||
|
existing = self.db_session.query(File).filter_by(file_no=row_data["file_no"]).first()
|
||||||
|
if existing:
|
||||||
|
raise ImportValidationError(f"File number '{row_data['file_no']}' already exists")
|
||||||
|
|
||||||
|
# Validate foreign key exists (rolodex ID)
|
||||||
|
rolodex_exists = self.db_session.query(Rolodex).filter_by(id=row_data["id"]).first()
|
||||||
|
if not rolodex_exists:
|
||||||
|
raise ImportValidationError(f"Rolodex ID '{row_data['id']}' does not exist")
|
||||||
|
|
||||||
|
# Parse dates
|
||||||
|
opened_date = None
|
||||||
|
closed_date = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
opened_date = self.parse_date(row_data["opened"])
|
||||||
|
except ValueError as e:
|
||||||
|
raise ImportValidationError(f"Invalid opened date: {e}")
|
||||||
|
|
||||||
|
if row_data.get("closed"):
|
||||||
|
try:
|
||||||
|
closed_date = self.parse_date(row_data["closed"])
|
||||||
|
except ValueError as e:
|
||||||
|
raise ImportValidationError(f"Invalid closed date: {e}")
|
||||||
|
|
||||||
|
# Parse financial fields
|
||||||
|
try:
|
||||||
|
rate_per_hour = self.parse_float(row_data["rate_per_hour"])
|
||||||
|
if rate_per_hour < 0:
|
||||||
|
raise ImportValidationError("Rate per hour cannot be negative")
|
||||||
|
except ValueError as e:
|
||||||
|
raise ImportValidationError(f"Invalid rate per hour: {e}")
|
||||||
|
|
||||||
|
# Parse all financial balance fields
|
||||||
|
financial_fields = [
|
||||||
|
"trust_bal_p", "hours_p", "hourly_fees_p", "flat_fees_p",
|
||||||
|
"disbursements_p", "credit_bal_p", "total_charges_p", "amount_owing_p",
|
||||||
|
"trust_bal", "hours", "hourly_fees", "flat_fees",
|
||||||
|
"disbursements", "credit_bal", "total_charges", "amount_owing", "transferable"
|
||||||
|
]
|
||||||
|
|
||||||
|
financial_data = {}
|
||||||
|
for field in financial_fields:
|
||||||
|
try:
|
||||||
|
financial_data[field] = self.parse_float(row_data.get(field, "0"))
|
||||||
|
except ValueError as e:
|
||||||
|
raise ImportValidationError(f"Invalid {field}: {e}")
|
||||||
|
|
||||||
|
# Create instance
|
||||||
|
files = File(
|
||||||
|
file_no=self.normalize_string(row_data["file_no"], 45),
|
||||||
|
id=self.normalize_string(row_data["id"], 80),
|
||||||
|
regarding=row_data.get("regarding", ""), # Text field
|
||||||
|
empl_num=self.normalize_string(row_data["empl_num"], 10),
|
||||||
|
file_type=self.normalize_string(row_data["file_type"], 45),
|
||||||
|
opened=opened_date,
|
||||||
|
closed=closed_date,
|
||||||
|
status=self.normalize_string(row_data["status"], 45),
|
||||||
|
footer_code=self.normalize_string(row_data.get("footer_code", ""), 45),
|
||||||
|
opposing=self.normalize_string(row_data.get("opposing", ""), 80),
|
||||||
|
rate_per_hour=rate_per_hour,
|
||||||
|
memo=row_data.get("memo", ""), # Text field
|
||||||
|
**financial_data # Unpack all financial fields
|
||||||
|
)
|
||||||
|
|
||||||
|
return files
|
||||||
206
app/import_export/import_service.py
Normal file
206
app/import_export/import_service.py
Normal file
@@ -0,0 +1,206 @@
|
|||||||
|
"""
|
||||||
|
Main Import Service - coordinates all CSV importers
|
||||||
|
"""
|
||||||
|
from typing import Dict, List, Any, Optional, Union
|
||||||
|
import logging
|
||||||
|
from enum import Enum
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from .base import ImportResult
|
||||||
|
from .rolodex_importer import RolodexCSVImporter
|
||||||
|
from .phone_importer import PhoneCSVImporter
|
||||||
|
from .files_importer import FilesCSVImporter
|
||||||
|
from .ledger_importer import LedgerCSVImporter
|
||||||
|
from .qdros_importer import QdrosCSVImporter
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class TableType(Enum):
|
||||||
|
"""Supported table types for import"""
|
||||||
|
ROLODEX = "rolodex"
|
||||||
|
PHONE = "phone"
|
||||||
|
FILES = "files"
|
||||||
|
LEDGER = "ledger"
|
||||||
|
QDROS = "qdros"
|
||||||
|
|
||||||
|
|
||||||
|
class ImportService:
|
||||||
|
"""Main service for handling CSV imports"""
|
||||||
|
|
||||||
|
def __init__(self, db_session: Session):
|
||||||
|
self.db_session = db_session
|
||||||
|
self._importers = {
|
||||||
|
TableType.ROLODEX: RolodexCSVImporter,
|
||||||
|
TableType.PHONE: PhoneCSVImporter,
|
||||||
|
TableType.FILES: FilesCSVImporter,
|
||||||
|
TableType.LEDGER: LedgerCSVImporter,
|
||||||
|
TableType.QDROS: QdrosCSVImporter
|
||||||
|
}
|
||||||
|
|
||||||
|
def get_supported_tables(self) -> List[str]:
|
||||||
|
"""Get list of supported table names"""
|
||||||
|
return [table.value for table in TableType]
|
||||||
|
|
||||||
|
def get_table_schema(self, table_name: str) -> Optional[Dict[str, Any]]:
|
||||||
|
"""Get schema information for a table"""
|
||||||
|
try:
|
||||||
|
table_type = TableType(table_name.lower())
|
||||||
|
importer_class = self._importers[table_type]
|
||||||
|
temp_importer = importer_class(self.db_session, "temp_schema_check")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"table_name": temp_importer.table_name,
|
||||||
|
"required_fields": temp_importer.required_fields,
|
||||||
|
"field_mapping": temp_importer.field_mapping,
|
||||||
|
"sample_headers": list(temp_importer.field_mapping.keys())
|
||||||
|
}
|
||||||
|
except (ValueError, KeyError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def import_csv(
|
||||||
|
self,
|
||||||
|
table_name: str,
|
||||||
|
csv_content: str,
|
||||||
|
encoding: str = "utf-8",
|
||||||
|
import_id: Optional[str] = None
|
||||||
|
) -> ImportResult:
|
||||||
|
"""Import CSV data to specified table"""
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Validate table name
|
||||||
|
table_type = TableType(table_name.lower())
|
||||||
|
except ValueError:
|
||||||
|
result = ImportResult()
|
||||||
|
result.add_error(f"Unsupported table: {table_name}")
|
||||||
|
return result
|
||||||
|
|
||||||
|
# Get appropriate importer
|
||||||
|
importer_class = self._importers[table_type]
|
||||||
|
importer = importer_class(self.db_session, import_id)
|
||||||
|
|
||||||
|
logger.info(f"Starting CSV import for table: {table_name} (import_id: {importer.import_id})")
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Process the CSV
|
||||||
|
result = importer.process_csv_content(csv_content, encoding)
|
||||||
|
|
||||||
|
if result.success:
|
||||||
|
logger.info(
|
||||||
|
f"Successfully imported {result.imported_rows} rows to {table_name}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.warning(
|
||||||
|
f"Import failed for {table_name}: {len(result.errors)} errors"
|
||||||
|
)
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Unexpected error during import to {table_name}: {str(e)}")
|
||||||
|
result = ImportResult()
|
||||||
|
result.add_error(f"Unexpected error: {str(e)}")
|
||||||
|
return result
|
||||||
|
|
||||||
|
def batch_import(
|
||||||
|
self,
|
||||||
|
imports: List[Dict[str, Any]]
|
||||||
|
) -> Dict[str, ImportResult]:
|
||||||
|
"""
|
||||||
|
Import multiple CSV files in a batch
|
||||||
|
|
||||||
|
Args:
|
||||||
|
imports: List of dicts with keys: table_name, csv_content, encoding
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict mapping table names to ImportResult objects
|
||||||
|
"""
|
||||||
|
results = {}
|
||||||
|
|
||||||
|
# Recommended import order (dependencies first)
|
||||||
|
import_order = [
|
||||||
|
TableType.ROLODEX, # No dependencies
|
||||||
|
TableType.PHONE, # Depends on ROLODEX
|
||||||
|
TableType.FILES, # Depends on ROLODEX
|
||||||
|
TableType.LEDGER, # Depends on FILES
|
||||||
|
TableType.QDROS # Depends on FILES
|
||||||
|
]
|
||||||
|
|
||||||
|
# Group imports by table type
|
||||||
|
imports_by_table = {}
|
||||||
|
for import_data in imports:
|
||||||
|
table_name = import_data["table_name"].lower()
|
||||||
|
if table_name not in imports_by_table:
|
||||||
|
imports_by_table[table_name] = []
|
||||||
|
imports_by_table[table_name].append(import_data)
|
||||||
|
|
||||||
|
# Process in dependency order
|
||||||
|
for table_type in import_order:
|
||||||
|
table_name = table_type.value
|
||||||
|
if table_name in imports_by_table:
|
||||||
|
table_imports = imports_by_table[table_name]
|
||||||
|
|
||||||
|
for import_data in table_imports:
|
||||||
|
result = self.import_csv(
|
||||||
|
table_name,
|
||||||
|
import_data["csv_content"],
|
||||||
|
import_data.get("encoding", "utf-8")
|
||||||
|
)
|
||||||
|
|
||||||
|
# Use a unique key if multiple imports for same table
|
||||||
|
key = table_name
|
||||||
|
counter = 1
|
||||||
|
while key in results:
|
||||||
|
counter += 1
|
||||||
|
key = f"{table_name}_{counter}"
|
||||||
|
|
||||||
|
results[key] = result
|
||||||
|
|
||||||
|
# Stop processing if critical import fails
|
||||||
|
if not result.success and table_type in [TableType.ROLODEX, TableType.FILES]:
|
||||||
|
logger.error(f"Critical import failed for {table_name}, stopping batch")
|
||||||
|
break
|
||||||
|
|
||||||
|
return results
|
||||||
|
|
||||||
|
def validate_csv_headers(self, table_name: str, csv_content: str) -> ImportResult:
|
||||||
|
"""Validate CSV headers without importing data"""
|
||||||
|
try:
|
||||||
|
table_type = TableType(table_name.lower())
|
||||||
|
except ValueError:
|
||||||
|
result = ImportResult()
|
||||||
|
result.add_error(f"Unsupported table: {table_name}")
|
||||||
|
return result
|
||||||
|
|
||||||
|
# Get appropriate importer
|
||||||
|
importer_class = self._importers[table_type]
|
||||||
|
importer = importer_class(self.db_session, "validation_check")
|
||||||
|
|
||||||
|
# Parse headers only
|
||||||
|
import csv
|
||||||
|
import io
|
||||||
|
|
||||||
|
try:
|
||||||
|
delimiter = importer.detect_delimiter(csv_content)
|
||||||
|
csv_reader = csv.DictReader(io.StringIO(csv_content), delimiter=delimiter)
|
||||||
|
headers = csv_reader.fieldnames or []
|
||||||
|
|
||||||
|
if not headers:
|
||||||
|
result = ImportResult()
|
||||||
|
result.add_error("No headers found in CSV file")
|
||||||
|
return result
|
||||||
|
|
||||||
|
# Validate headers
|
||||||
|
result = ImportResult()
|
||||||
|
is_valid = importer.validate_headers(headers)
|
||||||
|
result.success = is_valid
|
||||||
|
|
||||||
|
if is_valid:
|
||||||
|
result.add_warning(f"Headers validated successfully for {table_name}")
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
result = ImportResult()
|
||||||
|
result.add_error(f"Error validating headers: {str(e)}")
|
||||||
|
return result
|
||||||
113
app/import_export/ledger_importer.py
Normal file
113
app/import_export/ledger_importer.py
Normal file
@@ -0,0 +1,113 @@
|
|||||||
|
"""
|
||||||
|
LEDGER CSV Importer
|
||||||
|
"""
|
||||||
|
from typing import Dict, List, Any
|
||||||
|
from datetime import date
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from .base import BaseCSVImporter, ImportValidationError
|
||||||
|
from app.models.ledger import Ledger
|
||||||
|
from app.models.files import File
|
||||||
|
|
||||||
|
|
||||||
|
class LedgerCSVImporter(BaseCSVImporter):
|
||||||
|
"""CSV importer for LEDGER table"""
|
||||||
|
|
||||||
|
@property
|
||||||
|
def table_name(self) -> str:
|
||||||
|
return "ledger"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def required_fields(self) -> List[str]:
|
||||||
|
return ["file_no", "date", "t_code", "t_type", "empl_num", "amount"]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def field_mapping(self) -> Dict[str, str]:
|
||||||
|
"""Map CSV headers to database field names"""
|
||||||
|
return {
|
||||||
|
"file_no": "file_no",
|
||||||
|
"item_no": "item_no",
|
||||||
|
"date": "date",
|
||||||
|
"t_code": "t_code",
|
||||||
|
"t_type": "t_type",
|
||||||
|
"t_type_l": "t_type_l",
|
||||||
|
"empl_num": "empl_num",
|
||||||
|
"quantity": "quantity",
|
||||||
|
"rate": "rate",
|
||||||
|
"amount": "amount",
|
||||||
|
"billed": "billed",
|
||||||
|
"note": "note"
|
||||||
|
}
|
||||||
|
|
||||||
|
def create_model_instance(self, row_data: Dict[str, Any]) -> Ledger:
|
||||||
|
"""Create a Ledger instance from processed row data"""
|
||||||
|
|
||||||
|
# Validate required fields
|
||||||
|
required_checks = [
|
||||||
|
("file_no", "File number"),
|
||||||
|
("date", "Date"),
|
||||||
|
("t_code", "Transaction code"),
|
||||||
|
("t_type", "Transaction type"),
|
||||||
|
("empl_num", "Employee number"),
|
||||||
|
("amount", "Amount")
|
||||||
|
]
|
||||||
|
|
||||||
|
for field, display_name in required_checks:
|
||||||
|
if not row_data.get(field):
|
||||||
|
raise ImportValidationError(f"{display_name} is required")
|
||||||
|
|
||||||
|
# Validate foreign key exists (file number)
|
||||||
|
file_exists = self.db_session.query(File).filter_by(file_no=row_data["file_no"]).first()
|
||||||
|
if not file_exists:
|
||||||
|
raise ImportValidationError(f"File number '{row_data['file_no']}' does not exist")
|
||||||
|
|
||||||
|
# Parse date
|
||||||
|
try:
|
||||||
|
transaction_date = self.parse_date(row_data["date"])
|
||||||
|
except ValueError as e:
|
||||||
|
raise ImportValidationError(f"Invalid date: {e}")
|
||||||
|
|
||||||
|
# Parse numeric fields
|
||||||
|
try:
|
||||||
|
item_no = 1 # Default
|
||||||
|
if row_data.get("item_no"):
|
||||||
|
item_no = self.parse_int(row_data["item_no"])
|
||||||
|
if item_no < 1:
|
||||||
|
raise ImportValidationError("Item number must be positive")
|
||||||
|
except ValueError as e:
|
||||||
|
raise ImportValidationError(f"Invalid item number: {e}")
|
||||||
|
|
||||||
|
try:
|
||||||
|
quantity = self.parse_float(row_data.get("quantity", "0"))
|
||||||
|
rate = self.parse_float(row_data.get("rate", "0"))
|
||||||
|
amount = self.parse_float(row_data["amount"])
|
||||||
|
except ValueError as e:
|
||||||
|
raise ImportValidationError(f"Invalid numeric value: {e}")
|
||||||
|
|
||||||
|
# Validate transaction code and type
|
||||||
|
t_code = self.normalize_string(row_data["t_code"], 10)
|
||||||
|
t_type = self.normalize_string(row_data["t_type"], 1)
|
||||||
|
t_type_l = self.normalize_string(row_data.get("t_type_l", ""), 1)
|
||||||
|
|
||||||
|
# Validate billed field (Y/N)
|
||||||
|
billed = row_data.get("billed", "N").strip().upper()
|
||||||
|
if billed not in ["Y", "N", ""]:
|
||||||
|
billed = "N" # Default to N if invalid
|
||||||
|
|
||||||
|
# Create instance
|
||||||
|
ledger = Ledger(
|
||||||
|
file_no=self.normalize_string(row_data["file_no"], 45),
|
||||||
|
item_no=item_no,
|
||||||
|
date=transaction_date,
|
||||||
|
t_code=t_code,
|
||||||
|
t_type=t_type,
|
||||||
|
t_type_l=t_type_l,
|
||||||
|
empl_num=self.normalize_string(row_data["empl_num"], 10),
|
||||||
|
quantity=quantity,
|
||||||
|
rate=rate,
|
||||||
|
amount=amount,
|
||||||
|
billed=billed,
|
||||||
|
note=row_data.get("note", "") # Text field
|
||||||
|
)
|
||||||
|
|
||||||
|
return ledger
|
||||||
160
app/import_export/logging_config.py
Normal file
160
app/import_export/logging_config.py
Normal file
@@ -0,0 +1,160 @@
|
|||||||
|
"""
|
||||||
|
Enhanced logging configuration for import operations
|
||||||
|
"""
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Optional, Dict, Any
|
||||||
|
|
||||||
|
|
||||||
|
class ImportLogger:
|
||||||
|
"""Specialized logger for import operations"""
|
||||||
|
|
||||||
|
def __init__(self, import_id: str, table_name: str):
|
||||||
|
self.import_id = import_id
|
||||||
|
self.table_name = table_name
|
||||||
|
self.logger = logging.getLogger(f"import.{table_name}")
|
||||||
|
|
||||||
|
# Create logs directory if it doesn't exist
|
||||||
|
log_dir = "logs/imports"
|
||||||
|
os.makedirs(log_dir, exist_ok=True)
|
||||||
|
|
||||||
|
# Create file handler for this specific import
|
||||||
|
log_file = os.path.join(log_dir, f"{import_id}_{table_name}.log")
|
||||||
|
file_handler = logging.FileHandler(log_file)
|
||||||
|
file_handler.setLevel(logging.DEBUG)
|
||||||
|
|
||||||
|
# Create formatter
|
||||||
|
formatter = logging.Formatter(
|
||||||
|
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||||
|
)
|
||||||
|
file_handler.setFormatter(formatter)
|
||||||
|
|
||||||
|
# Add handler to logger
|
||||||
|
self.logger.addHandler(file_handler)
|
||||||
|
self.logger.setLevel(logging.DEBUG)
|
||||||
|
|
||||||
|
# Track import session details
|
||||||
|
self.session_start = datetime.utcnow()
|
||||||
|
self.row_count = 0
|
||||||
|
self.error_count = 0
|
||||||
|
|
||||||
|
def info(self, message: str, **kwargs):
|
||||||
|
"""Log info message with import context"""
|
||||||
|
self._log_with_context("info", message, **kwargs)
|
||||||
|
|
||||||
|
def warning(self, message: str, **kwargs):
|
||||||
|
"""Log warning message with import context"""
|
||||||
|
self._log_with_context("warning", message, **kwargs)
|
||||||
|
|
||||||
|
def error(self, message: str, **kwargs):
|
||||||
|
"""Log error message with import context"""
|
||||||
|
self.error_count += 1
|
||||||
|
self._log_with_context("error", message, **kwargs)
|
||||||
|
|
||||||
|
def debug(self, message: str, **kwargs):
|
||||||
|
"""Log debug message with import context"""
|
||||||
|
self._log_with_context("debug", message, **kwargs)
|
||||||
|
|
||||||
|
def _log_with_context(self, level: str, message: str, **kwargs):
|
||||||
|
"""Log message with import context"""
|
||||||
|
context = {
|
||||||
|
"import_id": self.import_id,
|
||||||
|
"table": self.table_name,
|
||||||
|
"row_count": self.row_count,
|
||||||
|
**kwargs
|
||||||
|
}
|
||||||
|
|
||||||
|
context_str = " | ".join([f"{k}={v}" for k, v in context.items()])
|
||||||
|
full_message = f"[{context_str}] {message}"
|
||||||
|
|
||||||
|
getattr(self.logger, level)(full_message)
|
||||||
|
|
||||||
|
def log_row_processed(self, row_number: int, success: bool = True):
|
||||||
|
"""Log that a row has been processed"""
|
||||||
|
self.row_count += 1
|
||||||
|
if success:
|
||||||
|
self.debug(f"Row {row_number} processed successfully")
|
||||||
|
else:
|
||||||
|
self.error(f"Row {row_number} failed to process")
|
||||||
|
|
||||||
|
def log_validation_error(self, row_number: int, field: str, value: Any, error: str):
|
||||||
|
"""Log validation error for specific field"""
|
||||||
|
self.error(
|
||||||
|
f"Validation error on row {row_number}",
|
||||||
|
field=field,
|
||||||
|
value=str(value)[:100], # Truncate long values
|
||||||
|
error=error
|
||||||
|
)
|
||||||
|
|
||||||
|
def log_import_summary(self, total_rows: int, imported_rows: int, error_rows: int):
|
||||||
|
"""Log final import summary"""
|
||||||
|
duration = datetime.utcnow() - self.session_start
|
||||||
|
|
||||||
|
self.info(
|
||||||
|
f"Import completed",
|
||||||
|
total_rows=total_rows,
|
||||||
|
imported_rows=imported_rows,
|
||||||
|
error_rows=error_rows,
|
||||||
|
duration_seconds=duration.total_seconds(),
|
||||||
|
success_rate=f"{(imported_rows/total_rows)*100:.1f}%" if total_rows > 0 else "0%"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def create_import_logger(import_id: str, table_name: str) -> ImportLogger:
|
||||||
|
"""Factory function to create import logger"""
|
||||||
|
return ImportLogger(import_id, table_name)
|
||||||
|
|
||||||
|
|
||||||
|
class ImportMetrics:
|
||||||
|
"""Track import performance metrics"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.start_time = datetime.utcnow()
|
||||||
|
self.end_time = None
|
||||||
|
self.total_rows = 0
|
||||||
|
self.processed_rows = 0
|
||||||
|
self.error_rows = 0
|
||||||
|
self.validation_errors = []
|
||||||
|
self.database_errors = []
|
||||||
|
|
||||||
|
def record_row_processed(self, success: bool = True):
|
||||||
|
"""Record that a row was processed"""
|
||||||
|
self.processed_rows += 1
|
||||||
|
if not success:
|
||||||
|
self.error_rows += 1
|
||||||
|
|
||||||
|
def record_validation_error(self, row_number: int, error: str):
|
||||||
|
"""Record a validation error"""
|
||||||
|
self.validation_errors.append({
|
||||||
|
"row": row_number,
|
||||||
|
"error": error,
|
||||||
|
"timestamp": datetime.utcnow()
|
||||||
|
})
|
||||||
|
|
||||||
|
def record_database_error(self, error: str):
|
||||||
|
"""Record a database error"""
|
||||||
|
self.database_errors.append({
|
||||||
|
"error": error,
|
||||||
|
"timestamp": datetime.utcnow()
|
||||||
|
})
|
||||||
|
|
||||||
|
def finalize(self):
|
||||||
|
"""Finalize metrics collection"""
|
||||||
|
self.end_time = datetime.utcnow()
|
||||||
|
|
||||||
|
def get_summary(self) -> Dict[str, Any]:
|
||||||
|
"""Get metrics summary"""
|
||||||
|
duration = (self.end_time or datetime.utcnow()) - self.start_time
|
||||||
|
|
||||||
|
return {
|
||||||
|
"start_time": self.start_time.isoformat(),
|
||||||
|
"end_time": self.end_time.isoformat() if self.end_time else None,
|
||||||
|
"duration_seconds": duration.total_seconds(),
|
||||||
|
"total_rows": self.total_rows,
|
||||||
|
"processed_rows": self.processed_rows,
|
||||||
|
"error_rows": self.error_rows,
|
||||||
|
"success_rate": (self.processed_rows / self.total_rows * 100) if self.total_rows > 0 else 0,
|
||||||
|
"validation_errors": len(self.validation_errors),
|
||||||
|
"database_errors": len(self.database_errors)
|
||||||
|
}
|
||||||
54
app/import_export/phone_importer.py
Normal file
54
app/import_export/phone_importer.py
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
"""
|
||||||
|
PHONE CSV Importer
|
||||||
|
"""
|
||||||
|
from typing import Dict, List, Any
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from .base import BaseCSVImporter, ImportValidationError
|
||||||
|
from app.models.rolodex import Phone, Rolodex
|
||||||
|
|
||||||
|
|
||||||
|
class PhoneCSVImporter(BaseCSVImporter):
|
||||||
|
"""CSV importer for PHONE table"""
|
||||||
|
|
||||||
|
@property
|
||||||
|
def table_name(self) -> str:
|
||||||
|
return "phone"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def required_fields(self) -> List[str]:
|
||||||
|
return ["rolodex_id", "phone"] # rolodex_id and phone number are required
|
||||||
|
|
||||||
|
@property
|
||||||
|
def field_mapping(self) -> Dict[str, str]:
|
||||||
|
"""Map CSV headers to database field names"""
|
||||||
|
return {
|
||||||
|
"rolodex_id": "rolodex_id",
|
||||||
|
"location": "location",
|
||||||
|
"phone": "phone"
|
||||||
|
}
|
||||||
|
|
||||||
|
def create_model_instance(self, row_data: Dict[str, Any]) -> Phone:
|
||||||
|
"""Create a Phone instance from processed row data"""
|
||||||
|
|
||||||
|
# Validate required fields
|
||||||
|
if not row_data.get("rolodex_id"):
|
||||||
|
raise ImportValidationError("Rolodex ID is required")
|
||||||
|
if not row_data.get("phone"):
|
||||||
|
raise ImportValidationError("Phone number is required")
|
||||||
|
|
||||||
|
# Validate foreign key exists
|
||||||
|
rolodex_exists = self.db_session.query(Rolodex).filter_by(
|
||||||
|
id=row_data["rolodex_id"]
|
||||||
|
).first()
|
||||||
|
if not rolodex_exists:
|
||||||
|
raise ImportValidationError(f"Rolodex ID '{row_data['rolodex_id']}' does not exist")
|
||||||
|
|
||||||
|
# Create instance with field length validation
|
||||||
|
phone = Phone(
|
||||||
|
rolodex_id=self.normalize_string(row_data["rolodex_id"], 80),
|
||||||
|
location=self.normalize_string(row_data.get("location", ""), 45),
|
||||||
|
phone=self.normalize_string(row_data["phone"], 45)
|
||||||
|
)
|
||||||
|
|
||||||
|
return phone
|
||||||
137
app/import_export/qdros_importer.py
Normal file
137
app/import_export/qdros_importer.py
Normal file
@@ -0,0 +1,137 @@
|
|||||||
|
"""
|
||||||
|
QDROS CSV Importer
|
||||||
|
"""
|
||||||
|
from typing import Dict, List, Any
|
||||||
|
from datetime import date
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from .base import BaseCSVImporter, ImportValidationError
|
||||||
|
from app.models.qdro import QDRO
|
||||||
|
from app.models.files import File
|
||||||
|
|
||||||
|
|
||||||
|
class QdrosCSVImporter(BaseCSVImporter):
|
||||||
|
"""CSV importer for QDROS table"""
|
||||||
|
|
||||||
|
@property
|
||||||
|
def table_name(self) -> str:
|
||||||
|
return "qdros"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def required_fields(self) -> List[str]:
|
||||||
|
return ["file_no"] # Only file_no is strictly required
|
||||||
|
|
||||||
|
@property
|
||||||
|
def field_mapping(self) -> Dict[str, str]:
|
||||||
|
"""Map CSV headers to database field names"""
|
||||||
|
return {
|
||||||
|
"file_no": "file_no",
|
||||||
|
"version": "version",
|
||||||
|
"plan_id": "plan_id",
|
||||||
|
# Legacy CSV fields
|
||||||
|
"field1": "field1",
|
||||||
|
"field2": "field2",
|
||||||
|
"part": "part",
|
||||||
|
"altp": "altp",
|
||||||
|
"pet": "pet",
|
||||||
|
"res": "res",
|
||||||
|
# Case information
|
||||||
|
"case_type": "case_type",
|
||||||
|
"case_code": "case_code",
|
||||||
|
"section": "section",
|
||||||
|
"case_number": "case_number",
|
||||||
|
# Dates
|
||||||
|
"judgment_date": "judgment_date",
|
||||||
|
"valuation_date": "valuation_date",
|
||||||
|
"married_on": "married_on",
|
||||||
|
# Award and venue
|
||||||
|
"percent_awarded": "percent_awarded",
|
||||||
|
"ven_city": "ven_city",
|
||||||
|
"ven_cnty": "ven_cnty",
|
||||||
|
"ven_st": "ven_st",
|
||||||
|
# Document status dates
|
||||||
|
"draft_out": "draft_out",
|
||||||
|
"draft_apr": "draft_apr",
|
||||||
|
"final_out": "final_out",
|
||||||
|
# Additional fields
|
||||||
|
"judge": "judge",
|
||||||
|
"form_name": "form_name",
|
||||||
|
"status": "status",
|
||||||
|
"content": "content",
|
||||||
|
"notes": "notes",
|
||||||
|
"approval_status": "approval_status",
|
||||||
|
"approved_date": "approved_date",
|
||||||
|
"filed_date": "filed_date"
|
||||||
|
}
|
||||||
|
|
||||||
|
def create_model_instance(self, row_data: Dict[str, Any]) -> QDRO:
|
||||||
|
"""Create a Qdro instance from processed row data"""
|
||||||
|
|
||||||
|
# Validate required fields
|
||||||
|
if not row_data.get("file_no"):
|
||||||
|
raise ImportValidationError("File number is required")
|
||||||
|
|
||||||
|
# Validate foreign key exists (file number)
|
||||||
|
file_exists = self.db_session.query(File).filter_by(file_no=row_data["file_no"]).first()
|
||||||
|
if not file_exists:
|
||||||
|
raise ImportValidationError(f"File number '{row_data['file_no']}' does not exist")
|
||||||
|
|
||||||
|
# Parse date fields
|
||||||
|
date_fields = [
|
||||||
|
"judgment_date", "valuation_date", "married_on",
|
||||||
|
"draft_out", "draft_apr", "final_out", "approved_date", "filed_date"
|
||||||
|
]
|
||||||
|
|
||||||
|
parsed_dates = {}
|
||||||
|
for field in date_fields:
|
||||||
|
if row_data.get(field):
|
||||||
|
try:
|
||||||
|
parsed_dates[field] = self.parse_date(row_data[field])
|
||||||
|
except ValueError as e:
|
||||||
|
raise ImportValidationError(f"Invalid {field}: {e}")
|
||||||
|
else:
|
||||||
|
parsed_dates[field] = None
|
||||||
|
|
||||||
|
# Validate state abbreviation length
|
||||||
|
ven_st = row_data.get("ven_st", "")
|
||||||
|
if ven_st and len(ven_st) > 2:
|
||||||
|
self.result.add_warning(f"State abbreviation truncated: {ven_st}")
|
||||||
|
ven_st = ven_st[:2]
|
||||||
|
|
||||||
|
# Set default status if not provided
|
||||||
|
status = row_data.get("status", "DRAFT")
|
||||||
|
|
||||||
|
# Create instance
|
||||||
|
qdro = QDRO(
|
||||||
|
file_no=self.normalize_string(row_data["file_no"], 45),
|
||||||
|
version=self.normalize_string(row_data.get("version", "01"), 10),
|
||||||
|
plan_id=self.normalize_string(row_data.get("plan_id", ""), 45),
|
||||||
|
# Legacy CSV fields
|
||||||
|
field1=self.normalize_string(row_data.get("field1", ""), 100),
|
||||||
|
field2=self.normalize_string(row_data.get("field2", ""), 100),
|
||||||
|
part=self.normalize_string(row_data.get("part", ""), 100),
|
||||||
|
altp=self.normalize_string(row_data.get("altp", ""), 100),
|
||||||
|
pet=self.normalize_string(row_data.get("pet", ""), 100),
|
||||||
|
res=self.normalize_string(row_data.get("res", ""), 100),
|
||||||
|
# Case information
|
||||||
|
case_type=self.normalize_string(row_data.get("case_type", ""), 45),
|
||||||
|
case_code=self.normalize_string(row_data.get("case_code", ""), 45),
|
||||||
|
section=self.normalize_string(row_data.get("section", ""), 45),
|
||||||
|
case_number=self.normalize_string(row_data.get("case_number", ""), 100),
|
||||||
|
# Dates
|
||||||
|
**parsed_dates,
|
||||||
|
# Award and venue
|
||||||
|
percent_awarded=self.normalize_string(row_data.get("percent_awarded", ""), 100),
|
||||||
|
ven_city=self.normalize_string(row_data.get("ven_city", ""), 50),
|
||||||
|
ven_cnty=self.normalize_string(row_data.get("ven_cnty", ""), 50),
|
||||||
|
ven_st=ven_st,
|
||||||
|
# Additional fields
|
||||||
|
judge=self.normalize_string(row_data.get("judge", ""), 100),
|
||||||
|
form_name=self.normalize_string(row_data.get("form_name", ""), 200),
|
||||||
|
status=self.normalize_string(status, 45),
|
||||||
|
content=row_data.get("content", ""), # Text field
|
||||||
|
notes=row_data.get("notes", ""), # Text field
|
||||||
|
approval_status=self.normalize_string(row_data.get("approval_status", ""), 45)
|
||||||
|
)
|
||||||
|
|
||||||
|
return qdro
|
||||||
93
app/import_export/rolodex_importer.py
Normal file
93
app/import_export/rolodex_importer.py
Normal file
@@ -0,0 +1,93 @@
|
|||||||
|
"""
|
||||||
|
ROLODEX CSV Importer
|
||||||
|
"""
|
||||||
|
from typing import Dict, List, Any
|
||||||
|
from datetime import date
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from .base import BaseCSVImporter, ImportValidationError
|
||||||
|
from app.models.rolodex import Rolodex
|
||||||
|
|
||||||
|
|
||||||
|
class RolodexCSVImporter(BaseCSVImporter):
|
||||||
|
"""CSV importer for ROLODEX table"""
|
||||||
|
|
||||||
|
@property
|
||||||
|
def table_name(self) -> str:
|
||||||
|
return "rolodex"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def required_fields(self) -> List[str]:
|
||||||
|
return ["id", "last"] # Only ID and last name are required
|
||||||
|
|
||||||
|
@property
|
||||||
|
def field_mapping(self) -> Dict[str, str]:
|
||||||
|
"""Map CSV headers to database field names"""
|
||||||
|
return {
|
||||||
|
"id": "id",
|
||||||
|
"last": "last",
|
||||||
|
"first": "first",
|
||||||
|
"middle": "middle",
|
||||||
|
"prefix": "prefix",
|
||||||
|
"suffix": "suffix",
|
||||||
|
"title": "title",
|
||||||
|
"group": "group",
|
||||||
|
"a1": "a1",
|
||||||
|
"a2": "a2",
|
||||||
|
"a3": "a3",
|
||||||
|
"city": "city",
|
||||||
|
"abrev": "abrev",
|
||||||
|
"zip": "zip",
|
||||||
|
"email": "email",
|
||||||
|
"dob": "dob",
|
||||||
|
"ss_number": "ss_number",
|
||||||
|
"legal_status": "legal_status",
|
||||||
|
"memo": "memo"
|
||||||
|
}
|
||||||
|
|
||||||
|
def create_model_instance(self, row_data: Dict[str, Any]) -> Rolodex:
|
||||||
|
"""Create a Rolodex instance from processed row data"""
|
||||||
|
|
||||||
|
# Validate required fields
|
||||||
|
if not row_data.get("id"):
|
||||||
|
raise ImportValidationError("ID is required")
|
||||||
|
if not row_data.get("last"):
|
||||||
|
raise ImportValidationError("Last name is required")
|
||||||
|
|
||||||
|
# Check for duplicate ID
|
||||||
|
existing = self.db_session.query(Rolodex).filter_by(id=row_data["id"]).first()
|
||||||
|
if existing:
|
||||||
|
raise ImportValidationError(f"Rolodex ID '{row_data['id']}' already exists")
|
||||||
|
|
||||||
|
# Parse date of birth
|
||||||
|
dob = None
|
||||||
|
if row_data.get("dob"):
|
||||||
|
try:
|
||||||
|
dob = self.parse_date(row_data["dob"])
|
||||||
|
except ValueError as e:
|
||||||
|
raise ImportValidationError(f"Invalid date of birth: {e}")
|
||||||
|
|
||||||
|
# Create instance with field length validation
|
||||||
|
rolodex = Rolodex(
|
||||||
|
id=self.normalize_string(row_data["id"], 80),
|
||||||
|
last=self.normalize_string(row_data["last"], 80),
|
||||||
|
first=self.normalize_string(row_data.get("first", ""), 45),
|
||||||
|
middle=self.normalize_string(row_data.get("middle", ""), 45),
|
||||||
|
prefix=self.normalize_string(row_data.get("prefix", ""), 45),
|
||||||
|
suffix=self.normalize_string(row_data.get("suffix", ""), 45),
|
||||||
|
title=self.normalize_string(row_data.get("title", ""), 45),
|
||||||
|
group=self.normalize_string(row_data.get("group", ""), 45),
|
||||||
|
a1=self.normalize_string(row_data.get("a1", ""), 45),
|
||||||
|
a2=self.normalize_string(row_data.get("a2", ""), 45),
|
||||||
|
a3=self.normalize_string(row_data.get("a3", ""), 45),
|
||||||
|
city=self.normalize_string(row_data.get("city", ""), 80),
|
||||||
|
abrev=self.normalize_string(row_data.get("abrev", ""), 45),
|
||||||
|
zip=self.normalize_string(row_data.get("zip", ""), 45),
|
||||||
|
email=self.normalize_string(row_data.get("email", ""), 100),
|
||||||
|
dob=dob,
|
||||||
|
ss_number=self.normalize_string(row_data.get("ss_number", ""), 20),
|
||||||
|
legal_status=self.normalize_string(row_data.get("legal_status", ""), 45),
|
||||||
|
memo=row_data.get("memo", "") # Text field, no length limit
|
||||||
|
)
|
||||||
|
|
||||||
|
return rolodex
|
||||||
11
app/main.py
11
app/main.py
@@ -175,6 +175,7 @@ from app.api.document_workflows import router as document_workflows_router
|
|||||||
from app.api.session_management import router as session_management_router
|
from app.api.session_management import router as session_management_router
|
||||||
from app.api.advanced_templates import router as advanced_templates_router
|
from app.api.advanced_templates import router as advanced_templates_router
|
||||||
from app.api.jobs import router as jobs_router
|
from app.api.jobs import router as jobs_router
|
||||||
|
from app.api.import_csv import router as import_csv_router
|
||||||
|
|
||||||
logger.info("Including API routers")
|
logger.info("Including API routers")
|
||||||
app.include_router(advanced_variables_router, prefix="/api/variables", tags=["advanced-variables"])
|
app.include_router(advanced_variables_router, prefix="/api/variables", tags=["advanced-variables"])
|
||||||
@@ -201,6 +202,7 @@ app.include_router(deadlines_router, prefix="/api/deadlines", tags=["deadlines"]
|
|||||||
app.include_router(document_workflows_router, prefix="/api/workflows", tags=["document-workflows"])
|
app.include_router(document_workflows_router, prefix="/api/workflows", tags=["document-workflows"])
|
||||||
app.include_router(labels_router, prefix="/api/labels", tags=["labels"])
|
app.include_router(labels_router, prefix="/api/labels", tags=["labels"])
|
||||||
app.include_router(jobs_router, prefix="/api/jobs", tags=["jobs"])
|
app.include_router(jobs_router, prefix="/api/jobs", tags=["jobs"])
|
||||||
|
app.include_router(import_csv_router, prefix="/api/admin/import", tags=["import"])
|
||||||
|
|
||||||
|
|
||||||
@app.get("/", response_class=HTMLResponse)
|
@app.get("/", response_class=HTMLResponse)
|
||||||
@@ -284,6 +286,15 @@ async def admin_page(request: Request):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/admin/import", response_class=HTMLResponse)
|
||||||
|
async def admin_import_page(request: Request):
|
||||||
|
"""CSV Import page (admin only)"""
|
||||||
|
return templates.TemplateResponse(
|
||||||
|
"admin_import.html",
|
||||||
|
{"request": request, "title": "CSV Import - " + settings.app_name}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ from .qdro import QDRO, QDROVersion, QDROCommunication
|
|||||||
from .audit import AuditLog, LoginAttempt, ImportAudit, ImportAuditFile
|
from .audit import AuditLog, LoginAttempt, ImportAudit, ImportAuditFile
|
||||||
from .auth import RefreshToken
|
from .auth import RefreshToken
|
||||||
from .additional import Deposit, Payment, FileNote, FormVariable, ReportVariable, Document
|
from .additional import Deposit, Payment, FileNote, FormVariable, ReportVariable, Document
|
||||||
from .flexible import FlexibleImport
|
|
||||||
from .support import SupportTicket, TicketResponse, TicketStatus, TicketPriority, TicketCategory
|
from .support import SupportTicket, TicketResponse, TicketStatus, TicketPriority, TicketCategory
|
||||||
from .pensions import (
|
from .pensions import (
|
||||||
Pension, PensionSchedule, MarriageHistory, DeathBenefit,
|
Pension, PensionSchedule, MarriageHistory, DeathBenefit,
|
||||||
@@ -52,7 +51,7 @@ from .lookups import (
|
|||||||
__all__ = [
|
__all__ = [
|
||||||
"BaseModel", "User", "Rolodex", "Phone", "File", "Ledger", "QDRO", "QDROVersion", "QDROCommunication",
|
"BaseModel", "User", "Rolodex", "Phone", "File", "Ledger", "QDRO", "QDROVersion", "QDROCommunication",
|
||||||
"AuditLog", "LoginAttempt", "ImportAudit", "ImportAuditFile", "RefreshToken",
|
"AuditLog", "LoginAttempt", "ImportAudit", "ImportAuditFile", "RefreshToken",
|
||||||
"Deposit", "Payment", "FileNote", "FormVariable", "ReportVariable", "Document", "FlexibleImport",
|
"Deposit", "Payment", "FileNote", "FormVariable", "ReportVariable", "Document",
|
||||||
"SupportTicket", "TicketResponse", "TicketStatus", "TicketPriority", "TicketCategory",
|
"SupportTicket", "TicketResponse", "TicketStatus", "TicketPriority", "TicketCategory",
|
||||||
"Pension", "PensionSchedule", "MarriageHistory", "DeathBenefit",
|
"Pension", "PensionSchedule", "MarriageHistory", "DeathBenefit",
|
||||||
"SeparationAgreement", "LifeTable", "NumberTable", "PensionResult",
|
"SeparationAgreement", "LifeTable", "NumberTable", "PensionResult",
|
||||||
|
|||||||
477
static/js/admin_import.js
Normal file
477
static/js/admin_import.js
Normal file
@@ -0,0 +1,477 @@
|
|||||||
|
/**
|
||||||
|
* Admin Import JavaScript
|
||||||
|
* Handles CSV file import functionality
|
||||||
|
*/
|
||||||
|
|
||||||
|
class ImportManager {
|
||||||
|
constructor() {
|
||||||
|
this.supportedTables = [];
|
||||||
|
this.batchFileCount = 0;
|
||||||
|
this.currentImportId = null;
|
||||||
|
this.pollInterval = null;
|
||||||
|
|
||||||
|
this.init();
|
||||||
|
}
|
||||||
|
|
||||||
|
async init() {
|
||||||
|
await this.loadSupportedTables();
|
||||||
|
this.setupEventListeners();
|
||||||
|
this.addInitialBatchFile();
|
||||||
|
}
|
||||||
|
|
||||||
|
async loadSupportedTables() {
|
||||||
|
try {
|
||||||
|
console.log('Loading supported tables...');
|
||||||
|
const response = await window.http.wrappedFetch('/api/admin/import/tables');
|
||||||
|
if (response.ok) {
|
||||||
|
const data = await response.json();
|
||||||
|
this.supportedTables = data.tables || [];
|
||||||
|
console.log('Supported tables loaded:', this.supportedTables);
|
||||||
|
} else {
|
||||||
|
console.error('Failed to load supported tables, status:', response.status);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to load supported tables:', error);
|
||||||
|
window.alerts.error('Failed to load supported tables');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
setupEventListeners() {
|
||||||
|
// Single import form
|
||||||
|
document.getElementById('importForm').addEventListener('submit', (e) => {
|
||||||
|
e.preventDefault();
|
||||||
|
this.handleSingleImport();
|
||||||
|
});
|
||||||
|
|
||||||
|
// Validate button
|
||||||
|
document.getElementById('validateBtn').addEventListener('click', () => {
|
||||||
|
this.validateHeaders();
|
||||||
|
});
|
||||||
|
|
||||||
|
// Table selection change
|
||||||
|
document.getElementById('tableSelect').addEventListener('change', (e) => {
|
||||||
|
this.onTableChange(e.target.value);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Batch import buttons
|
||||||
|
document.getElementById('addBatchFile').addEventListener('click', () => {
|
||||||
|
this.addBatchFile();
|
||||||
|
});
|
||||||
|
|
||||||
|
document.getElementById('batchImportBtn').addEventListener('click', () => {
|
||||||
|
this.handleBatchImport();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async onTableChange(tableName) {
|
||||||
|
const schemaInfo = document.getElementById('schemaInfo');
|
||||||
|
const schemaDetails = document.getElementById('schemaDetails');
|
||||||
|
|
||||||
|
if (!tableName) {
|
||||||
|
schemaInfo.classList.add('hidden');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
console.log('Loading schema for table:', tableName);
|
||||||
|
const response = await window.http.wrappedFetch(`/api/admin/import/tables/${tableName}/schema`);
|
||||||
|
if (response.ok) {
|
||||||
|
const data = await response.json();
|
||||||
|
const schema = data.schema;
|
||||||
|
console.log('Schema loaded for', tableName, ':', schema);
|
||||||
|
|
||||||
|
let html = '<div class="grid grid-cols-1 md:grid-cols-2 gap-4">';
|
||||||
|
html += '<div><h4 class="font-semibold mb-2">Required Fields:</h4>';
|
||||||
|
html += '<ul class="list-disc list-inside space-y-1">';
|
||||||
|
schema.required_fields.forEach(field => {
|
||||||
|
html += `<li><code class="bg-blue-100 px-1 rounded">${field}</code></li>`;
|
||||||
|
});
|
||||||
|
html += '</ul></div>';
|
||||||
|
|
||||||
|
html += '<div><h4 class="font-semibold mb-2">All Available Fields:</h4>';
|
||||||
|
html += '<div class="max-h-32 overflow-y-auto">';
|
||||||
|
html += '<div class="grid grid-cols-2 gap-1 text-xs">';
|
||||||
|
Object.keys(schema.field_mapping).forEach(field => {
|
||||||
|
html += `<code class="bg-gray-100 px-1 rounded">${field}</code>`;
|
||||||
|
});
|
||||||
|
html += '</div></div></div></div>';
|
||||||
|
|
||||||
|
schemaDetails.innerHTML = html;
|
||||||
|
schemaInfo.classList.remove('hidden');
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to load schema:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async validateHeaders() {
|
||||||
|
const tableSelect = document.getElementById('tableSelect');
|
||||||
|
const fileInput = document.getElementById('csvFile');
|
||||||
|
|
||||||
|
console.log('Starting header validation...');
|
||||||
|
|
||||||
|
if (!tableSelect.value) {
|
||||||
|
console.warn('No table selected for validation');
|
||||||
|
window.alerts.error('Please select a table type');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!fileInput.files[0]) {
|
||||||
|
console.warn('No file selected for validation');
|
||||||
|
window.alerts.error('Please select a CSV file');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('Validating headers for table:', tableSelect.value, 'file:', fileInput.files[0].name);
|
||||||
|
|
||||||
|
const formData = new FormData();
|
||||||
|
formData.append('table_name', tableSelect.value);
|
||||||
|
formData.append('file', fileInput.files[0]);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await window.http.wrappedFetch('/api/admin/import/validate', {
|
||||||
|
method: 'POST',
|
||||||
|
body: formData
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('Validation response status:', response.status);
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
const result = await response.json();
|
||||||
|
console.log('Validation result:', result);
|
||||||
|
if (result.success) {
|
||||||
|
window.alerts.success('CSV headers validated successfully!');
|
||||||
|
} else {
|
||||||
|
const errors = result.validation_result.errors.join('\\n');
|
||||||
|
console.error('Validation errors:', result.validation_result.errors);
|
||||||
|
window.alerts.error(`Validation failed:\\n${errors}`);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
const error = await response.json();
|
||||||
|
console.error('Validation failed with error:', error);
|
||||||
|
window.alerts.error(`Validation failed: ${error.detail}`);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Validation error:', error);
|
||||||
|
window.alerts.error('Failed to validate CSV headers');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async handleSingleImport() {
|
||||||
|
const tableSelect = document.getElementById('tableSelect');
|
||||||
|
const fileInput = document.getElementById('csvFile');
|
||||||
|
|
||||||
|
console.log('Starting single import...');
|
||||||
|
|
||||||
|
if (!tableSelect.value) {
|
||||||
|
console.warn('No table selected for import');
|
||||||
|
window.alerts.error('Please select a table type');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!fileInput.files[0]) {
|
||||||
|
console.warn('No file selected for import');
|
||||||
|
window.alerts.error('Please select a CSV file');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('Importing to table:', tableSelect.value, 'file:', fileInput.files[0].name);
|
||||||
|
|
||||||
|
const formData = new FormData();
|
||||||
|
formData.append('table_name', tableSelect.value);
|
||||||
|
formData.append('file', fileInput.files[0]);
|
||||||
|
|
||||||
|
// Show progress
|
||||||
|
this.showProgress();
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await window.http.wrappedFetch('/api/admin/import/csv', {
|
||||||
|
method: 'POST',
|
||||||
|
body: formData
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('Import response status:', response.status);
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
const result = await response.json();
|
||||||
|
console.log('Import started successfully:', result);
|
||||||
|
this.currentImportId = result.import_id;
|
||||||
|
this.updateProgress(`Import started for ${result.table_name} (ID: ${result.import_id})`, 'info');
|
||||||
|
this.startPolling();
|
||||||
|
} else {
|
||||||
|
const error = await response.json();
|
||||||
|
console.error('Import failed:', error);
|
||||||
|
this.updateProgress(`Import failed: ${error.detail}`, 'error');
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Import error:', error);
|
||||||
|
this.updateProgress('Import failed: Network error', 'error');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
addInitialBatchFile() {
|
||||||
|
this.addBatchFile();
|
||||||
|
}
|
||||||
|
|
||||||
|
addBatchFile() {
|
||||||
|
this.batchFileCount++;
|
||||||
|
const container = document.getElementById('batchFiles');
|
||||||
|
|
||||||
|
const fileDiv = document.createElement('div');
|
||||||
|
fileDiv.className = 'flex space-x-3 items-center';
|
||||||
|
fileDiv.id = `batchFile${this.batchFileCount}`;
|
||||||
|
|
||||||
|
fileDiv.innerHTML = `
|
||||||
|
<select name="batch_table_${this.batchFileCount}" class="flex-none w-40 px-3 py-2 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500">
|
||||||
|
<option value="">-- Select Table --</option>
|
||||||
|
<option value="rolodex">ROLODEX</option>
|
||||||
|
<option value="phone">PHONE</option>
|
||||||
|
<option value="files">FILES</option>
|
||||||
|
<option value="ledger">LEDGER</option>
|
||||||
|
<option value="qdros">QDROS</option>
|
||||||
|
</select>
|
||||||
|
<input type="file" name="batch_file_${this.batchFileCount}" accept=".csv"
|
||||||
|
class="flex-1 px-3 py-2 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500">
|
||||||
|
<button type="button" onclick="importManager.removeBatchFile(${this.batchFileCount})"
|
||||||
|
class="flex-none px-3 py-2 bg-red-600 text-white rounded-md hover:bg-red-700 focus:outline-none focus:ring-2 focus:ring-red-500">
|
||||||
|
Remove
|
||||||
|
</button>
|
||||||
|
`;
|
||||||
|
|
||||||
|
container.appendChild(fileDiv);
|
||||||
|
}
|
||||||
|
|
||||||
|
removeBatchFile(fileId) {
|
||||||
|
const fileDiv = document.getElementById(`batchFile${fileId}`);
|
||||||
|
if (fileDiv) {
|
||||||
|
fileDiv.remove();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async handleBatchImport() {
|
||||||
|
const batchFiles = document.getElementById('batchFiles');
|
||||||
|
const fileDivs = batchFiles.children;
|
||||||
|
|
||||||
|
const formData = new FormData();
|
||||||
|
const tableNames = [];
|
||||||
|
let hasFiles = false;
|
||||||
|
|
||||||
|
for (let i = 0; i < fileDivs.length; i++) {
|
||||||
|
const div = fileDivs[i];
|
||||||
|
const tableSelect = div.querySelector('select');
|
||||||
|
const fileInput = div.querySelector('input[type="file"]');
|
||||||
|
|
||||||
|
if (tableSelect.value && fileInput.files[0]) {
|
||||||
|
tableNames.push(tableSelect.value);
|
||||||
|
formData.append('files', fileInput.files[0]);
|
||||||
|
hasFiles = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!hasFiles) {
|
||||||
|
window.alerts.error('Please select at least one table and file');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add table names to form data
|
||||||
|
tableNames.forEach(name => {
|
||||||
|
formData.append('table_names', name);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Show progress
|
||||||
|
this.showProgress();
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await window.http.wrappedFetch('/api/admin/import/batch', {
|
||||||
|
method: 'POST',
|
||||||
|
body: formData
|
||||||
|
});
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
const result = await response.json();
|
||||||
|
this.currentImportIds = result.import_ids;
|
||||||
|
this.updateProgress(`Batch import started for ${result.total_files} files`, 'info');
|
||||||
|
this.startBatchPolling();
|
||||||
|
} else {
|
||||||
|
const error = await response.json();
|
||||||
|
this.updateProgress(`Batch import failed: ${error.detail}`, 'error');
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Batch import error:', error);
|
||||||
|
this.updateProgress('Batch import failed: Network error', 'error');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
showProgress() {
|
||||||
|
document.getElementById('importProgress').classList.remove('hidden');
|
||||||
|
document.getElementById('importResults').classList.add('hidden');
|
||||||
|
}
|
||||||
|
|
||||||
|
updateProgress(message, type = 'info') {
|
||||||
|
const progressDetails = document.getElementById('progressDetails');
|
||||||
|
const timestamp = new Date().toLocaleTimeString();
|
||||||
|
|
||||||
|
let colorClass = 'text-blue-600';
|
||||||
|
if (type === 'error') colorClass = 'text-red-600';
|
||||||
|
if (type === 'success') colorClass = 'text-green-600';
|
||||||
|
if (type === 'warning') colorClass = 'text-yellow-600';
|
||||||
|
|
||||||
|
progressDetails.innerHTML += `
|
||||||
|
<div class="flex items-center space-x-2 mb-2">
|
||||||
|
<span class="text-gray-500 text-sm">${timestamp}</span>
|
||||||
|
<span class="${colorClass}">${message}</span>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
|
||||||
|
// Scroll to bottom
|
||||||
|
progressDetails.scrollTop = progressDetails.scrollHeight;
|
||||||
|
}
|
||||||
|
|
||||||
|
startPolling() {
|
||||||
|
if (this.pollInterval) {
|
||||||
|
clearInterval(this.pollInterval);
|
||||||
|
}
|
||||||
|
|
||||||
|
this.pollInterval = setInterval(async () => {
|
||||||
|
await this.checkImportStatus();
|
||||||
|
}, 2000); // Poll every 2 seconds
|
||||||
|
}
|
||||||
|
|
||||||
|
startBatchPolling() {
|
||||||
|
if (this.pollInterval) {
|
||||||
|
clearInterval(this.pollInterval);
|
||||||
|
}
|
||||||
|
|
||||||
|
this.pollInterval = setInterval(async () => {
|
||||||
|
await this.checkBatchStatus();
|
||||||
|
}, 2000); // Poll every 2 seconds
|
||||||
|
}
|
||||||
|
|
||||||
|
async checkImportStatus() {
|
||||||
|
if (!this.currentImportId) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await window.http.wrappedFetch(`/api/admin/import/status/${this.currentImportId}`);
|
||||||
|
if (response.ok) {
|
||||||
|
const status = await response.json();
|
||||||
|
|
||||||
|
if (status.status === 'COMPLETED') {
|
||||||
|
clearInterval(this.pollInterval);
|
||||||
|
this.updateProgress('Import completed successfully!', 'success');
|
||||||
|
this.showResults(status.result);
|
||||||
|
} else if (status.status === 'FAILED') {
|
||||||
|
clearInterval(this.pollInterval);
|
||||||
|
this.updateProgress(`Import failed: ${status.error || 'Unknown error'}`, 'error');
|
||||||
|
if (status.result) {
|
||||||
|
this.showResults(status.result);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
this.updateProgress(`Import status: ${status.status}`, 'info');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Status check error:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async checkBatchStatus() {
|
||||||
|
if (!this.currentImportIds || !Array.isArray(this.currentImportIds)) return;
|
||||||
|
|
||||||
|
let allCompleted = true;
|
||||||
|
let anyFailed = false;
|
||||||
|
|
||||||
|
for (const importId of this.currentImportIds) {
|
||||||
|
try {
|
||||||
|
const response = await window.http.wrappedFetch(`/api/admin/import/status/${importId}`);
|
||||||
|
if (response.ok) {
|
||||||
|
const status = await response.json();
|
||||||
|
|
||||||
|
if (status.status === 'PROCESSING') {
|
||||||
|
allCompleted = false;
|
||||||
|
} else if (status.status === 'FAILED') {
|
||||||
|
anyFailed = true;
|
||||||
|
this.updateProgress(`${status.table_name} import failed: ${status.error || 'Unknown error'}`, 'error');
|
||||||
|
} else if (status.status === 'COMPLETED') {
|
||||||
|
this.updateProgress(`${status.table_name} import completed`, 'success');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Batch status check error:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (allCompleted) {
|
||||||
|
clearInterval(this.pollInterval);
|
||||||
|
const message = anyFailed ? 'Batch import completed with some failures' : 'Batch import completed successfully!';
|
||||||
|
const type = anyFailed ? 'warning' : 'success';
|
||||||
|
this.updateProgress(message, type);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
showResults(result) {
|
||||||
|
const resultsContent = document.getElementById('resultsContent');
|
||||||
|
const resultsDiv = document.getElementById('importResults');
|
||||||
|
|
||||||
|
let html = '<div class="space-y-4">';
|
||||||
|
|
||||||
|
// Summary
|
||||||
|
html += `
|
||||||
|
<div class="grid grid-cols-2 md:grid-cols-4 gap-4">
|
||||||
|
<div class="bg-blue-50 p-3 rounded">
|
||||||
|
<div class="text-2xl font-bold text-blue-600">${result.total_rows}</div>
|
||||||
|
<div class="text-sm text-blue-800">Total Rows</div>
|
||||||
|
</div>
|
||||||
|
<div class="bg-green-50 p-3 rounded">
|
||||||
|
<div class="text-2xl font-bold text-green-600">${result.imported_rows}</div>
|
||||||
|
<div class="text-sm text-green-800">Imported</div>
|
||||||
|
</div>
|
||||||
|
<div class="bg-yellow-50 p-3 rounded">
|
||||||
|
<div class="text-2xl font-bold text-yellow-600">${result.skipped_rows}</div>
|
||||||
|
<div class="text-sm text-yellow-800">Skipped</div>
|
||||||
|
</div>
|
||||||
|
<div class="bg-red-50 p-3 rounded">
|
||||||
|
<div class="text-2xl font-bold text-red-600">${result.error_rows}</div>
|
||||||
|
<div class="text-sm text-red-800">Errors</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
|
||||||
|
// Errors
|
||||||
|
if (result.errors && result.errors.length > 0) {
|
||||||
|
html += '<div class="bg-red-50 border border-red-200 rounded p-4">';
|
||||||
|
html += '<h4 class="font-semibold text-red-800 mb-2">Errors:</h4>';
|
||||||
|
html += '<div class="text-sm text-red-700 space-y-1 max-h-40 overflow-y-auto">';
|
||||||
|
result.errors.forEach(error => {
|
||||||
|
html += `<div>${this.escapeHtml(error)}</div>`;
|
||||||
|
});
|
||||||
|
html += '</div></div>';
|
||||||
|
}
|
||||||
|
|
||||||
|
// Warnings
|
||||||
|
if (result.warnings && result.warnings.length > 0) {
|
||||||
|
html += '<div class="bg-yellow-50 border border-yellow-200 rounded p-4">';
|
||||||
|
html += '<h4 class="font-semibold text-yellow-800 mb-2">Warnings:</h4>';
|
||||||
|
html += '<div class="text-sm text-yellow-700 space-y-1 max-h-40 overflow-y-auto">';
|
||||||
|
result.warnings.forEach(warning => {
|
||||||
|
html += `<div>${this.escapeHtml(warning)}</div>`;
|
||||||
|
});
|
||||||
|
html += '</div></div>';
|
||||||
|
}
|
||||||
|
|
||||||
|
html += '</div>';
|
||||||
|
|
||||||
|
resultsContent.innerHTML = html;
|
||||||
|
resultsDiv.classList.remove('hidden');
|
||||||
|
}
|
||||||
|
|
||||||
|
escapeHtml(text) {
|
||||||
|
const div = document.createElement('div');
|
||||||
|
div.textContent = text;
|
||||||
|
return div.innerHTML;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize when DOM is loaded
|
||||||
|
document.addEventListener('DOMContentLoaded', () => {
|
||||||
|
window.importManager = new ImportManager();
|
||||||
|
});
|
||||||
133
templates/admin_import.html
Normal file
133
templates/admin_import.html
Normal file
@@ -0,0 +1,133 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
<title>{{ title }}</title>
|
||||||
|
<link href="/static/css/output.css" rel="stylesheet">
|
||||||
|
<script src="/static/js/http.js"></script>
|
||||||
|
<script src="/static/js/alerts.js"></script>
|
||||||
|
<script src="/static/js/main.js"></script>
|
||||||
|
</head>
|
||||||
|
<body class="bg-gray-100">
|
||||||
|
<div id="main-container" class="min-h-screen">
|
||||||
|
<!-- Navigation -->
|
||||||
|
<nav class="bg-blue-600 text-white shadow-lg">
|
||||||
|
<div class="max-w-7xl mx-auto px-4">
|
||||||
|
<div class="flex justify-between h-16">
|
||||||
|
<div class="flex items-center">
|
||||||
|
<h1 class="text-xl font-bold">Data Import</h1>
|
||||||
|
</div>
|
||||||
|
<div class="flex items-center space-x-4">
|
||||||
|
<a href="/admin" class="hover:text-blue-200">← Back to Admin</a>
|
||||||
|
<div id="userDropdown" class="relative">
|
||||||
|
<button class="flex items-center space-x-2 hover:text-blue-200">
|
||||||
|
<span>Loading...</span>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
<!-- Main Content -->
|
||||||
|
<div class="max-w-4xl mx-auto py-8 px-4">
|
||||||
|
<!-- Page Header -->
|
||||||
|
<div class="mb-8">
|
||||||
|
<h2 class="text-3xl font-bold text-gray-900 mb-4">CSV Data Import</h2>
|
||||||
|
<p class="text-gray-600">
|
||||||
|
Import CSV files converted from legacy .sc files into the database.
|
||||||
|
Please ensure your CSV files follow the correct format for each table.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Import Form -->
|
||||||
|
<div class="bg-white rounded-lg shadow-md p-6 mb-8">
|
||||||
|
<h3 class="text-xl font-semibold mb-4">Single File Import</h3>
|
||||||
|
|
||||||
|
<form id="importForm" class="space-y-4">
|
||||||
|
<div>
|
||||||
|
<label for="tableSelect" class="block text-sm font-medium text-gray-700 mb-2">
|
||||||
|
Select Table Type
|
||||||
|
</label>
|
||||||
|
<select id="tableSelect" name="table_name"
|
||||||
|
class="w-full px-3 py-2 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent">
|
||||||
|
<option value="">-- Select Table --</option>
|
||||||
|
<option value="rolodex">ROLODEX (Contacts)</option>
|
||||||
|
<option value="phone">PHONE (Phone Numbers)</option>
|
||||||
|
<option value="files">FILES (Case Files)</option>
|
||||||
|
<option value="ledger">LEDGER (Financial Transactions)</option>
|
||||||
|
<option value="qdros">QDROS (QDRO Documents)</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<label for="csvFile" class="block text-sm font-medium text-gray-700 mb-2">
|
||||||
|
Select CSV File
|
||||||
|
</label>
|
||||||
|
<input type="file" id="csvFile" name="file" accept=".csv"
|
||||||
|
class="w-full px-3 py-2 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent">
|
||||||
|
<p class="text-sm text-gray-500 mt-1">Only CSV files are supported. Max file size: 100MB</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex space-x-4">
|
||||||
|
<button type="button" id="validateBtn"
|
||||||
|
class="px-4 py-2 bg-yellow-600 text-white rounded-md hover:bg-yellow-700 focus:outline-none focus:ring-2 focus:ring-yellow-500 disabled:opacity-50">
|
||||||
|
Validate Headers
|
||||||
|
</button>
|
||||||
|
<button type="submit" id="importBtn"
|
||||||
|
class="px-4 py-2 bg-blue-600 text-white rounded-md hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-blue-500 disabled:opacity-50">
|
||||||
|
Import Data
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Schema Information -->
|
||||||
|
<div id="schemaInfo" class="bg-blue-50 rounded-lg p-6 mb-8 hidden">
|
||||||
|
<h3 class="text-lg font-semibold text-blue-900 mb-3">Required Columns</h3>
|
||||||
|
<div id="schemaDetails" class="text-sm text-blue-800"></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Import Progress -->
|
||||||
|
<div id="importProgress" class="bg-white rounded-lg shadow-md p-6 mb-8 hidden">
|
||||||
|
<h3 class="text-xl font-semibold mb-4">Import Progress</h3>
|
||||||
|
<div id="progressDetails"></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Import Results -->
|
||||||
|
<div id="importResults" class="bg-white rounded-lg shadow-md p-6 hidden">
|
||||||
|
<h3 class="text-xl font-semibold mb-4">Import Results</h3>
|
||||||
|
<div id="resultsContent"></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Batch Import Section -->
|
||||||
|
<div class="bg-white rounded-lg shadow-md p-6">
|
||||||
|
<h3 class="text-xl font-semibold mb-4">Batch Import (Multiple Files)</h3>
|
||||||
|
<p class="text-gray-600 mb-4">
|
||||||
|
Import multiple CSV files at once. Files will be processed in dependency order
|
||||||
|
(ROLODEX → PHONE/FILES → LEDGER/QDROS).
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<div id="batchFiles" class="space-y-3">
|
||||||
|
<!-- Batch file inputs will be added here -->
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex space-x-4 mt-4">
|
||||||
|
<button type="button" id="addBatchFile"
|
||||||
|
class="px-4 py-2 bg-green-600 text-white rounded-md hover:bg-green-700 focus:outline-none focus:ring-2 focus:ring-green-500">
|
||||||
|
Add File
|
||||||
|
</button>
|
||||||
|
<button type="button" id="batchImportBtn"
|
||||||
|
class="px-4 py-2 bg-purple-600 text-white rounded-md hover:bg-purple-700 focus:outline-none focus:ring-2 focus:ring-purple-500 disabled:opacity-50">
|
||||||
|
Start Batch Import
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Include the import JavaScript -->
|
||||||
|
<script src="/static/js/admin_import.js"></script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
Reference in New Issue
Block a user