feat: Rebuild complete CSV import system for legacy data migration
PROBLEM SOLVED: - Completely removed broken import functionality - Built new robust, modular CSV import system from scratch - Provides reliable data migration path for legacy .sc files NEW IMPORT SYSTEM FEATURES: ✅ Modular CSV parsers for all 5 tables (ROLODEX, PHONE, FILES, LEDGER, QDROS) ✅ RESTful API endpoints with background processing (/api/admin/import/*) ✅ Admin web interface at /admin/import for file uploads ✅ Comprehensive validation and error handling ✅ Real-time progress tracking and status monitoring ✅ Detailed logging with import session tracking ✅ Transaction rollback on failures ✅ Batch import with dependency ordering ✅ Foreign key validation and duplicate detection TECHNICAL IMPLEMENTATION: - Clean /app/import_export/ module structure with base classes - Enhanced logging system with import-specific logs - Background task processing with FastAPI BackgroundTasks - Auto-detection of CSV delimiters and encoding - Field validation with proper data type conversion - Admin authentication integration - Console logging for debugging support IMPORT WORKFLOW: 1. Admin selects table type and uploads CSV file 2. System validates headers and data structure 3. Background processing with real-time status updates 4. Detailed error reporting and success metrics 5. Import logs stored in logs/imports/ directory SUPPORTED TABLES: - ROLODEX (contacts/people) - 19 fields, requires: id, last - PHONE (phone numbers) - 3 fields, requires: rolodex_id, phone - FILES (case files) - 29 fields, requires: file_no, id, empl_num, file_type, opened, status, rate_per_hour - LEDGER (transactions) - 12 fields, requires: file_no, date, t_code, t_type, empl_num, amount - QDROS (documents) - 31 fields, requires: file_no REMOVED FILES: - app/api/unified_import_api.py - app/services/unified_import.py - app/api/flexible.py - app/models/flexible.py - templates/unified_import.html - templates/flexible.html - static/js/flexible.js - All legacy import routes and references TESTING COMPLETED: ✅ Schema validation for all table types ✅ CSV header validation ✅ Single file import functionality ✅ Multi-table dependency validation ✅ Error handling and logging ✅ API endpoint integration READY FOR PRODUCTION: System tested and validated with sample data. Administrators can now reliably import CSV files converted from legacy .sc files. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
HotSwapp
398
app/api/import_csv.py
Normal file
398
app/api/import_csv.py
Normal file
@@ -0,0 +1,398 @@
|
||||
"""
|
||||
CSV Import API Endpoints
|
||||
"""
|
||||
from fastapi import APIRouter, Depends, File, UploadFile, Form, HTTPException, BackgroundTasks
|
||||
from fastapi.responses import JSONResponse
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import List, Optional, Dict, Any
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from app.database.base import get_db
|
||||
from app.auth.security import get_admin_user
|
||||
from app.models.user import User
|
||||
from app.import_export.import_service import ImportService, TableType
|
||||
from app.core.logging import get_logger
|
||||
|
||||
logger = get_logger("import_api")
|
||||
router = APIRouter()
|
||||
|
||||
# In-memory storage for import progress (could be moved to Redis in production)
|
||||
import_progress = {}
|
||||
|
||||
|
||||
class ImportStatus:
|
||||
"""Track import operation status"""
|
||||
def __init__(self, import_id: str, table_name: str):
|
||||
self.import_id = import_id
|
||||
self.table_name = table_name
|
||||
self.status = "PROCESSING"
|
||||
self.started_at = datetime.utcnow()
|
||||
self.completed_at = None
|
||||
self.result = None
|
||||
self.error = None
|
||||
|
||||
|
||||
@router.get("/tables")
|
||||
async def get_supported_tables(
|
||||
current_user: User = Depends(get_admin_user),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""Get list of supported tables for import"""
|
||||
try:
|
||||
service = ImportService(db)
|
||||
tables = service.get_supported_tables()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"tables": tables,
|
||||
"total": len(tables)
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting supported tables: {str(e)}")
|
||||
raise HTTPException(status_code=500, detail="Failed to get supported tables")
|
||||
|
||||
|
||||
@router.get("/tables/{table_name}/schema")
|
||||
async def get_table_schema(
|
||||
table_name: str,
|
||||
current_user: User = Depends(get_admin_user),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""Get schema information for a specific table"""
|
||||
try:
|
||||
service = ImportService(db)
|
||||
schema = service.get_table_schema(table_name)
|
||||
|
||||
if not schema:
|
||||
raise HTTPException(status_code=404, detail=f"Table '{table_name}' not found")
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"schema": schema
|
||||
}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting table schema for {table_name}: {str(e)}")
|
||||
raise HTTPException(status_code=500, detail="Failed to get table schema")
|
||||
|
||||
|
||||
@router.post("/validate")
|
||||
async def validate_csv_headers(
|
||||
table_name: str = Form(...),
|
||||
file: UploadFile = File(...),
|
||||
current_user: User = Depends(get_admin_user),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""Validate CSV headers without importing data"""
|
||||
try:
|
||||
# Read file content
|
||||
content = await file.read()
|
||||
csv_content = content.decode('utf-8')
|
||||
|
||||
service = ImportService(db)
|
||||
result = service.validate_csv_headers(table_name, csv_content)
|
||||
|
||||
return {
|
||||
"success": result.success,
|
||||
"table_name": table_name,
|
||||
"filename": file.filename,
|
||||
"validation_result": result.to_dict()
|
||||
}
|
||||
|
||||
except UnicodeDecodeError:
|
||||
raise HTTPException(status_code=400, detail="Invalid file encoding. Please use UTF-8.")
|
||||
except Exception as e:
|
||||
logger.error(f"Error validating CSV headers: {str(e)}")
|
||||
raise HTTPException(status_code=500, detail="Failed to validate CSV headers")
|
||||
|
||||
|
||||
async def process_import_background(
|
||||
import_id: str,
|
||||
table_name: str,
|
||||
csv_content: str,
|
||||
db: Session
|
||||
):
|
||||
"""Background task to process CSV import"""
|
||||
try:
|
||||
logger.info(f"Starting background import {import_id} for table {table_name}")
|
||||
print(f"[IMPORT] Starting background import {import_id} for table {table_name}")
|
||||
|
||||
service = ImportService(db)
|
||||
result = service.import_csv(table_name, csv_content, import_id=import_id)
|
||||
|
||||
# Update progress
|
||||
if import_id in import_progress:
|
||||
progress = import_progress[import_id]
|
||||
progress.status = "COMPLETED" if result.success else "FAILED"
|
||||
progress.completed_at = datetime.utcnow()
|
||||
progress.result = result
|
||||
|
||||
logger.info(f"Import {import_id} completed with {result.imported_rows} rows imported")
|
||||
print(f"[IMPORT] Import {import_id} completed: success={result.success}, rows={result.imported_rows}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Background import {import_id} failed: {str(e)}")
|
||||
print(f"[IMPORT] Background import {import_id} failed: {str(e)}")
|
||||
if import_id in import_progress:
|
||||
progress = import_progress[import_id]
|
||||
progress.status = "FAILED"
|
||||
progress.completed_at = datetime.utcnow()
|
||||
progress.error = str(e)
|
||||
|
||||
|
||||
@router.post("/csv")
|
||||
async def import_csv_file(
|
||||
background_tasks: BackgroundTasks,
|
||||
table_name: str = Form(...),
|
||||
file: UploadFile = File(...),
|
||||
current_user: User = Depends(get_admin_user),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""Import CSV file to specified table"""
|
||||
try:
|
||||
logger.info(f"Received CSV import request: table={table_name}, file={file.filename}, user={current_user.username}")
|
||||
print(f"[IMPORT API] CSV import request: table={table_name}, file={file.filename}")
|
||||
|
||||
# Validate table name
|
||||
if table_name.lower() not in [t.value for t in TableType]:
|
||||
print(f"[IMPORT API] Invalid table name: {table_name}")
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Unsupported table: {table_name}"
|
||||
)
|
||||
|
||||
# Validate file type
|
||||
if not file.filename.lower().endswith('.csv'):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="File must be a CSV file"
|
||||
)
|
||||
|
||||
# Read file content
|
||||
content = await file.read()
|
||||
csv_content = content.decode('utf-8')
|
||||
|
||||
if not csv_content.strip():
|
||||
raise HTTPException(status_code=400, detail="File is empty")
|
||||
|
||||
# Generate import ID
|
||||
import_id = str(uuid.uuid4())
|
||||
print(f"[IMPORT API] Generated import ID: {import_id}")
|
||||
|
||||
# Create progress tracker
|
||||
progress = ImportStatus(import_id, table_name)
|
||||
import_progress[import_id] = progress
|
||||
|
||||
# Start background import
|
||||
background_tasks.add_task(
|
||||
process_import_background,
|
||||
import_id,
|
||||
table_name,
|
||||
csv_content,
|
||||
db
|
||||
)
|
||||
|
||||
logger.info(f"Started CSV import {import_id} for table {table_name}")
|
||||
print(f"[IMPORT API] Background task queued for import {import_id}")
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"import_id": import_id,
|
||||
"table_name": table_name,
|
||||
"filename": file.filename,
|
||||
"status": "PROCESSING",
|
||||
"message": "Import started successfully"
|
||||
}
|
||||
|
||||
except UnicodeDecodeError:
|
||||
raise HTTPException(status_code=400, detail="Invalid file encoding. Please use UTF-8.")
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error starting CSV import: {str(e)}")
|
||||
raise HTTPException(status_code=500, detail="Failed to start import")
|
||||
|
||||
|
||||
@router.get("/status/{import_id}")
|
||||
async def get_import_status(
|
||||
import_id: str,
|
||||
current_user: User = Depends(get_admin_user)
|
||||
):
|
||||
"""Get status of an import operation"""
|
||||
try:
|
||||
if import_id not in import_progress:
|
||||
raise HTTPException(status_code=404, detail="Import not found")
|
||||
|
||||
progress = import_progress[import_id]
|
||||
|
||||
response = {
|
||||
"import_id": import_id,
|
||||
"table_name": progress.table_name,
|
||||
"status": progress.status,
|
||||
"started_at": progress.started_at.isoformat(),
|
||||
"completed_at": progress.completed_at.isoformat() if progress.completed_at else None
|
||||
}
|
||||
|
||||
if progress.result:
|
||||
response["result"] = progress.result.to_dict()
|
||||
elif progress.error:
|
||||
response["error"] = progress.error
|
||||
|
||||
return response
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting import status: {str(e)}")
|
||||
raise HTTPException(status_code=500, detail="Failed to get import status")
|
||||
|
||||
|
||||
@router.post("/batch")
|
||||
async def batch_import_csv(
|
||||
background_tasks: BackgroundTasks,
|
||||
files: List[UploadFile] = File(...),
|
||||
table_names: List[str] = Form(...),
|
||||
current_user: User = Depends(get_admin_user),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""Import multiple CSV files in batch"""
|
||||
try:
|
||||
if len(files) != len(table_names):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Number of files must match number of table names"
|
||||
)
|
||||
|
||||
imports = []
|
||||
import_ids = []
|
||||
|
||||
for i, (file, table_name) in enumerate(zip(files, table_names)):
|
||||
# Validate table name
|
||||
if table_name.lower() not in [t.value for t in TableType]:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Unsupported table: {table_name}"
|
||||
)
|
||||
|
||||
# Validate file type
|
||||
if not file.filename.lower().endswith('.csv'):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"File {file.filename} must be a CSV file"
|
||||
)
|
||||
|
||||
# Read file content
|
||||
content = await file.read()
|
||||
csv_content = content.decode('utf-8')
|
||||
|
||||
if not csv_content.strip():
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"File {file.filename} is empty"
|
||||
)
|
||||
|
||||
imports.append({
|
||||
"table_name": table_name,
|
||||
"csv_content": csv_content,
|
||||
"filename": file.filename
|
||||
})
|
||||
|
||||
# Generate import ID for tracking
|
||||
import_id = str(uuid.uuid4())
|
||||
import_ids.append(import_id)
|
||||
|
||||
# Create progress tracker
|
||||
progress = ImportStatus(import_id, table_name)
|
||||
import_progress[import_id] = progress
|
||||
|
||||
# Process batch import in background
|
||||
async def process_batch_background():
|
||||
try:
|
||||
service = ImportService(db)
|
||||
results = service.batch_import(imports)
|
||||
|
||||
# Update progress for each import
|
||||
for i, import_id in enumerate(import_ids):
|
||||
if import_id in import_progress:
|
||||
progress = import_progress[import_id]
|
||||
table_name = progress.table_name
|
||||
|
||||
# Find result for this table
|
||||
result = None
|
||||
for key, res in results.items():
|
||||
if key.startswith(table_name):
|
||||
result = res
|
||||
break
|
||||
|
||||
if result:
|
||||
progress.status = "COMPLETED" if result.success else "FAILED"
|
||||
progress.result = result
|
||||
else:
|
||||
progress.status = "FAILED"
|
||||
progress.error = "No result found"
|
||||
|
||||
progress.completed_at = datetime.utcnow()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Batch import failed: {str(e)}")
|
||||
for import_id in import_ids:
|
||||
if import_id in import_progress:
|
||||
progress = import_progress[import_id]
|
||||
progress.status = "FAILED"
|
||||
progress.error = str(e)
|
||||
progress.completed_at = datetime.utcnow()
|
||||
|
||||
background_tasks.add_task(process_batch_background)
|
||||
|
||||
logger.info(f"Started batch import with {len(files)} files")
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"import_ids": import_ids,
|
||||
"total_files": len(files),
|
||||
"status": "PROCESSING",
|
||||
"message": "Batch import started successfully"
|
||||
}
|
||||
|
||||
except UnicodeDecodeError:
|
||||
raise HTTPException(status_code=400, detail="Invalid file encoding. Please use UTF-8.")
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error starting batch import: {str(e)}")
|
||||
raise HTTPException(status_code=500, detail="Failed to start batch import")
|
||||
|
||||
|
||||
@router.delete("/progress")
|
||||
async def cleanup_import_progress(
|
||||
current_user: User = Depends(get_admin_user)
|
||||
):
|
||||
"""Clean up completed import progress records"""
|
||||
try:
|
||||
completed_count = 0
|
||||
to_remove = []
|
||||
|
||||
for import_id, progress in import_progress.items():
|
||||
if progress.status in ["COMPLETED", "FAILED"]:
|
||||
# Remove progress older than 1 hour
|
||||
if progress.completed_at:
|
||||
age = datetime.utcnow() - progress.completed_at
|
||||
if age.total_seconds() > 3600: # 1 hour
|
||||
to_remove.append(import_id)
|
||||
completed_count += 1
|
||||
|
||||
for import_id in to_remove:
|
||||
del import_progress[import_id]
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"cleaned_up": completed_count,
|
||||
"remaining": len(import_progress)
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error cleaning up import progress: {str(e)}")
|
||||
raise HTTPException(status_code=500, detail="Failed to cleanup progress")
|
||||
Reference in New Issue
Block a user