""" CSV Import API Endpoints """ from fastapi import APIRouter, Depends, File, UploadFile, Form, HTTPException, BackgroundTasks from fastapi.responses import JSONResponse from sqlalchemy.orm import Session from typing import List, Optional, Dict, Any import logging import uuid from datetime import datetime from app.database.base import get_db from app.auth.security import get_admin_user from app.models.user import User from app.import_export.import_service import ImportService, TableType from app.core.logging import get_logger logger = get_logger("import_api") router = APIRouter() # In-memory storage for import progress (could be moved to Redis in production) import_progress = {} class ImportStatus: """Track import operation status""" def __init__(self, import_id: str, table_name: str): self.import_id = import_id self.table_name = table_name self.status = "PROCESSING" self.started_at = datetime.utcnow() self.completed_at = None self.result = None self.error = None @router.get("/tables") async def get_supported_tables( current_user: User = Depends(get_admin_user), db: Session = Depends(get_db) ): """Get list of supported tables for import""" try: service = ImportService(db) tables = service.get_supported_tables() return { "success": True, "tables": tables, "total": len(tables) } except Exception as e: logger.error(f"Error getting supported tables: {str(e)}") raise HTTPException(status_code=500, detail="Failed to get supported tables") @router.get("/tables/{table_name}/schema") async def get_table_schema( table_name: str, current_user: User = Depends(get_admin_user), db: Session = Depends(get_db) ): """Get schema information for a specific table""" try: service = ImportService(db) schema = service.get_table_schema(table_name) if not schema: raise HTTPException(status_code=404, detail=f"Table '{table_name}' not found") return { "success": True, "schema": schema } except HTTPException: raise except Exception as e: logger.error(f"Error getting table schema for {table_name}: {str(e)}") raise HTTPException(status_code=500, detail="Failed to get table schema") @router.post("/validate") async def validate_csv_headers( table_name: str = Form(...), file: UploadFile = File(...), current_user: User = Depends(get_admin_user), db: Session = Depends(get_db) ): """Validate CSV headers without importing data""" try: # Read file content content = await file.read() csv_content = content.decode('utf-8') service = ImportService(db) result = service.validate_csv_headers(table_name, csv_content) return { "success": result.success, "table_name": table_name, "filename": file.filename, "validation_result": result.to_dict() } except UnicodeDecodeError: raise HTTPException(status_code=400, detail="Invalid file encoding. Please use UTF-8.") except Exception as e: logger.error(f"Error validating CSV headers: {str(e)}") raise HTTPException(status_code=500, detail="Failed to validate CSV headers") async def process_import_background( import_id: str, table_name: str, csv_content: str, db: Session ): """Background task to process CSV import""" try: logger.info(f"Starting background import {import_id} for table {table_name}") print(f"[IMPORT] Starting background import {import_id} for table {table_name}") service = ImportService(db) result = service.import_csv(table_name, csv_content, import_id=import_id) # Update progress if import_id in import_progress: progress = import_progress[import_id] progress.status = "COMPLETED" if result.success else "FAILED" progress.completed_at = datetime.utcnow() progress.result = result logger.info(f"Import {import_id} completed with {result.imported_rows} rows imported") print(f"[IMPORT] Import {import_id} completed: success={result.success}, rows={result.imported_rows}") except Exception as e: logger.error(f"Background import {import_id} failed: {str(e)}") print(f"[IMPORT] Background import {import_id} failed: {str(e)}") if import_id in import_progress: progress = import_progress[import_id] progress.status = "FAILED" progress.completed_at = datetime.utcnow() progress.error = str(e) @router.post("/csv") async def import_csv_file( background_tasks: BackgroundTasks, table_name: str = Form(...), file: UploadFile = File(...), current_user: User = Depends(get_admin_user), db: Session = Depends(get_db) ): """Import CSV file to specified table""" try: logger.info(f"Received CSV import request: table={table_name}, file={file.filename}, user={current_user.username}") print(f"[IMPORT API] CSV import request: table={table_name}, file={file.filename}") # Validate table name if table_name.lower() not in [t.value for t in TableType]: print(f"[IMPORT API] Invalid table name: {table_name}") raise HTTPException( status_code=400, detail=f"Unsupported table: {table_name}" ) # Validate file type if not file.filename.lower().endswith('.csv'): raise HTTPException( status_code=400, detail="File must be a CSV file" ) # Read file content content = await file.read() csv_content = content.decode('utf-8') if not csv_content.strip(): raise HTTPException(status_code=400, detail="File is empty") # Generate import ID import_id = str(uuid.uuid4()) print(f"[IMPORT API] Generated import ID: {import_id}") # Create progress tracker progress = ImportStatus(import_id, table_name) import_progress[import_id] = progress # Start background import background_tasks.add_task( process_import_background, import_id, table_name, csv_content, db ) logger.info(f"Started CSV import {import_id} for table {table_name}") print(f"[IMPORT API] Background task queued for import {import_id}") return { "success": True, "import_id": import_id, "table_name": table_name, "filename": file.filename, "status": "PROCESSING", "message": "Import started successfully" } except UnicodeDecodeError: raise HTTPException(status_code=400, detail="Invalid file encoding. Please use UTF-8.") except HTTPException: raise except Exception as e: logger.error(f"Error starting CSV import: {str(e)}") raise HTTPException(status_code=500, detail="Failed to start import") @router.get("/status/{import_id}") async def get_import_status( import_id: str, current_user: User = Depends(get_admin_user) ): """Get status of an import operation""" try: if import_id not in import_progress: raise HTTPException(status_code=404, detail="Import not found") progress = import_progress[import_id] response = { "import_id": import_id, "table_name": progress.table_name, "status": progress.status, "started_at": progress.started_at.isoformat(), "completed_at": progress.completed_at.isoformat() if progress.completed_at else None } if progress.result: response["result"] = progress.result.to_dict() elif progress.error: response["error"] = progress.error return response except HTTPException: raise except Exception as e: logger.error(f"Error getting import status: {str(e)}") raise HTTPException(status_code=500, detail="Failed to get import status") @router.post("/batch") async def batch_import_csv( background_tasks: BackgroundTasks, files: List[UploadFile] = File(...), table_names: List[str] = Form(...), current_user: User = Depends(get_admin_user), db: Session = Depends(get_db) ): """Import multiple CSV files in batch""" try: if len(files) != len(table_names): raise HTTPException( status_code=400, detail="Number of files must match number of table names" ) imports = [] import_ids = [] for i, (file, table_name) in enumerate(zip(files, table_names)): # Validate table name if table_name.lower() not in [t.value for t in TableType]: raise HTTPException( status_code=400, detail=f"Unsupported table: {table_name}" ) # Validate file type if not file.filename.lower().endswith('.csv'): raise HTTPException( status_code=400, detail=f"File {file.filename} must be a CSV file" ) # Read file content content = await file.read() csv_content = content.decode('utf-8') if not csv_content.strip(): raise HTTPException( status_code=400, detail=f"File {file.filename} is empty" ) imports.append({ "table_name": table_name, "csv_content": csv_content, "filename": file.filename }) # Generate import ID for tracking import_id = str(uuid.uuid4()) import_ids.append(import_id) # Create progress tracker progress = ImportStatus(import_id, table_name) import_progress[import_id] = progress # Process batch import in background async def process_batch_background(): try: service = ImportService(db) results = service.batch_import(imports) # Update progress for each import for i, import_id in enumerate(import_ids): if import_id in import_progress: progress = import_progress[import_id] table_name = progress.table_name # Find result for this table result = None for key, res in results.items(): if key.startswith(table_name): result = res break if result: progress.status = "COMPLETED" if result.success else "FAILED" progress.result = result else: progress.status = "FAILED" progress.error = "No result found" progress.completed_at = datetime.utcnow() except Exception as e: logger.error(f"Batch import failed: {str(e)}") for import_id in import_ids: if import_id in import_progress: progress = import_progress[import_id] progress.status = "FAILED" progress.error = str(e) progress.completed_at = datetime.utcnow() background_tasks.add_task(process_batch_background) logger.info(f"Started batch import with {len(files)} files") return { "success": True, "import_ids": import_ids, "total_files": len(files), "status": "PROCESSING", "message": "Batch import started successfully" } except UnicodeDecodeError: raise HTTPException(status_code=400, detail="Invalid file encoding. Please use UTF-8.") except HTTPException: raise except Exception as e: logger.error(f"Error starting batch import: {str(e)}") raise HTTPException(status_code=500, detail="Failed to start batch import") @router.delete("/progress") async def cleanup_import_progress( current_user: User = Depends(get_admin_user) ): """Clean up completed import progress records""" try: completed_count = 0 to_remove = [] for import_id, progress in import_progress.items(): if progress.status in ["COMPLETED", "FAILED"]: # Remove progress older than 1 hour if progress.completed_at: age = datetime.utcnow() - progress.completed_at if age.total_seconds() > 3600: # 1 hour to_remove.append(import_id) completed_count += 1 for import_id in to_remove: del import_progress[import_id] return { "success": True, "cleaned_up": completed_count, "remaining": len(import_progress) } except Exception as e: logger.error(f"Error cleaning up import progress: {str(e)}") raise HTTPException(status_code=500, detail="Failed to cleanup progress")