771 lines
30 KiB
Python
771 lines
30 KiB
Python
"""
|
|
CSV Import API Endpoints
|
|
"""
|
|
from fastapi import APIRouter, Depends, File, UploadFile, Form, HTTPException, BackgroundTasks, Body
|
|
from fastapi.responses import JSONResponse
|
|
from sqlalchemy.orm import Session
|
|
from typing import List, Optional, Dict, Any
|
|
import logging
|
|
import uuid
|
|
from datetime import datetime
|
|
|
|
from app.database.base import get_db
|
|
from app.auth.security import get_admin_user
|
|
from app.models.user import User
|
|
from app.import_export.import_service import ImportService, TableType
|
|
from app.core.logging import get_logger
|
|
|
|
logger = get_logger("import_api")
|
|
router = APIRouter()
|
|
|
|
# In-memory storage for import progress (could be moved to Redis in production)
|
|
import_progress = {}
|
|
|
|
|
|
class ImportStatus:
|
|
"""Track import operation status"""
|
|
def __init__(self, import_id: str, table_name: str):
|
|
self.import_id = import_id
|
|
self.table_name = table_name
|
|
self.status = "PROCESSING"
|
|
self.started_at = datetime.utcnow()
|
|
self.completed_at = None
|
|
self.result = None
|
|
self.error = None
|
|
|
|
|
|
@router.get("/tables")
|
|
async def get_supported_tables(
|
|
current_user: User = Depends(get_admin_user),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Get list of supported tables for import"""
|
|
try:
|
|
service = ImportService(db)
|
|
tables = service.get_supported_tables()
|
|
|
|
return {
|
|
"success": True,
|
|
"tables": tables,
|
|
"total": len(tables)
|
|
}
|
|
except Exception as e:
|
|
logger.error(f"Error getting supported tables: {str(e)}")
|
|
raise HTTPException(status_code=500, detail="Failed to get supported tables")
|
|
|
|
|
|
@router.get("/discover-files")
|
|
async def discover_csv_files(
|
|
current_user: User = Depends(get_admin_user)
|
|
):
|
|
"""Discover available CSV files in the old database directory"""
|
|
try:
|
|
import os
|
|
import glob
|
|
from pathlib import Path
|
|
|
|
# Look for CSV files in the old database directory
|
|
base_dir = Path("old database/Office")
|
|
csv_files = []
|
|
|
|
if base_dir.exists():
|
|
# Find all CSV files
|
|
for csv_file in glob.glob(str(base_dir / "**/*.csv"), recursive=True):
|
|
file_path = Path(csv_file)
|
|
relative_path = file_path.relative_to(base_dir)
|
|
|
|
# Try to map to known table types
|
|
filename = file_path.stem.upper()
|
|
table_mapping = {
|
|
"ROLODEX": "rolodex",
|
|
"ROLEX_V": "rolodex", # ROLEX_V variant
|
|
"PHONE": "phone",
|
|
"FILES": "files",
|
|
"FILES_R": "files",
|
|
"FILES_V": "files",
|
|
"LEDGER": "ledger",
|
|
"QDROS": "qdros",
|
|
"PAYMENTS": "ledger",
|
|
"DEPOSITS": "ledger",
|
|
"EMPLOYEE": "employee",
|
|
"SETUP": "setup",
|
|
"FILETYPE": "filetype",
|
|
"TRNSTYPE": "trnstype",
|
|
"TRNSACTN": "trnsactn",
|
|
"TRNSLKUP": "trnslkup",
|
|
"PENSIONS": "pensions"
|
|
}
|
|
|
|
suggested_table = table_mapping.get(filename, "unknown")
|
|
|
|
csv_files.append({
|
|
"filename": file_path.name,
|
|
"path": str(relative_path),
|
|
"full_path": str(file_path),
|
|
"suggested_table": suggested_table,
|
|
"size": file_path.stat().st_size if file_path.exists() else 0
|
|
})
|
|
|
|
# Sort by filename
|
|
csv_files.sort(key=lambda x: x["filename"])
|
|
|
|
return {
|
|
"success": True,
|
|
"files": csv_files,
|
|
"total": len(csv_files),
|
|
"base_directory": str(base_dir)
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error discovering CSV files: {str(e)}")
|
|
raise HTTPException(status_code=500, detail="Failed to discover CSV files")
|
|
|
|
|
|
@router.get("/tables/{table_name}/schema")
|
|
async def get_table_schema(
|
|
table_name: str,
|
|
current_user: User = Depends(get_admin_user),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Get schema information for a specific table"""
|
|
try:
|
|
service = ImportService(db)
|
|
schema = service.get_table_schema(table_name)
|
|
|
|
if not schema:
|
|
raise HTTPException(status_code=404, detail=f"Table '{table_name}' not found")
|
|
|
|
return {
|
|
"success": True,
|
|
"schema": schema
|
|
}
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Error getting table schema for {table_name}: {str(e)}")
|
|
raise HTTPException(status_code=500, detail="Failed to get table schema")
|
|
|
|
|
|
@router.post("/validate")
|
|
async def validate_csv_headers(
|
|
table_name: str = Form(...),
|
|
file: UploadFile = File(...),
|
|
current_user: User = Depends(get_admin_user),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Validate CSV headers without importing data"""
|
|
try:
|
|
# Read file content with encoding detection
|
|
content = await file.read()
|
|
|
|
# Try multiple encodings
|
|
encodings = ['utf-8', 'windows-1252', 'iso-8859-1', 'cp1252', 'latin-1']
|
|
csv_content = None
|
|
used_encoding = None
|
|
|
|
for encoding in encodings:
|
|
try:
|
|
csv_content = content.decode(encoding)
|
|
used_encoding = encoding
|
|
break
|
|
except UnicodeDecodeError:
|
|
continue
|
|
|
|
if csv_content is None:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail="Could not decode file. Please ensure it's a valid text file."
|
|
)
|
|
|
|
service = ImportService(db)
|
|
result = service.validate_csv_headers(table_name, csv_content)
|
|
|
|
return {
|
|
"success": result.success,
|
|
"table_name": table_name,
|
|
"filename": file.filename,
|
|
"validation_result": result.to_dict()
|
|
}
|
|
|
|
except UnicodeDecodeError:
|
|
raise HTTPException(status_code=400, detail="Could not decode file. Please ensure it's a valid text file.")
|
|
except Exception as e:
|
|
logger.error(f"Error validating CSV headers: {str(e)}")
|
|
raise HTTPException(status_code=500, detail="Failed to validate CSV headers")
|
|
|
|
|
|
async def process_import_background(
|
|
import_id: str,
|
|
table_name: str,
|
|
csv_content: str,
|
|
db: Session
|
|
):
|
|
"""Background task to process CSV import"""
|
|
try:
|
|
logger.info(f"Starting background import {import_id} for table {table_name}")
|
|
print(f"[IMPORT] Starting background import {import_id} for table {table_name}")
|
|
|
|
service = ImportService(db)
|
|
result = service.import_csv(table_name, csv_content, import_id=import_id)
|
|
|
|
# Update progress
|
|
if import_id in import_progress:
|
|
progress = import_progress[import_id]
|
|
progress.status = "COMPLETED" if result.success else "FAILED"
|
|
progress.completed_at = datetime.utcnow()
|
|
progress.result = result
|
|
|
|
logger.info(f"Import {import_id} completed with {result.imported_rows} rows imported")
|
|
print(f"[IMPORT] Import {import_id} completed: success={result.success}, rows={result.imported_rows}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Background import {import_id} failed: {str(e)}")
|
|
print(f"[IMPORT] Background import {import_id} failed: {str(e)}")
|
|
if import_id in import_progress:
|
|
progress = import_progress[import_id]
|
|
progress.status = "FAILED"
|
|
progress.completed_at = datetime.utcnow()
|
|
progress.error = str(e)
|
|
|
|
|
|
@router.post("/csv")
|
|
async def import_csv_file(
|
|
background_tasks: BackgroundTasks,
|
|
table_name: str = Form(...),
|
|
file: UploadFile = File(...),
|
|
current_user: User = Depends(get_admin_user),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Import CSV file to specified table"""
|
|
try:
|
|
logger.info(f"Received CSV import request: table={table_name}, file={file.filename}, user={current_user.username}")
|
|
print(f"[IMPORT API] CSV import request: table={table_name}, file={file.filename}")
|
|
|
|
# Validate table name
|
|
if table_name.lower() not in [t.value for t in TableType]:
|
|
print(f"[IMPORT API] Invalid table name: {table_name}")
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail=f"Unsupported table: {table_name}"
|
|
)
|
|
|
|
# Validate file type
|
|
if not file.filename.lower().endswith('.csv'):
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail="File must be a CSV file"
|
|
)
|
|
|
|
# Read file content with encoding detection
|
|
content = await file.read()
|
|
|
|
# Try multiple encodings
|
|
encodings = ['utf-8', 'windows-1252', 'iso-8859-1', 'cp1252', 'latin-1']
|
|
csv_content = None
|
|
used_encoding = None
|
|
|
|
for encoding in encodings:
|
|
try:
|
|
csv_content = content.decode(encoding)
|
|
used_encoding = encoding
|
|
break
|
|
except UnicodeDecodeError:
|
|
continue
|
|
|
|
if csv_content is None:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail="Could not decode file. Please ensure it's a valid text file."
|
|
)
|
|
|
|
if not csv_content.strip():
|
|
raise HTTPException(status_code=400, detail="File is empty")
|
|
|
|
# Generate import ID
|
|
import_id = str(uuid.uuid4())
|
|
print(f"[IMPORT API] Generated import ID: {import_id}")
|
|
|
|
# Create progress tracker
|
|
progress = ImportStatus(import_id, table_name)
|
|
import_progress[import_id] = progress
|
|
|
|
# Start background import
|
|
background_tasks.add_task(
|
|
process_import_background,
|
|
import_id,
|
|
table_name,
|
|
csv_content,
|
|
db
|
|
)
|
|
|
|
logger.info(f"Started CSV import {import_id} for table {table_name}")
|
|
print(f"[IMPORT API] Background task queued for import {import_id}")
|
|
|
|
return {
|
|
"success": True,
|
|
"import_id": import_id,
|
|
"table_name": table_name,
|
|
"filename": file.filename,
|
|
"status": "PROCESSING",
|
|
"message": "Import started successfully"
|
|
}
|
|
|
|
except UnicodeDecodeError:
|
|
raise HTTPException(status_code=400, detail="Could not decode file. Please ensure it's a valid text file.")
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Error starting CSV import: {str(e)}")
|
|
raise HTTPException(status_code=500, detail="Failed to start import")
|
|
|
|
|
|
@router.get("/status/{import_id}")
|
|
async def get_import_status(
|
|
import_id: str,
|
|
current_user: User = Depends(get_admin_user)
|
|
):
|
|
"""Get status of an import operation"""
|
|
try:
|
|
if import_id not in import_progress:
|
|
raise HTTPException(status_code=404, detail="Import not found")
|
|
|
|
progress = import_progress[import_id]
|
|
|
|
response = {
|
|
"import_id": import_id,
|
|
"table_name": progress.table_name,
|
|
"status": progress.status,
|
|
"started_at": progress.started_at.isoformat(),
|
|
"completed_at": progress.completed_at.isoformat() if progress.completed_at else None
|
|
}
|
|
|
|
if progress.result:
|
|
response["result"] = progress.result.to_dict()
|
|
# Also include error details if the import failed
|
|
if not progress.result.success and progress.result.errors:
|
|
response["error"] = "; ".join(progress.result.errors[:3])
|
|
elif progress.error:
|
|
response["error"] = progress.error
|
|
|
|
return response
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Error getting import status: {str(e)}")
|
|
raise HTTPException(status_code=500, detail="Failed to get import status")
|
|
|
|
|
|
@router.post("/batch")
|
|
async def batch_import_csv(
|
|
background_tasks: BackgroundTasks,
|
|
files: List[UploadFile] = File(...),
|
|
table_names: List[str] = Form(...),
|
|
current_user: User = Depends(get_admin_user),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Import multiple CSV files in batch"""
|
|
try:
|
|
if len(files) != len(table_names):
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail="Number of files must match number of table names"
|
|
)
|
|
|
|
imports = []
|
|
import_ids = []
|
|
|
|
for i, (file, table_name) in enumerate(zip(files, table_names)):
|
|
# Validate table name
|
|
if table_name.lower() not in [t.value for t in TableType]:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail=f"Unsupported table: {table_name}"
|
|
)
|
|
|
|
# Validate file type
|
|
if not file.filename.lower().endswith('.csv'):
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail=f"File {file.filename} must be a CSV file"
|
|
)
|
|
|
|
# Read file content with encoding detection
|
|
content = await file.read()
|
|
|
|
# Try multiple encodings
|
|
encodings = ['utf-8', 'windows-1252', 'iso-8859-1', 'cp1252', 'latin-1']
|
|
csv_content = None
|
|
used_encoding = None
|
|
|
|
for encoding in encodings:
|
|
try:
|
|
csv_content = content.decode(encoding)
|
|
used_encoding = encoding
|
|
break
|
|
except UnicodeDecodeError:
|
|
continue
|
|
|
|
if csv_content is None:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail="Could not decode file. Please ensure it's a valid text file."
|
|
)
|
|
|
|
if not csv_content.strip():
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail=f"File {file.filename} is empty"
|
|
)
|
|
|
|
imports.append({
|
|
"table_name": table_name,
|
|
"csv_content": csv_content,
|
|
"filename": file.filename
|
|
})
|
|
|
|
# Generate import ID for tracking
|
|
import_id = str(uuid.uuid4())
|
|
import_ids.append(import_id)
|
|
|
|
# Create progress tracker
|
|
progress = ImportStatus(import_id, table_name)
|
|
import_progress[import_id] = progress
|
|
|
|
# Process batch import in background
|
|
async def process_batch_background():
|
|
try:
|
|
service = ImportService(db)
|
|
results = service.batch_import(imports)
|
|
|
|
# Update progress for each import
|
|
for i, import_id in enumerate(import_ids):
|
|
if import_id in import_progress:
|
|
progress = import_progress[import_id]
|
|
table_name = progress.table_name
|
|
|
|
# Find result for this table
|
|
result = None
|
|
for key, res in results.items():
|
|
if key.startswith(table_name):
|
|
result = res
|
|
break
|
|
|
|
if result:
|
|
progress.status = "COMPLETED" if result.success else "FAILED"
|
|
progress.result = result
|
|
# If import failed, capture the error details
|
|
if not result.success and result.errors:
|
|
progress.error = "; ".join(result.errors[:3]) # Show first 3 errors
|
|
else:
|
|
progress.status = "FAILED"
|
|
progress.error = "No result found"
|
|
|
|
progress.completed_at = datetime.utcnow()
|
|
|
|
except Exception as e:
|
|
logger.error(f"Batch import failed: {str(e)}")
|
|
for import_id in import_ids:
|
|
if import_id in import_progress:
|
|
progress = import_progress[import_id]
|
|
progress.status = "FAILED"
|
|
progress.error = str(e)
|
|
progress.completed_at = datetime.utcnow()
|
|
|
|
background_tasks.add_task(process_batch_background)
|
|
|
|
logger.info(f"Started batch import with {len(files)} files")
|
|
|
|
return {
|
|
"success": True,
|
|
"import_ids": import_ids,
|
|
"total_files": len(files),
|
|
"status": "PROCESSING",
|
|
"message": "Batch import started successfully"
|
|
}
|
|
|
|
except UnicodeDecodeError:
|
|
raise HTTPException(status_code=400, detail="Invalid file encoding. Please use UTF-8.")
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Error starting batch import: {str(e)}")
|
|
raise HTTPException(status_code=500, detail="Failed to start batch import")
|
|
|
|
|
|
@router.post("/batch-from-files")
|
|
async def batch_import_from_files(
|
|
background_tasks: BackgroundTasks,
|
|
file_mappings: List[Dict[str, str]] = Body(...),
|
|
current_user: User = Depends(get_admin_user),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Import multiple CSV files from discovered file paths"""
|
|
try:
|
|
from pathlib import Path
|
|
|
|
if not file_mappings:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail="No file mappings provided"
|
|
)
|
|
|
|
imports = []
|
|
import_ids = []
|
|
|
|
for mapping in file_mappings:
|
|
file_path = mapping.get("file_path")
|
|
table_name = mapping.get("table_name")
|
|
|
|
if not file_path or not table_name:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail="Each mapping must have file_path and table_name"
|
|
)
|
|
|
|
# Validate table name
|
|
if table_name.lower() not in [t.value for t in TableType]:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail=f"Unsupported table: {table_name}"
|
|
)
|
|
|
|
# Read file content
|
|
full_path = Path(file_path)
|
|
if not full_path.exists():
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail=f"File not found: {file_path}"
|
|
)
|
|
|
|
# Read file content with encoding detection
|
|
encodings = ['utf-8', 'windows-1252', 'iso-8859-1', 'cp1252', 'latin-1']
|
|
csv_content = None
|
|
used_encoding = None
|
|
|
|
for encoding in encodings:
|
|
try:
|
|
with open(full_path, 'r', encoding=encoding) as f:
|
|
csv_content = f.read()
|
|
used_encoding = encoding
|
|
break
|
|
except UnicodeDecodeError:
|
|
continue
|
|
|
|
if csv_content is None:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail=f"Could not decode file {file_path}. Please ensure it's a valid text file."
|
|
)
|
|
|
|
if not csv_content.strip():
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail=f"File {file_path} is empty"
|
|
)
|
|
|
|
imports.append({
|
|
"table_name": table_name,
|
|
"csv_content": csv_content,
|
|
"filename": full_path.name
|
|
})
|
|
|
|
# Generate import ID for tracking
|
|
import_id = str(uuid.uuid4())
|
|
import_ids.append(import_id)
|
|
|
|
# Create progress tracker
|
|
progress = ImportStatus(import_id, table_name)
|
|
import_progress[import_id] = progress
|
|
|
|
# Process batch import in background
|
|
async def process_batch_background():
|
|
try:
|
|
service = ImportService(db)
|
|
results = service.batch_import(imports)
|
|
|
|
# Update progress for each import
|
|
for i, import_id in enumerate(import_ids):
|
|
if import_id in import_progress:
|
|
progress = import_progress[import_id]
|
|
table_name = progress.table_name
|
|
|
|
# Find result for this table
|
|
result = None
|
|
for key, res in results.items():
|
|
if key.startswith(table_name):
|
|
result = res
|
|
break
|
|
|
|
if result:
|
|
progress.status = "COMPLETED" if result.success else "FAILED"
|
|
progress.result = result
|
|
# If import failed, capture the error details
|
|
if not result.success and result.errors:
|
|
progress.error = "; ".join(result.errors[:3]) # Show first 3 errors
|
|
else:
|
|
progress.status = "FAILED"
|
|
progress.error = "No result found"
|
|
|
|
progress.completed_at = datetime.utcnow()
|
|
|
|
except Exception as e:
|
|
logger.error(f"Batch import failed: {str(e)}")
|
|
for import_id in import_ids:
|
|
if import_id in import_progress:
|
|
progress = import_progress[import_id]
|
|
progress.status = "FAILED"
|
|
progress.error = str(e)
|
|
progress.completed_at = datetime.utcnow()
|
|
|
|
background_tasks.add_task(process_batch_background)
|
|
|
|
logger.info(f"Started batch import from files with {len(imports)} files")
|
|
|
|
return {
|
|
"success": True,
|
|
"import_ids": import_ids,
|
|
"total_files": len(imports),
|
|
"status": "PROCESSING",
|
|
"message": "Batch import from files started successfully"
|
|
}
|
|
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Error starting batch import from files: {str(e)}")
|
|
raise HTTPException(status_code=500, detail="Failed to start batch import")
|
|
|
|
|
|
@router.get("/status")
|
|
async def get_import_status_overview(
|
|
current_user: User = Depends(get_admin_user),
|
|
db: Session = Depends(get_db)
|
|
):
|
|
"""Get overview of which tables have been successfully imported"""
|
|
try:
|
|
from sqlalchemy import inspect, text
|
|
|
|
# Get list of all tables in the database
|
|
inspector = inspect(db.bind)
|
|
existing_tables = inspector.get_table_names()
|
|
|
|
# Define all possible table types and their display names
|
|
table_info = {
|
|
"rolodex": {"name": "ROLODEX (Contacts)", "category": "Core", "expected_files": ["ROLODEX.csv", "ROLEX_V.csv"]},
|
|
"phone": {"name": "PHONE (Phone Numbers)", "category": "Core", "expected_files": ["PHONE.csv"]},
|
|
"files": {"name": "FILES (Case Files)", "category": "Core", "expected_files": ["FILES.csv", "FILES_R.csv", "FILES_V.csv"]},
|
|
"ledger": {"name": "LEDGER (Financial)", "category": "Core", "expected_files": ["LEDGER.csv"]},
|
|
"qdros": {"name": "QDROS (Documents)", "category": "Core", "expected_files": ["QDROS.csv"]},
|
|
"gruplkup": {"name": "GRUPLKUP", "category": "Lookup", "expected_files": ["GRUPLKUP.csv"]},
|
|
"employee": {"name": "EMPLOYEE", "category": "Lookup", "expected_files": ["EMPLOYEE.csv"]},
|
|
"filetype": {"name": "FILETYPE", "category": "Lookup", "expected_files": ["FILETYPE.csv"]},
|
|
"trnstype": {"name": "TRNSTYPE", "category": "Lookup", "expected_files": ["TRNSTYPE.csv"]},
|
|
"trnslkup": {"name": "TRNSLKUP", "category": "Lookup", "expected_files": ["TRNSLKUP.csv"]},
|
|
"rvarlkup": {"name": "RVARLKUP", "category": "Lookup", "expected_files": ["RVARLKUP.csv"]},
|
|
"fvarlkup": {"name": "FVARLKUP", "category": "Lookup", "expected_files": ["FVARLKUP.csv"]},
|
|
"filenots": {"name": "FILENOTS", "category": "Lookup", "expected_files": ["FILENOTS.csv"]},
|
|
"planinfo": {"name": "PLANINFO", "category": "Lookup", "expected_files": ["PLANINFO.csv"]},
|
|
"setup": {"name": "SETUP", "category": "Configuration", "expected_files": ["SETUP.csv"]},
|
|
"deposits": {"name": "DEPOSITS", "category": "Financial", "expected_files": ["DEPOSITS.csv"]},
|
|
"payments": {"name": "PAYMENTS", "category": "Financial", "expected_files": ["PAYMENTS.csv"]},
|
|
"trnsactn": {"name": "TRNSACTN", "category": "Financial", "expected_files": ["TRNSACTN.csv"]},
|
|
"pensions": {"name": "PENSIONS", "category": "Pension", "expected_files": ["PENSIONS.csv"]},
|
|
"marriage": {"name": "MARRIAGE", "category": "Pension", "expected_files": ["MARRIAGE.csv"]},
|
|
"death": {"name": "DEATH", "category": "Pension", "expected_files": ["DEATH.csv"]},
|
|
"separate": {"name": "SEPARATE", "category": "Pension", "expected_files": ["SEPARATE.csv"]},
|
|
"schedule": {"name": "SCHEDULE", "category": "Pension", "expected_files": ["SCHEDULE.csv"]},
|
|
"numberal": {"name": "NUMBERAL", "category": "Forms", "expected_files": ["NUMBERAL.csv"]},
|
|
"inx_lkup": {"name": "INX_LKUP", "category": "Forms", "expected_files": ["INX_LKUP.csv"]},
|
|
"form_lst": {"name": "FORM_LST", "category": "Forms", "expected_files": ["FORM_LST.csv"]},
|
|
"form_inx": {"name": "FORM_INX", "category": "Forms", "expected_files": ["FORM_INX.csv"]},
|
|
"lifetabl": {"name": "LIFETABL", "category": "Forms", "expected_files": ["LIFETABL.csv"]}
|
|
}
|
|
|
|
# Check status of each table
|
|
table_status = []
|
|
for table_name, info in table_info.items():
|
|
status = {
|
|
"table_name": table_name,
|
|
"display_name": info["name"],
|
|
"category": info["category"],
|
|
"expected_files": info["expected_files"],
|
|
"exists": table_name in existing_tables,
|
|
"row_count": 0,
|
|
"imported": False
|
|
}
|
|
|
|
if status["exists"]:
|
|
try:
|
|
# Get row count
|
|
result = db.execute(text(f"SELECT COUNT(*) FROM {table_name}"))
|
|
status["row_count"] = result.scalar()
|
|
status["imported"] = status["row_count"] > 0
|
|
except Exception as e:
|
|
logger.warning(f"Could not get row count for {table_name}: {e}")
|
|
status["row_count"] = -1 # Error getting count
|
|
|
|
table_status.append(status)
|
|
|
|
# Group by category
|
|
categories = {}
|
|
for status in table_status:
|
|
category = status["category"]
|
|
if category not in categories:
|
|
categories[category] = []
|
|
categories[category].append(status)
|
|
|
|
# Calculate summary stats
|
|
total_tables = len(table_status)
|
|
imported_tables = len([s for s in table_status if s["imported"]])
|
|
total_rows = sum(s["row_count"] for s in table_status if s["row_count"] > 0)
|
|
|
|
return {
|
|
"success": True,
|
|
"summary": {
|
|
"total_tables": total_tables,
|
|
"imported_tables": imported_tables,
|
|
"empty_tables": len([s for s in table_status if s["exists"] and s["row_count"] == 0]),
|
|
"missing_tables": len([s for s in table_status if not s["exists"]]),
|
|
"total_rows": total_rows,
|
|
"completion_percentage": round((imported_tables / total_tables) * 100, 1) if total_tables > 0 else 0
|
|
},
|
|
"categories": categories,
|
|
"tables": table_status
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error getting import status overview: {str(e)}")
|
|
raise HTTPException(status_code=500, detail="Failed to get import status")
|
|
|
|
|
|
@router.delete("/progress")
|
|
async def cleanup_import_progress(
|
|
current_user: User = Depends(get_admin_user)
|
|
):
|
|
"""Clean up completed import progress records"""
|
|
try:
|
|
completed_count = 0
|
|
to_remove = []
|
|
|
|
for import_id, progress in import_progress.items():
|
|
if progress.status in ["COMPLETED", "FAILED"]:
|
|
# Remove progress older than 1 hour
|
|
if progress.completed_at:
|
|
age = datetime.utcnow() - progress.completed_at
|
|
if age.total_seconds() > 3600: # 1 hour
|
|
to_remove.append(import_id)
|
|
completed_count += 1
|
|
|
|
for import_id in to_remove:
|
|
del import_progress[import_id]
|
|
|
|
return {
|
|
"success": True,
|
|
"cleaned_up": completed_count,
|
|
"remaining": len(import_progress)
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error cleaning up import progress: {str(e)}")
|
|
raise HTTPException(status_code=500, detail="Failed to cleanup progress") |