working on new system for importing
This commit is contained in:
HotSwapp
@@ -1,281 +0,0 @@
|
||||
"""
|
||||
Flexible Imports admin API: list, filter, and export unmapped rows captured during CSV imports.
|
||||
"""
|
||||
from typing import Optional, Dict, Any, List
|
||||
from datetime import datetime
|
||||
import csv
|
||||
import io
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, HTTPException
|
||||
from fastapi.responses import StreamingResponse
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import func, or_, cast, String
|
||||
|
||||
from app.database.base import get_db
|
||||
from app.auth.security import get_admin_user
|
||||
from app.models.flexible import FlexibleImport
|
||||
|
||||
|
||||
router = APIRouter(prefix="/flexible", tags=["flexible"])
|
||||
|
||||
|
||||
@router.get("/imports")
|
||||
async def list_flexible_imports(
|
||||
file_type: Optional[str] = Query(None, description="Filter by CSV file type (e.g., FILES.csv)"),
|
||||
target_table: Optional[str] = Query(None, description="Filter by target model table name"),
|
||||
q: Optional[str] = Query(None, description="Quick text search across file type, target table, and unmapped data"),
|
||||
has_keys: Optional[List[str]] = Query(
|
||||
None,
|
||||
description="Filter rows where extra_data (or its 'unmapped' payload) contains these keys. Repeat param for multiple keys.",
|
||||
),
|
||||
skip: int = Query(0, ge=0),
|
||||
limit: int = Query(50, ge=1, le=500),
|
||||
db: Session = Depends(get_db),
|
||||
current_user=Depends(get_admin_user),
|
||||
):
|
||||
"""List flexible import rows with optional filtering, quick search, and pagination."""
|
||||
query = db.query(FlexibleImport)
|
||||
if file_type:
|
||||
query = query.filter(FlexibleImport.file_type == file_type)
|
||||
if target_table:
|
||||
query = query.filter(FlexibleImport.target_table == target_table)
|
||||
if q:
|
||||
pattern = f"%{q.strip()}%"
|
||||
# Search across file_type, target_table, and serialized JSON extra_data
|
||||
query = query.filter(
|
||||
or_(
|
||||
FlexibleImport.file_type.ilike(pattern),
|
||||
FlexibleImport.target_table.ilike(pattern),
|
||||
cast(FlexibleImport.extra_data, String).ilike(pattern),
|
||||
)
|
||||
)
|
||||
|
||||
# Filter by key presence inside JSON payload by string matching of the serialized JSON
|
||||
# This is DB-agnostic and works across SQLite/Postgres, though not as precise as JSON operators.
|
||||
if has_keys:
|
||||
for k in [k for k in has_keys if k is not None and str(k).strip() != ""]:
|
||||
key = str(k).strip()
|
||||
# Look for the JSON key token followed by a colon, e.g. "key":
|
||||
query = query.filter(cast(FlexibleImport.extra_data, String).ilike(f'%"{key}":%'))
|
||||
|
||||
total = query.count()
|
||||
items = (
|
||||
query.order_by(FlexibleImport.id.desc())
|
||||
.offset(skip)
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
|
||||
def serialize(item: FlexibleImport) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": item.id,
|
||||
"file_type": item.file_type,
|
||||
"target_table": item.target_table,
|
||||
"primary_key_field": item.primary_key_field,
|
||||
"primary_key_value": item.primary_key_value,
|
||||
"extra_data": item.extra_data,
|
||||
}
|
||||
|
||||
return {
|
||||
"total": total,
|
||||
"skip": skip,
|
||||
"limit": limit,
|
||||
"items": [serialize(i) for i in items],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/options")
|
||||
async def flexible_options(
|
||||
db: Session = Depends(get_db),
|
||||
current_user=Depends(get_admin_user),
|
||||
):
|
||||
"""Return distinct file types and target tables for filter dropdowns."""
|
||||
file_types: List[str] = [
|
||||
ft for (ft,) in db.query(func.distinct(FlexibleImport.file_type)).order_by(FlexibleImport.file_type.asc()).all()
|
||||
if ft is not None
|
||||
]
|
||||
target_tables: List[str] = [
|
||||
tt for (tt,) in db.query(func.distinct(FlexibleImport.target_table)).order_by(FlexibleImport.target_table.asc()).all()
|
||||
if tt is not None and tt != ""
|
||||
]
|
||||
return {"file_types": file_types, "target_tables": target_tables}
|
||||
|
||||
|
||||
@router.get("/export")
|
||||
async def export_unmapped_csv(
|
||||
file_type: Optional[str] = Query(None, description="Filter by CSV file type (e.g., FILES.csv)"),
|
||||
target_table: Optional[str] = Query(None, description="Filter by target model table name"),
|
||||
has_keys: Optional[List[str]] = Query(
|
||||
None,
|
||||
description="Filter rows where extra_data (or its 'unmapped' payload) contains these keys. Repeat param for multiple keys.",
|
||||
),
|
||||
db: Session = Depends(get_db),
|
||||
current_user=Depends(get_admin_user),
|
||||
):
|
||||
"""Export unmapped rows as CSV for review. Includes basic metadata columns and unmapped fields.
|
||||
|
||||
If FlexibleImport.extra_data contains a nested 'unmapped' dict, those keys are exported.
|
||||
Otherwise, all keys of extra_data are exported.
|
||||
"""
|
||||
query = db.query(FlexibleImport)
|
||||
if file_type:
|
||||
query = query.filter(FlexibleImport.file_type == file_type)
|
||||
if target_table:
|
||||
query = query.filter(FlexibleImport.target_table == target_table)
|
||||
if has_keys:
|
||||
for k in [k for k in has_keys if k is not None and str(k).strip() != ""]:
|
||||
key = str(k).strip()
|
||||
query = query.filter(cast(FlexibleImport.extra_data, String).ilike(f'%"{key}":%'))
|
||||
|
||||
rows: List[FlexibleImport] = query.order_by(FlexibleImport.id.asc()).all()
|
||||
if not rows:
|
||||
raise HTTPException(status_code=404, detail="No matching flexible imports to export")
|
||||
|
||||
# Determine union of unmapped keys across all rows
|
||||
unmapped_keys: List[str] = []
|
||||
key_set = set()
|
||||
for r in rows:
|
||||
data = r.extra_data or {}
|
||||
payload = data.get("unmapped") if isinstance(data, dict) and isinstance(data.get("unmapped"), dict) else data
|
||||
if isinstance(payload, dict):
|
||||
for k in payload.keys():
|
||||
if k not in key_set:
|
||||
key_set.add(k)
|
||||
unmapped_keys.append(k)
|
||||
|
||||
# Prepare CSV
|
||||
meta_headers = [
|
||||
"id",
|
||||
"file_type",
|
||||
"target_table",
|
||||
"primary_key_field",
|
||||
"primary_key_value",
|
||||
]
|
||||
fieldnames = meta_headers + unmapped_keys
|
||||
|
||||
output = io.StringIO()
|
||||
writer = csv.DictWriter(output, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
|
||||
for r in rows:
|
||||
row_out: Dict[str, Any] = {
|
||||
"id": r.id,
|
||||
"file_type": r.file_type,
|
||||
"target_table": r.target_table or "",
|
||||
"primary_key_field": r.primary_key_field or "",
|
||||
"primary_key_value": r.primary_key_value or "",
|
||||
}
|
||||
data = r.extra_data or {}
|
||||
payload = data.get("unmapped") if isinstance(data, dict) and isinstance(data.get("unmapped"), dict) else data
|
||||
if isinstance(payload, dict):
|
||||
for k in unmapped_keys:
|
||||
v = payload.get(k)
|
||||
# Normalize lists/dicts to JSON strings for CSV safety
|
||||
if isinstance(v, (dict, list)):
|
||||
try:
|
||||
import json as _json
|
||||
row_out[k] = _json.dumps(v, ensure_ascii=False)
|
||||
except Exception:
|
||||
row_out[k] = str(v)
|
||||
else:
|
||||
row_out[k] = v if v is not None else ""
|
||||
writer.writerow(row_out)
|
||||
|
||||
output.seek(0)
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
filename_parts = ["flexible_unmapped"]
|
||||
if file_type:
|
||||
filename_parts.append(file_type.replace("/", "-").replace(" ", "_"))
|
||||
if target_table:
|
||||
filename_parts.append(target_table.replace("/", "-").replace(" ", "_"))
|
||||
filename = "_".join(filename_parts) + f"_{timestamp}.csv"
|
||||
|
||||
return StreamingResponse(
|
||||
iter([output.getvalue()]),
|
||||
media_type="text/csv",
|
||||
headers={
|
||||
"Content-Disposition": f"attachment; filename=\"{filename}\"",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/export/{row_id}")
|
||||
async def export_single_row_csv(
|
||||
row_id: int,
|
||||
db: Session = Depends(get_db),
|
||||
current_user=Depends(get_admin_user),
|
||||
):
|
||||
"""Export a single flexible import row as CSV.
|
||||
|
||||
Includes metadata columns plus keys from the row's unmapped payload.
|
||||
If FlexibleImport.extra_data contains a nested 'unmapped' dict, those keys are exported;
|
||||
otherwise, all keys of extra_data are exported.
|
||||
"""
|
||||
row: Optional[FlexibleImport] = (
|
||||
db.query(FlexibleImport).filter(FlexibleImport.id == row_id).first()
|
||||
)
|
||||
if not row:
|
||||
raise HTTPException(status_code=404, detail="Flexible import row not found")
|
||||
|
||||
data = row.extra_data or {}
|
||||
payload = (
|
||||
data.get("unmapped")
|
||||
if isinstance(data, dict) and isinstance(data.get("unmapped"), dict)
|
||||
else data
|
||||
)
|
||||
|
||||
unmapped_keys: List[str] = []
|
||||
if isinstance(payload, dict):
|
||||
for k in payload.keys():
|
||||
unmapped_keys.append(k)
|
||||
|
||||
meta_headers = [
|
||||
"id",
|
||||
"file_type",
|
||||
"target_table",
|
||||
"primary_key_field",
|
||||
"primary_key_value",
|
||||
]
|
||||
fieldnames = meta_headers + unmapped_keys
|
||||
|
||||
output = io.StringIO()
|
||||
writer = csv.DictWriter(output, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
|
||||
row_out: Dict[str, Any] = {
|
||||
"id": row.id,
|
||||
"file_type": row.file_type,
|
||||
"target_table": row.target_table or "",
|
||||
"primary_key_field": row.primary_key_field or "",
|
||||
"primary_key_value": row.primary_key_value or "",
|
||||
}
|
||||
if isinstance(payload, dict):
|
||||
for k in unmapped_keys:
|
||||
v = payload.get(k)
|
||||
if isinstance(v, (dict, list)):
|
||||
try:
|
||||
import json as _json
|
||||
row_out[k] = _json.dumps(v, ensure_ascii=False)
|
||||
except Exception:
|
||||
row_out[k] = str(v)
|
||||
else:
|
||||
row_out[k] = v if v is not None else ""
|
||||
|
||||
writer.writerow(row_out)
|
||||
output.seek(0)
|
||||
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
filename = (
|
||||
f"flexible_row_{row.id}_{row.file_type.replace('/', '-').replace(' ', '_')}_{timestamp}.csv"
|
||||
if row.file_type
|
||||
else f"flexible_row_{row.id}_{timestamp}.csv"
|
||||
)
|
||||
|
||||
return StreamingResponse(
|
||||
iter([output.getvalue()]),
|
||||
media_type="text/csv",
|
||||
headers={
|
||||
"Content-Disposition": f"attachment; filename=\"{filename}\"",
|
||||
},
|
||||
)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -20,6 +20,7 @@ pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
|
||||
|
||||
# JWT Security
|
||||
security = HTTPBearer()
|
||||
optional_security = HTTPBearer(auto_error=False)
|
||||
|
||||
|
||||
def verify_password(plain_password: str, hashed_password: str) -> bool:
|
||||
@@ -190,6 +191,31 @@ def get_current_user(
|
||||
return user
|
||||
|
||||
|
||||
def get_optional_current_user(
|
||||
credentials: Optional[HTTPAuthorizationCredentials] = Depends(optional_security),
|
||||
db: Session = Depends(get_db)
|
||||
) -> Optional[User]:
|
||||
"""Get current authenticated user, but allow None if not authenticated"""
|
||||
if not credentials:
|
||||
return None
|
||||
|
||||
try:
|
||||
token = credentials.credentials
|
||||
username = verify_token(token)
|
||||
|
||||
if username is None:
|
||||
return None
|
||||
|
||||
user = db.query(User).filter(User.username == username).first()
|
||||
if user is None or not user.is_active:
|
||||
return None
|
||||
|
||||
return user
|
||||
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def get_admin_user(current_user: User = Depends(get_current_user)) -> User:
|
||||
"""Require admin privileges"""
|
||||
if not current_user.is_admin:
|
||||
|
||||
@@ -9,7 +9,15 @@ from app.config import settings
|
||||
|
||||
engine = create_engine(
|
||||
settings.database_url,
|
||||
connect_args={"check_same_thread": False} if "sqlite" in settings.database_url else {}
|
||||
connect_args={
|
||||
"check_same_thread": False,
|
||||
# SQLite performance optimizations for bulk imports
|
||||
"timeout": 30,
|
||||
} if "sqlite" in settings.database_url else {},
|
||||
# Performance settings for bulk operations
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=3600, # Recycle connections after 1 hour
|
||||
echo=False # Set to True for SQL debugging
|
||||
)
|
||||
|
||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
|
||||
6
app/import_export/__init__.py
Normal file
6
app/import_export/__init__.py
Normal file
@@ -0,0 +1,6 @@
|
||||
"""
|
||||
Import/Export module for Delphi Database System
|
||||
|
||||
This module provides clean, modular CSV import functionality
|
||||
for all database tables.
|
||||
"""
|
||||
20
app/main.py
20
app/main.py
@@ -160,8 +160,6 @@ from app.api.documents import router as documents_router
|
||||
from app.api.billing import router as billing_router
|
||||
from app.api.search import router as search_router
|
||||
from app.api.admin import router as admin_router
|
||||
from app.api.import_data import router as import_router
|
||||
from app.api.flexible import router as flexible_router
|
||||
from app.api.support import router as support_router
|
||||
from app.api.settings import router as settings_router
|
||||
from app.api.mortality import router as mortality_router
|
||||
@@ -189,10 +187,8 @@ app.include_router(billing_router, prefix="/api/billing", tags=["billing"])
|
||||
app.include_router(documents_router, prefix="/api/documents", tags=["documents"])
|
||||
app.include_router(search_router, prefix="/api/search", tags=["search"])
|
||||
app.include_router(admin_router, prefix="/api/admin", tags=["admin"])
|
||||
app.include_router(import_router, prefix="/api/import", tags=["import"])
|
||||
app.include_router(support_router, prefix="/api/support", tags=["support"])
|
||||
app.include_router(settings_router, prefix="/api/settings", tags=["settings"])
|
||||
app.include_router(flexible_router, prefix="/api")
|
||||
app.include_router(mortality_router, prefix="/api/mortality", tags=["mortality"])
|
||||
app.include_router(pensions_router, prefix="/api/pensions", tags=["pensions"])
|
||||
app.include_router(pension_valuation_router, prefix="/api/pensions", tags=["pensions-valuation"])
|
||||
@@ -288,22 +284,10 @@ async def admin_page(request: Request):
|
||||
)
|
||||
|
||||
|
||||
@app.get("/import", response_class=HTMLResponse)
|
||||
async def import_page(request: Request):
|
||||
"""Data import management page (admin only)"""
|
||||
return templates.TemplateResponse(
|
||||
"import.html",
|
||||
{"request": request, "title": "Data Import - " + settings.app_name}
|
||||
)
|
||||
|
||||
|
||||
@app.get("/flexible", response_class=HTMLResponse)
|
||||
async def flexible_page(request: Request):
|
||||
"""Flexible imports admin page (admin only)."""
|
||||
return templates.TemplateResponse(
|
||||
"flexible.html",
|
||||
{"request": request, "title": "Flexible Imports - " + settings.app_name}
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
"""
|
||||
Flexible storage for unmapped CSV columns during import
|
||||
"""
|
||||
from sqlalchemy import Column, Integer, String
|
||||
from sqlalchemy.types import JSON
|
||||
|
||||
from app.models.base import BaseModel
|
||||
|
||||
|
||||
class FlexibleImport(BaseModel):
|
||||
"""Stores per-row extra/unmapped data for any import, without persisting mapping patterns."""
|
||||
|
||||
__tablename__ = "flexible_imports"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
|
||||
# The CSV filename used by the importer (e.g., "FILES.csv" or arbitrary names in flexible mode)
|
||||
file_type = Column(String(120), nullable=False, index=True)
|
||||
|
||||
# The SQLAlchemy model table this extra data is associated with (if any)
|
||||
target_table = Column(String(120), nullable=True, index=True)
|
||||
|
||||
# Optional link to the primary record created in the target table
|
||||
primary_key_field = Column(String(120), nullable=True)
|
||||
primary_key_value = Column(String(255), nullable=True, index=True)
|
||||
|
||||
# Extra unmapped columns from the CSV row
|
||||
extra_data = Column(JSON, nullable=False)
|
||||
|
||||
def __repr__(self) -> str: # pragma: no cover - repr utility
|
||||
return (
|
||||
f"<FlexibleImport(id={self.id}, file_type='{self.file_type}', "
|
||||
f"target_table='{self.target_table}', pk_field='{self.primary_key_field}', "
|
||||
f"pk_value='{self.primary_key_value}')>"
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user