coming together
This commit is contained in:
HotSwapp
281
app/api/flexible.py
Normal file
281
app/api/flexible.py
Normal file
@@ -0,0 +1,281 @@
|
||||
"""
|
||||
Flexible Imports admin API: list, filter, and export unmapped rows captured during CSV imports.
|
||||
"""
|
||||
from typing import Optional, Dict, Any, List
|
||||
from datetime import datetime
|
||||
import csv
|
||||
import io
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, HTTPException
|
||||
from fastapi.responses import StreamingResponse
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import func, or_, cast, String
|
||||
|
||||
from app.database.base import get_db
|
||||
from app.auth.security import get_admin_user
|
||||
from app.models.flexible import FlexibleImport
|
||||
|
||||
|
||||
router = APIRouter(prefix="/flexible", tags=["flexible"])
|
||||
|
||||
|
||||
@router.get("/imports")
|
||||
async def list_flexible_imports(
|
||||
file_type: Optional[str] = Query(None, description="Filter by CSV file type (e.g., FILES.csv)"),
|
||||
target_table: Optional[str] = Query(None, description="Filter by target model table name"),
|
||||
q: Optional[str] = Query(None, description="Quick text search across file type, target table, and unmapped data"),
|
||||
has_keys: Optional[List[str]] = Query(
|
||||
None,
|
||||
description="Filter rows where extra_data (or its 'unmapped' payload) contains these keys. Repeat param for multiple keys.",
|
||||
),
|
||||
skip: int = Query(0, ge=0),
|
||||
limit: int = Query(50, ge=1, le=500),
|
||||
db: Session = Depends(get_db),
|
||||
current_user=Depends(get_admin_user),
|
||||
):
|
||||
"""List flexible import rows with optional filtering, quick search, and pagination."""
|
||||
query = db.query(FlexibleImport)
|
||||
if file_type:
|
||||
query = query.filter(FlexibleImport.file_type == file_type)
|
||||
if target_table:
|
||||
query = query.filter(FlexibleImport.target_table == target_table)
|
||||
if q:
|
||||
pattern = f"%{q.strip()}%"
|
||||
# Search across file_type, target_table, and serialized JSON extra_data
|
||||
query = query.filter(
|
||||
or_(
|
||||
FlexibleImport.file_type.ilike(pattern),
|
||||
FlexibleImport.target_table.ilike(pattern),
|
||||
cast(FlexibleImport.extra_data, String).ilike(pattern),
|
||||
)
|
||||
)
|
||||
|
||||
# Filter by key presence inside JSON payload by string matching of the serialized JSON
|
||||
# This is DB-agnostic and works across SQLite/Postgres, though not as precise as JSON operators.
|
||||
if has_keys:
|
||||
for k in [k for k in has_keys if k is not None and str(k).strip() != ""]:
|
||||
key = str(k).strip()
|
||||
# Look for the JSON key token followed by a colon, e.g. "key":
|
||||
query = query.filter(cast(FlexibleImport.extra_data, String).ilike(f'%"{key}":%'))
|
||||
|
||||
total = query.count()
|
||||
items = (
|
||||
query.order_by(FlexibleImport.id.desc())
|
||||
.offset(skip)
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
|
||||
def serialize(item: FlexibleImport) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": item.id,
|
||||
"file_type": item.file_type,
|
||||
"target_table": item.target_table,
|
||||
"primary_key_field": item.primary_key_field,
|
||||
"primary_key_value": item.primary_key_value,
|
||||
"extra_data": item.extra_data,
|
||||
}
|
||||
|
||||
return {
|
||||
"total": total,
|
||||
"skip": skip,
|
||||
"limit": limit,
|
||||
"items": [serialize(i) for i in items],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/options")
|
||||
async def flexible_options(
|
||||
db: Session = Depends(get_db),
|
||||
current_user=Depends(get_admin_user),
|
||||
):
|
||||
"""Return distinct file types and target tables for filter dropdowns."""
|
||||
file_types: List[str] = [
|
||||
ft for (ft,) in db.query(func.distinct(FlexibleImport.file_type)).order_by(FlexibleImport.file_type.asc()).all()
|
||||
if ft is not None
|
||||
]
|
||||
target_tables: List[str] = [
|
||||
tt for (tt,) in db.query(func.distinct(FlexibleImport.target_table)).order_by(FlexibleImport.target_table.asc()).all()
|
||||
if tt is not None and tt != ""
|
||||
]
|
||||
return {"file_types": file_types, "target_tables": target_tables}
|
||||
|
||||
|
||||
@router.get("/export")
|
||||
async def export_unmapped_csv(
|
||||
file_type: Optional[str] = Query(None, description="Filter by CSV file type (e.g., FILES.csv)"),
|
||||
target_table: Optional[str] = Query(None, description="Filter by target model table name"),
|
||||
has_keys: Optional[List[str]] = Query(
|
||||
None,
|
||||
description="Filter rows where extra_data (or its 'unmapped' payload) contains these keys. Repeat param for multiple keys.",
|
||||
),
|
||||
db: Session = Depends(get_db),
|
||||
current_user=Depends(get_admin_user),
|
||||
):
|
||||
"""Export unmapped rows as CSV for review. Includes basic metadata columns and unmapped fields.
|
||||
|
||||
If FlexibleImport.extra_data contains a nested 'unmapped' dict, those keys are exported.
|
||||
Otherwise, all keys of extra_data are exported.
|
||||
"""
|
||||
query = db.query(FlexibleImport)
|
||||
if file_type:
|
||||
query = query.filter(FlexibleImport.file_type == file_type)
|
||||
if target_table:
|
||||
query = query.filter(FlexibleImport.target_table == target_table)
|
||||
if has_keys:
|
||||
for k in [k for k in has_keys if k is not None and str(k).strip() != ""]:
|
||||
key = str(k).strip()
|
||||
query = query.filter(cast(FlexibleImport.extra_data, String).ilike(f'%"{key}":%'))
|
||||
|
||||
rows: List[FlexibleImport] = query.order_by(FlexibleImport.id.asc()).all()
|
||||
if not rows:
|
||||
raise HTTPException(status_code=404, detail="No matching flexible imports to export")
|
||||
|
||||
# Determine union of unmapped keys across all rows
|
||||
unmapped_keys: List[str] = []
|
||||
key_set = set()
|
||||
for r in rows:
|
||||
data = r.extra_data or {}
|
||||
payload = data.get("unmapped") if isinstance(data, dict) and isinstance(data.get("unmapped"), dict) else data
|
||||
if isinstance(payload, dict):
|
||||
for k in payload.keys():
|
||||
if k not in key_set:
|
||||
key_set.add(k)
|
||||
unmapped_keys.append(k)
|
||||
|
||||
# Prepare CSV
|
||||
meta_headers = [
|
||||
"id",
|
||||
"file_type",
|
||||
"target_table",
|
||||
"primary_key_field",
|
||||
"primary_key_value",
|
||||
]
|
||||
fieldnames = meta_headers + unmapped_keys
|
||||
|
||||
output = io.StringIO()
|
||||
writer = csv.DictWriter(output, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
|
||||
for r in rows:
|
||||
row_out: Dict[str, Any] = {
|
||||
"id": r.id,
|
||||
"file_type": r.file_type,
|
||||
"target_table": r.target_table or "",
|
||||
"primary_key_field": r.primary_key_field or "",
|
||||
"primary_key_value": r.primary_key_value or "",
|
||||
}
|
||||
data = r.extra_data or {}
|
||||
payload = data.get("unmapped") if isinstance(data, dict) and isinstance(data.get("unmapped"), dict) else data
|
||||
if isinstance(payload, dict):
|
||||
for k in unmapped_keys:
|
||||
v = payload.get(k)
|
||||
# Normalize lists/dicts to JSON strings for CSV safety
|
||||
if isinstance(v, (dict, list)):
|
||||
try:
|
||||
import json as _json
|
||||
row_out[k] = _json.dumps(v, ensure_ascii=False)
|
||||
except Exception:
|
||||
row_out[k] = str(v)
|
||||
else:
|
||||
row_out[k] = v if v is not None else ""
|
||||
writer.writerow(row_out)
|
||||
|
||||
output.seek(0)
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
filename_parts = ["flexible_unmapped"]
|
||||
if file_type:
|
||||
filename_parts.append(file_type.replace("/", "-").replace(" ", "_"))
|
||||
if target_table:
|
||||
filename_parts.append(target_table.replace("/", "-").replace(" ", "_"))
|
||||
filename = "_".join(filename_parts) + f"_{timestamp}.csv"
|
||||
|
||||
return StreamingResponse(
|
||||
iter([output.getvalue()]),
|
||||
media_type="text/csv",
|
||||
headers={
|
||||
"Content-Disposition": f"attachment; filename=\"{filename}\"",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/export/{row_id}")
|
||||
async def export_single_row_csv(
|
||||
row_id: int,
|
||||
db: Session = Depends(get_db),
|
||||
current_user=Depends(get_admin_user),
|
||||
):
|
||||
"""Export a single flexible import row as CSV.
|
||||
|
||||
Includes metadata columns plus keys from the row's unmapped payload.
|
||||
If FlexibleImport.extra_data contains a nested 'unmapped' dict, those keys are exported;
|
||||
otherwise, all keys of extra_data are exported.
|
||||
"""
|
||||
row: Optional[FlexibleImport] = (
|
||||
db.query(FlexibleImport).filter(FlexibleImport.id == row_id).first()
|
||||
)
|
||||
if not row:
|
||||
raise HTTPException(status_code=404, detail="Flexible import row not found")
|
||||
|
||||
data = row.extra_data or {}
|
||||
payload = (
|
||||
data.get("unmapped")
|
||||
if isinstance(data, dict) and isinstance(data.get("unmapped"), dict)
|
||||
else data
|
||||
)
|
||||
|
||||
unmapped_keys: List[str] = []
|
||||
if isinstance(payload, dict):
|
||||
for k in payload.keys():
|
||||
unmapped_keys.append(k)
|
||||
|
||||
meta_headers = [
|
||||
"id",
|
||||
"file_type",
|
||||
"target_table",
|
||||
"primary_key_field",
|
||||
"primary_key_value",
|
||||
]
|
||||
fieldnames = meta_headers + unmapped_keys
|
||||
|
||||
output = io.StringIO()
|
||||
writer = csv.DictWriter(output, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
|
||||
row_out: Dict[str, Any] = {
|
||||
"id": row.id,
|
||||
"file_type": row.file_type,
|
||||
"target_table": row.target_table or "",
|
||||
"primary_key_field": row.primary_key_field or "",
|
||||
"primary_key_value": row.primary_key_value or "",
|
||||
}
|
||||
if isinstance(payload, dict):
|
||||
for k in unmapped_keys:
|
||||
v = payload.get(k)
|
||||
if isinstance(v, (dict, list)):
|
||||
try:
|
||||
import json as _json
|
||||
row_out[k] = _json.dumps(v, ensure_ascii=False)
|
||||
except Exception:
|
||||
row_out[k] = str(v)
|
||||
else:
|
||||
row_out[k] = v if v is not None else ""
|
||||
|
||||
writer.writerow(row_out)
|
||||
output.seek(0)
|
||||
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
filename = (
|
||||
f"flexible_row_{row.id}_{row.file_type.replace('/', '-').replace(' ', '_')}_{timestamp}.csv"
|
||||
if row.file_type
|
||||
else f"flexible_row_{row.id}_{timestamp}.csv"
|
||||
)
|
||||
|
||||
return StreamingResponse(
|
||||
iter([output.getvalue()]),
|
||||
media_type="text/csv",
|
||||
headers={
|
||||
"Content-Disposition": f"attachment; filename=\"{filename}\"",
|
||||
},
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user