delphi-database/app/api/flexible.py

"""
Flexible Imports admin API: list, filter, and export unmapped rows captured during CSV imports.
"""
from typing import Optional, Dict, Any, List
from datetime import datetime
import csv
import io

from fastapi import APIRouter, Depends, Query, HTTPException
from fastapi.responses import StreamingResponse
from sqlalchemy.orm import Session
from sqlalchemy import func, or_, cast, String

from app.database.base import get_db
from app.auth.security import get_admin_user
from app.models.flexible import FlexibleImport


router = APIRouter(prefix="/flexible", tags=["flexible"])


@router.get("/imports")
async def list_flexible_imports(
    file_type: Optional[str] = Query(None, description="Filter by CSV file type (e.g., FILES.csv)"),
    target_table: Optional[str] = Query(None, description="Filter by target model table name"),
    q: Optional[str] = Query(None, description="Quick text search across file type, target table, and unmapped data"),
    has_keys: Optional[List[str]] = Query(
        None,
        description="Filter rows where extra_data (or its 'unmapped' payload) contains these keys. Repeat param for multiple keys.",
    ),
    skip: int = Query(0, ge=0),
    limit: int = Query(50, ge=1, le=500),
    db: Session = Depends(get_db),
    current_user=Depends(get_admin_user),
):
    """List flexible import rows with optional filtering, quick search, and pagination."""
    query = db.query(FlexibleImport)
    if file_type:
        query = query.filter(FlexibleImport.file_type == file_type)
    if target_table:
        query = query.filter(FlexibleImport.target_table == target_table)
    if q:
        pattern = f"%{q.strip()}%"
        # Search across file_type, target_table, and serialized JSON extra_data
        query = query.filter(
            or_(
                FlexibleImport.file_type.ilike(pattern),
                FlexibleImport.target_table.ilike(pattern),
                cast(FlexibleImport.extra_data, String).ilike(pattern),
            )
        )

    # Filter by key presence inside JSON payload by string matching of the serialized JSON
    # This is DB-agnostic and works across SQLite/Postgres, though not as precise as JSON operators.
    if has_keys:
        for k in [k for k in has_keys if k is not None and str(k).strip() != ""]:
            key = str(k).strip()
            # Look for the JSON key token followed by a colon, e.g. "key":
            query = query.filter(cast(FlexibleImport.extra_data, String).ilike(f'%"{key}":%'))

    total = query.count()
    items = (
        query.order_by(FlexibleImport.id.desc())
        .offset(skip)
        .limit(limit)
        .all()
    )

    def serialize(item: FlexibleImport) -> Dict[str, Any]:
        return {
            "id": item.id,
            "file_type": item.file_type,
            "target_table": item.target_table,
            "primary_key_field": item.primary_key_field,
            "primary_key_value": item.primary_key_value,
            "extra_data": item.extra_data,
        }

    return {
        "total": total,
        "skip": skip,
        "limit": limit,
        "items": [serialize(i) for i in items],
    }


@router.get("/options")
async def flexible_options(
    db: Session = Depends(get_db),
    current_user=Depends(get_admin_user),
):
    """Return distinct file types and target tables for filter dropdowns."""
    file_types: List[str] = [
        ft for (ft,) in db.query(func.distinct(FlexibleImport.file_type)).order_by(FlexibleImport.file_type.asc()).all()
        if ft is not None
    ]
    target_tables: List[str] = [
        tt for (tt,) in db.query(func.distinct(FlexibleImport.target_table)).order_by(FlexibleImport.target_table.asc()).all()
        if tt is not None and tt != ""
    ]
    return {"file_types": file_types, "target_tables": target_tables}


@router.get("/export")
async def export_unmapped_csv(
    file_type: Optional[str] = Query(None, description="Filter by CSV file type (e.g., FILES.csv)"),
    target_table: Optional[str] = Query(None, description="Filter by target model table name"),
    has_keys: Optional[List[str]] = Query(
        None,
        description="Filter rows where extra_data (or its 'unmapped' payload) contains these keys. Repeat param for multiple keys.",
    ),
    db: Session = Depends(get_db),
    current_user=Depends(get_admin_user),
):
    """Export unmapped rows as CSV for review. Includes basic metadata columns and unmapped fields.

    If FlexibleImport.extra_data contains a nested 'unmapped' dict, those keys are exported.
    Otherwise, all keys of extra_data are exported.
    """
    query = db.query(FlexibleImport)
    if file_type:
        query = query.filter(FlexibleImport.file_type == file_type)
    if target_table:
        query = query.filter(FlexibleImport.target_table == target_table)
    if has_keys:
        for k in [k for k in has_keys if k is not None and str(k).strip() != ""]:
            key = str(k).strip()
            query = query.filter(cast(FlexibleImport.extra_data, String).ilike(f'%"{key}":%'))

    rows: List[FlexibleImport] = query.order_by(FlexibleImport.id.asc()).all()
    if not rows:
        raise HTTPException(status_code=404, detail="No matching flexible imports to export")

    # Determine union of unmapped keys across all rows
    unmapped_keys: List[str] = []
    key_set = set()
    for r in rows:
        data = r.extra_data or {}
        payload = data.get("unmapped") if isinstance(data, dict) and isinstance(data.get("unmapped"), dict) else data
        if isinstance(payload, dict):
            for k in payload.keys():
                if k not in key_set:
                    key_set.add(k)
                    unmapped_keys.append(k)

    # Prepare CSV
    meta_headers = [
        "id",
        "file_type",
        "target_table",
        "primary_key_field",
        "primary_key_value",
    ]
    fieldnames = meta_headers + unmapped_keys

    output = io.StringIO()
    writer = csv.DictWriter(output, fieldnames=fieldnames)
    writer.writeheader()

    for r in rows:
        row_out: Dict[str, Any] = {
            "id": r.id,
            "file_type": r.file_type,
            "target_table": r.target_table or "",
            "primary_key_field": r.primary_key_field or "",
            "primary_key_value": r.primary_key_value or "",
        }
        data = r.extra_data or {}
        payload = data.get("unmapped") if isinstance(data, dict) and isinstance(data.get("unmapped"), dict) else data
        if isinstance(payload, dict):
            for k in unmapped_keys:
                v = payload.get(k)
                # Normalize lists/dicts to JSON strings for CSV safety
                if isinstance(v, (dict, list)):
                    try:
                        import json as _json
                        row_out[k] = _json.dumps(v, ensure_ascii=False)
                    except Exception:
                        row_out[k] = str(v)
                else:
                    row_out[k] = v if v is not None else ""
        writer.writerow(row_out)

    output.seek(0)
    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    filename_parts = ["flexible_unmapped"]
    if file_type:
        filename_parts.append(file_type.replace("/", "-").replace(" ", "_"))
    if target_table:
        filename_parts.append(target_table.replace("/", "-").replace(" ", "_"))
    filename = "_".join(filename_parts) + f"_{timestamp}.csv"

    return StreamingResponse(
        iter([output.getvalue()]),
        media_type="text/csv",
        headers={
            "Content-Disposition": f"attachment; filename=\"{filename}\"",
        },
    )


@router.get("/export/{row_id}")
async def export_single_row_csv(
    row_id: int,
    db: Session = Depends(get_db),
    current_user=Depends(get_admin_user),
):
    """Export a single flexible import row as CSV.

    Includes metadata columns plus keys from the row's unmapped payload.
    If FlexibleImport.extra_data contains a nested 'unmapped' dict, those keys are exported;
    otherwise, all keys of extra_data are exported.
    """
    row: Optional[FlexibleImport] = (
        db.query(FlexibleImport).filter(FlexibleImport.id == row_id).first()
    )
    if not row:
        raise HTTPException(status_code=404, detail="Flexible import row not found")

    data = row.extra_data or {}
    payload = (
        data.get("unmapped")
        if isinstance(data, dict) and isinstance(data.get("unmapped"), dict)
        else data
    )

    unmapped_keys: List[str] = []
    if isinstance(payload, dict):
        for k in payload.keys():
            unmapped_keys.append(k)

    meta_headers = [
        "id",
        "file_type",
        "target_table",
        "primary_key_field",
        "primary_key_value",
    ]
    fieldnames = meta_headers + unmapped_keys

    output = io.StringIO()
    writer = csv.DictWriter(output, fieldnames=fieldnames)
    writer.writeheader()

    row_out: Dict[str, Any] = {
        "id": row.id,
        "file_type": row.file_type,
        "target_table": row.target_table or "",
        "primary_key_field": row.primary_key_field or "",
        "primary_key_value": row.primary_key_value or "",
    }
    if isinstance(payload, dict):
        for k in unmapped_keys:
            v = payload.get(k)
            if isinstance(v, (dict, list)):
                try:
                    import json as _json
                    row_out[k] = _json.dumps(v, ensure_ascii=False)
                except Exception:
                    row_out[k] = str(v)
            else:
                row_out[k] = v if v is not None else ""

    writer.writerow(row_out)
    output.seek(0)

    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    filename = (
        f"flexible_row_{row.id}_{row.file_type.replace('/', '-').replace(' ', '_')}_{timestamp}.csv"
        if row.file_type
        else f"flexible_row_{row.id}_{timestamp}.csv"
    )

    return StreamingResponse(
        iter([output.getvalue()]),
        media_type="text/csv",
        headers={
            "Content-Disposition": f"attachment; filename=\"{filename}\"",
        },
    )