diff --git a/app/main.py b/app/main.py index b0c44b3..7794179 100644 --- a/app/main.py +++ b/app/main.py @@ -29,7 +29,7 @@ import structlog from structlog import contextvars as structlog_contextvars from .database import create_tables, get_db, get_database_url -from .models import User, Case, Client, Phone, Transaction, Document, Payment, ImportLog, Qdros +from .models import User, Case, Client, Phone, Transaction, Document, Payment, ImportLog, Qdros, LegacyFile from .auth import authenticate_user, get_current_user_from_session from .reporting import ( build_phone_book_pdf, @@ -578,7 +578,13 @@ def import_rolodex_data(db: Session, file_path: str) -> Dict[str, Any]: result = { 'success': 0, 'errors': [], - 'total_rows': 0 + 'total_rows': 0, + 'memo_imported': 0, + 'memo_missing': 0, + 'email_imported': 0, + 'email_missing': 0, + 'skipped_duplicates': 0, + 'encoding_used': None, } expected_fields = { @@ -606,6 +612,7 @@ def import_rolodex_data(db: Session, file_path: str) -> Dict[str, Any]: try: f, used_encoding = open_text_with_fallbacks(file_path) + result['encoding_used'] = used_encoding with f as file: reader = csv.DictReader(file) @@ -630,7 +637,13 @@ def import_rolodex_data(db: Session, file_path: str) -> Dict[str, Any]: # Check for existing client existing = db.query(Client).filter(Client.rolodex_id == rolodex_id).first() if existing: - result['errors'].append(f"Row {row_num}: Client with ID '{rolodex_id}' already exists") + result['skipped_duplicates'] += 1 + logger.warning( + "rolodex_import_duplicate", + row=row_num, + rolodex_id=rolodex_id, + file=file_path, + ) continue # Parse DOB (YYYY-MM-DD or MM/DD/YY variants) @@ -645,6 +658,21 @@ def import_rolodex_data(db: Session, file_path: str) -> Dict[str, Any]: except ValueError: continue + email_val = row.get('Email', '').strip() or None + memo_val = row.get('Memo', '') + memo_clean = memo_val.strip() if memo_val is not None else '' + memo_val_clean = memo_clean or None + + if email_val: + result['email_imported'] += 1 + else: + result['email_missing'] += 1 + + if memo_val_clean: + result['memo_imported'] += 1 + else: + result['memo_missing'] += 1 + client = Client( rolodex_id=rolodex_id, prefix=row.get('Prefix', '').strip() or None, @@ -659,20 +687,41 @@ def import_rolodex_data(db: Session, file_path: str) -> Dict[str, Any]: state=(row.get('Abrev', '').strip() or row.get('St', '').strip() or None), zip_code=row.get('Zip', '').strip() or None, group=row.get('Group', '').strip() or None, - email=row.get('Email', '').strip() or None, + email=email_val, dob=dob_val, ssn=row.get('SS#', '').strip() or None, legal_status=row.get('Legal_Status', '').strip() or None, - memo=row.get('Memo', '').strip() or None, + memo=memo_val_clean, ) db.add(client) result['success'] += 1 + logger.info( + "rolodex_import_row", + row=row_num, + rolodex_id=rolodex_id, + email_present=bool(email_val), + memo_present=bool(memo_val_clean), + ) + except Exception as e: result['errors'].append(f"Row {row_num}: {str(e)}") db.commit() + logger.info( + "rolodex_import_complete", + file=file_path, + encoding=used_encoding, + total_rows=result['total_rows'], + success=result['success'], + memo_imported=result['memo_imported'], + memo_missing=result['memo_missing'], + email_imported=result['email_imported'], + email_missing=result['email_missing'], + skipped_duplicates=result['skipped_duplicates'], + errors=len(result['errors']), + ) except Exception as e: logger.error("rolodex_import_failed", file=file_path, error=str(e)) @@ -757,7 +806,11 @@ def import_files_data(db: Session, file_path: str) -> Dict[str, Any]: result = { 'success': 0, 'errors': [], - 'total_rows': 0 + 'total_rows': 0, + 'client_linked': 0, + 'client_missing': 0, + 'encoding_used': None, + 'skipped_duplicates': 0, } expected_fields = { @@ -795,6 +848,7 @@ def import_files_data(db: Session, file_path: str) -> Dict[str, Any]: f = None try: f, used_encoding = open_text_with_fallbacks(file_path) + result['encoding_used'] = used_encoding reader = csv.DictReader(f) headers = reader.fieldnames or [] @@ -816,7 +870,13 @@ def import_files_data(db: Session, file_path: str) -> Dict[str, Any]: # Check for existing case existing = db.query(Case).filter(Case.file_no == file_no).first() if existing: - result['errors'].append(f"Row {row_num}: Case with file number '{file_no}' already exists") + result['skipped_duplicates'] += 1 + logger.warning( + "files_import_duplicate", + row=row_num, + file_no=file_no, + file=file_path, + ) continue # Find client by ID @@ -825,8 +885,16 @@ def import_files_data(db: Session, file_path: str) -> Dict[str, Any]: if client_id: client = db.query(Client).filter(Client.rolodex_id == client_id).first() if not client: + result['client_missing'] += 1 + logger.warning( + "files_import_missing_client", + row=row_num, + file_no=file_no, + legacy_client_id=client_id, + ) result['errors'].append(f"Row {row_num}: Client with ID '{client_id}' not found") continue + result['client_linked'] += 1 case = Case( file_no=file_no, @@ -835,16 +903,35 @@ def import_files_data(db: Session, file_path: str) -> Dict[str, Any]: case_type=row.get('File_Type', '').strip() or None, description=row.get('Regarding', '').strip() or None, open_date=parse_date(row.get('Opened', '')), - close_date=parse_date(row.get('Closed', '')) + close_date=parse_date(row.get('Closed', '')), ) db.add(case) result['success'] += 1 + logger.info( + "files_import_row", + row=row_num, + file_no=file_no, + client_id=client.id if client else None, + status=case.status, + ) + except Exception as e: result['errors'].append(f"Row {row_num}: {str(e)}") db.commit() + logger.info( + "files_import_complete", + file=file_path, + encoding=used_encoding, + total_rows=result['total_rows'], + success=result['success'], + client_linked=result['client_linked'], + client_missing=result['client_missing'], + skipped_duplicates=result['skipped_duplicates'], + errors=len(result['errors']), + ) except Exception as e: result['errors'].append(f"Import failed: {str(e)}") diff --git a/app/templates/rolodex_view.html b/app/templates/rolodex_view.html index 9dac4fa..2fb2d27 100644 --- a/app/templates/rolodex_view.html +++ b/app/templates/rolodex_view.html @@ -59,7 +59,13 @@
Email
-
{{ client.email or '' }}
+
+ {% if client.email %} + {{ client.email }} + {% else %} + No email + {% endif %} +
DOB
@@ -77,7 +83,13 @@
Memo / Notes
-
{{ client.memo or '' }}
+
+ {% if client.memo %} + {{ client.memo }} + {% else %} + No notes available + {% endif %} +
@@ -171,8 +183,9 @@ - {% if client.cases and client.cases|length > 0 %} - {% for c in client.cases %} + {% set sorted_cases = client.cases | sort(attribute='open_date', reverse=True) %} + {% if sorted_cases and sorted_cases|length > 0 %} + {% for c in sorted_cases %} {% if status_filter == 'all' or (status_filter == 'open' and (c.status != 'closed')) or (status_filter == 'closed' and c.status == 'closed') %} {{ c.file_no }} @@ -184,7 +197,7 @@ Open {% endif %} - {{ c.open_date.strftime('%Y-%m-%d') if c.open_date else '' }} + {{ c.open_date.strftime('%Y-%m-%d') if c.open_date else '—' }} @@ -197,7 +210,7 @@ {% endif %} {% endfor %} {% else %} - No related cases. + No related cases linked. {% endif %} diff --git a/delphi.db b/delphi.db index 33698af..347ce1c 100644 Binary files a/delphi.db and b/delphi.db differ diff --git a/scripts/fix_case_links.py b/scripts/fix_case_links.py new file mode 100644 index 0000000..4e25a62 --- /dev/null +++ b/scripts/fix_case_links.py @@ -0,0 +1,94 @@ +"""Utility script to repair orphaned Case records. + +This script attempts to link Case records to Client entries based on legacy +identifiers. It should be executed inside the running Docker container. + +Usage: + docker compose exec delphi-db python scripts/fix_case_links.py + +Safety: + - Prints a summary before making changes. + - Requires confirmation unless run with --yes. +""" + +from __future__ import annotations + +import argparse +from collections import defaultdict + +from sqlalchemy.orm import joinedload + +from app.database import SessionLocal +from app.models import Case, Client + + +def build_client_maps(session) -> tuple[dict[str, int], dict[str, list[int]]]: + rolodex_to_client: dict[str, int] = {} + file_to_client: dict[str, list[int]] = defaultdict(list) + + for client in session.query(Client).options(joinedload(Client.cases)).all(): + if client.rolodex_id: + rolodex_to_client[client.rolodex_id] = client.id + for case in client.cases: + file_to_client[case.file_no].append(client.id) + + return rolodex_to_client, file_to_client + + +def find_orphaned_cases(session): + return session.query(Case).filter(Case.client_id == None).all() # noqa: E711 + + +def main(confirm: bool) -> int: + session = SessionLocal() + try: + rolodex_map, file_map = build_client_maps(session) + orphans = find_orphaned_cases(session) + + if not orphans: + print("No orphaned cases found. 🎉") + return 0 + + assignments: list[tuple[Case, int]] = [] + for case in orphans: + candidate_client_id = None + if case.file_no in file_map and file_map[case.file_no]: + candidate_client_id = file_map[case.file_no][0] + elif case.file_no in rolodex_map: + candidate_client_id = rolodex_map[case.file_no] + + if candidate_client_id: + assignments.append((case, candidate_client_id)) + + if not assignments: + print("No matching clients found for orphaned cases. Nothing to do.") + return 1 + + print("Orphaned cases detected:") + for case, client_id in assignments: + print(f" Case {case.file_no} (id={case.id}) → Client id {client_id}") + + if not confirm: + response = input("Apply these fixes? [y/N]: ").strip().lower() + if response not in {"y", "yes"}: + print("Aborting with no changes.") + return 1 + + updated = 0 + for case, client_id in assignments: + case.client_id = client_id + updated += 1 + + session.commit() + print(f"Updated {updated} cases with matching clients.") + return 0 + finally: + session.close() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Repair orphaned Case links.") + parser.add_argument("--yes", action="store_true", help="Apply changes without confirmation") + args = parser.parse_args() + raise SystemExit(main(confirm=args.yes)) + diff --git a/static/js/custom.js b/static/js/custom.js index 117dfcc..d0b32bc 100644 --- a/static/js/custom.js +++ b/static/js/custom.js @@ -2,10 +2,12 @@ document.addEventListener('DOMContentLoaded', function() { // Initialize tooltips if any - var tooltipTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="tooltip"]')); - var tooltipList = tooltipTriggerList.map(function (tooltipTriggerEl) { - return new bootstrap.Tooltip(tooltipTriggerEl); - }); + if (window.bootstrap && bootstrap.Tooltip) { + var tooltipTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="tooltip"]')); + tooltipTriggerList.forEach(function (tooltipTriggerEl) { + new bootstrap.Tooltip(tooltipTriggerEl); + }); + } // Auto-hide alerts after 5 seconds var alerts = document.querySelectorAll('.alert:not(.alert-permanent)'); diff --git a/tests/test_import_data.py b/tests/test_import_data.py new file mode 100644 index 0000000..0cb519d --- /dev/null +++ b/tests/test_import_data.py @@ -0,0 +1,130 @@ +"""Tests for Rolodex and Files CSV import helpers.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from app.main import import_files_data, import_rolodex_data +from app.models import Base, Case, Client + + +def _make_engine(db_path: Path): + engine = create_engine(f"sqlite:///{db_path}") + Base.metadata.create_all(engine) + return engine + + +@pytest.fixture() +def session(tmp_path): + engine = _make_engine(tmp_path / "import-tests.db") + SessionLocal = sessionmaker(bind=engine) + db = SessionLocal() + try: + yield db + finally: + db.close() + engine.dispose() + + +def _write_csv(path: Path, content: str) -> Path: + path.write_text(content, encoding="utf-8") + return path + + +def test_import_rolodex_captures_email_and_memo(session, tmp_path): + csv_path = _write_csv( + tmp_path / "ROLODEX.csv", + "Id,Prefix,First,Middle,Last,Suffix,Title,A1,A2,A3,City,Abrev,St,Zip,Email,DOB,SS#,Legal_Status,Group,Memo\n" + "1001,,Ada,,Lovelace,,Countess,123 Main,,,London,,UK,12345,ada@example.com,12/10/1815,XXX-XX-1111,Active,VIP,Top client notes\n" + "1002,,Alan,,Turing,,,43 Park,,Bletchley,,UK,67890,,06/23/1912,XXX-XX-2222,Active,VIP, \n", + ) + + result = import_rolodex_data(session, str(csv_path)) + + assert result["success"] == 2 + assert result["email_imported"] == 1 + assert result["email_missing"] == 1 + assert result["memo_imported"] == 1 + assert result["memo_missing"] == 1 + assert not result["errors"] + + clients = {c.rolodex_id: c for c in session.query(Client).all()} + assert clients["1001"].email == "ada@example.com" + assert clients["1001"].memo == "Top client notes" + assert clients["1002"].email is None + assert clients["1002"].memo is None + + +def test_import_rolodex_handles_duplicates(session, tmp_path): + csv_path = _write_csv( + tmp_path / "ROLODEX.csv", + "Id,Prefix,First,Middle,Last,Suffix,Title,A1,A2,A3,City,Abrev,St,Zip,Email,DOB,SS#,Legal_Status,Group,Memo\n" + "3001,,Grace,,Hopper,,,,,Arlington,VA,Virginia,22202,grace@example.com,12/09/1906,XXX-XX-3333,Active,VIP,Notes\n" + "3001,,Grace,,Hopper,,,,,Arlington,VA,Virginia,22202,grace@example.com,12/09/1906,XXX-XX-3333,Active,VIP,Duplicate\n", + ) + + result = import_rolodex_data(session, str(csv_path)) + + assert result["success"] == 1 + assert result["skipped_duplicates"] == 1 + assert len(result["errors"]) == 0 + + +def test_import_files_links_clients_and_detects_missing(session, tmp_path): + _write_csv( + tmp_path / "ROLODEX.csv", + "Id,Prefix,First,Middle,Last,Suffix,Title,A1,A2,A3,City,Abrev,St,Zip,Email,DOB,SS#,Legal_Status,Group,Memo\n" + "2001,,Grace,,Hopper,,,,,Arlington,VA,Virginia,22202,grace@example.com,12/09/1906,XXX-XX-3333,Active,VIP,Notes\n", + ) + import_rolodex_data(session, str(tmp_path / "ROLODEX.csv")) + + files_path = _write_csv( + tmp_path / "FILES.csv", + "File_No,Id,File_Type,Regarding,Opened,Closed,Empl_Num,Rate_Per_Hour,Status,Footer_Code,Opposing,Hours,Hours_P,Trust_Bal,Trust_Bal_P,Hourly_Fees,Hourly_Fees_P,Flat_Fees,Flat_Fees_P,Disbursements,Disbursements_P,Credit_Bal,Credit_Bal_P,Total_Charges,Total_Charges_P,Amount_Owing,Amount_Owing_P,Transferable,Memo\n" + "F-001,2001,Divorce,Important matter,01/01/2020,,E1,175,Open,F1,,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,No,Initial legacy memo\n" + "F-002,9999,Divorce,Missing client,01/01/2020,,E1,175,Open,F1,,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,No,Should be skipped\n", + ) + + result = import_files_data(session, str(files_path)) + + assert result["success"] == 1 + assert result["client_linked"] == 1 + assert result["client_missing"] == 1 + assert result["errors"], "Expected missing client to produce an error" + + case = session.query(Case).filter(Case.file_no == "F-001").one() + assert case.client is not None + assert case.client.rolodex_id == "2001" + + missing_case = session.query(Case).filter(Case.file_no == "F-002").first() + assert missing_case is None + + +def test_import_files_respects_duplicates(session, tmp_path): + _write_csv( + tmp_path / "ROLODEX.csv", + "Id,Prefix,First,Middle,Last,Suffix,Title,A1,A2,A3,City,Abrev,St,Zip,Email,DOB,SS#,Legal_Status,Group,Memo\n" + "4001,,Marie,,Curie,,,,,Paris,,France,75000,marie@example.com,11/07/1867,XXX-XX-4444,Active,VIP,Notes\n", + ) + import_rolodex_data(session, str(tmp_path / "ROLODEX.csv")) + + files_csv = _write_csv( + tmp_path / "FILES.csv", + "File_No,Id,File_Type,Regarding,Opened,Closed,Empl_Num,Rate_Per_Hour,Status,Footer_Code,Opposing,Hours,Hours_P,Trust_Bal,Trust_Bal_P,Hourly_Fees,Hourly_Fees_P,Flat_Fees,Flat_Fees_P,Disbursements,Disbursements_P,Credit_Bal,Credit_Bal_P,Total_Charges,Total_Charges_P,Amount_Owing,Amount_Owing_P,Transferable,Memo\n" + "F-100,4001,Divorce,Legacy matter,01/01/2020,,E1,175,Open,F1,,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,No,First import\n" + "F-100,4001,Divorce,Legacy matter,01/01/2020,,E1,175,Open,F1,,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,No,Duplicate row\n", + ) + + first_result = import_files_data(session, str(files_csv)) + assert first_result["success"] == 1 + assert first_result["skipped_duplicates"] == 0 + + second_result = import_files_data(session, str(files_csv)) + assert second_result["success"] == 0 + assert second_result["skipped_duplicates"] == 1 + assert not session.query(Case).filter(Case.file_no == "F-100").all()[1:] +