Improve Rolodex imports, display, and add repair script

app/main.py

@@ -29,7 +29,7 @@ import structlog
 from structlog import contextvars as structlog_contextvars
 
 from .database import create_tables, get_db, get_database_url
-from .models import User, Case, Client, Phone, Transaction, Document, Payment, ImportLog, Qdros
+from .models import User, Case, Client, Phone, Transaction, Document, Payment, ImportLog, Qdros, LegacyFile
 from .auth import authenticate_user, get_current_user_from_session
 from .reporting import (
     build_phone_book_pdf,
@@ -578,7 +578,13 @@ def import_rolodex_data(db: Session, file_path: str) -> Dict[str, Any]:
     result = {
         'success': 0,
         'errors': [],
-        'total_rows': 0
+        'total_rows': 0,
+        'memo_imported': 0,
+        'memo_missing': 0,
+        'email_imported': 0,
+        'email_missing': 0,
+        'skipped_duplicates': 0,
+        'encoding_used': None,
     }
 
     expected_fields = {
@@ -606,6 +612,7 @@ def import_rolodex_data(db: Session, file_path: str) -> Dict[str, Any]:
 
     try:
         f, used_encoding = open_text_with_fallbacks(file_path)
+        result['encoding_used'] = used_encoding
         with f as file:
             reader = csv.DictReader(file)
 
@@ -630,7 +637,13 @@ def import_rolodex_data(db: Session, file_path: str) -> Dict[str, Any]:
                     # Check for existing client
                     existing = db.query(Client).filter(Client.rolodex_id == rolodex_id).first()
                     if existing:
-                        result['errors'].append(f"Row {row_num}: Client with ID '{rolodex_id}' already exists")
+                        result['skipped_duplicates'] += 1
+                        logger.warning(
+                            "rolodex_import_duplicate",
+                            row=row_num,
+                            rolodex_id=rolodex_id,
+                            file=file_path,
+                        )
                         continue
 
                     # Parse DOB (YYYY-MM-DD or MM/DD/YY variants)
@@ -645,6 +658,21 @@ def import_rolodex_data(db: Session, file_path: str) -> Dict[str, Any]:
                         except ValueError:
                             continue
 
+                    email_val = row.get('Email', '').strip() or None
+                    memo_val = row.get('Memo', '')
+                    memo_clean = memo_val.strip() if memo_val is not None else ''
+                    memo_val_clean = memo_clean or None
+
+                    if email_val:
+                        result['email_imported'] += 1
+                    else:
+                        result['email_missing'] += 1
+
+                    if memo_val_clean:
+                        result['memo_imported'] += 1
+                    else:
+                        result['memo_missing'] += 1
+
                     client = Client(
                         rolodex_id=rolodex_id,
                         prefix=row.get('Prefix', '').strip() or None,
@@ -659,20 +687,41 @@ def import_rolodex_data(db: Session, file_path: str) -> Dict[str, Any]:
                         state=(row.get('Abrev', '').strip() or row.get('St', '').strip() or None),
                         zip_code=row.get('Zip', '').strip() or None,
                         group=row.get('Group', '').strip() or None,
-                        email=row.get('Email', '').strip() or None,
+                        email=email_val,
                         dob=dob_val,
                         ssn=row.get('SS#', '').strip() or None,
                         legal_status=row.get('Legal_Status', '').strip() or None,
-                        memo=row.get('Memo', '').strip() or None,
+                        memo=memo_val_clean,
                     )
 
                     db.add(client)
                     result['success'] += 1
 
+                    logger.info(
+                        "rolodex_import_row",
+                        row=row_num,
+                        rolodex_id=rolodex_id,
+                        email_present=bool(email_val),
+                        memo_present=bool(memo_val_clean),
+                    )
+
                 except Exception as e:
                     result['errors'].append(f"Row {row_num}: {str(e)}")
 
         db.commit()
+        logger.info(
+            "rolodex_import_complete",
+            file=file_path,
+            encoding=used_encoding,
+            total_rows=result['total_rows'],
+            success=result['success'],
+            memo_imported=result['memo_imported'],
+            memo_missing=result['memo_missing'],
+            email_imported=result['email_imported'],
+            email_missing=result['email_missing'],
+            skipped_duplicates=result['skipped_duplicates'],
+            errors=len(result['errors']),
+        )
 
     except Exception as e:
         logger.error("rolodex_import_failed", file=file_path, error=str(e))
@@ -757,7 +806,11 @@ def import_files_data(db: Session, file_path: str) -> Dict[str, Any]:
     result = {
         'success': 0,
         'errors': [],
-        'total_rows': 0
+        'total_rows': 0,
+        'client_linked': 0,
+        'client_missing': 0,
+        'encoding_used': None,
+        'skipped_duplicates': 0,
     }
 
     expected_fields = {
@@ -795,6 +848,7 @@ def import_files_data(db: Session, file_path: str) -> Dict[str, Any]:
     f = None
     try:
         f, used_encoding = open_text_with_fallbacks(file_path)
+        result['encoding_used'] = used_encoding
         reader = csv.DictReader(f)
 
         headers = reader.fieldnames or []
@@ -816,7 +870,13 @@ def import_files_data(db: Session, file_path: str) -> Dict[str, Any]:
                 # Check for existing case
                 existing = db.query(Case).filter(Case.file_no == file_no).first()
                 if existing:
-                    result['errors'].append(f"Row {row_num}: Case with file number '{file_no}' already exists")
+                    result['skipped_duplicates'] += 1
+                    logger.warning(
+                        "files_import_duplicate",
+                        row=row_num,
+                        file_no=file_no,
+                        file=file_path,
+                    )
                     continue
 
                 # Find client by ID
@@ -825,8 +885,16 @@ def import_files_data(db: Session, file_path: str) -> Dict[str, Any]:
                 if client_id:
                     client = db.query(Client).filter(Client.rolodex_id == client_id).first()
                     if not client:
+                        result['client_missing'] += 1
+                        logger.warning(
+                            "files_import_missing_client",
+                            row=row_num,
+                            file_no=file_no,
+                            legacy_client_id=client_id,
+                        )
                         result['errors'].append(f"Row {row_num}: Client with ID '{client_id}' not found")
                         continue
+                    result['client_linked'] += 1
 
                 case = Case(
                     file_no=file_no,
@@ -835,16 +903,35 @@ def import_files_data(db: Session, file_path: str) -> Dict[str, Any]:
                     case_type=row.get('File_Type', '').strip() or None,
                     description=row.get('Regarding', '').strip() or None,
                     open_date=parse_date(row.get('Opened', '')),
-                    close_date=parse_date(row.get('Closed', ''))
+                    close_date=parse_date(row.get('Closed', '')),
                 )
 
                 db.add(case)
                 result['success'] += 1
 
+                logger.info(
+                    "files_import_row",
+                    row=row_num,
+                    file_no=file_no,
+                    client_id=client.id if client else None,
+                    status=case.status,
+                )
+
             except Exception as e:
                 result['errors'].append(f"Row {row_num}: {str(e)}")
 
         db.commit()
+        logger.info(
+            "files_import_complete",
+            file=file_path,
+            encoding=used_encoding,
+            total_rows=result['total_rows'],
+            success=result['success'],
+            client_linked=result['client_linked'],
+            client_missing=result['client_missing'],
+            skipped_duplicates=result['skipped_duplicates'],
+            errors=len(result['errors']),
+        )
 
     except Exception as e:
         result['errors'].append(f"Import failed: {str(e)}")

app/templates/rolodex_view.html

@@ -59,7 +59,13 @@
           </div>
           <div class="col-md-3">
             <div class="text-muted small">Email</div>
-            <div>{{ client.email or '' }}</div>
+            <div>
+              {% if client.email %}
+                <a href="mailto:{{ client.email }}">{{ client.email }}</a>
+              {% else %}
+                <span class="text-muted">No email</span>
+              {% endif %}
+            </div>
           </div>
           <div class="col-md-3">
             <div class="text-muted small">DOB</div>
@@ -77,7 +83,13 @@
           </div>
           <div class="col-md-9">
             <div class="text-muted small">Memo / Notes</div>
-            <div>{{ client.memo or '' }}</div>
+            <div>
+              {% if client.memo %}
+                {{ client.memo }}
+              {% else %}
+                <span class="text-muted">No notes available</span>
+              {% endif %}
+            </div>
           </div>
         </div>
 
@@ -171,8 +183,9 @@
               </tr>
             </thead>
             <tbody>
-              {% if client.cases and client.cases|length > 0 %}
+              {% set sorted_cases = client.cases | sort(attribute='open_date', reverse=True) %}
-                {% for c in client.cases %}
+              {% if sorted_cases and sorted_cases|length > 0 %}
+                {% for c in sorted_cases %}
                 {% if status_filter == 'all' or (status_filter == 'open' and (c.status != 'closed')) or (status_filter == 'closed' and c.status == 'closed') %}
                 <tr>
                   <td>{{ c.file_no }}</td>
@@ -184,7 +197,7 @@
                       <span class="badge bg-success">Open</span>
                     {% endif %}
                   </td>
-                  <td>{{ c.open_date.strftime('%Y-%m-%d') if c.open_date else '' }}</td>
+                  <td>{{ c.open_date.strftime('%Y-%m-%d') if c.open_date else '—' }}</td>
                   <td class="text-end">
                     <a class="btn btn-sm btn-outline-primary" href="/case/{{ c.id }}">
                       <i class="bi bi-folder2-open"></i>
@@ -197,7 +210,7 @@
                 {% endif %}
                 {% endfor %}
               {% else %}
-                <tr><td colspan="5" class="text-center text-muted py-3">No related cases.</td></tr>
+                <tr><td colspan="5" class="text-center text-muted py-3">No related cases linked.</td></tr>
               {% endif %}
             </tbody>
           </table>

scripts/fix_case_links.py

@@ -0,0 +1,94 @@
+"""Utility script to repair orphaned Case records.
+
+This script attempts to link Case records to Client entries based on legacy
+identifiers. It should be executed inside the running Docker container.
+
+Usage:
+    docker compose exec delphi-db python scripts/fix_case_links.py
+
+Safety:
+    - Prints a summary before making changes.
+    - Requires confirmation unless run with --yes.
+"""
+
+from __future__ import annotations
+
+import argparse
+from collections import defaultdict
+
+from sqlalchemy.orm import joinedload
+
+from app.database import SessionLocal
+from app.models import Case, Client
+
+
+def build_client_maps(session) -> tuple[dict[str, int], dict[str, list[int]]]:
+    rolodex_to_client: dict[str, int] = {}
+    file_to_client: dict[str, list[int]] = defaultdict(list)
+
+    for client in session.query(Client).options(joinedload(Client.cases)).all():
+        if client.rolodex_id:
+            rolodex_to_client[client.rolodex_id] = client.id
+        for case in client.cases:
+            file_to_client[case.file_no].append(client.id)
+
+    return rolodex_to_client, file_to_client
+
+
+def find_orphaned_cases(session):
+    return session.query(Case).filter(Case.client_id == None).all()  # noqa: E711
+
+
+def main(confirm: bool) -> int:
+    session = SessionLocal()
+    try:
+        rolodex_map, file_map = build_client_maps(session)
+        orphans = find_orphaned_cases(session)
+
+        if not orphans:
+            print("No orphaned cases found. 🎉")
+            return 0
+
+        assignments: list[tuple[Case, int]] = []
+        for case in orphans:
+            candidate_client_id = None
+            if case.file_no in file_map and file_map[case.file_no]:
+                candidate_client_id = file_map[case.file_no][0]
+            elif case.file_no in rolodex_map:
+                candidate_client_id = rolodex_map[case.file_no]
+
+            if candidate_client_id:
+                assignments.append((case, candidate_client_id))
+
+        if not assignments:
+            print("No matching clients found for orphaned cases. Nothing to do.")
+            return 1
+
+        print("Orphaned cases detected:")
+        for case, client_id in assignments:
+            print(f"  Case {case.file_no} (id={case.id}) → Client id {client_id}")
+
+        if not confirm:
+            response = input("Apply these fixes? [y/N]: ").strip().lower()
+            if response not in {"y", "yes"}:
+                print("Aborting with no changes.")
+                return 1
+
+        updated = 0
+        for case, client_id in assignments:
+            case.client_id = client_id
+            updated += 1
+
+        session.commit()
+        print(f"Updated {updated} cases with matching clients.")
+        return 0
+    finally:
+        session.close()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Repair orphaned Case links.")
+    parser.add_argument("--yes", action="store_true", help="Apply changes without confirmation")
+    args = parser.parse_args()
+    raise SystemExit(main(confirm=args.yes))
+

static/js/custom.js

@@ -2,10 +2,12 @@
 
 document.addEventListener('DOMContentLoaded', function() {
     // Initialize tooltips if any
+    if (window.bootstrap && bootstrap.Tooltip) {
         var tooltipTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="tooltip"]'));
-    var tooltipList = tooltipTriggerList.map(function (tooltipTriggerEl) {
+        tooltipTriggerList.forEach(function (tooltipTriggerEl) {
-        return new bootstrap.Tooltip(tooltipTriggerEl);
+            new bootstrap.Tooltip(tooltipTriggerEl);
         });
+    }
 
     // Auto-hide alerts after 5 seconds
     var alerts = document.querySelectorAll('.alert:not(.alert-permanent)');

tests/test_import_data.py

@@ -0,0 +1,130 @@
+"""Tests for Rolodex and Files CSV import helpers."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from app.main import import_files_data, import_rolodex_data
+from app.models import Base, Case, Client
+
+
+def _make_engine(db_path: Path):
+    engine = create_engine(f"sqlite:///{db_path}")
+    Base.metadata.create_all(engine)
+    return engine
+
+
+@pytest.fixture()
+def session(tmp_path):
+    engine = _make_engine(tmp_path / "import-tests.db")
+    SessionLocal = sessionmaker(bind=engine)
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()
+        engine.dispose()
+
+
+def _write_csv(path: Path, content: str) -> Path:
+    path.write_text(content, encoding="utf-8")
+    return path
+
+
+def test_import_rolodex_captures_email_and_memo(session, tmp_path):
+    csv_path = _write_csv(
+        tmp_path / "ROLODEX.csv",
+        "Id,Prefix,First,Middle,Last,Suffix,Title,A1,A2,A3,City,Abrev,St,Zip,Email,DOB,SS#,Legal_Status,Group,Memo\n"
+        "1001,,Ada,,Lovelace,,Countess,123 Main,,,London,,UK,12345,ada@example.com,12/10/1815,XXX-XX-1111,Active,VIP,Top client notes\n"
+        "1002,,Alan,,Turing,,,43 Park,,Bletchley,,UK,67890,,06/23/1912,XXX-XX-2222,Active,VIP,   \n",
+    )
+
+    result = import_rolodex_data(session, str(csv_path))
+
+    assert result["success"] == 2
+    assert result["email_imported"] == 1
+    assert result["email_missing"] == 1
+    assert result["memo_imported"] == 1
+    assert result["memo_missing"] == 1
+    assert not result["errors"]
+
+    clients = {c.rolodex_id: c for c in session.query(Client).all()}
+    assert clients["1001"].email == "ada@example.com"
+    assert clients["1001"].memo == "Top client notes"
+    assert clients["1002"].email is None
+    assert clients["1002"].memo is None
+
+
+def test_import_rolodex_handles_duplicates(session, tmp_path):
+    csv_path = _write_csv(
+        tmp_path / "ROLODEX.csv",
+        "Id,Prefix,First,Middle,Last,Suffix,Title,A1,A2,A3,City,Abrev,St,Zip,Email,DOB,SS#,Legal_Status,Group,Memo\n"
+        "3001,,Grace,,Hopper,,,,,Arlington,VA,Virginia,22202,grace@example.com,12/09/1906,XXX-XX-3333,Active,VIP,Notes\n"
+        "3001,,Grace,,Hopper,,,,,Arlington,VA,Virginia,22202,grace@example.com,12/09/1906,XXX-XX-3333,Active,VIP,Duplicate\n",
+    )
+
+    result = import_rolodex_data(session, str(csv_path))
+
+    assert result["success"] == 1
+    assert result["skipped_duplicates"] == 1
+    assert len(result["errors"]) == 0
+
+
+def test_import_files_links_clients_and_detects_missing(session, tmp_path):
+    _write_csv(
+        tmp_path / "ROLODEX.csv",
+        "Id,Prefix,First,Middle,Last,Suffix,Title,A1,A2,A3,City,Abrev,St,Zip,Email,DOB,SS#,Legal_Status,Group,Memo\n"
+        "2001,,Grace,,Hopper,,,,,Arlington,VA,Virginia,22202,grace@example.com,12/09/1906,XXX-XX-3333,Active,VIP,Notes\n",
+    )
+    import_rolodex_data(session, str(tmp_path / "ROLODEX.csv"))
+
+    files_path = _write_csv(
+        tmp_path / "FILES.csv",
+        "File_No,Id,File_Type,Regarding,Opened,Closed,Empl_Num,Rate_Per_Hour,Status,Footer_Code,Opposing,Hours,Hours_P,Trust_Bal,Trust_Bal_P,Hourly_Fees,Hourly_Fees_P,Flat_Fees,Flat_Fees_P,Disbursements,Disbursements_P,Credit_Bal,Credit_Bal_P,Total_Charges,Total_Charges_P,Amount_Owing,Amount_Owing_P,Transferable,Memo\n"
+        "F-001,2001,Divorce,Important matter,01/01/2020,,E1,175,Open,F1,,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,No,Initial legacy memo\n"
+        "F-002,9999,Divorce,Missing client,01/01/2020,,E1,175,Open,F1,,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,No,Should be skipped\n",
+    )
+
+    result = import_files_data(session, str(files_path))
+
+    assert result["success"] == 1
+    assert result["client_linked"] == 1
+    assert result["client_missing"] == 1
+    assert result["errors"], "Expected missing client to produce an error"
+
+    case = session.query(Case).filter(Case.file_no == "F-001").one()
+    assert case.client is not None
+    assert case.client.rolodex_id == "2001"
+
+    missing_case = session.query(Case).filter(Case.file_no == "F-002").first()
+    assert missing_case is None
+
+
+def test_import_files_respects_duplicates(session, tmp_path):
+    _write_csv(
+        tmp_path / "ROLODEX.csv",
+        "Id,Prefix,First,Middle,Last,Suffix,Title,A1,A2,A3,City,Abrev,St,Zip,Email,DOB,SS#,Legal_Status,Group,Memo\n"
+        "4001,,Marie,,Curie,,,,,Paris,,France,75000,marie@example.com,11/07/1867,XXX-XX-4444,Active,VIP,Notes\n",
+    )
+    import_rolodex_data(session, str(tmp_path / "ROLODEX.csv"))
+
+    files_csv = _write_csv(
+        tmp_path / "FILES.csv",
+        "File_No,Id,File_Type,Regarding,Opened,Closed,Empl_Num,Rate_Per_Hour,Status,Footer_Code,Opposing,Hours,Hours_P,Trust_Bal,Trust_Bal_P,Hourly_Fees,Hourly_Fees_P,Flat_Fees,Flat_Fees_P,Disbursements,Disbursements_P,Credit_Bal,Credit_Bal_P,Total_Charges,Total_Charges_P,Amount_Owing,Amount_Owing_P,Transferable,Memo\n"
+        "F-100,4001,Divorce,Legacy matter,01/01/2020,,E1,175,Open,F1,,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,No,First import\n"
+        "F-100,4001,Divorce,Legacy matter,01/01/2020,,E1,175,Open,F1,,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,No,Duplicate row\n",
+    )
+
+    first_result = import_files_data(session, str(files_csv))
+    assert first_result["success"] == 1
+    assert first_result["skipped_duplicates"] == 0
+
+    second_result = import_files_data(session, str(files_csv))
+    assert second_result["success"] == 0
+    assert second_result["skipped_duplicates"] == 1
+    assert not session.query(Case).filter(Case.file_no == "F-100").all()[1:]
+