hotswap/delphi-database-v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve Rolodex imports, display, and add repair script

Browse Source
This commit is contained in:
HotSwapp
2025-10-13 15:00:13 -05:00
parent 2e2380552e
commit 84c3dac83a
6 changed files with 344 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
app/main.py
View File

@@ -29,7 +29,7 @@ import structlog
from structlog import contextvars as structlog_contextvars
from .database import create_tables, get_db, get_database_url
from .models import User, Case, Client, Phone, Transaction, Document, Payment, ImportLog, Qdros
from .models import User, Case, Client, Phone, Transaction, Document, Payment, ImportLog, Qdros, LegacyFile
from .auth import authenticate_user, get_current_user_from_session
from .reporting import (
build_phone_book_pdf,
@@ -578,7 +578,13 @@ def import_rolodex_data(db: Session, file_path: str) -> Dict[str, Any]:
result = {
'success': 0,
'errors': [],
'total_rows': 0
'total_rows': 0,
'memo_imported': 0,
'memo_missing': 0,
'email_imported': 0,
'email_missing': 0,
'skipped_duplicates': 0,
'encoding_used': None,
}
expected_fields = {
@@ -606,6 +612,7 @@ def import_rolodex_data(db: Session, file_path: str) -> Dict[str, Any]:
try:
f, used_encoding = open_text_with_fallbacks(file_path)
result['encoding_used'] = used_encoding
with f as file:
reader = csv.DictReader(file)
@@ -630,7 +637,13 @@ def import_rolodex_data(db: Session, file_path: str) -> Dict[str, Any]:
# Check for existing client
existing = db.query(Client).filter(Client.rolodex_id == rolodex_id).first()
if existing:
result['errors'].append(f"Row {row_num}: Client with ID '{rolodex_id}' already exists")
result['skipped_duplicates'] += 1
logger.warning(
"rolodex_import_duplicate",
row=row_num,
rolodex_id=rolodex_id,
file=file_path,
)
continue
# Parse DOB (YYYY-MM-DD or MM/DD/YY variants)
@@ -645,6 +658,21 @@ def import_rolodex_data(db: Session, file_path: str) -> Dict[str, Any]:
except ValueError:
continue
email_val = row.get('Email', '').strip() or None
memo_val = row.get('Memo', '')
memo_clean = memo_val.strip() if memo_val is not None else ''
memo_val_clean = memo_clean or None
if email_val:
result['email_imported'] += 1
else:
result['email_missing'] += 1
if memo_val_clean:
result['memo_imported'] += 1
else:
result['memo_missing'] += 1
client = Client(
rolodex_id=rolodex_id,
prefix=row.get('Prefix', '').strip() or None,
@@ -659,20 +687,41 @@ def import_rolodex_data(db: Session, file_path: str) -> Dict[str, Any]:
state=(row.get('Abrev', '').strip() or row.get('St', '').strip() or None),
zip_code=row.get('Zip', '').strip() or None,
group=row.get('Group', '').strip() or None,
email=row.get('Email', '').strip() or None,
email=email_val,
dob=dob_val,
ssn=row.get('SS#', '').strip() or None,
legal_status=row.get('Legal_Status', '').strip() or None,
memo=row.get('Memo', '').strip() or None,
memo=memo_val_clean,
)
db.add(client)
result['success'] += 1
logger.info(
"rolodex_import_row",
row=row_num,
rolodex_id=rolodex_id,
email_present=bool(email_val),
memo_present=bool(memo_val_clean),
)
except Exception as e:
result['errors'].append(f"Row {row_num}: {str(e)}")
db.commit()
logger.info(
"rolodex_import_complete",
file=file_path,
encoding=used_encoding,
total_rows=result['total_rows'],
success=result['success'],
memo_imported=result['memo_imported'],
memo_missing=result['memo_missing'],
email_imported=result['email_imported'],
email_missing=result['email_missing'],
skipped_duplicates=result['skipped_duplicates'],
errors=len(result['errors']),
)
except Exception as e:
logger.error("rolodex_import_failed", file=file_path, error=str(e))
@@ -757,7 +806,11 @@ def import_files_data(db: Session, file_path: str) -> Dict[str, Any]:
result = {
'success': 0,
'errors': [],
'total_rows': 0
'total_rows': 0,
'client_linked': 0,
'client_missing': 0,
'encoding_used': None,
'skipped_duplicates': 0,
}
expected_fields = {
@@ -795,6 +848,7 @@ def import_files_data(db: Session, file_path: str) -> Dict[str, Any]:
f = None
try:
f, used_encoding = open_text_with_fallbacks(file_path)
result['encoding_used'] = used_encoding
reader = csv.DictReader(f)
headers = reader.fieldnames or []
@@ -816,7 +870,13 @@ def import_files_data(db: Session, file_path: str) -> Dict[str, Any]:
# Check for existing case
existing = db.query(Case).filter(Case.file_no == file_no).first()
if existing:
result['errors'].append(f"Row {row_num}: Case with file number '{file_no}' already exists")
result['skipped_duplicates'] += 1
logger.warning(
"files_import_duplicate",
row=row_num,
file_no=file_no,
file=file_path,
)
continue
# Find client by ID
@@ -825,8 +885,16 @@ def import_files_data(db: Session, file_path: str) -> Dict[str, Any]:
if client_id:
client = db.query(Client).filter(Client.rolodex_id == client_id).first()
if not client:
result['client_missing'] += 1
logger.warning(
"files_import_missing_client",
row=row_num,
file_no=file_no,
legacy_client_id=client_id,
)
result['errors'].append(f"Row {row_num}: Client with ID '{client_id}' not found")
continue
result['client_linked'] += 1
case = Case(
file_no=file_no,
@@ -835,16 +903,35 @@ def import_files_data(db: Session, file_path: str) -> Dict[str, Any]:
case_type=row.get('File_Type', '').strip() or None,
description=row.get('Regarding', '').strip() or None,
open_date=parse_date(row.get('Opened', '')),
close_date=parse_date(row.get('Closed', ''))
close_date=parse_date(row.get('Closed', '')),
)
db.add(case)
result['success'] += 1
logger.info(
"files_import_row",
row=row_num,
file_no=file_no,
client_id=client.id if client else None,
status=case.status,
)
except Exception as e:
result['errors'].append(f"Row {row_num}: {str(e)}")
db.commit()
logger.info(
"files_import_complete",
file=file_path,
encoding=used_encoding,
total_rows=result['total_rows'],
success=result['success'],
client_linked=result['client_linked'],
client_missing=result['client_missing'],
skipped_duplicates=result['skipped_duplicates'],
errors=len(result['errors']),
)
except Exception as e:
result['errors'].append(f"Import failed: {str(e)}")

25
app/templates/rolodex_view.html
View File

@@ -59,7 +59,13 @@
</div>
<div class="col-md-3">
<div class="text-muted small">Email</div>
<div>{{ client.email or '' }}</div>
<div>
{% if client.email %}
<a href="mailto:{{ client.email }}">{{ client.email }}</a>
{% else %}
<span class="text-muted">No email</span>
{% endif %}
</div>
</div>
<div class="col-md-3">
<div class="text-muted small">DOB</div>
@@ -77,7 +83,13 @@
</div>
<div class="col-md-9">
<div class="text-muted small">Memo / Notes</div>
<div>{{ client.memo or '' }}</div>
<div>
{% if client.memo %}
{{ client.memo }}
{% else %}
<span class="text-muted">No notes available</span>
{% endif %}
</div>
</div>
</div>
@@ -171,8 +183,9 @@
</tr>
</thead>
<tbody>
{% if client.cases and client.cases|length > 0 %}
{% for c in client.cases %}
{% set sorted_cases = client.cases | sort(attribute='open_date', reverse=True) %}
{% if sorted_cases and sorted_cases|length > 0 %}
{% for c in sorted_cases %}
{% if status_filter == 'all' or (status_filter == 'open' and (c.status != 'closed')) or (status_filter == 'closed' and c.status == 'closed') %}
<tr>
<td>{{ c.file_no }}</td>
@@ -184,7 +197,7 @@
<span class="badge bg-success">Open</span>
{% endif %}
</td>
<td>{{ c.open_date.strftime('%Y-%m-%d') if c.open_date else '' }}</td>
<td>{{ c.open_date.strftime('%Y-%m-%d') if c.open_date else '—' }}</td>
<td class="text-end">
<a class="btn btn-sm btn-outline-primary" href="/case/{{ c.id }}">
<i class="bi bi-folder2-open"></i>
@@ -197,7 +210,7 @@
{% endif %}
{% endfor %}
{% else %}
<tr><td colspan="5" class="text-center text-muted py-3">No related cases.</td></tr>
<tr><td colspan="5" class="text-center text-muted py-3">No related cases linked.</td></tr>
{% endif %}
</tbody>
</table>

BIN
delphi.db
View File

Binary file not shown.

94
scripts/fix_case_links.py Normal file
View File

@@ -0,0 +1,94 @@
"""Utility script to repair orphaned Case records.
This script attempts to link Case records to Client entries based on legacy
identifiers. It should be executed inside the running Docker container.
Usage:
docker compose exec delphi-db python scripts/fix_case_links.py
Safety:
- Prints a summary before making changes.
- Requires confirmation unless run with --yes.
"""
from __future__ import annotations
import argparse
from collections import defaultdict
from sqlalchemy.orm import joinedload
from app.database import SessionLocal
from app.models import Case, Client
def build_client_maps(session) -> tuple[dict[str, int], dict[str, list[int]]]:
rolodex_to_client: dict[str, int] = {}
file_to_client: dict[str, list[int]] = defaultdict(list)
for client in session.query(Client).options(joinedload(Client.cases)).all():
if client.rolodex_id:
rolodex_to_client[client.rolodex_id] = client.id
for case in client.cases:
file_to_client[case.file_no].append(client.id)
return rolodex_to_client, file_to_client
def find_orphaned_cases(session):
return session.query(Case).filter(Case.client_id == None).all() # noqa: E711
def main(confirm: bool) -> int:
session = SessionLocal()
try:
rolodex_map, file_map = build_client_maps(session)
orphans = find_orphaned_cases(session)
if not orphans:
print("No orphaned cases found. 🎉")
return 0
assignments: list[tuple[Case, int]] = []
for case in orphans:
candidate_client_id = None
if case.file_no in file_map and file_map[case.file_no]:
candidate_client_id = file_map[case.file_no][0]
elif case.file_no in rolodex_map:
candidate_client_id = rolodex_map[case.file_no]
if candidate_client_id:
assignments.append((case, candidate_client_id))
if not assignments:
print("No matching clients found for orphaned cases. Nothing to do.")
return 1
print("Orphaned cases detected:")
for case, client_id in assignments:
print(f" Case {case.file_no} (id={case.id}) → Client id {client_id}")
if not confirm:
response = input("Apply these fixes? [y/N]: ").strip().lower()
if response not in {"y", "yes"}:
print("Aborting with no changes.")
return 1
updated = 0
for case, client_id in assignments:
case.client_id = client_id
updated += 1
session.commit()
print(f"Updated {updated} cases with matching clients.")
return 0
finally:
session.close()
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Repair orphaned Case links.")
parser.add_argument("--yes", action="store_true", help="Apply changes without confirmation")
args = parser.parse_args()
raise SystemExit(main(confirm=args.yes))

10
static/js/custom.js
View File

@@ -2,10 +2,12 @@
document.addEventListener('DOMContentLoaded', function() {
// Initialize tooltips if any
var tooltipTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="tooltip"]'));
var tooltipList = tooltipTriggerList.map(function (tooltipTriggerEl) {
return new bootstrap.Tooltip(tooltipTriggerEl);
});
if (window.bootstrap && bootstrap.Tooltip) {
var tooltipTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="tooltip"]'));
tooltipTriggerList.forEach(function (tooltipTriggerEl) {
new bootstrap.Tooltip(tooltipTriggerEl);
});
}
// Auto-hide alerts after 5 seconds
var alerts = document.querySelectorAll('.alert:not(.alert-permanent)');

130
tests/test_import_data.py Normal file
View File

@@ -0,0 +1,130 @@
"""Tests for Rolodex and Files CSV import helpers."""
from __future__ import annotations
from pathlib import Path
import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from app.main import import_files_data, import_rolodex_data
from app.models import Base, Case, Client
def _make_engine(db_path: Path):
engine = create_engine(f"sqlite:///{db_path}")
Base.metadata.create_all(engine)
return engine
@pytest.fixture()
def session(tmp_path):
engine = _make_engine(tmp_path / "import-tests.db")
SessionLocal = sessionmaker(bind=engine)
db = SessionLocal()
try:
yield db
finally:
db.close()
engine.dispose()
def _write_csv(path: Path, content: str) -> Path:
path.write_text(content, encoding="utf-8")
return path
def test_import_rolodex_captures_email_and_memo(session, tmp_path):
csv_path = _write_csv(
tmp_path / "ROLODEX.csv",
"Id,Prefix,First,Middle,Last,Suffix,Title,A1,A2,A3,City,Abrev,St,Zip,Email,DOB,SS#,Legal_Status,Group,Memo\n"
"1001,,Ada,,Lovelace,,Countess,123 Main,,,London,,UK,12345,ada@example.com,12/10/1815,XXX-XX-1111,Active,VIP,Top client notes\n"
"1002,,Alan,,Turing,,,43 Park,,Bletchley,,UK,67890,,06/23/1912,XXX-XX-2222,Active,VIP, \n",
)
result = import_rolodex_data(session, str(csv_path))
assert result["success"] == 2
assert result["email_imported"] == 1
assert result["email_missing"] == 1
assert result["memo_imported"] == 1
assert result["memo_missing"] == 1
assert not result["errors"]
clients = {c.rolodex_id: c for c in session.query(Client).all()}
assert clients["1001"].email == "ada@example.com"
assert clients["1001"].memo == "Top client notes"
assert clients["1002"].email is None
assert clients["1002"].memo is None
def test_import_rolodex_handles_duplicates(session, tmp_path):
csv_path = _write_csv(
tmp_path / "ROLODEX.csv",
"Id,Prefix,First,Middle,Last,Suffix,Title,A1,A2,A3,City,Abrev,St,Zip,Email,DOB,SS#,Legal_Status,Group,Memo\n"
"3001,,Grace,,Hopper,,,,,Arlington,VA,Virginia,22202,grace@example.com,12/09/1906,XXX-XX-3333,Active,VIP,Notes\n"
"3001,,Grace,,Hopper,,,,,Arlington,VA,Virginia,22202,grace@example.com,12/09/1906,XXX-XX-3333,Active,VIP,Duplicate\n",
)
result = import_rolodex_data(session, str(csv_path))
assert result["success"] == 1
assert result["skipped_duplicates"] == 1
assert len(result["errors"]) == 0
def test_import_files_links_clients_and_detects_missing(session, tmp_path):
_write_csv(
tmp_path / "ROLODEX.csv",
"Id,Prefix,First,Middle,Last,Suffix,Title,A1,A2,A3,City,Abrev,St,Zip,Email,DOB,SS#,Legal_Status,Group,Memo\n"
"2001,,Grace,,Hopper,,,,,Arlington,VA,Virginia,22202,grace@example.com,12/09/1906,XXX-XX-3333,Active,VIP,Notes\n",
)
import_rolodex_data(session, str(tmp_path / "ROLODEX.csv"))
files_path = _write_csv(
tmp_path / "FILES.csv",
"File_No,Id,File_Type,Regarding,Opened,Closed,Empl_Num,Rate_Per_Hour,Status,Footer_Code,Opposing,Hours,Hours_P,Trust_Bal,Trust_Bal_P,Hourly_Fees,Hourly_Fees_P,Flat_Fees,Flat_Fees_P,Disbursements,Disbursements_P,Credit_Bal,Credit_Bal_P,Total_Charges,Total_Charges_P,Amount_Owing,Amount_Owing_P,Transferable,Memo\n"
"F-001,2001,Divorce,Important matter,01/01/2020,,E1,175,Open,F1,,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,No,Initial legacy memo\n"
"F-002,9999,Divorce,Missing client,01/01/2020,,E1,175,Open,F1,,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,No,Should be skipped\n",
)
result = import_files_data(session, str(files_path))
assert result["success"] == 1
assert result["client_linked"] == 1
assert result["client_missing"] == 1
assert result["errors"], "Expected missing client to produce an error"
case = session.query(Case).filter(Case.file_no == "F-001").one()
assert case.client is not None
assert case.client.rolodex_id == "2001"
missing_case = session.query(Case).filter(Case.file_no == "F-002").first()
assert missing_case is None
def test_import_files_respects_duplicates(session, tmp_path):
_write_csv(
tmp_path / "ROLODEX.csv",
"Id,Prefix,First,Middle,Last,Suffix,Title,A1,A2,A3,City,Abrev,St,Zip,Email,DOB,SS#,Legal_Status,Group,Memo\n"
"4001,,Marie,,Curie,,,,,Paris,,France,75000,marie@example.com,11/07/1867,XXX-XX-4444,Active,VIP,Notes\n",
)
import_rolodex_data(session, str(tmp_path / "ROLODEX.csv"))
files_csv = _write_csv(
tmp_path / "FILES.csv",
"File_No,Id,File_Type,Regarding,Opened,Closed,Empl_Num,Rate_Per_Hour,Status,Footer_Code,Opposing,Hours,Hours_P,Trust_Bal,Trust_Bal_P,Hourly_Fees,Hourly_Fees_P,Flat_Fees,Flat_Fees_P,Disbursements,Disbursements_P,Credit_Bal,Credit_Bal_P,Total_Charges,Total_Charges_P,Amount_Owing,Amount_Owing_P,Transferable,Memo\n"
"F-100,4001,Divorce,Legacy matter,01/01/2020,,E1,175,Open,F1,,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,No,First import\n"
"F-100,4001,Divorce,Legacy matter,01/01/2020,,E1,175,Open,F1,,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,No,Duplicate row\n",
)
first_result = import_files_data(session, str(files_csv))
assert first_result["success"] == 1
assert first_result["skipped_duplicates"] == 0
second_result = import_files_data(session, str(files_csv))
assert second_result["success"] == 0
assert second_result["skipped_duplicates"] == 1
assert not session.query(Case).filter(Case.file_no == "F-100").all()[1:]