hotswap/delphi-database-v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve Rolodex imports, display, and add repair script

Browse Source
This commit is contained in:
HotSwapp
2025-10-13 15:00:13 -05:00
parent 2e2380552e
commit 84c3dac83a
6 changed files with 344 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
app/main.py
View File

@@ -29,7 +29,7 @@ import structlog
from structlog import contextvars as structlog_contextvars
from .database import create_tables, get_db, get_database_url
from .models import User, Case, Client, Phone, Transaction, Document, Payment, ImportLog, Qdros
from .models import User, Case, Client, Phone, Transaction, Document, Payment, ImportLog, Qdros, LegacyFile
from .auth import authenticate_user, get_current_user_from_session
from .reporting import (
build_phone_book_pdf,
@@ -578,7 +578,13 @@ def import_rolodex_data(db: Session, file_path: str) -> Dict[str, Any]:
result = {
'success': 0,
'errors': [],
'total_rows': 0
'total_rows': 0,
'memo_imported': 0,
'memo_missing': 0,
'email_imported': 0,
'email_missing': 0,
'skipped_duplicates': 0,
'encoding_used': None,
}
expected_fields = {
@@ -606,6 +612,7 @@ def import_rolodex_data(db: Session, file_path: str) -> Dict[str, Any]:
try:
f, used_encoding = open_text_with_fallbacks(file_path)
result['encoding_used'] = used_encoding
with f as file:
reader = csv.DictReader(file)
@@ -630,7 +637,13 @@ def import_rolodex_data(db: Session, file_path: str) -> Dict[str, Any]:
# Check for existing client
existing = db.query(Client).filter(Client.rolodex_id == rolodex_id).first()
if existing:
result['errors'].append(f"Row {row_num}: Client with ID '{rolodex_id}' already exists")
result['skipped_duplicates'] += 1
logger.warning(
"rolodex_import_duplicate",
row=row_num,
rolodex_id=rolodex_id,
file=file_path,
)
continue
# Parse DOB (YYYY-MM-DD or MM/DD/YY variants)
@@ -645,6 +658,21 @@ def import_rolodex_data(db: Session, file_path: str) -> Dict[str, Any]:
except ValueError:
continue
email_val = row.get('Email', '').strip() or None
memo_val = row.get('Memo', '')
memo_clean = memo_val.strip() if memo_val is not None else ''
memo_val_clean = memo_clean or None
if email_val:
result['email_imported'] += 1
else:
result['email_missing'] += 1
if memo_val_clean:
result['memo_imported'] += 1
else:
result['memo_missing'] += 1
client = Client(
rolodex_id=rolodex_id,
prefix=row.get('Prefix', '').strip() or None,
@@ -659,20 +687,41 @@ def import_rolodex_data(db: Session, file_path: str) -> Dict[str, Any]:
state=(row.get('Abrev', '').strip() or row.get('St', '').strip() or None),
zip_code=row.get('Zip', '').strip() or None,
group=row.get('Group', '').strip() or None,
email=row.get('Email', '').strip() or None,
email=email_val,
dob=dob_val,
ssn=row.get('SS#', '').strip() or None,
legal_status=row.get('Legal_Status', '').strip() or None,
memo=row.get('Memo', '').strip() or None,
memo=memo_val_clean,
)
db.add(client)
result['success'] += 1
logger.info(
"rolodex_import_row",
row=row_num,
rolodex_id=rolodex_id,
email_present=bool(email_val),
memo_present=bool(memo_val_clean),
)
except Exception as e:
result['errors'].append(f"Row {row_num}: {str(e)}")
db.commit()
logger.info(
"rolodex_import_complete",
file=file_path,
encoding=used_encoding,
total_rows=result['total_rows'],
success=result['success'],
memo_imported=result['memo_imported'],
memo_missing=result['memo_missing'],
email_imported=result['email_imported'],
email_missing=result['email_missing'],
skipped_duplicates=result['skipped_duplicates'],
errors=len(result['errors']),
)
except Exception as e:
logger.error("rolodex_import_failed", file=file_path, error=str(e))
@@ -757,7 +806,11 @@ def import_files_data(db: Session, file_path: str) -> Dict[str, Any]:
result = {
'success': 0,
'errors': [],
'total_rows': 0
'total_rows': 0,
'client_linked': 0,
'client_missing': 0,
'encoding_used': None,
'skipped_duplicates': 0,
}
expected_fields = {
@@ -795,6 +848,7 @@ def import_files_data(db: Session, file_path: str) -> Dict[str, Any]:
f = None
try:
f, used_encoding = open_text_with_fallbacks(file_path)
result['encoding_used'] = used_encoding
reader = csv.DictReader(f)
headers = reader.fieldnames or []
@@ -816,7 +870,13 @@ def import_files_data(db: Session, file_path: str) -> Dict[str, Any]:
# Check for existing case
existing = db.query(Case).filter(Case.file_no == file_no).first()
if existing:
result['errors'].append(f"Row {row_num}: Case with file number '{file_no}' already exists")
result['skipped_duplicates'] += 1
logger.warning(
"files_import_duplicate",
row=row_num,
file_no=file_no,
file=file_path,
)
continue
# Find client by ID
@@ -825,8 +885,16 @@ def import_files_data(db: Session, file_path: str) -> Dict[str, Any]:
if client_id:
client = db.query(Client).filter(Client.rolodex_id == client_id).first()
if not client:
result['client_missing'] += 1
logger.warning(
"files_import_missing_client",
row=row_num,
file_no=file_no,
legacy_client_id=client_id,
)
result['errors'].append(f"Row {row_num}: Client with ID '{client_id}' not found")
continue
result['client_linked'] += 1
case = Case(
file_no=file_no,
@@ -835,16 +903,35 @@ def import_files_data(db: Session, file_path: str) -> Dict[str, Any]:
case_type=row.get('File_Type', '').strip() or None,
description=row.get('Regarding', '').strip() or None,
open_date=parse_date(row.get('Opened', '')),
close_date=parse_date(row.get('Closed', ''))
close_date=parse_date(row.get('Closed', '')),
)
db.add(case)
result['success'] += 1
logger.info(
"files_import_row",
row=row_num,
file_no=file_no,
client_id=client.id if client else None,
status=case.status,
)
except Exception as e:
result['errors'].append(f"Row {row_num}: {str(e)}")
db.commit()
logger.info(
"files_import_complete",
file=file_path,
encoding=used_encoding,
total_rows=result['total_rows'],
success=result['success'],
client_linked=result['client_linked'],
client_missing=result['client_missing'],
skipped_duplicates=result['skipped_duplicates'],
errors=len(result['errors']),
)
except Exception as e:
result['errors'].append(f"Import failed: {str(e)}")