hotswap/delphi-database-v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix upload detection for model class names and add States/Printers/Setup import

Browse Source 
- Enhanced get_import_type_from_filename() to recognize model class names (LegacyFile, FilesR, etc.) in addition to legacy CSV names
- Added import functions for States, Printers, and Setup reference tables
- Updated VALID_IMPORT_TYPES and IMPORT_ORDER to include new tables
- Updated admin panel table counts to display new reference tables
- Created UPLOAD_FIX.md documentation explaining the changes and how to handle existing unknown files

This fixes the issue where files uploaded with model class names (e.g., LegacyFile.csv) were being categorized as 'unknown' instead of being properly detected.
This commit is contained in:
HotSwapp
2025-10-13 09:08:06 -05:00
parent e6a78221e6
commit 69f1043be3
3 changed files with 380 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

217
app/import_legacy.py
View File

@@ -20,7 +20,7 @@ from .models import (
Footers, FileStat, Employee, GroupLkup, FileType,
Qdros, PlanInfo, Pensions, PensionMarriage, PensionDeath,
PensionSchedule, PensionSeparate, PensionResults,
RolexV, FVarLkup, RVarLkup
RolexV, FVarLkup, RVarLkup, States, Printers, Setup
)
logger = structlog.get_logger(__name__)
@@ -660,6 +660,221 @@ def import_rvarlkup(db: Session, file_path: str) -> Dict[str, Any]:
return result
def import_states(db: Session, file_path: str) -> Dict[str, Any]:
"""Import STATES.csv → States model with upsert logic."""
result = {'success': 0, 'errors': [], 'total_rows': 0, 'updated': 0, 'inserted': 0}
try:
f, encoding = open_text_with_fallbacks(file_path)
reader = csv.DictReader(f)
for row_num, row in enumerate(reader, start=2):
result['total_rows'] += 1
try:
abrev = clean_string(row.get('Abrev'))
if not abrev:
continue
# Check if record already exists
existing = db.query(States).filter(States.abrev == abrev).first()
if existing:
# Update existing record
existing.st = clean_string(row.get('St'))
result['updated'] += 1
else:
# Insert new record
record = States(
abrev=abrev,
st=clean_string(row.get('St'))
)
db.add(record)
result['inserted'] += 1
result['success'] += 1
# Commit in batches for performance
if result['success'] % BATCH_SIZE == 0:
db.commit()
except Exception as e:
result['errors'].append(f"Row {row_num}: {str(e)}")
db.rollback()
# Commit any remaining changes
db.commit()
f.close()
logger.info("import_states_complete", **result)
except Exception as e:
db.rollback()
result['errors'].append(f"Fatal error: {str(e)}")
logger.error("import_states_failed", error=str(e))
return result
def import_printers(db: Session, file_path: str) -> Dict[str, Any]:
"""Import PRINTERS.csv → Printers model with upsert logic."""
result = {'success': 0, 'errors': [], 'total_rows': 0, 'updated': 0, 'inserted': 0}
try:
f, encoding = open_text_with_fallbacks(file_path)
reader = csv.DictReader(f)
for row_num, row in enumerate(reader, start=2):
result['total_rows'] += 1
try:
number_str = clean_string(row.get('Number'))
if not number_str:
continue
try:
number = int(number_str)
except ValueError:
result['errors'].append(f"Row {row_num}: Invalid Number '{number_str}'")
continue
# Check if record already exists
existing = db.query(Printers).filter(Printers.number == number).first()
if existing:
# Update existing record
existing.name = clean_string(row.get('Name'))
existing.port = clean_string(row.get('Port'))
existing.page_break = clean_string(row.get('Page_Break'))
existing.setup_st = clean_string(row.get('Setup_St'))
existing.phone_book = clean_string(row.get('Phone_Book'))
existing.rolodex_info = clean_string(row.get('Rolodex_Info'))
existing.envelope = clean_string(row.get('Envelope'))
existing.file_cabinet = clean_string(row.get('File_Cabinet'))
existing.accounts = clean_string(row.get('Accounts'))
existing.statements = clean_string(row.get('Statements'))
existing.calendar = clean_string(row.get('Calendar'))
existing.reset_st = clean_string(row.get('Reset_St'))
existing.b_underline = clean_string(row.get('B_Underline'))
existing.e_underline = clean_string(row.get('E_Underline'))
existing.b_bold = clean_string(row.get('B_Bold'))
existing.e_bold = clean_string(row.get('E_Bold'))
result['updated'] += 1
else:
# Insert new record
record = Printers(
number=number,
name=clean_string(row.get('Name')),
port=clean_string(row.get('Port')),
page_break=clean_string(row.get('Page_Break')),
setup_st=clean_string(row.get('Setup_St')),
phone_book=clean_string(row.get('Phone_Book')),
rolodex_info=clean_string(row.get('Rolodex_Info')),
envelope=clean_string(row.get('Envelope')),
file_cabinet=clean_string(row.get('File_Cabinet')),
accounts=clean_string(row.get('Accounts')),
statements=clean_string(row.get('Statements')),
calendar=clean_string(row.get('Calendar')),
reset_st=clean_string(row.get('Reset_St')),
b_underline=clean_string(row.get('B_Underline')),
e_underline=clean_string(row.get('E_Underline')),
b_bold=clean_string(row.get('B_Bold')),
e_bold=clean_string(row.get('E_Bold'))
)
db.add(record)
result['inserted'] += 1
result['success'] += 1
# Commit in batches for performance
if result['success'] % BATCH_SIZE == 0:
db.commit()
except Exception as e:
result['errors'].append(f"Row {row_num}: {str(e)}")
db.rollback()
# Commit any remaining changes
db.commit()
f.close()
logger.info("import_printers_complete", **result)
except Exception as e:
db.rollback()
result['errors'].append(f"Fatal error: {str(e)}")
logger.error("import_printers_failed", error=str(e))
return result
def import_setup(db: Session, file_path: str) -> Dict[str, Any]:
"""Import SETUP.csv → Setup model (clears and re-inserts)."""
result = {'success': 0, 'errors': [], 'total_rows': 0}
try:
# Clear existing setup records (typically only one row in legacy system)
db.query(Setup).delete()
db.commit()
f, encoding = open_text_with_fallbacks(file_path)
reader = csv.DictReader(f)
batch = []
for row_num, row in enumerate(reader, start=2):
result['total_rows'] += 1
try:
# Parse default_printer as integer if present
default_printer = None
default_printer_str = clean_string(row.get('Default_Printer'))
if default_printer_str:
try:
default_printer = int(default_printer_str)
except ValueError:
result['errors'].append(f"Row {row_num}: Invalid Default_Printer '{default_printer_str}'")
record = Setup(
appl_title=clean_string(row.get('Appl_Title')),
l_head1=clean_string(row.get('L_Head1')),
l_head2=clean_string(row.get('L_Head2')),
l_head3=clean_string(row.get('L_Head3')),
l_head4=clean_string(row.get('L_Head4')),
l_head5=clean_string(row.get('L_Head5')),
l_head6=clean_string(row.get('L_Head6')),
l_head7=clean_string(row.get('L_Head7')),
l_head8=clean_string(row.get('L_Head8')),
l_head9=clean_string(row.get('L_Head9')),
l_head10=clean_string(row.get('L_Head10')),
default_printer=default_printer
)
batch.append(record)
if len(batch) >= BATCH_SIZE:
db.bulk_save_objects(batch)
db.commit()
result['success'] += len(batch)
batch = []
except Exception as e:
result['errors'].append(f"Row {row_num}: {str(e)}")
if batch:
db.bulk_save_objects(batch)
db.commit()
result['success'] += len(batch)
f.close()
logger.info("import_setup_complete", **result)
except Exception as e:
db.rollback()
result['errors'].append(f"Fatal error: {str(e)}")
logger.error("import_setup_failed", error=str(e))
return result
# ============================================================================
# Core Data Table Imports
# ============================================================================

81
app/main.py
View File

@@ -246,7 +246,7 @@ app.mount("/static", StaticFiles(directory="static"), name="static")
VALID_IMPORT_TYPES: List[str] = [
# Reference tables
'trnstype', 'trnslkup', 'footers', 'filestat', 'employee',
'gruplkup', 'filetype', 'fvarlkup', 'rvarlkup',
'gruplkup', 'filetype', 'fvarlkup', 'rvarlkup', 'states', 'printers', 'setup',
# Core data tables
'rolodex', 'phone', 'rolex_v', 'files', 'files_r', 'files_v',
'filenots', 'ledger', 'deposits', 'payments',
@@ -259,11 +259,11 @@ VALID_IMPORT_TYPES: List[str] = [
# Centralized import order for auto-import after upload
# Reference tables first, then core tables, then specialized tables
IMPORT_ORDER: List[str] = [
# Reference tables
'trnstype', 'trnslkup', 'footers', 'filestat', 'employee', 'gruplkup', 'filetype', 'fvarlkup', 'rvarlkup',
# Core tables
# Reference tables - import these first
'trnstype', 'trnslkup', 'footers', 'filestat', 'employee', 'gruplkup', 'filetype', 'fvarlkup', 'rvarlkup', 'states', 'printers', 'setup',
# Core tables - import after reference tables
'rolodex', 'phone', 'rolex_v', 'files', 'files_r', 'files_v', 'filenots', 'ledger', 'deposits', 'payments',
# Specialized tables
# Specialized tables - import last
'planinfo', 'qdros', 'pensions', 'pension_marriage', 'pension_death', 'pension_schedule', 'pension_separate', 'pension_results',
]
ORDER_INDEX: Dict[str, int] = {t: i for i, t in enumerate(IMPORT_ORDER)}
@@ -272,6 +272,9 @@ ORDER_INDEX: Dict[str, int] = {t: i for i, t in enumerate(IMPORT_ORDER)}
def get_import_type_from_filename(filename: str) -> str:
"""
Determine import type based on filename pattern for legacy CSV files.
Supports both legacy CSV naming (e.g., FILES.csv, LEDGER.csv) and
model class naming (e.g., LegacyFile.csv, Ledger.csv).
Args:
filename: Name of the uploaded CSV file
@@ -294,7 +297,7 @@ def get_import_type_from_filename(filename: str) -> str:
return 'filestat'
if 'EMPLOYEE' in base:
return 'employee'
if 'GRUPLKUP' in base or 'GROUPLKUP' in base:
if 'GRUPLKUP' in base or 'GROUPLKUP' in base or base == 'GROUPLKUP':
return 'gruplkup'
if 'FILETYPE' in base:
return 'filetype'
@@ -302,27 +305,53 @@ def get_import_type_from_filename(filename: str) -> str:
return 'fvarlkup'
if 'RVARLKUP' in base:
return 'rvarlkup'
if 'STATES' in base or base == 'STATES':
return 'states'
if 'PRINTERS' in base or base == 'PRINTERS':
return 'printers'
if 'SETUP' in base or base == 'SETUP':
return 'setup'
# Core data tables
# Core data tables - check most specific patterns first
# Check for ROLEX_V and ROLEXV before ROLEX
if 'ROLEX_V' in base or 'ROLEXV' in base:
return 'rolex_v'
if 'ROLODEX' in base or 'ROLEX' in base:
return 'rolodex'
# Check for FILES_R, FILES_V, FILENOTS before generic FILES
if 'FILES_R' in base or 'FILESR' in base:
return 'files_r'
if 'FILES_V' in base or 'FILESV' in base:
return 'files_v'
if 'FILENOTS' in base or 'FILE_NOTS' in base:
if 'FILENOTS' in base or 'FILE_NOTS' in base or base == 'FILENOTS':
return 'filenots'
if 'FILES' in base or 'FILE' in base:
# Check for model class name "LEGACYFILE" before generic "FILE"
if base == 'LEGACYFILE':
return 'files'
if 'PHONE' in base:
if 'FILES' in base:
return 'files'
# Only match generic "FILE" if it's the exact base name or starts with it
if base == 'FILE' or base.startswith('FILE_'):
return 'files'
# ROLODEX variations
if 'ROLODEX' in base or base == 'ROLEX':
return 'rolodex'
# PHONE variations (including model class name LEGACYPHONE)
if 'PHONE' in base or base == 'LEGACYPHONE':
return 'phone'
# LEDGER
if 'LEDGER' in base:
return 'ledger'
# DEPOSITS
if 'DEPOSITS' in base or 'DEPOSIT' in base:
return 'deposits'
if 'PAYMENTS' in base or 'PAYMENT' in base:
# PAYMENTS (including model class name LEGACYPAYMENT)
if 'PAYMENTS' in base or 'PAYMENT' in base or base == 'LEGACYPAYMENT':
return 'payments'
# Specialized tables
@@ -330,17 +359,23 @@ def get_import_type_from_filename(filename: str) -> str:
return 'planinfo'
if 'QDROS' in base or 'QDRO' in base:
return 'qdros'
if 'MARRIAGE' in base:
# Pension sub-tables - check most specific first
if 'MARRIAGE' in base or base == 'PENSIONMARRIAGE':
return 'pension_marriage'
if 'DEATH' in base:
if 'DEATH' in base or base == 'PENSIONDEATH':
return 'pension_death'
if 'SCHEDULE' in base:
if 'SCHEDULE' in base or base == 'PENSIONSCHEDULE':
return 'pension_schedule'
if 'SEPARATE' in base:
if 'SEPARATE' in base or base == 'PENSIONSEPARATE':
return 'pension_separate'
if 'RESULTS' in base:
if 'RESULTS' in base or base == 'PENSIONRESULTS':
return 'pension_results'
if 'PENSIONS' in base or 'PENSION' in base:
# Generic PENSIONS - check last after specific pension tables
if 'PENSIONS' in base or base == 'PENSIONS':
return 'pensions'
if base == 'PENSION':
return 'pensions'
raise ValueError(f"Unknown file type for filename: {filename}")
@@ -1050,6 +1085,9 @@ def process_csv_import(db: Session, import_type: str, file_path: str) -> Dict[st
'filetype': import_legacy.import_filetype,
'fvarlkup': import_legacy.import_fvarlkup,
'rvarlkup': import_legacy.import_rvarlkup,
'states': import_legacy.import_states,
'printers': import_legacy.import_printers,
'setup': import_legacy.import_setup,
# Core data tables
'rolodex': import_legacy.import_rolodex,
@@ -2146,7 +2184,7 @@ async def admin_panel(request: Request, db: Session = Depends(get_db)):
Footers, FileStat, Employee, GroupLkup, FileType,
Qdros, PlanInfo, Pensions, PensionMarriage, PensionDeath,
PensionSchedule, PensionSeparate, PensionResults,
RolexV, FVarLkup, RVarLkup,
RolexV, FVarLkup, RVarLkup, States, Printers, Setup,
# Modern tables
Client, Phone, Case, Transaction, Payment, Document
)
@@ -2162,6 +2200,9 @@ async def admin_panel(request: Request, db: Session = Depends(get_db)):
'FileType': db.query(FileType).count(),
'FVarLkup': db.query(FVarLkup).count(),
'RVarLkup': db.query(RVarLkup).count(),
'States': db.query(States).count(),
'Printers': db.query(Printers).count(),
'Setup': db.query(Setup).count(),
},
'core': {
'Rolodex': db.query(Rolodex).count(),