Implement comprehensive CSV import system for legacy database migration

- Added 5 new legacy models to app/models.py (FileType, FileNots, RolexV, FVarLkup, RVarLkup)
- Created app/import_legacy.py with import functions for all legacy tables:
  * Reference tables: TRNSTYPE, TRNSLKUP, FOOTERS, FILESTAT, EMPLOYEE, GRUPLKUP, FILETYPE, FVARLKUP, RVARLKUP
  * Core tables: ROLODEX, PHONE, ROLEX_V, FILES, FILES_R, FILES_V, FILENOTS, LEDGER, DEPOSITS, PAYMENTS
  * Specialized: PLANINFO, QDROS, PENSIONS and all pension-related tables
- Created app/sync_legacy_to_modern.py with sync functions to populate modern models from legacy data
- Updated admin routes in app/main.py:
  * Extended process_csv_import to support all new import types
  * Added /admin/sync endpoint for syncing legacy to modern models
  * Updated get_import_type_from_filename to recognize all CSV file patterns
- Enhanced app/templates/admin.html with:
  * Import Order Guide showing recommended import sequence
  * Sync to Modern Models section with confirmation dialog
  * Sync results display with detailed per-table statistics
  * Updated supported file formats list
- All import functions use batch processing (500 rows), proper error handling, and structured logging
- Sync functions maintain foreign key integrity and skip orphaned records with warnings
This commit is contained in:
HotSwapp
2025-10-08 09:41:38 -05:00
parent 2efbf14940
commit 4030dbd88e
6 changed files with 2545 additions and 38 deletions

View File

@@ -683,3 +683,68 @@ class PensionSeparate(Base):
__table_args__ = (
ForeignKeyConstraint(["file_no", "version"], ["pensions.file_no", "pensions.version"], ondelete="CASCADE"),
)
class FileType(Base):
"""FILETYPE reference table for file/case types."""
__tablename__ = "filetype"
file_type = Column(String, primary_key=True)
def __repr__(self):
return f"<FileType(file_type='{self.file_type}')>"
class FileNots(Base):
"""FILENOTS table for file memos/notes."""
__tablename__ = "filenots"
file_no = Column(String, ForeignKey("files.file_no", ondelete="CASCADE"), primary_key=True)
memo_date = Column(Date, primary_key=True)
memo_note = Column(Text)
__table_args__ = (
Index("ix_filenots_file_no", "file_no"),
)
def __repr__(self):
return f"<FileNots(file_no='{self.file_no}', date='{self.memo_date}')>"
class RolexV(Base):
"""ROLEX_V variables per rolodex entry."""
__tablename__ = "rolex_v"
id = Column(String, ForeignKey("rolodex.id", ondelete="CASCADE"), primary_key=True)
identifier = Column(String, primary_key=True)
response = Column(Text)
__table_args__ = (
Index("ix_rolex_v_id", "id"),
)
def __repr__(self):
return f"<RolexV(id='{self.id}', identifier='{self.identifier}')>"
class FVarLkup(Base):
"""FVARLKUP file variable lookup table."""
__tablename__ = "fvarlkup"
identifier = Column(String, primary_key=True)
query = Column(Text)
response = Column(Text)
def __repr__(self):
return f"<FVarLkup(identifier='{self.identifier}')>"
class RVarLkup(Base):
"""RVARLKUP rolodex variable lookup table."""
__tablename__ = "rvarlkup"
identifier = Column(String, primary_key=True)
query = Column(Text)
def __repr__(self):
return f"<RVarLkup(identifier='{self.identifier}')>"