hotswap/delphi-database
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

work on import

Browse Source
This commit is contained in:
HotSwapp
2025-09-22 22:07:50 -04:00
parent 830ddcc4d1
commit 1116658d40
7 changed files with 1426 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

146
app/import_export/import_service.py
View File

@@ -12,6 +12,7 @@ from .phone_importer import PhoneCSVImporter
from .files_importer import FilesCSVImporter
from .ledger_importer import LedgerCSVImporter
from .qdros_importer import QdrosCSVImporter
from .generic_importer import GenericCSVImporter
logger = logging.getLogger(__name__)
@@ -23,6 +24,32 @@ class TableType(Enum):
FILES = "files"
LEDGER = "ledger"
QDROS = "qdros"
# Generic table types for all other CSV files
GRUPLKUP = "gruplkup"
EMPLOYEE = "employee"
SETUP = "setup"
FILETYPE = "filetype"
TRNSTYPE = "trnstype"
TRNSACTN = "trnsactn"
TRNSLKUP = "trnslkup"
RVARLKUP = "rvarlkup"
FVARLKUP = "fvarlkup"
FILENOTS = "filenots"
DEPOSITS = "deposits"
PAYMENTS = "payments"
PENSIONS = "pensions"
PLANINFO = "planinfo"
# Form tables
NUMBERAL = "numberal"
INX_LKUP = "inx_lkup"
FORM_LST = "form_lst"
FORM_INX = "form_inx"
LIFETABL = "lifetabl"
# Pension tables
MARRIAGE = "marriage"
DEATH = "death"
SEPARATE = "separate"
SCHEDULE = "schedule"
class ImportService:
@@ -31,11 +58,36 @@ class ImportService:
def __init__(self, db_session: Session):
self.db_session = db_session
self._importers = {
TableType.ROLODEX: RolodexCSVImporter,
TableType.PHONE: PhoneCSVImporter,
TableType.FILES: FilesCSVImporter,
TableType.LEDGER: LedgerCSVImporter,
TableType.QDROS: QdrosCSVImporter
# Use generic importers for all tables to handle legacy CSV structure variations
TableType.ROLODEX: GenericCSVImporter, # Use generic importer for rolodex (more flexible)
TableType.PHONE: GenericCSVImporter, # Use generic importer for phone
TableType.FILES: GenericCSVImporter, # Use generic importer for files
TableType.LEDGER: GenericCSVImporter, # Use generic importer for ledger (to avoid FK issues)
TableType.QDROS: GenericCSVImporter, # Use generic importer for qdros (to avoid FK issues)
# Generic importer for all other tables
TableType.GRUPLKUP: GenericCSVImporter,
TableType.EMPLOYEE: GenericCSVImporter,
TableType.SETUP: GenericCSVImporter,
TableType.FILETYPE: GenericCSVImporter,
TableType.TRNSTYPE: GenericCSVImporter,
TableType.TRNSACTN: GenericCSVImporter,
TableType.TRNSLKUP: GenericCSVImporter,
TableType.RVARLKUP: GenericCSVImporter,
TableType.FVARLKUP: GenericCSVImporter,
TableType.FILENOTS: GenericCSVImporter,
TableType.DEPOSITS: GenericCSVImporter,
TableType.PAYMENTS: GenericCSVImporter,
TableType.PENSIONS: GenericCSVImporter,
TableType.PLANINFO: GenericCSVImporter,
TableType.NUMBERAL: GenericCSVImporter,
TableType.INX_LKUP: GenericCSVImporter,
TableType.FORM_LST: GenericCSVImporter,
TableType.FORM_INX: GenericCSVImporter,
TableType.LIFETABL: GenericCSVImporter,
TableType.MARRIAGE: GenericCSVImporter,
TableType.DEATH: GenericCSVImporter,
TableType.SEPARATE: GenericCSVImporter,
TableType.SCHEDULE: GenericCSVImporter,
}
def get_supported_tables(self) -> List[str]:
@@ -47,7 +99,12 @@ class ImportService:
try:
table_type = TableType(table_name.lower())
importer_class = self._importers[table_type]
temp_importer = importer_class(self.db_session, "temp_schema_check")
# Handle generic importer differently
if importer_class == GenericCSVImporter:
temp_importer = importer_class(self.db_session, table_name, "temp_schema_check")
else:
temp_importer = importer_class(self.db_session, "temp_schema_check")
return {
"table_name": temp_importer.table_name,
@@ -77,7 +134,12 @@ class ImportService:
# Get appropriate importer
importer_class = self._importers[table_type]
importer = importer_class(self.db_session, import_id)
# Handle generic importer differently
if importer_class == GenericCSVImporter:
importer = importer_class(self.db_session, table_name, import_id)
else:
importer = importer_class(self.db_session, import_id)
logger.info(f"Starting CSV import for table: {table_name} (import_id: {importer.import_id})")
@@ -119,11 +181,39 @@ class ImportService:
# Recommended import order (dependencies first)
import_order = [
# Core tables with dependencies
TableType.ROLODEX, # No dependencies
TableType.PHONE, # Depends on ROLODEX
TableType.FILES, # Depends on ROLODEX
TableType.LEDGER, # Depends on FILES
TableType.QDROS # Depends on FILES
TableType.QDROS, # Depends on FILES
# Lookup and reference tables (no dependencies)
TableType.GRUPLKUP,
TableType.EMPLOYEE,
TableType.SETUP,
TableType.FILETYPE,
TableType.TRNSTYPE,
TableType.TRNSACTN,
TableType.TRNSLKUP,
TableType.RVARLKUP,
TableType.FVARLKUP,
TableType.FILENOTS,
TableType.PLANINFO,
# Financial tables
TableType.DEPOSITS,
TableType.PAYMENTS,
TableType.PENSIONS,
# Form tables
TableType.NUMBERAL,
TableType.INX_LKUP,
TableType.FORM_LST,
TableType.FORM_INX,
TableType.LIFETABL,
# Pension tables
TableType.MARRIAGE,
TableType.DEATH,
TableType.SEPARATE,
TableType.SCHEDULE
]
# Group imports by table type
@@ -134,11 +224,15 @@ class ImportService:
imports_by_table[table_name] = []
imports_by_table[table_name].append(import_data)
# Track processed tables
processed_tables = set()
# Process in dependency order
for table_type in import_order:
table_name = table_type.value
if table_name in imports_by_table:
table_imports = imports_by_table[table_name]
processed_tables.add(table_name)
for import_data in table_imports:
result = self.import_csv(
@@ -160,6 +254,35 @@ class ImportService:
if not result.success and table_type in [TableType.ROLODEX, TableType.FILES]:
logger.error(f"Critical import failed for {table_name}, stopping batch")
break
# Small delay to reduce database lock contention
import time
time.sleep(0.1)
# Process any remaining tables not in the explicit order
for table_name, table_imports in imports_by_table.items():
if table_name not in processed_tables:
logger.info(f"Processing table {table_name} (not in explicit order)")
for import_data in table_imports:
result = self.import_csv(
table_name,
import_data["csv_content"],
import_data.get("encoding", "utf-8")
)
# Use a unique key if multiple imports for same table
key = table_name
counter = 1
while key in results:
counter += 1
key = f"{table_name}_{counter}"
results[key] = result
# Small delay to reduce database lock contention
import time
time.sleep(0.1)
return results
@@ -174,7 +297,12 @@ class ImportService:
# Get appropriate importer
importer_class = self._importers[table_type]
importer = importer_class(self.db_session, "validation_check")
# Handle generic importer differently
if importer_class == GenericCSVImporter:
importer = importer_class(self.db_session, table_name, "validation_check")
else:
importer = importer_class(self.db_session, "validation_check")
# Parse headers only
import csv