63809d46fb
- Implement upsert logic in import_phone() function - Check for existing (id, phone) combinations before insert - Track duplicates within CSV to skip gracefully - Update existing records instead of failing on duplicates - Add detailed statistics: inserted, updated, skipped counts - Align with upsert pattern used in other import functions - Add documentation in docs/PHONE_IMPORT_FIX.md Fixes: UNIQUE constraint failed: phone.id, phone.phone error when re-importing or uploading CSV with duplicate entries
1797 lines
65 KiB
Python
1797 lines
65 KiB
Python
"""
|
|
Legacy CSV import functions for Delphi Database.
|
|
|
|
This module provides import functions for all legacy database tables,
|
|
reading from old-csv files and populating the legacy SQLAlchemy models.
|
|
"""
|
|
|
|
import csv
|
|
import os
|
|
from datetime import datetime
|
|
from decimal import Decimal, InvalidOperation
|
|
from typing import Dict, Any, Optional
|
|
from sqlalchemy.orm import Session
|
|
from sqlalchemy.exc import IntegrityError
|
|
import structlog
|
|
|
|
from .models import (
|
|
Rolodex, LegacyPhone, LegacyFile, FilesR, FilesV, FileNots,
|
|
Ledger, Deposits, LegacyPayment, TrnsType, TrnsLkup,
|
|
Footers, FileStat, Employee, GroupLkup, FileType,
|
|
Qdros, PlanInfo, Pensions, PensionMarriage, PensionDeath,
|
|
PensionSchedule, PensionSeparate, PensionResults,
|
|
RolexV, FVarLkup, RVarLkup
|
|
)
|
|
|
|
logger = structlog.get_logger(__name__)
|
|
|
|
# Batch size for commits
|
|
BATCH_SIZE = 500
|
|
|
|
|
|
def open_text_with_fallbacks(file_path: str):
|
|
"""
|
|
Open a text file trying multiple encodings commonly seen in legacy CSVs.
|
|
|
|
Returns a tuple of (file_object, encoding_used).
|
|
"""
|
|
# First try strict mode with common encodings
|
|
# Try latin-1/iso-8859-1 earlier as they are more forgiving and commonly used in legacy data
|
|
encodings = ["utf-8", "utf-8-sig", "iso-8859-1", "latin-1", "cp1252", "windows-1252", "cp1250"]
|
|
last_error = None
|
|
for enc in encodings:
|
|
try:
|
|
f = open(file_path, 'r', encoding=enc, errors='strict', newline='')
|
|
# Read more than 1KB to catch encoding issues deeper in the file
|
|
# Many legacy CSVs have issues beyond the first few rows
|
|
_ = f.read(10240) # Read 10KB to test
|
|
f.seek(0)
|
|
logger.info("csv_open_encoding_selected", file=file_path, encoding=enc)
|
|
return f, enc
|
|
except Exception as e:
|
|
last_error = e
|
|
logger.warning("encoding_fallback_failed", file=file_path, encoding=enc, error=str(e))
|
|
try:
|
|
f.close()
|
|
except:
|
|
pass
|
|
continue
|
|
|
|
# If strict mode fails, try with error replacement for robustness
|
|
logger.warning("strict_encoding_failed", file=file_path, trying_with_replace=True)
|
|
try:
|
|
# Try UTF-8 with error replacement first (most common case)
|
|
f = open(file_path, 'r', encoding='utf-8', errors='replace', newline='')
|
|
_ = f.read(1024)
|
|
f.seek(0)
|
|
logger.info("csv_open_encoding_with_replace", file=file_path, encoding="utf-8-replace")
|
|
return f, "utf-8-replace"
|
|
except Exception as e:
|
|
logger.warning("utf8_replace_failed", file=file_path, error=str(e))
|
|
|
|
# Final fallback: use latin-1 with replace (handles any byte sequence)
|
|
try:
|
|
f = open(file_path, 'r', encoding='latin-1', errors='replace', newline='')
|
|
_ = f.read(1024)
|
|
f.seek(0)
|
|
logger.info("csv_open_encoding_fallback", file=file_path, encoding="latin-1-replace")
|
|
return f, "latin-1-replace"
|
|
except Exception as e:
|
|
last_error = e
|
|
|
|
error_msg = f"Unable to open file '{file_path}' with any supported encodings"
|
|
if last_error:
|
|
error_msg += f". Last error: {str(last_error)}"
|
|
raise RuntimeError(error_msg)
|
|
|
|
|
|
def parse_date(date_str: str) -> Optional[datetime]:
|
|
"""Parse date string in various formats, return None if blank/invalid."""
|
|
if not date_str or not date_str.strip():
|
|
return None
|
|
|
|
date_str = date_str.strip()
|
|
|
|
# Try common date formats
|
|
formats = [
|
|
"%m/%d/%Y",
|
|
"%m/%d/%y",
|
|
"%Y-%m-%d",
|
|
"%m-%d-%Y",
|
|
"%m-%d-%y",
|
|
]
|
|
|
|
for fmt in formats:
|
|
try:
|
|
return datetime.strptime(date_str, fmt)
|
|
except ValueError:
|
|
continue
|
|
|
|
logger.warning("date_parse_failed", date_string=date_str)
|
|
return None
|
|
|
|
|
|
def parse_decimal(value: str) -> Optional[Decimal]:
|
|
"""Parse decimal string, return None if blank/invalid."""
|
|
if not value or not value.strip():
|
|
return None
|
|
|
|
try:
|
|
return Decimal(value.strip())
|
|
except (InvalidOperation, ValueError):
|
|
logger.warning("decimal_parse_failed", value=value)
|
|
return None
|
|
|
|
|
|
def clean_string(value: str) -> Optional[str]:
|
|
"""Clean string value, return None if blank."""
|
|
if not value or not value.strip():
|
|
return None
|
|
return value.strip()
|
|
|
|
|
|
# ============================================================================
|
|
# Reference Table Imports (should be imported first)
|
|
# ============================================================================
|
|
|
|
def import_trnstype(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import TRNSTYPE.csv → TrnsType model with upsert logic."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0, 'updated': 0, 'inserted': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
t_type = clean_string(row.get('T_Type'))
|
|
if not t_type:
|
|
continue
|
|
|
|
# Check if record already exists
|
|
existing = db.query(TrnsType).filter(TrnsType.t_type == t_type).first()
|
|
|
|
if existing:
|
|
# Update existing record
|
|
existing.t_type_l = clean_string(row.get('T_Type_L'))
|
|
existing.header = clean_string(row.get('Header'))
|
|
existing.footer = clean_string(row.get('Footer'))
|
|
result['updated'] += 1
|
|
else:
|
|
# Insert new record
|
|
record = TrnsType(
|
|
t_type=t_type,
|
|
t_type_l=clean_string(row.get('T_Type_L')),
|
|
header=clean_string(row.get('Header')),
|
|
footer=clean_string(row.get('Footer'))
|
|
)
|
|
db.add(record)
|
|
result['inserted'] += 1
|
|
|
|
result['success'] += 1
|
|
|
|
# Commit in batches for performance
|
|
if result['success'] % BATCH_SIZE == 0:
|
|
db.commit()
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
db.rollback()
|
|
|
|
# Commit any remaining changes
|
|
db.commit()
|
|
|
|
f.close()
|
|
logger.info("import_trnstype_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_trnstype_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_trnslkup(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import TRNSLKUP.csv → TrnsLkup model with upsert logic."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0, 'updated': 0, 'inserted': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
t_code = clean_string(row.get('T_Code'))
|
|
if not t_code:
|
|
continue
|
|
|
|
# Check if record already exists
|
|
existing = db.query(TrnsLkup).filter(TrnsLkup.t_code == t_code).first()
|
|
|
|
if existing:
|
|
# Update existing record
|
|
existing.t_type = clean_string(row.get('T_Type'))
|
|
existing.t_type_l = clean_string(row.get('T_Type_L'))
|
|
existing.amount = parse_decimal(row.get('Amount'))
|
|
existing.description = clean_string(row.get('Description'))
|
|
result['updated'] += 1
|
|
else:
|
|
# Insert new record
|
|
record = TrnsLkup(
|
|
t_code=t_code,
|
|
t_type=clean_string(row.get('T_Type')),
|
|
t_type_l=clean_string(row.get('T_Type_L')),
|
|
amount=parse_decimal(row.get('Amount')),
|
|
description=clean_string(row.get('Description'))
|
|
)
|
|
db.add(record)
|
|
result['inserted'] += 1
|
|
|
|
result['success'] += 1
|
|
|
|
# Commit in batches for performance
|
|
if result['success'] % BATCH_SIZE == 0:
|
|
db.commit()
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
db.rollback()
|
|
|
|
# Commit any remaining changes
|
|
db.commit()
|
|
|
|
f.close()
|
|
logger.info("import_trnslkup_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_trnslkup_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_footers(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import FOOTERS.csv → Footers model with upsert logic."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0, 'updated': 0, 'inserted': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
f_code = clean_string(row.get('F_Code'))
|
|
if not f_code:
|
|
continue
|
|
|
|
# Check if record already exists
|
|
existing = db.query(Footers).filter(Footers.f_code == f_code).first()
|
|
|
|
if existing:
|
|
# Update existing record
|
|
existing.f_footer = clean_string(row.get('F_Footer'))
|
|
result['updated'] += 1
|
|
else:
|
|
# Insert new record
|
|
record = Footers(
|
|
f_code=f_code,
|
|
f_footer=clean_string(row.get('F_Footer'))
|
|
)
|
|
db.add(record)
|
|
result['inserted'] += 1
|
|
|
|
result['success'] += 1
|
|
|
|
# Commit in batches for performance
|
|
if result['success'] % BATCH_SIZE == 0:
|
|
db.commit()
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
db.rollback()
|
|
|
|
# Commit any remaining changes
|
|
db.commit()
|
|
|
|
f.close()
|
|
logger.info("import_footers_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_footers_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_filestat(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import FILESTAT.csv → FileStat model with upsert logic."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0, 'updated': 0, 'inserted': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
status = clean_string(row.get('Status'))
|
|
if not status:
|
|
continue
|
|
|
|
# Check if record already exists
|
|
existing = db.query(FileStat).filter(FileStat.status == status).first()
|
|
|
|
if existing:
|
|
# Update existing record
|
|
existing.definition = clean_string(row.get('Definition'))
|
|
existing.send = clean_string(row.get('Send'))
|
|
existing.footer_code = clean_string(row.get('Footer_Code'))
|
|
result['updated'] += 1
|
|
else:
|
|
# Insert new record
|
|
record = FileStat(
|
|
status=status,
|
|
definition=clean_string(row.get('Definition')),
|
|
send=clean_string(row.get('Send')),
|
|
footer_code=clean_string(row.get('Footer_Code'))
|
|
)
|
|
db.add(record)
|
|
result['inserted'] += 1
|
|
|
|
result['success'] += 1
|
|
|
|
# Commit in batches for performance
|
|
if result['success'] % BATCH_SIZE == 0:
|
|
db.commit()
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
db.rollback()
|
|
|
|
# Commit any remaining changes
|
|
db.commit()
|
|
|
|
f.close()
|
|
logger.info("import_filestat_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_filestat_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_employee(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import EMPLOYEE.csv → Employee model with upsert logic."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0, 'updated': 0, 'inserted': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
empl_num = clean_string(row.get('Empl_Num'))
|
|
if not empl_num:
|
|
continue
|
|
|
|
# Check if record already exists
|
|
existing = db.query(Employee).filter(Employee.empl_num == empl_num).first()
|
|
|
|
if existing:
|
|
# Update existing record
|
|
existing.empl_id = clean_string(row.get('Empl_Id'))
|
|
existing.rate_per_hour = parse_decimal(row.get('Rate_Per_Hour'))
|
|
result['updated'] += 1
|
|
else:
|
|
# Insert new record
|
|
record = Employee(
|
|
empl_num=empl_num,
|
|
empl_id=clean_string(row.get('Empl_Id')),
|
|
rate_per_hour=parse_decimal(row.get('Rate_Per_Hour'))
|
|
)
|
|
db.add(record)
|
|
result['inserted'] += 1
|
|
|
|
result['success'] += 1
|
|
|
|
# Commit in batches for performance
|
|
if result['success'] % BATCH_SIZE == 0:
|
|
db.commit()
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
db.rollback()
|
|
|
|
# Commit any remaining changes
|
|
db.commit()
|
|
|
|
f.close()
|
|
logger.info("import_employee_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_employee_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_gruplkup(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import GRUPLKUP.csv → GroupLkup model with upsert logic."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0, 'updated': 0, 'inserted': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
code = clean_string(row.get('Code'))
|
|
if not code:
|
|
continue
|
|
|
|
# Check if record already exists
|
|
existing = db.query(GroupLkup).filter(GroupLkup.code == code).first()
|
|
|
|
if existing:
|
|
# Update existing record
|
|
existing.description = clean_string(row.get('Description'))
|
|
existing.title = clean_string(row.get('Title'))
|
|
result['updated'] += 1
|
|
else:
|
|
# Insert new record
|
|
record = GroupLkup(
|
|
code=code,
|
|
description=clean_string(row.get('Description')),
|
|
title=clean_string(row.get('Title'))
|
|
)
|
|
db.add(record)
|
|
result['inserted'] += 1
|
|
|
|
result['success'] += 1
|
|
|
|
# Commit in batches for performance
|
|
if result['success'] % BATCH_SIZE == 0:
|
|
db.commit()
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
db.rollback()
|
|
|
|
# Commit any remaining changes
|
|
db.commit()
|
|
|
|
f.close()
|
|
logger.info("import_gruplkup_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_gruplkup_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_filetype(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import FILETYPE.csv → FileType model with upsert logic."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0, 'updated': 0, 'inserted': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
# Track seen file types in-memory to avoid duplicates within the same CSV
|
|
seen_in_batch = set()
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
file_type = clean_string(row.get('File_Type'))
|
|
if not file_type:
|
|
continue
|
|
|
|
# Skip if we've already processed this file_type in current import
|
|
if file_type in seen_in_batch:
|
|
continue
|
|
seen_in_batch.add(file_type)
|
|
|
|
# Check if it already exists in DB
|
|
existing = db.query(FileType).filter(FileType.file_type == file_type).first()
|
|
if existing:
|
|
# No fields to update for FileType (only has PK), just count it
|
|
result['updated'] += 1
|
|
else:
|
|
# Insert new record
|
|
record = FileType(file_type=file_type)
|
|
db.add(record)
|
|
result['inserted'] += 1
|
|
|
|
result['success'] += 1
|
|
|
|
# Commit in batches for performance
|
|
if result['success'] % BATCH_SIZE == 0:
|
|
db.commit()
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
db.rollback()
|
|
|
|
# Commit any remaining changes
|
|
db.commit()
|
|
|
|
f.close()
|
|
logger.info("import_filetype_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_filetype_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_fvarlkup(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import FVARLKUP.csv → FVarLkup model with upsert logic."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0, 'updated': 0, 'inserted': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
identifier = clean_string(row.get('Identifier'))
|
|
if not identifier:
|
|
continue
|
|
|
|
# Check if record already exists
|
|
existing = db.query(FVarLkup).filter(FVarLkup.identifier == identifier).first()
|
|
|
|
if existing:
|
|
# Update existing record
|
|
existing.query = clean_string(row.get('Query'))
|
|
existing.response = clean_string(row.get('Response'))
|
|
result['updated'] += 1
|
|
else:
|
|
# Insert new record
|
|
record = FVarLkup(
|
|
identifier=identifier,
|
|
query=clean_string(row.get('Query')),
|
|
response=clean_string(row.get('Response'))
|
|
)
|
|
db.add(record)
|
|
result['inserted'] += 1
|
|
|
|
result['success'] += 1
|
|
|
|
# Commit in batches for performance
|
|
if result['success'] % BATCH_SIZE == 0:
|
|
db.commit()
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
db.rollback()
|
|
|
|
# Commit any remaining changes
|
|
db.commit()
|
|
|
|
f.close()
|
|
logger.info("import_fvarlkup_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_fvarlkup_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_rvarlkup(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import RVARLKUP.csv → RVarLkup model with upsert logic."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0, 'updated': 0, 'inserted': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
identifier = clean_string(row.get('Identifier'))
|
|
if not identifier:
|
|
continue
|
|
|
|
# Check if record already exists
|
|
existing = db.query(RVarLkup).filter(RVarLkup.identifier == identifier).first()
|
|
|
|
if existing:
|
|
# Update existing record
|
|
existing.query = clean_string(row.get('Query'))
|
|
result['updated'] += 1
|
|
else:
|
|
# Insert new record
|
|
record = RVarLkup(
|
|
identifier=identifier,
|
|
query=clean_string(row.get('Query'))
|
|
)
|
|
db.add(record)
|
|
result['inserted'] += 1
|
|
|
|
result['success'] += 1
|
|
|
|
# Commit in batches for performance
|
|
if result['success'] % BATCH_SIZE == 0:
|
|
db.commit()
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
db.rollback()
|
|
|
|
# Commit any remaining changes
|
|
db.commit()
|
|
|
|
f.close()
|
|
logger.info("import_rvarlkup_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_rvarlkup_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
# ============================================================================
|
|
# Core Data Table Imports
|
|
# ============================================================================
|
|
|
|
def import_rolodex(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import ROLODEX.csv → Rolodex model."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0, 'skipped': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
# Track IDs we've seen in this import to handle duplicates
|
|
seen_in_import = set()
|
|
batch = []
|
|
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
rolodex_id = clean_string(row.get('Id'))
|
|
if not rolodex_id:
|
|
continue
|
|
|
|
# Skip if we've already processed this ID in current import
|
|
if rolodex_id in seen_in_import:
|
|
result['skipped'] += 1
|
|
continue
|
|
|
|
# Skip if it already exists in database
|
|
if db.query(Rolodex).filter(Rolodex.id == rolodex_id).first():
|
|
result['skipped'] += 1
|
|
seen_in_import.add(rolodex_id)
|
|
continue
|
|
|
|
record = Rolodex(
|
|
id=rolodex_id,
|
|
prefix=clean_string(row.get('Prefix')),
|
|
first=clean_string(row.get('First')),
|
|
middle=clean_string(row.get('Middle')),
|
|
last=clean_string(row.get('Last')),
|
|
suffix=clean_string(row.get('Suffix')),
|
|
title=clean_string(row.get('Title')),
|
|
a1=clean_string(row.get('A1')),
|
|
a2=clean_string(row.get('A2')),
|
|
a3=clean_string(row.get('A3')),
|
|
city=clean_string(row.get('City')),
|
|
abrev=clean_string(row.get('Abrev')),
|
|
st=clean_string(row.get('St')),
|
|
zip=clean_string(row.get('Zip')),
|
|
email=clean_string(row.get('Email')),
|
|
dob=parse_date(row.get('DOB')),
|
|
ss=clean_string(row.get('SS#')),
|
|
legal_status=clean_string(row.get('Legal_Status')),
|
|
group=clean_string(row.get('Group')),
|
|
memo=clean_string(row.get('Memo'))
|
|
)
|
|
batch.append(record)
|
|
seen_in_import.add(rolodex_id)
|
|
|
|
if len(batch) >= BATCH_SIZE:
|
|
try:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
batch = []
|
|
except IntegrityError as ie:
|
|
db.rollback()
|
|
# Handle any remaining duplicates by inserting one at a time
|
|
for record in batch:
|
|
try:
|
|
db.add(record)
|
|
db.commit()
|
|
result['success'] += 1
|
|
except IntegrityError:
|
|
db.rollback()
|
|
result['skipped'] += 1
|
|
batch = []
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
|
|
# Save remaining batch
|
|
if batch:
|
|
try:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
except IntegrityError:
|
|
db.rollback()
|
|
# Handle any remaining duplicates by inserting one at a time
|
|
for record in batch:
|
|
try:
|
|
db.add(record)
|
|
db.commit()
|
|
result['success'] += 1
|
|
except IntegrityError:
|
|
db.rollback()
|
|
result['skipped'] += 1
|
|
|
|
f.close()
|
|
logger.info("import_rolodex_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_rolodex_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_phone(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import PHONE.csv → LegacyPhone model with upsert logic."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0, 'updated': 0, 'inserted': 0, 'skipped': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
# Track seen combinations in this import to handle duplicates within the CSV
|
|
seen_in_import = set()
|
|
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
rolodex_id = clean_string(row.get('Id'))
|
|
phone = clean_string(row.get('Phone'))
|
|
|
|
if not rolodex_id or not phone:
|
|
continue
|
|
|
|
# Create a composite key for tracking
|
|
composite_key = (rolodex_id, phone)
|
|
|
|
# Skip if we've already processed this combination in current import
|
|
if composite_key in seen_in_import:
|
|
result['skipped'] += 1
|
|
continue
|
|
|
|
seen_in_import.add(composite_key)
|
|
|
|
# Check if record already exists in database
|
|
existing = db.query(LegacyPhone).filter(
|
|
LegacyPhone.id == rolodex_id,
|
|
LegacyPhone.phone == phone
|
|
).first()
|
|
|
|
if existing:
|
|
# Update existing record
|
|
existing.location = clean_string(row.get('Location'))
|
|
result['updated'] += 1
|
|
else:
|
|
# Insert new record
|
|
record = LegacyPhone(
|
|
id=rolodex_id,
|
|
phone=phone,
|
|
location=clean_string(row.get('Location'))
|
|
)
|
|
db.add(record)
|
|
result['inserted'] += 1
|
|
|
|
result['success'] += 1
|
|
|
|
# Commit in batches for performance
|
|
if result['success'] % BATCH_SIZE == 0:
|
|
db.commit()
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
db.rollback()
|
|
|
|
# Commit any remaining changes
|
|
db.commit()
|
|
|
|
f.close()
|
|
logger.info("import_phone_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_phone_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_rolex_v(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import ROLEX_V.csv → RolexV model."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
batch = []
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
rolodex_id = clean_string(row.get('Id'))
|
|
identifier = clean_string(row.get('Identifier'))
|
|
|
|
if not rolodex_id or not identifier:
|
|
continue
|
|
|
|
record = RolexV(
|
|
id=rolodex_id,
|
|
identifier=identifier,
|
|
response=clean_string(row.get('Response'))
|
|
)
|
|
batch.append(record)
|
|
|
|
if len(batch) >= BATCH_SIZE:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
batch = []
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
|
|
if batch:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
|
|
f.close()
|
|
logger.info("import_rolex_v_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_rolex_v_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_files(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import FILES.csv → LegacyFile model."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
batch = []
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
file_no = clean_string(row.get('File_No'))
|
|
if not file_no:
|
|
continue
|
|
|
|
record = LegacyFile(
|
|
file_no=file_no,
|
|
id=clean_string(row.get('Id')),
|
|
file_type=clean_string(row.get('File_Type')),
|
|
regarding=clean_string(row.get('Regarding')),
|
|
opened=parse_date(row.get('Opened')),
|
|
closed=parse_date(row.get('Closed')),
|
|
empl_num=clean_string(row.get('Empl_Num')),
|
|
rate_per_hour=parse_decimal(row.get('Rate_Per_Hour')),
|
|
status=clean_string(row.get('Status')),
|
|
footer_code=clean_string(row.get('Footer_Code')),
|
|
opposing=clean_string(row.get('Opposing')),
|
|
hours=parse_decimal(row.get('Hours')),
|
|
hours_p=parse_decimal(row.get('Hours_P')),
|
|
trust_bal=parse_decimal(row.get('Trust_Bal')),
|
|
trust_bal_p=parse_decimal(row.get('Trust_Bal_P')),
|
|
hourly_fees=parse_decimal(row.get('Hourly_Fees')),
|
|
hourly_fees_p=parse_decimal(row.get('Hourly_Fees_P')),
|
|
flat_fees=parse_decimal(row.get('Flat_Fees')),
|
|
flat_fees_p=parse_decimal(row.get('Flat_Fees_P')),
|
|
disbursements=parse_decimal(row.get('Disbursements')),
|
|
disbursements_p=parse_decimal(row.get('Disbursements_P')),
|
|
credit_bal=parse_decimal(row.get('Credit_Bal')),
|
|
credit_bal_p=parse_decimal(row.get('Credit_Bal_P')),
|
|
total_charges=parse_decimal(row.get('Total_Charges')),
|
|
total_charges_p=parse_decimal(row.get('Total_Charges_P')),
|
|
amount_owing=parse_decimal(row.get('Amount_Owing')),
|
|
amount_owing_p=parse_decimal(row.get('Amount_Owing_P')),
|
|
transferable=parse_decimal(row.get('Transferable')),
|
|
memo=clean_string(row.get('Memo'))
|
|
)
|
|
batch.append(record)
|
|
|
|
if len(batch) >= BATCH_SIZE:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
batch = []
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
|
|
if batch:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
|
|
f.close()
|
|
logger.info("import_files_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_files_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_files_r(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import FILES_R.csv → FilesR model."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
batch = []
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
file_no = clean_string(row.get('File_No'))
|
|
relationship = clean_string(row.get('Relationship'))
|
|
rolodex_id = clean_string(row.get('Rolodex_Id'))
|
|
|
|
if not file_no or not relationship or not rolodex_id:
|
|
continue
|
|
|
|
record = FilesR(
|
|
file_no=file_no,
|
|
relationship=relationship,
|
|
rolodex_id=rolodex_id
|
|
)
|
|
batch.append(record)
|
|
|
|
if len(batch) >= BATCH_SIZE:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
batch = []
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
|
|
if batch:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
|
|
f.close()
|
|
logger.info("import_files_r_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_files_r_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_files_v(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import FILES_V.csv → FilesV model."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
batch = []
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
file_no = clean_string(row.get('File_No'))
|
|
identifier = clean_string(row.get('Identifier'))
|
|
|
|
if not file_no or not identifier:
|
|
continue
|
|
|
|
record = FilesV(
|
|
file_no=file_no,
|
|
identifier=identifier,
|
|
response=clean_string(row.get('Response'))
|
|
)
|
|
batch.append(record)
|
|
|
|
if len(batch) >= BATCH_SIZE:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
batch = []
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
|
|
if batch:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
|
|
f.close()
|
|
logger.info("import_files_v_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_files_v_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_filenots(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import FILENOTS.csv → FileNots model."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
batch = []
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
file_no = clean_string(row.get('File_No'))
|
|
memo_date = parse_date(row.get('Memo_Date'))
|
|
|
|
if not file_no or not memo_date:
|
|
continue
|
|
|
|
record = FileNots(
|
|
file_no=file_no,
|
|
memo_date=memo_date,
|
|
memo_note=clean_string(row.get('Memo_Note'))
|
|
)
|
|
batch.append(record)
|
|
|
|
if len(batch) >= BATCH_SIZE:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
batch = []
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
|
|
if batch:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
|
|
f.close()
|
|
logger.info("import_filenots_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_filenots_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_ledger(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import LEDGER.csv → Ledger model."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
batch = []
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
file_no = clean_string(row.get('File_No'))
|
|
item_no_str = clean_string(row.get('Item_No'))
|
|
|
|
if not file_no or not item_no_str:
|
|
continue
|
|
|
|
try:
|
|
item_no = int(item_no_str)
|
|
except ValueError:
|
|
result['errors'].append(f"Row {row_num}: Invalid Item_No '{item_no_str}'")
|
|
continue
|
|
|
|
record = Ledger(
|
|
file_no=file_no,
|
|
date=parse_date(row.get('Date')),
|
|
item_no=item_no,
|
|
empl_num=clean_string(row.get('Empl_Num')),
|
|
t_code=clean_string(row.get('T_Code')),
|
|
t_type=clean_string(row.get('T_Type')),
|
|
t_type_l=clean_string(row.get('T_Type_L')),
|
|
quantity=parse_decimal(row.get('Quantity')),
|
|
rate=parse_decimal(row.get('Rate')),
|
|
amount=parse_decimal(row.get('Amount')),
|
|
billed=clean_string(row.get('Billed')),
|
|
note=clean_string(row.get('Note'))
|
|
)
|
|
batch.append(record)
|
|
|
|
if len(batch) >= BATCH_SIZE:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
batch = []
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
|
|
if batch:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
|
|
f.close()
|
|
logger.info("import_ledger_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_ledger_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_deposits(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import DEPOSITS.csv → Deposits model."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
batch = []
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
deposit_date = parse_date(row.get('Deposit_Date'))
|
|
|
|
if not deposit_date:
|
|
continue
|
|
|
|
record = Deposits(
|
|
deposit_date=deposit_date,
|
|
total=parse_decimal(row.get('Total'))
|
|
)
|
|
batch.append(record)
|
|
|
|
if len(batch) >= BATCH_SIZE:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
batch = []
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
|
|
if batch:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
|
|
f.close()
|
|
logger.info("import_deposits_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_deposits_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_payments(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import PAYMENTS.csv → LegacyPayment model."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
batch = []
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
# LegacyPayment has auto-increment id, so we don't need to check for it
|
|
record = LegacyPayment(
|
|
deposit_date=parse_date(row.get('Deposit_Date')),
|
|
file_no=clean_string(row.get('File_No')),
|
|
rolodex_id=clean_string(row.get('Id')),
|
|
regarding=clean_string(row.get('Regarding')),
|
|
amount=parse_decimal(row.get('Amount')),
|
|
note=clean_string(row.get('Note'))
|
|
)
|
|
batch.append(record)
|
|
|
|
if len(batch) >= BATCH_SIZE:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
batch = []
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
|
|
if batch:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
|
|
f.close()
|
|
logger.info("import_payments_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_payments_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
# ============================================================================
|
|
# Specialized Imports (QDRO, Pensions, Plan Info)
|
|
# ============================================================================
|
|
|
|
def import_planinfo(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import PLANINFO.csv → PlanInfo model."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
batch = []
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
plan_id = clean_string(row.get('Plan_Id'))
|
|
if not plan_id:
|
|
continue
|
|
|
|
record = PlanInfo(
|
|
plan_id=plan_id,
|
|
plan_name=clean_string(row.get('Plan_Name')),
|
|
plan_type=clean_string(row.get('Plan_Type')),
|
|
empl_id_no=clean_string(row.get('Empl_Id_No')),
|
|
plan_no=clean_string(row.get('Plan_No')),
|
|
nra=clean_string(row.get('NRA')),
|
|
era=clean_string(row.get('ERA')),
|
|
errf=clean_string(row.get('ERRF')),
|
|
colas=clean_string(row.get('COLAS')),
|
|
divided_by=clean_string(row.get('Divided_By')),
|
|
drafted=clean_string(row.get('Drafted')),
|
|
benefit_c=clean_string(row.get('Benefit_C')),
|
|
qdro_c=clean_string(row.get('QDRO_C')),
|
|
rev=clean_string(row.get('^REV')),
|
|
pa=clean_string(row.get('^PA')),
|
|
form_name=clean_string(row.get('Form_Name')),
|
|
drafted_on=parse_date(row.get('Drafted_On')),
|
|
memo=clean_string(row.get('Memo'))
|
|
)
|
|
batch.append(record)
|
|
|
|
if len(batch) >= BATCH_SIZE:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
batch = []
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
|
|
if batch:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
|
|
f.close()
|
|
logger.info("import_planinfo_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_planinfo_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_qdros(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import QDROS.csv → Qdros model."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
batch = []
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
file_no = clean_string(row.get('File_No'))
|
|
version = clean_string(row.get('Version'))
|
|
|
|
if not file_no or not version:
|
|
continue
|
|
|
|
record = Qdros(
|
|
file_no=file_no,
|
|
version=version,
|
|
plan_id=clean_string(row.get('Plan_Id')),
|
|
_1=clean_string(row.get('^1')),
|
|
_2=clean_string(row.get('^2')),
|
|
part=clean_string(row.get('^Part')),
|
|
altp=clean_string(row.get('^AltP')),
|
|
pet=clean_string(row.get('^Pet')),
|
|
res=clean_string(row.get('^Res')),
|
|
case_type=clean_string(row.get('Case_Type')),
|
|
case_code=clean_string(row.get('Case_Code')),
|
|
section=clean_string(row.get('Section')),
|
|
case_number=clean_string(row.get('Case_Number')),
|
|
judgment_date=parse_date(row.get('Judgment_Date')),
|
|
valuation_date=parse_date(row.get('Valuation_Date')),
|
|
married_on=parse_date(row.get('Married_On')),
|
|
percent_awarded=parse_decimal(row.get('Percent_Awarded')),
|
|
ven_city=clean_string(row.get('Ven_City')),
|
|
ven_cnty=clean_string(row.get('Ven_Cnty')),
|
|
ven_st=clean_string(row.get('Ven_St')),
|
|
draft_out=parse_date(row.get('Draft_Out')),
|
|
draft_apr=parse_date(row.get('Draft_Apr')),
|
|
final_out=parse_date(row.get('Final_Out')),
|
|
judge=clean_string(row.get('Judge')),
|
|
form_name=clean_string(row.get('Form_Name'))
|
|
)
|
|
batch.append(record)
|
|
|
|
if len(batch) >= BATCH_SIZE:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
batch = []
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
|
|
if batch:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
|
|
f.close()
|
|
logger.info("import_qdros_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_qdros_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_pensions(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import PENSIONS.csv → Pensions model."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
batch = []
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
file_no = clean_string(row.get('File_No'))
|
|
version = clean_string(row.get('Version'))
|
|
|
|
if not file_no or not version:
|
|
continue
|
|
|
|
record = Pensions(
|
|
file_no=file_no,
|
|
version=version,
|
|
plan_id=clean_string(row.get('Plan_Id')),
|
|
plan_name=clean_string(row.get('Plan_Name')),
|
|
title=clean_string(row.get('Title')),
|
|
first=clean_string(row.get('First')),
|
|
last=clean_string(row.get('Last')),
|
|
birth=parse_date(row.get('Birth')),
|
|
race=clean_string(row.get('Race')),
|
|
sex=clean_string(row.get('Sex')),
|
|
info=parse_date(row.get('Info')),
|
|
valu=parse_date(row.get('Valu')),
|
|
accrued=parse_decimal(row.get('Accrued')),
|
|
vested_per=parse_decimal(row.get('Vested_Per')),
|
|
start_age=parse_decimal(row.get('Start_Age')),
|
|
cola=parse_decimal(row.get('COLA')),
|
|
max_cola=parse_decimal(row.get('Max_COLA')),
|
|
withdrawal=parse_decimal(row.get('Withdrawal')),
|
|
pre_dr=parse_decimal(row.get('Pre_DR')),
|
|
post_dr=parse_decimal(row.get('Post_DR')),
|
|
tax_rate=parse_decimal(row.get('Tax_Rate'))
|
|
)
|
|
batch.append(record)
|
|
|
|
if len(batch) >= BATCH_SIZE:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
batch = []
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
|
|
if batch:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
|
|
f.close()
|
|
logger.info("import_pensions_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_pensions_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_pension_marriage(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import Pensions/MARRIAGE.csv → PensionMarriage model."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
batch = []
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
file_no = clean_string(row.get('File_No'))
|
|
version = clean_string(row.get('Version'))
|
|
|
|
if not file_no or not version:
|
|
continue
|
|
|
|
record = PensionMarriage(
|
|
file_no=file_no,
|
|
version=version,
|
|
married_from=parse_date(row.get('Married_From')),
|
|
married_to=parse_date(row.get('Married_To')),
|
|
married_years=parse_decimal(row.get('Married_Years')),
|
|
service_from=parse_date(row.get('Service_From')),
|
|
service_to=parse_date(row.get('Service_To')),
|
|
service_years=parse_decimal(row.get('Service_Years')),
|
|
marital_pct=parse_decimal(row.get('Marital_%'))
|
|
)
|
|
batch.append(record)
|
|
|
|
if len(batch) >= BATCH_SIZE:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
batch = []
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
|
|
if batch:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
|
|
f.close()
|
|
logger.info("import_pension_marriage_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_pension_marriage_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_pension_death(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import Pensions/DEATH.csv → PensionDeath model."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
batch = []
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
file_no = clean_string(row.get('File_No'))
|
|
version = clean_string(row.get('Version'))
|
|
|
|
if not file_no or not version:
|
|
continue
|
|
|
|
record = PensionDeath(
|
|
file_no=file_no,
|
|
version=version,
|
|
lump1=parse_decimal(row.get('Lump1')),
|
|
lump2=parse_decimal(row.get('Lump2')),
|
|
growth1=parse_decimal(row.get('Growth1')),
|
|
growth2=parse_decimal(row.get('Growth2')),
|
|
disc1=parse_decimal(row.get('Disc1')),
|
|
disc2=parse_decimal(row.get('Disc2'))
|
|
)
|
|
batch.append(record)
|
|
|
|
if len(batch) >= BATCH_SIZE:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
batch = []
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
|
|
if batch:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
|
|
f.close()
|
|
logger.info("import_pension_death_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_pension_death_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_pension_schedule(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import Pensions/SCHEDULE.csv → PensionSchedule model."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
batch = []
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
file_no = clean_string(row.get('File_No'))
|
|
version = clean_string(row.get('Version'))
|
|
|
|
if not file_no or not version:
|
|
continue
|
|
|
|
record = PensionSchedule(
|
|
file_no=file_no,
|
|
version=version,
|
|
vests_on=parse_date(row.get('Vests_On')),
|
|
vests_at=parse_decimal(row.get('Vests_At'))
|
|
)
|
|
batch.append(record)
|
|
|
|
if len(batch) >= BATCH_SIZE:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
batch = []
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
|
|
if batch:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
|
|
f.close()
|
|
logger.info("import_pension_schedule_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_pension_schedule_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_pension_separate(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import Pensions/SEPARATE.csv → PensionSeparate model."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
batch = []
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
file_no = clean_string(row.get('File_No'))
|
|
version = clean_string(row.get('Version'))
|
|
|
|
if not file_no or not version:
|
|
continue
|
|
|
|
record = PensionSeparate(
|
|
file_no=file_no,
|
|
version=version,
|
|
separation_rate=parse_decimal(row.get('Separation_Rate'))
|
|
)
|
|
batch.append(record)
|
|
|
|
if len(batch) >= BATCH_SIZE:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
batch = []
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
|
|
if batch:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
|
|
f.close()
|
|
logger.info("import_pension_separate_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_pension_separate_failed", error=str(e))
|
|
|
|
return result
|
|
|
|
|
|
def import_pension_results(db: Session, file_path: str) -> Dict[str, Any]:
|
|
"""Import Pensions/RESULTS.csv → PensionResults model."""
|
|
result = {'success': 0, 'errors': [], 'total_rows': 0}
|
|
|
|
try:
|
|
f, encoding = open_text_with_fallbacks(file_path)
|
|
reader = csv.DictReader(f)
|
|
|
|
batch = []
|
|
for row_num, row in enumerate(reader, start=2):
|
|
result['total_rows'] += 1
|
|
|
|
try:
|
|
# Note: RESULTS.csv might not have explicit file_no/version columns
|
|
# May need to derive from associated pension record
|
|
# For now, skip if missing
|
|
file_no = clean_string(row.get('File_No'))
|
|
version = clean_string(row.get('Version'))
|
|
|
|
if not file_no or not version:
|
|
continue
|
|
|
|
record = PensionResults(
|
|
file_no=file_no,
|
|
version=version,
|
|
accrued=parse_decimal(row.get('Accrued')),
|
|
start_age=parse_decimal(row.get('Start_Age')),
|
|
cola=parse_decimal(row.get('COLA')),
|
|
withdrawal=parse_decimal(row.get('Withdrawal')),
|
|
pre_dr=parse_decimal(row.get('Pre_DR')),
|
|
post_dr=parse_decimal(row.get('Post_DR')),
|
|
tax_rate=parse_decimal(row.get('Tax_Rate')),
|
|
age=parse_decimal(row.get('Age')),
|
|
years_from=parse_decimal(row.get('Years_From')),
|
|
life_exp=parse_decimal(row.get('Life_Exp')),
|
|
ev_monthly=parse_decimal(row.get('EV_Monthly')),
|
|
payments=parse_decimal(row.get('Payments')),
|
|
pay_out=parse_decimal(row.get('Pay_Out')),
|
|
fund_value=parse_decimal(row.get('Fund_Value')),
|
|
pv=parse_decimal(row.get('PV')),
|
|
mortality=parse_decimal(row.get('Mortality')),
|
|
pv_am=parse_decimal(row.get('PV_AM')),
|
|
pv_amt=parse_decimal(row.get('PV_AMT')),
|
|
pv_pre_db=parse_decimal(row.get('PV_Pre_DB')),
|
|
pv_annuity=parse_decimal(row.get('PV_Annuity')),
|
|
wv_at=parse_decimal(row.get('WV_AT')),
|
|
pv_plan=parse_decimal(row.get('PV_Plan')),
|
|
years_married=parse_decimal(row.get('Years_Married')),
|
|
years_service=parse_decimal(row.get('Years_Service')),
|
|
marr_per=parse_decimal(row.get('Marr_Per')),
|
|
marr_amt=parse_decimal(row.get('Marr_Amt'))
|
|
)
|
|
batch.append(record)
|
|
|
|
if len(batch) >= BATCH_SIZE:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
batch = []
|
|
|
|
except Exception as e:
|
|
result['errors'].append(f"Row {row_num}: {str(e)}")
|
|
|
|
if batch:
|
|
db.bulk_save_objects(batch)
|
|
db.commit()
|
|
result['success'] += len(batch)
|
|
|
|
f.close()
|
|
logger.info("import_pension_results_complete", **result)
|
|
|
|
except Exception as e:
|
|
db.rollback()
|
|
result['errors'].append(f"Fatal error: {str(e)}")
|
|
logger.error("import_pension_results_failed", error=str(e))
|
|
|
|
return result
|
|
|