Fix PHONE.csv import duplicate constraint error

- Implement upsert logic in import_phone() function
- Check for existing (id, phone) combinations before insert
- Track duplicates within CSV to skip gracefully
- Update existing records instead of failing on duplicates
- Add detailed statistics: inserted, updated, skipped counts
- Align with upsert pattern used in other import functions
- Add documentation in docs/PHONE_IMPORT_FIX.md

Fixes: UNIQUE constraint failed: phone.id, phone.phone error
when re-importing or uploading CSV with duplicate entries
This commit is contained in:
HotSwapp
2025-10-12 21:45:30 -05:00
parent 22e99d27ed
commit 63809d46fb
62 changed files with 500808 additions and 4269 deletions

View File

@@ -772,14 +772,16 @@ def import_rolodex(db: Session, file_path: str) -> Dict[str, Any]:
def import_phone(db: Session, file_path: str) -> Dict[str, Any]:
"""Import PHONE.csv → LegacyPhone model."""
result = {'success': 0, 'errors': [], 'total_rows': 0}
"""Import PHONE.csv → LegacyPhone model with upsert logic."""
result = {'success': 0, 'errors': [], 'total_rows': 0, 'updated': 0, 'inserted': 0, 'skipped': 0}
try:
f, encoding = open_text_with_fallbacks(file_path)
reader = csv.DictReader(f)
batch = []
# Track seen combinations in this import to handle duplicates within the CSV
seen_in_import = set()
for row_num, row in enumerate(reader, start=2):
result['total_rows'] += 1
@@ -790,26 +792,48 @@ def import_phone(db: Session, file_path: str) -> Dict[str, Any]:
if not rolodex_id or not phone:
continue
record = LegacyPhone(
id=rolodex_id,
phone=phone,
location=clean_string(row.get('Location'))
)
batch.append(record)
# Create a composite key for tracking
composite_key = (rolodex_id, phone)
if len(batch) >= BATCH_SIZE:
db.bulk_save_objects(batch)
# Skip if we've already processed this combination in current import
if composite_key in seen_in_import:
result['skipped'] += 1
continue
seen_in_import.add(composite_key)
# Check if record already exists in database
existing = db.query(LegacyPhone).filter(
LegacyPhone.id == rolodex_id,
LegacyPhone.phone == phone
).first()
if existing:
# Update existing record
existing.location = clean_string(row.get('Location'))
result['updated'] += 1
else:
# Insert new record
record = LegacyPhone(
id=rolodex_id,
phone=phone,
location=clean_string(row.get('Location'))
)
db.add(record)
result['inserted'] += 1
result['success'] += 1
# Commit in batches for performance
if result['success'] % BATCH_SIZE == 0:
db.commit()
result['success'] += len(batch)
batch = []
except Exception as e:
result['errors'].append(f"Row {row_num}: {str(e)}")
db.rollback()
if batch:
db.bulk_save_objects(batch)
db.commit()
result['success'] += len(batch)
# Commit any remaining changes
db.commit()
f.close()
logger.info("import_phone_complete", **result)