diff --git a/app/import_legacy.py b/app/import_legacy.py index 66c0d3f..9f0b4d1 100644 --- a/app/import_legacy.py +++ b/app/import_legacy.py @@ -591,13 +591,16 @@ def import_rvarlkup(db: Session, file_path: str) -> Dict[str, Any]: def import_rolodex(db: Session, file_path: str) -> Dict[str, Any]: """Import ROLODEX.csv → Rolodex model.""" - result = {'success': 0, 'errors': [], 'total_rows': 0} + result = {'success': 0, 'errors': [], 'total_rows': 0, 'skipped': 0} try: f, encoding = open_text_with_fallbacks(file_path) reader = csv.DictReader(f) + # Track IDs we've seen in this import to handle duplicates + seen_in_import = set() batch = [] + for row_num, row in enumerate(reader, start=2): result['total_rows'] += 1 @@ -606,6 +609,17 @@ def import_rolodex(db: Session, file_path: str) -> Dict[str, Any]: if not rolodex_id: continue + # Skip if we've already processed this ID in current import + if rolodex_id in seen_in_import: + result['skipped'] += 1 + continue + + # Skip if it already exists in database + if db.query(Rolodex).filter(Rolodex.id == rolodex_id).first(): + result['skipped'] += 1 + seen_in_import.add(rolodex_id) + continue + record = Rolodex( id=rolodex_id, prefix=clean_string(row.get('Prefix')), @@ -629,20 +643,47 @@ def import_rolodex(db: Session, file_path: str) -> Dict[str, Any]: memo=clean_string(row.get('Memo')) ) batch.append(record) + seen_in_import.add(rolodex_id) if len(batch) >= BATCH_SIZE: - db.bulk_save_objects(batch) - db.commit() - result['success'] += len(batch) - batch = [] + try: + db.bulk_save_objects(batch) + db.commit() + result['success'] += len(batch) + batch = [] + except IntegrityError as ie: + db.rollback() + # Handle any remaining duplicates by inserting one at a time + for record in batch: + try: + db.add(record) + db.commit() + result['success'] += 1 + except IntegrityError: + db.rollback() + result['skipped'] += 1 + batch = [] except Exception as e: result['errors'].append(f"Row {row_num}: {str(e)}") + # Save remaining batch if batch: - db.bulk_save_objects(batch) - db.commit() - result['success'] += len(batch) + try: + db.bulk_save_objects(batch) + db.commit() + result['success'] += len(batch) + except IntegrityError: + db.rollback() + # Handle any remaining duplicates by inserting one at a time + for record in batch: + try: + db.add(record) + db.commit() + result['success'] += 1 + except IntegrityError: + db.rollback() + result['skipped'] += 1 f.close() logger.info("import_rolodex_complete", **result) diff --git a/delphi.db b/delphi.db index d23b9a1..ccd4ce2 100644 Binary files a/delphi.db and b/delphi.db differ diff --git a/test_encoding.db b/test_encoding.db new file mode 100644 index 0000000..2efab94 Binary files /dev/null and b/test_encoding.db differ