Implement comprehensive CSV import system for legacy database migration
- Added 5 new legacy models to app/models.py (FileType, FileNots, RolexV, FVarLkup, RVarLkup) - Created app/import_legacy.py with import functions for all legacy tables: * Reference tables: TRNSTYPE, TRNSLKUP, FOOTERS, FILESTAT, EMPLOYEE, GRUPLKUP, FILETYPE, FVARLKUP, RVARLKUP * Core tables: ROLODEX, PHONE, ROLEX_V, FILES, FILES_R, FILES_V, FILENOTS, LEDGER, DEPOSITS, PAYMENTS * Specialized: PLANINFO, QDROS, PENSIONS and all pension-related tables - Created app/sync_legacy_to_modern.py with sync functions to populate modern models from legacy data - Updated admin routes in app/main.py: * Extended process_csv_import to support all new import types * Added /admin/sync endpoint for syncing legacy to modern models * Updated get_import_type_from_filename to recognize all CSV file patterns - Enhanced app/templates/admin.html with: * Import Order Guide showing recommended import sequence * Sync to Modern Models section with confirmation dialog * Sync results display with detailed per-table statistics * Updated supported file formats list - All import functions use batch processing (500 rows), proper error handling, and structured logging - Sync functions maintain foreign key integrity and skip orphaned records with warnings
This commit is contained in:
HotSwapp
@@ -49,7 +49,8 @@
|
||||
</label>
|
||||
<input type="file" class="form-control" id="files" name="files" multiple accept=".csv">
|
||||
<div class="form-text">
|
||||
Supported formats: ROLODEX*.csv, PHONE*.csv, FILES*.csv, LEDGER*.csv, QDROS*.csv, PAYMENTS*.csv
|
||||
<strong>Supported formats:</strong> ROLODEX, PHONE, FILES, LEDGER, PAYMENTS, DEPOSITS, QDROS, PENSIONS, PLANINFO,
|
||||
TRNSTYPE, TRNSLKUP, FOOTERS, FILESTAT, EMPLOYEE, GRUPLKUP, FILETYPE, and all related tables (*.csv)
|
||||
</div>
|
||||
</div>
|
||||
<button type="submit" class="btn btn-primary">
|
||||
@@ -136,6 +137,156 @@
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<!-- Import Order Guide -->
|
||||
<div class="card mb-4">
|
||||
<div class="card-header bg-info text-white">
|
||||
<h5 class="mb-0">
|
||||
<i class="bi bi-list-ol me-2"></i>Import Order Guide
|
||||
</h5>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<p class="mb-3">For best results, import tables in this recommended order:</p>
|
||||
<div class="row">
|
||||
<div class="col-md-6">
|
||||
<h6 class="text-primary"><i class="bi bi-1-circle me-2"></i>Reference Tables (Import First)</h6>
|
||||
<ul class="list-unstyled ms-3">
|
||||
<li><i class="bi bi-arrow-right me-2"></i>TRNSTYPE</li>
|
||||
<li><i class="bi bi-arrow-right me-2"></i>TRNSLKUP</li>
|
||||
<li><i class="bi bi-arrow-right me-2"></i>FOOTERS</li>
|
||||
<li><i class="bi bi-arrow-right me-2"></i>FILESTAT</li>
|
||||
<li><i class="bi bi-arrow-right me-2"></i>EMPLOYEE</li>
|
||||
<li><i class="bi bi-arrow-right me-2"></i>GRUPLKUP</li>
|
||||
<li><i class="bi bi-arrow-right me-2"></i>FILETYPE</li>
|
||||
<li><i class="bi bi-arrow-right me-2"></i>FVARLKUP, RVARLKUP</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<h6 class="text-success"><i class="bi bi-2-circle me-2"></i>Core Data Tables</h6>
|
||||
<ul class="list-unstyled ms-3">
|
||||
<li><i class="bi bi-arrow-right me-2"></i>ROLODEX</li>
|
||||
<li><i class="bi bi-arrow-right me-2"></i>PHONE, ROLEX_V</li>
|
||||
<li><i class="bi bi-arrow-right me-2"></i>FILES (+ FILES_R, FILES_V, FILENOTS)</li>
|
||||
<li><i class="bi bi-arrow-right me-2"></i>LEDGER</li>
|
||||
<li><i class="bi bi-arrow-right me-2"></i>DEPOSITS, PAYMENTS</li>
|
||||
<li><i class="bi bi-arrow-right me-2"></i>PLANINFO</li>
|
||||
<li><i class="bi bi-arrow-right me-2"></i>QDROS, PENSIONS (+ related tables)</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
<div class="alert alert-warning mt-3 mb-0">
|
||||
<i class="bi bi-exclamation-triangle me-2"></i>
|
||||
<strong>Important:</strong> Reference tables must be imported before core data to avoid foreign key errors.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Sync to Modern Models -->
|
||||
<div class="card mb-4">
|
||||
<div class="card-header bg-success text-white">
|
||||
<h5 class="mb-0">
|
||||
<i class="bi bi-arrow-repeat me-2"></i>Sync to Modern Models
|
||||
</h5>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<p>After importing legacy CSV data, sync it to the simplified modern application models (Client, Phone, Case, Transaction, Payment, Document).</p>
|
||||
<form action="/admin/sync" method="post" id="syncForm">
|
||||
<div class="mb-3">
|
||||
<div class="form-check">
|
||||
<input class="form-check-input" type="checkbox" id="clearExisting" name="clear_existing" value="true">
|
||||
<label class="form-check-label" for="clearExisting">
|
||||
<strong>Clear existing modern data before sync</strong>
|
||||
<br>
|
||||
<small class="text-muted">Warning: This will delete all current Client, Phone, Case, Transaction, Payment, and Document records!</small>
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
<button type="button" class="btn btn-success" onclick="confirmSync()">
|
||||
<i class="bi bi-arrow-repeat me-2"></i>Start Sync Process
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Sync Results -->
|
||||
{% if show_sync_results and sync_results %}
|
||||
<div class="card mb-4">
|
||||
<div class="card-header bg-success text-white">
|
||||
<h5 class="mb-0">
|
||||
<i class="bi bi-check-circle me-2"></i>Sync Results
|
||||
</h5>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<div class="row mb-3">
|
||||
<div class="col-md-3">
|
||||
<div class="card bg-light">
|
||||
<div class="card-body text-center">
|
||||
<h3 class="mb-0 text-success">{{ total_synced or 0 }}</h3>
|
||||
<small class="text-muted">Records Synced</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<div class="card bg-light">
|
||||
<div class="card-body text-center">
|
||||
<h3 class="mb-0 text-warning">{{ total_skipped or 0 }}</h3>
|
||||
<small class="text-muted">Records Skipped</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<div class="card bg-light">
|
||||
<div class="card-body text-center">
|
||||
<h3 class="mb-0 text-danger">{{ total_sync_errors or 0 }}</h3>
|
||||
<small class="text-muted">Errors</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<h6 class="mb-3">Detailed Results by Table:</h6>
|
||||
<div class="table-responsive">
|
||||
<table class="table table-sm table-bordered">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Modern Table</th>
|
||||
<th>Synced</th>
|
||||
<th>Skipped</th>
|
||||
<th>Errors</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for table_name, result in sync_results.items() %}
|
||||
<tr>
|
||||
<td><strong>{{ table_name.title() }}</strong></td>
|
||||
<td class="text-success">{{ result.success }}</td>
|
||||
<td class="text-warning">{{ result.skipped }}</td>
|
||||
<td class="text-danger">{{ result.errors|length }}</td>
|
||||
</tr>
|
||||
{% if result.errors %}
|
||||
<tr>
|
||||
<td colspan="4">
|
||||
<details>
|
||||
<summary class="text-danger">View Errors ({{ result.errors|length }})</summary>
|
||||
<ul class="mt-2 mb-0">
|
||||
{% for error in result.errors[:10] %}
|
||||
<li><small>{{ error }}</small></li>
|
||||
{% endfor %}
|
||||
{% if result.errors|length > 10 %}
|
||||
<li><small><em>... and {{ result.errors|length - 10 }} more errors</em></small></li>
|
||||
{% endif %}
|
||||
</ul>
|
||||
</details>
|
||||
</td>
|
||||
</tr>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<!-- Import Section -->
|
||||
<div class="card mb-4">
|
||||
<div class="card-header bg-warning">
|
||||
@@ -425,5 +576,20 @@ document.addEventListener('DOMContentLoaded', function() {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Sync confirmation function
|
||||
function confirmSync() {
|
||||
const clearCheckbox = document.getElementById('clearExisting');
|
||||
const clearExisting = clearCheckbox.checked;
|
||||
|
||||
let message = "Are you sure you want to sync legacy data to modern models?";
|
||||
if (clearExisting) {
|
||||
message += "\n\n⚠️ WARNING: This will DELETE all existing Client, Phone, Case, Transaction, Payment, and Document records before syncing!";
|
||||
}
|
||||
|
||||
if (confirm(message)) {
|
||||
document.getElementById('syncForm').submit();
|
||||
}
|
||||
}
|
||||
</script>
|
||||
{% endblock %}
|
||||
|
||||
Reference in New Issue
Block a user