Files
delphi-database-v2/app/templates/admin.html
HotSwapp 4030dbd88e Implement comprehensive CSV import system for legacy database migration
- Added 5 new legacy models to app/models.py (FileType, FileNots, RolexV, FVarLkup, RVarLkup)
- Created app/import_legacy.py with import functions for all legacy tables:
  * Reference tables: TRNSTYPE, TRNSLKUP, FOOTERS, FILESTAT, EMPLOYEE, GRUPLKUP, FILETYPE, FVARLKUP, RVARLKUP
  * Core tables: ROLODEX, PHONE, ROLEX_V, FILES, FILES_R, FILES_V, FILENOTS, LEDGER, DEPOSITS, PAYMENTS
  * Specialized: PLANINFO, QDROS, PENSIONS and all pension-related tables
- Created app/sync_legacy_to_modern.py with sync functions to populate modern models from legacy data
- Updated admin routes in app/main.py:
  * Extended process_csv_import to support all new import types
  * Added /admin/sync endpoint for syncing legacy to modern models
  * Updated get_import_type_from_filename to recognize all CSV file patterns
- Enhanced app/templates/admin.html with:
  * Import Order Guide showing recommended import sequence
  * Sync to Modern Models section with confirmation dialog
  * Sync results display with detailed per-table statistics
  * Updated supported file formats list
- All import functions use batch processing (500 rows), proper error handling, and structured logging
- Sync functions maintain foreign key integrity and skip orphaned records with warnings
2025-10-08 09:41:38 -05:00

596 lines
30 KiB
HTML
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
{% extends "base.html" %}
{% block title %}Admin Panel - Delphi Database{% endblock %}
{% block content %}
<div class="container-fluid">
<div class="row">
<div class="col-12">
<h1 class="mb-4">
<i class="bi bi-gear me-2"></i>Admin Panel
</h1>
<!-- Alert Messages -->
{% if error %}
<div class="alert alert-danger alert-dismissible fade show" role="alert">
<i class="bi bi-exclamation-triangle me-2"></i>{{ error }}
<button type="button" class="btn-close" data-bs-dismiss="alert"></button>
</div>
{% endif %}
{% if show_upload_results %}
<div class="alert alert-info alert-dismissible fade show" role="alert">
<i class="bi bi-info-circle me-2"></i>
Files uploaded successfully. Review the results below and select files to import.
<button type="button" class="btn-close" data-bs-dismiss="alert"></button>
</div>
{% endif %}
{% if show_import_results %}
<div class="alert alert-success alert-dismissible fade show" role="alert">
<i class="bi bi-check-circle me-2"></i>
Import completed. Check the results below for details.
<button type="button" class="btn-close" data-bs-dismiss="alert"></button>
</div>
{% endif %}
<!-- Upload Section -->
<div class="card mb-4">
<div class="card-header bg-primary text-white">
<h5 class="mb-0">
<i class="bi bi-upload me-2"></i>File Upload
</h5>
</div>
<div class="card-body">
<form action="/admin/upload" method="post" enctype="multipart/form-data">
<div class="mb-3">
<label for="files" class="form-label">
<i class="bi bi-file-earmark-spreadsheet me-2"></i>Select CSV Files
</label>
<input type="file" class="form-control" id="files" name="files" multiple accept=".csv">
<div class="form-text">
<strong>Supported formats:</strong> ROLODEX, PHONE, FILES, LEDGER, PAYMENTS, DEPOSITS, QDROS, PENSIONS, PLANINFO,
TRNSTYPE, TRNSLKUP, FOOTERS, FILESTAT, EMPLOYEE, GRUPLKUP, FILETYPE, and all related tables (*.csv)
</div>
</div>
<button type="submit" class="btn btn-primary">
<i class="bi bi-cloud-upload me-2"></i>Upload Files
</button>
</form>
</div>
</div>
<!-- Upload Results -->
{% if upload_results %}
<div class="card mb-4">
<div class="card-header bg-success text-white">
<h5 class="mb-0">
<i class="bi bi-check-circle me-2"></i>Upload Results
</h5>
</div>
<div class="card-body">
<div class="row">
<div class="col-md-8">
<div class="table-responsive">
<table class="table table-sm">
<thead>
<tr>
<th>Original Filename</th>
<th>Stored Filename</th>
<th>Import Type</th>
<th>Size</th>
<th>Status</th>
</tr>
</thead>
<tbody>
{% for result in upload_results %}
<tr>
<td>
<strong>{{ result.filename }}</strong>
<br>
<small class="text-muted">Original name</small>
</td>
<td>
<code class="small">{{ result.stored_filename }}</code>
<br>
<small class="text-muted">Stored as</small>
</td>
<td>
<span class="badge bg-primary">{{ result.import_type }}</span>
</td>
<td>{{ result.size }} bytes</td>
<td><i class="bi bi-check-circle text-success"></i> Uploaded</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
<div class="col-md-4">
<div class="alert alert-success">
<h6><i class="bi bi-info-circle me-2"></i>Ready for Import</h6>
<p class="mb-0">Files have been uploaded and validated. Use the import section below to process the data.</p>
</div>
</div>
</div>
</div>
</div>
{% endif %}
<!-- Upload Errors -->
{% if upload_errors %}
<div class="card mb-4">
<div class="card-header bg-danger text-white">
<h5 class="mb-0">
<i class="bi bi-exclamation-triangle me-2"></i>Upload Errors
</h5>
</div>
<div class="card-body">
<ul class="list-group list-group-flush">
{% for error in upload_errors %}
<li class="list-group-item text-danger">
<i class="bi bi-x-circle me-2"></i>{{ error }}
</li>
{% endfor %}
</ul>
</div>
</div>
{% endif %}
<!-- Import Order Guide -->
<div class="card mb-4">
<div class="card-header bg-info text-white">
<h5 class="mb-0">
<i class="bi bi-list-ol me-2"></i>Import Order Guide
</h5>
</div>
<div class="card-body">
<p class="mb-3">For best results, import tables in this recommended order:</p>
<div class="row">
<div class="col-md-6">
<h6 class="text-primary"><i class="bi bi-1-circle me-2"></i>Reference Tables (Import First)</h6>
<ul class="list-unstyled ms-3">
<li><i class="bi bi-arrow-right me-2"></i>TRNSTYPE</li>
<li><i class="bi bi-arrow-right me-2"></i>TRNSLKUP</li>
<li><i class="bi bi-arrow-right me-2"></i>FOOTERS</li>
<li><i class="bi bi-arrow-right me-2"></i>FILESTAT</li>
<li><i class="bi bi-arrow-right me-2"></i>EMPLOYEE</li>
<li><i class="bi bi-arrow-right me-2"></i>GRUPLKUP</li>
<li><i class="bi bi-arrow-right me-2"></i>FILETYPE</li>
<li><i class="bi bi-arrow-right me-2"></i>FVARLKUP, RVARLKUP</li>
</ul>
</div>
<div class="col-md-6">
<h6 class="text-success"><i class="bi bi-2-circle me-2"></i>Core Data Tables</h6>
<ul class="list-unstyled ms-3">
<li><i class="bi bi-arrow-right me-2"></i>ROLODEX</li>
<li><i class="bi bi-arrow-right me-2"></i>PHONE, ROLEX_V</li>
<li><i class="bi bi-arrow-right me-2"></i>FILES (+ FILES_R, FILES_V, FILENOTS)</li>
<li><i class="bi bi-arrow-right me-2"></i>LEDGER</li>
<li><i class="bi bi-arrow-right me-2"></i>DEPOSITS, PAYMENTS</li>
<li><i class="bi bi-arrow-right me-2"></i>PLANINFO</li>
<li><i class="bi bi-arrow-right me-2"></i>QDROS, PENSIONS (+ related tables)</li>
</ul>
</div>
</div>
<div class="alert alert-warning mt-3 mb-0">
<i class="bi bi-exclamation-triangle me-2"></i>
<strong>Important:</strong> Reference tables must be imported before core data to avoid foreign key errors.
</div>
</div>
</div>
<!-- Sync to Modern Models -->
<div class="card mb-4">
<div class="card-header bg-success text-white">
<h5 class="mb-0">
<i class="bi bi-arrow-repeat me-2"></i>Sync to Modern Models
</h5>
</div>
<div class="card-body">
<p>After importing legacy CSV data, sync it to the simplified modern application models (Client, Phone, Case, Transaction, Payment, Document).</p>
<form action="/admin/sync" method="post" id="syncForm">
<div class="mb-3">
<div class="form-check">
<input class="form-check-input" type="checkbox" id="clearExisting" name="clear_existing" value="true">
<label class="form-check-label" for="clearExisting">
<strong>Clear existing modern data before sync</strong>
<br>
<small class="text-muted">Warning: This will delete all current Client, Phone, Case, Transaction, Payment, and Document records!</small>
</label>
</div>
</div>
<button type="button" class="btn btn-success" onclick="confirmSync()">
<i class="bi bi-arrow-repeat me-2"></i>Start Sync Process
</button>
</form>
</div>
</div>
<!-- Sync Results -->
{% if show_sync_results and sync_results %}
<div class="card mb-4">
<div class="card-header bg-success text-white">
<h5 class="mb-0">
<i class="bi bi-check-circle me-2"></i>Sync Results
</h5>
</div>
<div class="card-body">
<div class="row mb-3">
<div class="col-md-3">
<div class="card bg-light">
<div class="card-body text-center">
<h3 class="mb-0 text-success">{{ total_synced or 0 }}</h3>
<small class="text-muted">Records Synced</small>
</div>
</div>
</div>
<div class="col-md-3">
<div class="card bg-light">
<div class="card-body text-center">
<h3 class="mb-0 text-warning">{{ total_skipped or 0 }}</h3>
<small class="text-muted">Records Skipped</small>
</div>
</div>
</div>
<div class="col-md-3">
<div class="card bg-light">
<div class="card-body text-center">
<h3 class="mb-0 text-danger">{{ total_sync_errors or 0 }}</h3>
<small class="text-muted">Errors</small>
</div>
</div>
</div>
</div>
<h6 class="mb-3">Detailed Results by Table:</h6>
<div class="table-responsive">
<table class="table table-sm table-bordered">
<thead>
<tr>
<th>Modern Table</th>
<th>Synced</th>
<th>Skipped</th>
<th>Errors</th>
</tr>
</thead>
<tbody>
{% for table_name, result in sync_results.items() %}
<tr>
<td><strong>{{ table_name.title() }}</strong></td>
<td class="text-success">{{ result.success }}</td>
<td class="text-warning">{{ result.skipped }}</td>
<td class="text-danger">{{ result.errors|length }}</td>
</tr>
{% if result.errors %}
<tr>
<td colspan="4">
<details>
<summary class="text-danger">View Errors ({{ result.errors|length }})</summary>
<ul class="mt-2 mb-0">
{% for error in result.errors[:10] %}
<li><small>{{ error }}</small></li>
{% endfor %}
{% if result.errors|length > 10 %}
<li><small><em>... and {{ result.errors|length - 10 }} more errors</em></small></li>
{% endif %}
</ul>
</details>
</td>
</tr>
{% endif %}
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
{% endif %}
<!-- Import Section -->
<div class="card mb-4">
<div class="card-header bg-warning">
<h5 class="mb-0">
<i class="bi bi-arrow-down-circle me-2"></i>Data Import
</h5>
</div>
<div class="card-body">
{% if files_by_type %}
<div class="row">
{% for import_type, files in files_by_type.items() %}
<div class="col-md-6 mb-4">
<div class="card h-100">
<div class="card-header">
<h6 class="mb-0">
<i class="bi bi-database me-2"></i>{{ import_type.title() }} Data
<span class="badge bg-secondary ms-2">{{ files|length }}</span>
</h6>
</div>
<div class="card-body">
<form action="/admin/import/{{ import_type }}" method="post">
<div class="mb-3">
<div class="d-flex justify-content-between align-items-center mb-2">
<label class="form-label mb-0">Available Files:</label>
<button type="button" class="btn btn-outline-primary btn-sm select-all-btn"
data-import-type="{{ import_type }}">
<i class="bi bi-check-all me-1"></i>Select All
</button>
</div>
<div class="list-group">
{% for file in files %}
<label class="list-group-item d-flex justify-content-between align-items-center">
<div>
<input class="form-check-input me-2 file-checkbox" type="checkbox"
name="selected_files" value="{{ file.filename }}" id="{{ file.filename }}">
<small class="text-muted">{{ file.filename }}</small>
<br>
<small class="text-muted">{{ file.size }} bytes • {{ file.modified.strftime('%Y-%m-%d %H:%M') }}</small>
</div>
</label>
{% endfor %}
</div>
</div>
<button type="submit" class="btn btn-success btn-sm">
<i class="bi bi-download me-2"></i>Import {{ import_type.title() }} Data
</button>
</form>
</div>
</div>
</div>
{% endfor %}
</div>
{% else %}
<div class="alert alert-info">
<i class="bi bi-info-circle me-2"></i>No CSV files available for import. Upload files first.
</div>
{% endif %}
</div>
</div>
<!-- Import Results -->
{% if import_results %}
<div class="card mb-4">
<div class="card-header bg-info text-white">
<h5 class="mb-0">
<i class="bi bi-graph-up me-2"></i>Import Results
</h5>
</div>
<div class="card-body">
<div class="row mb-3">
<div class="col-md-3">
<div class="card bg-success text-white">
<div class="card-body text-center">
<h3 class="mb-0">{{ total_success }}</h3>
<small>Successful</small>
</div>
</div>
</div>
<div class="col-md-3">
<div class="card bg-danger text-white">
<div class="card-body text-center">
<h3 class="mb-0">{{ total_errors }}</h3>
<small>Errors</small>
</div>
</div>
</div>
<div class="col-md-3">
<div class="card bg-primary text-white">
<div class="card-body text-center">
<h3 class="mb-0">{{ import_results|length }}</h3>
<small>Files</small>
</div>
</div>
</div>
<div class="col-md-3">
<div class="card bg-warning text-white">
<div class="card-body text-center">
<h3 class="mb-0">{{ total_success + total_errors }}</h3>
<small>Total Records</small>
</div>
</div>
</div>
</div>
<div class="table-responsive">
<table class="table table-striped">
<thead>
<tr>
<th>Filename</th>
<th>Status</th>
<th>Total Rows</th>
<th>Success</th>
<th>Errors</th>
<th>Details</th>
</tr>
</thead>
<tbody>
{% for result in import_results %}
<tr>
<td>{{ result.filename }}</td>
<td>
{% if result.status == 'success' %}
<span class="badge bg-success">Success</span>
{% else %}
<span class="badge bg-danger">Error</span>
{% endif %}
</td>
<td>{{ result.total_rows }}</td>
<td class="text-success">{{ result.success_count }}</td>
<td class="text-danger">{{ result.error_count }}</td>
<td>
{% if result.errors %}
<button class="btn btn-sm btn-outline-danger" type="button"
data-bs-toggle="collapse" data-bs-target="#errors-{{ loop.index }}">
View Errors ({{ result.errors|length }})
</button>
<div class="collapse mt-2" id="errors-{{ loop.index }}">
<div class="card card-body">
<ul class="list-unstyled mb-0">
{% for error in result.errors %}
<li class="text-danger small">
<i class="bi bi-x-circle me-1"></i>{{ error }}
</li>
{% endfor %}
</ul>
</div>
</div>
{% else %}
<span class="text-muted">No errors</span>
{% endif %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
{% endif %}
<!-- Recent Import History -->
{% if recent_imports %}
<div class="card">
<div class="card-header bg-secondary text-white">
<h5 class="mb-0">
<i class="bi bi-clock-history me-2"></i>Recent Import History
</h5>
</div>
<div class="card-body">
<div class="table-responsive">
<table class="table table-hover">
<thead>
<tr>
<th>Date/Time</th>
<th>Type</th>
<th>File</th>
<th>Status</th>
<th>Records</th>
<th>Success</th>
<th>Errors</th>
</tr>
</thead>
<tbody>
{% for import_log in recent_imports %}
<tr>
<td>{{ import_log.created_at.strftime('%Y-%m-%d %H:%M') }}</td>
<td>
<span class="badge bg-primary">{{ import_log.import_type }}</span>
</td>
<td>{{ import_log.file_name }}</td>
<td>
{% if import_log.status == 'completed' %}
<span class="badge bg-success">Completed</span>
{% elif import_log.status == 'failed' %}
<span class="badge bg-danger">Failed</span>
{% elif import_log.status == 'running' %}
<span class="badge bg-warning">Running</span>
{% else %}
<span class="badge bg-secondary">{{ import_log.status }}</span>
{% endif %}
</td>
<td>{{ import_log.total_rows }}</td>
<td class="text-success">{{ import_log.success_count }}</td>
<td class="text-danger">{{ import_log.error_count }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
{% endif %}
</div>
</div>
</div>
<script>
document.addEventListener('DOMContentLoaded', function() {
// Auto-refresh import status for running imports
function refreshRunningImports() {
const runningImports = document.querySelectorAll('span.badge.bg-warning');
if (runningImports.length > 0) {
// In a real application, you might implement WebSocket or polling here
setTimeout(refreshRunningImports, 5000); // Check every 5 seconds
}
}
// Start refresh cycle if there are running imports
refreshRunningImports();
// Select All functionality
document.querySelectorAll('.select-all-btn').forEach(button => {
button.addEventListener('click', function() {
const importType = this.getAttribute('data-import-type');
const form = this.closest('form');
const checkboxes = form.querySelectorAll('.file-checkbox');
const submitBtn = form.querySelector('button[type="submit"]');
// Toggle all checkboxes in this form
const allChecked = Array.from(checkboxes).every(cb => cb.checked);
checkboxes.forEach(checkbox => {
checkbox.checked = !allChecked;
});
// Update button text
this.innerHTML = allChecked ?
'<i class="bi bi-check-all me-1"></i>Select All' :
'<i class="bi bi-dash-square me-1"></i>Deselect All';
// Update submit button state
const hasSelection = Array.from(checkboxes).some(cb => cb.checked);
submitBtn.disabled = !hasSelection;
});
});
// File selection helpers
document.querySelectorAll('.file-checkbox').forEach(checkbox => {
checkbox.addEventListener('change', function() {
const form = this.closest('form');
const checkboxes = form.querySelectorAll('.file-checkbox');
const submitBtn = form.querySelector('button[type="submit"]');
const selectAllBtn = form.querySelector('.select-all-btn');
// Enable/disable submit button based on selection
const hasSelection = Array.from(checkboxes).some(cb => cb.checked);
submitBtn.disabled = !hasSelection;
// Update select all button state
const allChecked = Array.from(checkboxes).every(cb => cb.checked);
const noneChecked = Array.from(checkboxes).every(cb => !cb.checked);
if (allChecked) {
selectAllBtn.innerHTML = '<i class="bi bi-dash-square me-1"></i>Deselect All';
} else if (noneChecked) {
selectAllBtn.innerHTML = '<i class="bi bi-check-all me-1"></i>Select All';
} else {
selectAllBtn.innerHTML = '<i class="bi bi-check-square me-1"></i>Select All';
}
});
});
// Initialize submit buttons as disabled
document.querySelectorAll('form[action*="/admin/import/"]').forEach(form => {
const submitBtn = form.querySelector('button[type="submit"]');
if (submitBtn) {
submitBtn.disabled = true;
}
});
});
// Sync confirmation function
function confirmSync() {
const clearCheckbox = document.getElementById('clearExisting');
const clearExisting = clearCheckbox.checked;
let message = "Are you sure you want to sync legacy data to modern models?";
if (clearExisting) {
message += "\n\n⚠ WARNING: This will DELETE all existing Client, Phone, Case, Transaction, Payment, and Document records before syncing!";
}
if (confirm(message)) {
document.getElementById('syncForm').submit();
}
}
</script>
{% endblock %}