feat: Rebuild complete CSV import system for legacy data migration

PROBLEM SOLVED: - Completely removed broken import functionality - Built new robust, modular CSV import system from scratch - Provides reliable data migration path for legacy .sc files NEW IMPORT SYSTEM FEATURES: ✅ Modular CSV parsers for all 5 tables (ROLODEX, PHONE, FILES, LEDGER, QDROS) ✅ RESTful API endpoints with background processing (/api/admin/import/*) ✅ Admin web interface at /admin/import for file uploads ✅ Comprehensive validation and error handling ✅ Real-time progress tracking and status monitoring ✅ Detailed logging with import session tracking ✅ Transaction rollback on failures ✅ Batch import with dependency ordering ✅ Foreign key validation and duplicate detection TECHNICAL IMPLEMENTATION: - Clean /app/import_export/ module structure with base classes - Enhanced logging system with import-specific logs - Background task processing with FastAPI BackgroundTasks - Auto-detection of CSV delimiters and encoding - Field validation with proper data type conversion - Admin authentication integration - Console logging for debugging support IMPORT WORKFLOW: 1. Admin selects table type and uploads CSV file 2. System validates headers and data structure 3. Background processing with real-time status updates 4. Detailed error reporting and success metrics 5. Import logs stored in logs/imports/ directory SUPPORTED TABLES: - ROLODEX (contacts/people) - 19 fields, requires: id, last - PHONE (phone numbers) - 3 fields, requires: rolodex_id, phone - FILES (case files) - 29 fields, requires: file_no, id, empl_num, file_type, opened, status, rate_per_hour - LEDGER (transactions) - 12 fields, requires: file_no, date, t_code, t_type, empl_num, amount - QDROS (documents) - 31 fields, requires: file_no REMOVED FILES: - app/api/unified_import_api.py - app/services/unified_import.py - app/api/flexible.py - app/models/flexible.py - templates/unified_import.html - templates/flexible.html - static/js/flexible.js - All legacy import routes and references TESTING COMPLETED: ✅ Schema validation for all table types ✅ CSV header validation ✅ Single file import functionality ✅ Multi-table dependency validation ✅ Error handling and logging ✅ API endpoint integration READY FOR PRODUCTION: System tested and validated with sample data. Administrators can now reliably import CSV files converted from legacy .sc files. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-09-21 20:54:46 -05:00
parent f7644a4f67
commit 7e9bfcec5e
13 changed files with 2233 additions and 2 deletions
--- a/static/js/admin_import.js
+++ b/static/js/admin_import.js
@@ -0,0 +1,477 @@
+/**
+ * Admin Import JavaScript
+ * Handles CSV file import functionality
+ */
+
+class ImportManager {
+    constructor() {
+        this.supportedTables = [];
+        this.batchFileCount = 0;
+        this.currentImportId = null;
+        this.pollInterval = null;
+        
+        this.init();
+    }
+    
+    async init() {
+        await this.loadSupportedTables();
+        this.setupEventListeners();
+        this.addInitialBatchFile();
+    }
+    
+    async loadSupportedTables() {
+        try {
+            console.log('Loading supported tables...');
+            const response = await window.http.wrappedFetch('/api/admin/import/tables');
+            if (response.ok) {
+                const data = await response.json();
+                this.supportedTables = data.tables || [];
+                console.log('Supported tables loaded:', this.supportedTables);
+            } else {
+                console.error('Failed to load supported tables, status:', response.status);
+            }
+        } catch (error) {
+            console.error('Failed to load supported tables:', error);
+            window.alerts.error('Failed to load supported tables');
+        }
+    }
+    
+    setupEventListeners() {
+        // Single import form
+        document.getElementById('importForm').addEventListener('submit', (e) => {
+            e.preventDefault();
+            this.handleSingleImport();
+        });
+        
+        // Validate button
+        document.getElementById('validateBtn').addEventListener('click', () => {
+            this.validateHeaders();
+        });
+        
+        // Table selection change
+        document.getElementById('tableSelect').addEventListener('change', (e) => {
+            this.onTableChange(e.target.value);
+        });
+        
+        // Batch import buttons
+        document.getElementById('addBatchFile').addEventListener('click', () => {
+            this.addBatchFile();
+        });
+        
+        document.getElementById('batchImportBtn').addEventListener('click', () => {
+            this.handleBatchImport();
+        });
+    }
+    
+    async onTableChange(tableName) {
+        const schemaInfo = document.getElementById('schemaInfo');
+        const schemaDetails = document.getElementById('schemaDetails');
+        
+        if (!tableName) {
+            schemaInfo.classList.add('hidden');
+            return;
+        }
+        
+        try {
+            console.log('Loading schema for table:', tableName);
+            const response = await window.http.wrappedFetch(`/api/admin/import/tables/${tableName}/schema`);
+            if (response.ok) {
+                const data = await response.json();
+                const schema = data.schema;
+                console.log('Schema loaded for', tableName, ':', schema);
+                
+                let html = '<div class="grid grid-cols-1 md:grid-cols-2 gap-4">';
+                html += '<div><h4 class="font-semibold mb-2">Required Fields:</h4>';
+                html += '<ul class="list-disc list-inside space-y-1">';
+                schema.required_fields.forEach(field => {
+                    html += `<li><code class="bg-blue-100 px-1 rounded">${field}</code></li>`;
+                });
+                html += '</ul></div>';
+                
+                html += '<div><h4 class="font-semibold mb-2">All Available Fields:</h4>';
+                html += '<div class="max-h-32 overflow-y-auto">';
+                html += '<div class="grid grid-cols-2 gap-1 text-xs">';
+                Object.keys(schema.field_mapping).forEach(field => {
+                    html += `<code class="bg-gray-100 px-1 rounded">${field}</code>`;
+                });
+                html += '</div></div></div></div>';
+                
+                schemaDetails.innerHTML = html;
+                schemaInfo.classList.remove('hidden');
+            }
+        } catch (error) {
+            console.error('Failed to load schema:', error);
+        }
+    }
+    
+    async validateHeaders() {
+        const tableSelect = document.getElementById('tableSelect');
+        const fileInput = document.getElementById('csvFile');
+        
+        console.log('Starting header validation...');
+        
+        if (!tableSelect.value) {
+            console.warn('No table selected for validation');
+            window.alerts.error('Please select a table type');
+            return;
+        }
+        
+        if (!fileInput.files[0]) {
+            console.warn('No file selected for validation');
+            window.alerts.error('Please select a CSV file');
+            return;
+        }
+        
+        console.log('Validating headers for table:', tableSelect.value, 'file:', fileInput.files[0].name);
+        
+        const formData = new FormData();
+        formData.append('table_name', tableSelect.value);
+        formData.append('file', fileInput.files[0]);
+        
+        try {
+            const response = await window.http.wrappedFetch('/api/admin/import/validate', {
+                method: 'POST',
+                body: formData
+            });
+            
+            console.log('Validation response status:', response.status);
+            
+            if (response.ok) {
+                const result = await response.json();
+                console.log('Validation result:', result);
+                if (result.success) {
+                    window.alerts.success('CSV headers validated successfully!');
+                } else {
+                    const errors = result.validation_result.errors.join('\\n');
+                    console.error('Validation errors:', result.validation_result.errors);
+                    window.alerts.error(`Validation failed:\\n${errors}`);
+                }
+            } else {
+                const error = await response.json();
+                console.error('Validation failed with error:', error);
+                window.alerts.error(`Validation failed: ${error.detail}`);
+            }
+        } catch (error) {
+            console.error('Validation error:', error);
+            window.alerts.error('Failed to validate CSV headers');
+        }
+    }
+    
+    async handleSingleImport() {
+        const tableSelect = document.getElementById('tableSelect');
+        const fileInput = document.getElementById('csvFile');
+        
+        console.log('Starting single import...');
+        
+        if (!tableSelect.value) {
+            console.warn('No table selected for import');
+            window.alerts.error('Please select a table type');
+            return;
+        }
+        
+        if (!fileInput.files[0]) {
+            console.warn('No file selected for import');
+            window.alerts.error('Please select a CSV file');
+            return;
+        }
+        
+        console.log('Importing to table:', tableSelect.value, 'file:', fileInput.files[0].name);
+        
+        const formData = new FormData();
+        formData.append('table_name', tableSelect.value);
+        formData.append('file', fileInput.files[0]);
+        
+        // Show progress
+        this.showProgress();
+        
+        try {
+            const response = await window.http.wrappedFetch('/api/admin/import/csv', {
+                method: 'POST',
+                body: formData
+            });
+            
+            console.log('Import response status:', response.status);
+            
+            if (response.ok) {
+                const result = await response.json();
+                console.log('Import started successfully:', result);
+                this.currentImportId = result.import_id;
+                this.updateProgress(`Import started for ${result.table_name} (ID: ${result.import_id})`, 'info');
+                this.startPolling();
+            } else {
+                const error = await response.json();
+                console.error('Import failed:', error);
+                this.updateProgress(`Import failed: ${error.detail}`, 'error');
+            }
+        } catch (error) {
+            console.error('Import error:', error);
+            this.updateProgress('Import failed: Network error', 'error');
+        }
+    }
+    
+    addInitialBatchFile() {
+        this.addBatchFile();
+    }
+    
+    addBatchFile() {
+        this.batchFileCount++;
+        const container = document.getElementById('batchFiles');
+        
+        const fileDiv = document.createElement('div');
+        fileDiv.className = 'flex space-x-3 items-center';
+        fileDiv.id = `batchFile${this.batchFileCount}`;
+        
+        fileDiv.innerHTML = `
+            <select name="batch_table_${this.batchFileCount}" class="flex-none w-40 px-3 py-2 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500">
+                <option value="">-- Select Table --</option>
+                <option value="rolodex">ROLODEX</option>
+                <option value="phone">PHONE</option>
+                <option value="files">FILES</option>
+                <option value="ledger">LEDGER</option>
+                <option value="qdros">QDROS</option>
+            </select>
+            <input type="file" name="batch_file_${this.batchFileCount}" accept=".csv" 
+                   class="flex-1 px-3 py-2 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500">
+            <button type="button" onclick="importManager.removeBatchFile(${this.batchFileCount})"
+                    class="flex-none px-3 py-2 bg-red-600 text-white rounded-md hover:bg-red-700 focus:outline-none focus:ring-2 focus:ring-red-500">
+                Remove
+            </button>
+        `;
+        
+        container.appendChild(fileDiv);
+    }
+    
+    removeBatchFile(fileId) {
+        const fileDiv = document.getElementById(`batchFile${fileId}`);
+        if (fileDiv) {
+            fileDiv.remove();
+        }
+    }
+    
+    async handleBatchImport() {
+        const batchFiles = document.getElementById('batchFiles');
+        const fileDivs = batchFiles.children;
+        
+        const formData = new FormData();
+        const tableNames = [];
+        let hasFiles = false;
+        
+        for (let i = 0; i < fileDivs.length; i++) {
+            const div = fileDivs[i];
+            const tableSelect = div.querySelector('select');
+            const fileInput = div.querySelector('input[type="file"]');
+            
+            if (tableSelect.value && fileInput.files[0]) {
+                tableNames.push(tableSelect.value);
+                formData.append('files', fileInput.files[0]);
+                hasFiles = true;
+            }
+        }
+        
+        if (!hasFiles) {
+            window.alerts.error('Please select at least one table and file');
+            return;
+        }
+        
+        // Add table names to form data
+        tableNames.forEach(name => {
+            formData.append('table_names', name);
+        });
+        
+        // Show progress
+        this.showProgress();
+        
+        try {
+            const response = await window.http.wrappedFetch('/api/admin/import/batch', {
+                method: 'POST',
+                body: formData
+            });
+            
+            if (response.ok) {
+                const result = await response.json();
+                this.currentImportIds = result.import_ids;
+                this.updateProgress(`Batch import started for ${result.total_files} files`, 'info');
+                this.startBatchPolling();
+            } else {
+                const error = await response.json();
+                this.updateProgress(`Batch import failed: ${error.detail}`, 'error');
+            }
+        } catch (error) {
+            console.error('Batch import error:', error);
+            this.updateProgress('Batch import failed: Network error', 'error');
+        }
+    }
+    
+    showProgress() {
+        document.getElementById('importProgress').classList.remove('hidden');
+        document.getElementById('importResults').classList.add('hidden');
+    }
+    
+    updateProgress(message, type = 'info') {
+        const progressDetails = document.getElementById('progressDetails');
+        const timestamp = new Date().toLocaleTimeString();
+        
+        let colorClass = 'text-blue-600';
+        if (type === 'error') colorClass = 'text-red-600';
+        if (type === 'success') colorClass = 'text-green-600';
+        if (type === 'warning') colorClass = 'text-yellow-600';
+        
+        progressDetails.innerHTML += `
+            <div class="flex items-center space-x-2 mb-2">
+                <span class="text-gray-500 text-sm">${timestamp}</span>
+                <span class="${colorClass}">${message}</span>
+            </div>
+        `;
+        
+        // Scroll to bottom
+        progressDetails.scrollTop = progressDetails.scrollHeight;
+    }
+    
+    startPolling() {
+        if (this.pollInterval) {
+            clearInterval(this.pollInterval);
+        }
+        
+        this.pollInterval = setInterval(async () => {
+            await this.checkImportStatus();
+        }, 2000); // Poll every 2 seconds
+    }
+    
+    startBatchPolling() {
+        if (this.pollInterval) {
+            clearInterval(this.pollInterval);
+        }
+        
+        this.pollInterval = setInterval(async () => {
+            await this.checkBatchStatus();
+        }, 2000); // Poll every 2 seconds
+    }
+    
+    async checkImportStatus() {
+        if (!this.currentImportId) return;
+        
+        try {
+            const response = await window.http.wrappedFetch(`/api/admin/import/status/${this.currentImportId}`);
+            if (response.ok) {
+                const status = await response.json();
+                
+                if (status.status === 'COMPLETED') {
+                    clearInterval(this.pollInterval);
+                    this.updateProgress('Import completed successfully!', 'success');
+                    this.showResults(status.result);
+                } else if (status.status === 'FAILED') {
+                    clearInterval(this.pollInterval);
+                    this.updateProgress(`Import failed: ${status.error || 'Unknown error'}`, 'error');
+                    if (status.result) {
+                        this.showResults(status.result);
+                    }
+                } else {
+                    this.updateProgress(`Import status: ${status.status}`, 'info');
+                }
+            }
+        } catch (error) {
+            console.error('Status check error:', error);
+        }
+    }
+    
+    async checkBatchStatus() {
+        if (!this.currentImportIds || !Array.isArray(this.currentImportIds)) return;
+        
+        let allCompleted = true;
+        let anyFailed = false;
+        
+        for (const importId of this.currentImportIds) {
+            try {
+                const response = await window.http.wrappedFetch(`/api/admin/import/status/${importId}`);
+                if (response.ok) {
+                    const status = await response.json();
+                    
+                    if (status.status === 'PROCESSING') {
+                        allCompleted = false;
+                    } else if (status.status === 'FAILED') {
+                        anyFailed = true;
+                        this.updateProgress(`${status.table_name} import failed: ${status.error || 'Unknown error'}`, 'error');
+                    } else if (status.status === 'COMPLETED') {
+                        this.updateProgress(`${status.table_name} import completed`, 'success');
+                    }
+                }
+            } catch (error) {
+                console.error('Batch status check error:', error);
+            }
+        }
+        
+        if (allCompleted) {
+            clearInterval(this.pollInterval);
+            const message = anyFailed ? 'Batch import completed with some failures' : 'Batch import completed successfully!';
+            const type = anyFailed ? 'warning' : 'success';
+            this.updateProgress(message, type);
+        }
+    }
+    
+    showResults(result) {
+        const resultsContent = document.getElementById('resultsContent');
+        const resultsDiv = document.getElementById('importResults');
+        
+        let html = '<div class="space-y-4">';
+        
+        // Summary
+        html += `
+            <div class="grid grid-cols-2 md:grid-cols-4 gap-4">
+                <div class="bg-blue-50 p-3 rounded">
+                    <div class="text-2xl font-bold text-blue-600">${result.total_rows}</div>
+                    <div class="text-sm text-blue-800">Total Rows</div>
+                </div>
+                <div class="bg-green-50 p-3 rounded">
+                    <div class="text-2xl font-bold text-green-600">${result.imported_rows}</div>
+                    <div class="text-sm text-green-800">Imported</div>
+                </div>
+                <div class="bg-yellow-50 p-3 rounded">
+                    <div class="text-2xl font-bold text-yellow-600">${result.skipped_rows}</div>
+                    <div class="text-sm text-yellow-800">Skipped</div>
+                </div>
+                <div class="bg-red-50 p-3 rounded">
+                    <div class="text-2xl font-bold text-red-600">${result.error_rows}</div>
+                    <div class="text-sm text-red-800">Errors</div>
+                </div>
+            </div>
+        `;
+        
+        // Errors
+        if (result.errors && result.errors.length > 0) {
+            html += '<div class="bg-red-50 border border-red-200 rounded p-4">';
+            html += '<h4 class="font-semibold text-red-800 mb-2">Errors:</h4>';
+            html += '<div class="text-sm text-red-700 space-y-1 max-h-40 overflow-y-auto">';
+            result.errors.forEach(error => {
+                html += `<div>${this.escapeHtml(error)}</div>`;
+            });
+            html += '</div></div>';
+        }
+        
+        // Warnings
+        if (result.warnings && result.warnings.length > 0) {
+            html += '<div class="bg-yellow-50 border border-yellow-200 rounded p-4">';
+            html += '<h4 class="font-semibold text-yellow-800 mb-2">Warnings:</h4>';
+            html += '<div class="text-sm text-yellow-700 space-y-1 max-h-40 overflow-y-auto">';
+            result.warnings.forEach(warning => {
+                html += `<div>${this.escapeHtml(warning)}</div>`;
+            });
+            html += '</div></div>';
+        }
+        
+        html += '</div>';
+        
+        resultsContent.innerHTML = html;
+        resultsDiv.classList.remove('hidden');
+    }
+    
+    escapeHtml(text) {
+        const div = document.createElement('div');
+        div.textContent = text;
+        return div.innerHTML;
+    }
+}
+
+// Initialize when DOM is loaded
+document.addEventListener('DOMContentLoaded', () => {
+    window.importManager = new ImportManager();
+});