/** * Admin Import JavaScript * Handles CSV file import functionality */ class ImportManager { constructor() { this.supportedTables = []; this.batchFileCount = 0; this.currentImportId = null; this.pollInterval = null; this.bulkFiles = []; this.init(); } async init() { await this.loadSupportedTables(); await this.loadImportStatus(); this.setupEventListeners(); } async loadSupportedTables() { try { console.log('Loading supported tables...'); const response = await window.http.wrappedFetch('/api/admin/import/tables'); if (response.ok) { const data = await response.json(); this.supportedTables = data.tables || []; console.log('Supported tables loaded:', this.supportedTables); } else { console.error('Failed to load supported tables, status:', response.status); } } catch (error) { console.error('Failed to load supported tables:', error); window.alerts.error('Failed to load supported tables'); } } setupEventListeners() { // Single import form document.getElementById('importForm').addEventListener('submit', (e) => { e.preventDefault(); this.handleSingleImport(); }); // Validate button document.getElementById('validateBtn').addEventListener('click', () => { this.validateHeaders(); }); // Table selection change document.getElementById('tableSelect').addEventListener('change', (e) => { this.onTableChange(e.target.value); }); // Bulk file upload document.getElementById('bulkFiles').addEventListener('change', (e) => { this.handleBulkFileSelection(e); }); document.getElementById('autoMapBtn').addEventListener('click', () => { this.autoMapTables(); }); document.getElementById('clearAllBtn').addEventListener('click', () => { this.clearAllFiles(); }); document.getElementById('bulkUploadBtn').addEventListener('click', () => { this.handleBulkUpload(); }); // Status refresh button document.getElementById('refreshStatusBtn').addEventListener('click', () => { this.loadImportStatus(); }); } async onTableChange(tableName) { const schemaInfo = document.getElementById('schemaInfo'); const schemaDetails = document.getElementById('schemaDetails'); if (!tableName) { schemaInfo.classList.add('hidden'); return; } try { console.log('Loading schema for table:', tableName); const response = await window.http.wrappedFetch(`/api/admin/import/tables/${tableName}/schema`); if (response.ok) { const data = await response.json(); const schema = data.schema; console.log('Schema loaded for', tableName, ':', schema); let html = '
'; html += '

Required Fields:

'; html += '
'; html += '

All Available Fields:

'; html += '
'; html += '
'; Object.keys(schema.field_mapping).forEach(field => { html += `${field}`; }); html += '
'; schemaDetails.innerHTML = html; schemaInfo.classList.remove('hidden'); } } catch (error) { console.error('Failed to load schema:', error); } } async validateHeaders() { const tableSelect = document.getElementById('tableSelect'); const fileInput = document.getElementById('csvFile'); console.log('Starting header validation...'); if (!tableSelect.value) { console.warn('No table selected for validation'); window.alerts.error('Please select a table type'); return; } if (!fileInput.files[0]) { console.warn('No file selected for validation'); window.alerts.error('Please select a CSV file'); return; } console.log('Validating headers for table:', tableSelect.value, 'file:', fileInput.files[0].name); const formData = new FormData(); formData.append('table_name', tableSelect.value); formData.append('file', fileInput.files[0]); try { const response = await window.http.wrappedFetch('/api/admin/import/validate', { method: 'POST', body: formData }); console.log('Validation response status:', response.status); if (response.ok) { const result = await response.json(); console.log('Validation result:', result); if (result.success) { window.alerts.success('CSV headers validated successfully!'); } else { const errors = result.validation_result.errors.join('\\n'); console.error('Validation errors:', result.validation_result.errors); window.alerts.error(`Validation failed:\\n${errors}`); } } else { const error = await response.json(); console.error('Validation failed with error:', error); window.alerts.error(`Validation failed: ${error.detail}`); } } catch (error) { console.error('Validation error:', error); window.alerts.error('Failed to validate CSV headers'); } } async handleSingleImport() { const tableSelect = document.getElementById('tableSelect'); const fileInput = document.getElementById('csvFile'); console.log('Starting single import...'); if (!tableSelect.value) { console.warn('No table selected for import'); window.alerts.error('Please select a table type'); return; } if (!fileInput.files[0]) { console.warn('No file selected for import'); window.alerts.error('Please select a CSV file'); return; } console.log('Importing to table:', tableSelect.value, 'file:', fileInput.files[0].name); const formData = new FormData(); formData.append('table_name', tableSelect.value); formData.append('file', fileInput.files[0]); // Show progress this.showProgress(); try { const response = await window.http.wrappedFetch('/api/admin/import/csv', { method: 'POST', body: formData }); console.log('Import response status:', response.status); if (response.ok) { const result = await response.json(); console.log('Import started successfully:', result); this.currentImportId = result.import_id; this.updateProgress(`Import started for ${result.table_name} (ID: ${result.import_id})`, 'info'); this.startPolling(); } else { const error = await response.json(); console.error('Import failed:', error); this.updateProgress(`Import failed: ${error.detail}`, 'error'); } } catch (error) { console.error('Import error:', error); this.updateProgress('Import failed: Network error', 'error'); } } showProgress() { document.getElementById('importProgress').classList.remove('hidden'); document.getElementById('importResults').classList.add('hidden'); } updateProgress(message, type = 'info') { const progressDetails = document.getElementById('progressDetails'); const timestamp = new Date().toLocaleTimeString(); let colorClass = 'text-blue-600'; if (type === 'error') colorClass = 'text-red-600'; if (type === 'success') colorClass = 'text-green-600'; if (type === 'warning') colorClass = 'text-yellow-600'; progressDetails.innerHTML += `
${timestamp} ${message}
`; // Scroll to bottom progressDetails.scrollTop = progressDetails.scrollHeight; } startPolling() { if (this.pollInterval) { clearInterval(this.pollInterval); } this.pollInterval = setInterval(async () => { await this.checkImportStatus(); }, 2000); // Poll every 2 seconds } startBatchPolling() { if (this.pollInterval) { clearInterval(this.pollInterval); } this.pollInterval = setInterval(async () => { await this.checkBatchStatus(); }, 2000); // Poll every 2 seconds } async checkImportStatus() { if (!this.currentImportId) return; try { const response = await window.http.wrappedFetch(`/api/admin/import/status/${this.currentImportId}`); if (response.ok) { const status = await response.json(); if (status.status === 'COMPLETED') { clearInterval(this.pollInterval); this.updateProgress('Import completed successfully!', 'success'); this.showResults(status.result); // Refresh status after successful import setTimeout(() => { this.loadImportStatus(); }, 1000); } else if (status.status === 'FAILED') { clearInterval(this.pollInterval); this.updateProgress(`Import failed: ${status.error || 'Unknown error'}`, 'error'); if (status.result) { this.showResults(status.result); } } else { this.updateProgress(`Import status: ${status.status}`, 'info'); } } } catch (error) { console.error('Status check error:', error); } } async checkBatchStatus() { if (!this.currentImportIds || !Array.isArray(this.currentImportIds)) return; let allCompleted = true; let anyFailed = false; for (const importId of this.currentImportIds) { try { const response = await window.http.wrappedFetch(`/api/admin/import/status/${importId}`); if (response.ok) { const status = await response.json(); if (status.status === 'PROCESSING') { allCompleted = false; } else if (status.status === 'FAILED') { anyFailed = true; this.updateProgress(`${status.table_name} import failed: ${status.error || 'Unknown error'}`, 'error'); } else if (status.status === 'COMPLETED') { this.updateProgress(`${status.table_name} import completed`, 'success'); } } } catch (error) { console.error('Batch status check error:', error); } } if (allCompleted) { clearInterval(this.pollInterval); const message = anyFailed ? 'Batch import completed with some failures' : 'Batch import completed successfully!'; const type = anyFailed ? 'warning' : 'success'; this.updateProgress(message, type); // Refresh the import status after batch completion setTimeout(() => { this.loadImportStatus(); }, 1000); } } showResults(result) { const resultsContent = document.getElementById('resultsContent'); const resultsDiv = document.getElementById('importResults'); let html = '
'; // Summary html += `
${result.total_rows}
Total Rows
${result.imported_rows}
Imported
${result.skipped_rows}
Skipped
${result.error_rows}
Errors
`; // Errors if (result.errors && result.errors.length > 0) { html += '
'; html += '

Errors:

'; html += '
'; result.errors.forEach(error => { html += `
${this.escapeHtml(error)}
`; }); html += '
'; } // Warnings if (result.warnings && result.warnings.length > 0) { html += '
'; html += '

Warnings:

'; html += '
'; result.warnings.forEach(warning => { html += `
${this.escapeHtml(warning)}
`; }); html += '
'; } html += '
'; resultsContent.innerHTML = html; resultsDiv.classList.remove('hidden'); } handleBulkFileSelection(event) { const files = Array.from(event.target.files); console.log('Selected files:', files); this.bulkFiles = files.map(file => { const suggestion = this.suggestTableType(file.name); return { file: file, name: file.name, size: file.size, suggested_table: suggestion || 'skip' // Default to skip for unknown files }; }); this.renderBulkFiles(); this.updateBulkControls(); } suggestTableType(filename) { const name = filename.toUpperCase().replace('.CSV', ''); const mapping = { // Core supported tables 'ROLODEX': 'rolodex', 'ROLEX_V': 'rolodex', // ROLEX_V variant 'PHONE': 'phone', 'FILES': 'files', 'FILES_R': 'files', 'FILES_V': 'files', 'LEDGER': 'ledger', 'QDROS': 'qdros', // Lookup and reference tables 'GRUPLKUP': 'gruplkup', 'PLANINFO': 'planinfo', 'RVARLKUP': 'rvarlkup', 'FVARLKUP': 'fvarlkup', 'TRNSLKUP': 'trnslkup', 'EMPLOYEE': 'employee', 'FILETYPE': 'filetype', 'TRNSTYPE': 'trnstype', 'TRNSACTN': 'trnsactn', 'FILENOTS': 'filenots', 'SETUP': 'setup', 'PENSIONS': 'pensions', 'PAYMENTS': 'payments', 'DEPOSITS': 'deposits', // Form tables 'NUMBERAL': 'numberal', 'INX_LKUP': 'inx_lkup', 'FORM_LST': 'form_lst', 'FORM_INX': 'form_inx', 'LIFETABL': 'lifetabl', // Pension tables 'MARRIAGE': 'marriage', 'DEATH': 'death', 'SEPARATE': 'separate', 'SCHEDULE': 'schedule' }; return mapping[name] || ''; } renderBulkFiles() { const filesDisplay = document.getElementById('bulkFilesDisplay'); const filesList = document.getElementById('bulkFilesList'); const fileCount = document.getElementById('bulkFileCount'); fileCount.textContent = this.bulkFiles.length; if (this.bulkFiles.length === 0) { filesDisplay.classList.add('hidden'); return; } filesDisplay.classList.remove('hidden'); let html = ''; this.bulkFiles.forEach((fileObj, index) => { const sizeKB = Math.round(fileObj.size / 1024); html += `
${fileObj.name}

${sizeKB} KB

`; }); filesList.innerHTML = html; } updateBulkFileTableMapping(index, tableType) { if (this.bulkFiles[index]) { this.bulkFiles[index].suggested_table = tableType; } this.updateBulkControls(); } removeBulkFile(index) { this.bulkFiles.splice(index, 1); this.renderBulkFiles(); this.updateBulkControls(); } autoMapTables() { this.bulkFiles.forEach(fileObj => { if (!fileObj.suggested_table || fileObj.suggested_table === '') { const suggestion = this.suggestTableType(fileObj.name); // If no mapping found, default to 'skip' for unknown files fileObj.suggested_table = suggestion || 'skip'; } }); this.renderBulkFiles(); this.updateBulkControls(); } clearAllFiles() { this.bulkFiles = []; document.getElementById('bulkFiles').value = ''; this.renderBulkFiles(); this.updateBulkControls(); } updateBulkControls() { const autoMapBtn = document.getElementById('autoMapBtn'); const clearAllBtn = document.getElementById('clearAllBtn'); const uploadBtn = document.getElementById('bulkUploadBtn'); const hasFiles = this.bulkFiles.length > 0; const allMapped = this.bulkFiles.every(f => f.suggested_table && f.suggested_table !== ''); autoMapBtn.disabled = !hasFiles; clearAllBtn.disabled = !hasFiles; uploadBtn.disabled = !hasFiles || !allMapped; if (hasFiles) { const mappedCount = this.bulkFiles.filter(f => f.suggested_table && f.suggested_table !== '').length; const importCount = this.bulkFiles.filter(f => f.suggested_table && f.suggested_table !== 'skip').length; uploadBtn.innerHTML = ` Upload & Import ${importCount} Files (${mappedCount} mapped) `; } else { uploadBtn.innerHTML = ` Upload & Import All `; } } async handleBulkUpload() { if (this.bulkFiles.length === 0) { window.alerts.error('Please select files to upload'); return; } // Validate all files have table mappings const unmappedFiles = this.bulkFiles.filter(f => !f.suggested_table || f.suggested_table === ''); if (unmappedFiles.length > 0) { window.alerts.error(`Please select table types for: ${unmappedFiles.map(f => f.name).join(', ')}`); return; } // Filter out files marked as 'skip' const filesToImport = this.bulkFiles.filter(f => f.suggested_table !== 'skip'); if (filesToImport.length === 0) { window.alerts.error('No files selected for import (all marked as skip)'); return; } console.log('Starting bulk upload:', filesToImport); // Show progress this.showProgress(); try { // Use the existing batch endpoint that handles FormData const formData = new FormData(); // Add only files that are not marked as 'skip' filesToImport.forEach(fileObj => { formData.append('files', fileObj.file); }); // Add corresponding table names filesToImport.forEach(fileObj => { formData.append('table_names', fileObj.suggested_table); }); const response = await window.http.wrappedFetch('/api/admin/import/batch', { method: 'POST', body: formData }); if (response.ok) { const result = await response.json(); this.currentImportIds = result.import_ids; this.updateProgress(`Bulk upload started for ${result.total_files} files`, 'info'); this.startBatchPolling(); } else { const error = await response.json(); this.updateProgress(`Bulk upload failed: ${error.detail}`, 'error'); } } catch (error) { console.error('Bulk upload error:', error); this.updateProgress('Bulk upload failed: Network error', 'error'); } } async loadImportStatus() { try { console.log('Loading import status...'); const response = await window.http.wrappedFetch('/api/admin/import/status'); if (response.ok) { const data = await response.json(); this.displayImportStatus(data); console.log('Import status loaded:', data); } else { console.error('Failed to load import status, status:', response.status); window.alerts.error('Failed to load import status'); } } catch (error) { console.error('Failed to load import status:', error); window.alerts.error('Failed to load import status'); } } displayImportStatus(data) { const summary = data.summary; const categories = data.categories; // Update summary stats document.getElementById('totalTables').textContent = summary.total_tables; document.getElementById('importedTables').textContent = summary.imported_tables; document.getElementById('emptyTables').textContent = summary.empty_tables; document.getElementById('missingTables').textContent = summary.missing_tables; document.getElementById('totalRows').textContent = summary.total_rows.toLocaleString(); document.getElementById('completionPercentage').textContent = `${summary.completion_percentage}%`; // Update progress bar const progressBar = document.getElementById('progressBar'); progressBar.style.width = `${summary.completion_percentage}%`; // Display categories const categoryContainer = document.getElementById('statusByCategory'); categoryContainer.innerHTML = ''; Object.keys(categories).forEach(categoryName => { const tables = categories[categoryName]; const categoryDiv = document.createElement('div'); categoryDiv.className = 'bg-neutral-50 dark:bg-neutral-900 rounded-lg p-4'; const importedCount = tables.filter(t => t.imported).length; const totalCount = tables.length; categoryDiv.innerHTML = `

${categoryName} Tables ${importedCount}/${totalCount} imported

${tables.map(table => `
${table.imported ? '' : table.exists ? '' : '' }
${table.display_name}
${table.imported ? `${table.row_count.toLocaleString()} rows` : table.exists ? 'Empty table' : 'Not created' }
Expected: ${table.expected_files.join(', ')}
`).join('')}
`; categoryContainer.appendChild(categoryDiv); }); } escapeHtml(text) { const div = document.createElement('div'); div.textContent = text; return div.innerHTML; } } // Initialize when DOM is loaded document.addEventListener('DOMContentLoaded', () => { window.importManager = new ImportManager(); });