hotswap/delphi-database
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

work on import

Browse Source
This commit is contained in:
HotSwapp
2025-09-22 22:07:50 -04:00
parent 830ddcc4d1
commit 1116658d40
7 changed files with 1426 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

482
static/js/admin_import.js
View File

@@ -9,14 +9,15 @@ class ImportManager {
this.batchFileCount = 0;
this.currentImportId = null;
this.pollInterval = null;
this.bulkFiles = [];
this.init();
}
async init() {
await this.loadSupportedTables();
await this.loadImportStatus();
this.setupEventListeners();
this.addInitialBatchFile();
}
async loadSupportedTables() {
@@ -53,13 +54,26 @@ class ImportManager {
this.onTableChange(e.target.value);
});
// Batch import buttons
document.getElementById('addBatchFile').addEventListener('click', () => {
this.addBatchFile();
// Bulk file upload
document.getElementById('bulkFiles').addEventListener('change', (e) => {
this.handleBulkFileSelection(e);
});
document.getElementById('batchImportBtn').addEventListener('click', () => {
this.handleBatchImport();
document.getElementById('autoMapBtn').addEventListener('click', () => {
this.autoMapTables();
});
document.getElementById('clearAllBtn').addEventListener('click', () => {
this.clearAllFiles();
});
document.getElementById('bulkUploadBtn').addEventListener('click', () => {
this.handleBulkUpload();
});
// Status refresh button
document.getElementById('refreshStatusBtn').addEventListener('click', () => {
this.loadImportStatus();
});
}
@@ -209,98 +223,6 @@ class ImportManager {
}
}
addInitialBatchFile() {
this.addBatchFile();
}
addBatchFile() {
this.batchFileCount++;
const container = document.getElementById('batchFiles');
const fileDiv = document.createElement('div');
fileDiv.className = 'flex space-x-3 items-center';
fileDiv.id = `batchFile${this.batchFileCount}`;
fileDiv.innerHTML = `
<select name="batch_table_${this.batchFileCount}" class="flex-none w-40 px-3 py-2 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500">
<option value="">-- Select Table --</option>
<option value="rolodex">ROLODEX</option>
<option value="phone">PHONE</option>
<option value="files">FILES</option>
<option value="ledger">LEDGER</option>
<option value="qdros">QDROS</option>
</select>
<input type="file" name="batch_file_${this.batchFileCount}" accept=".csv"
class="flex-1 px-3 py-2 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500">
<button type="button" onclick="importManager.removeBatchFile(${this.batchFileCount})"
class="flex-none px-3 py-2 bg-red-600 text-white rounded-md hover:bg-red-700 focus:outline-none focus:ring-2 focus:ring-red-500">
Remove
</button>
`;
container.appendChild(fileDiv);
}
removeBatchFile(fileId) {
const fileDiv = document.getElementById(`batchFile${fileId}`);
if (fileDiv) {
fileDiv.remove();
}
}
async handleBatchImport() {
const batchFiles = document.getElementById('batchFiles');
const fileDivs = batchFiles.children;
const formData = new FormData();
const tableNames = [];
let hasFiles = false;
for (let i = 0; i < fileDivs.length; i++) {
const div = fileDivs[i];
const tableSelect = div.querySelector('select');
const fileInput = div.querySelector('input[type="file"]');
if (tableSelect.value && fileInput.files[0]) {
tableNames.push(tableSelect.value);
formData.append('files', fileInput.files[0]);
hasFiles = true;
}
}
if (!hasFiles) {
window.alerts.error('Please select at least one table and file');
return;
}
// Add table names to form data
tableNames.forEach(name => {
formData.append('table_names', name);
});
// Show progress
this.showProgress();
try {
const response = await window.http.wrappedFetch('/api/admin/import/batch', {
method: 'POST',
body: formData
});
if (response.ok) {
const result = await response.json();
this.currentImportIds = result.import_ids;
this.updateProgress(`Batch import started for ${result.total_files} files`, 'info');
this.startBatchPolling();
} else {
const error = await response.json();
this.updateProgress(`Batch import failed: ${error.detail}`, 'error');
}
} catch (error) {
console.error('Batch import error:', error);
this.updateProgress('Batch import failed: Network error', 'error');
}
}
showProgress() {
document.getElementById('importProgress').classList.remove('hidden');
@@ -359,6 +281,10 @@ class ImportManager {
clearInterval(this.pollInterval);
this.updateProgress('Import completed successfully!', 'success');
this.showResults(status.result);
// Refresh status after successful import
setTimeout(() => {
this.loadImportStatus();
}, 1000);
} else if (status.status === 'FAILED') {
clearInterval(this.pollInterval);
this.updateProgress(`Import failed: ${status.error || 'Unknown error'}`, 'error');
@@ -405,6 +331,11 @@ class ImportManager {
const message = anyFailed ? 'Batch import completed with some failures' : 'Batch import completed successfully!';
const type = anyFailed ? 'warning' : 'success';
this.updateProgress(message, type);
// Refresh the import status after batch completion
setTimeout(() => {
this.loadImportStatus();
}, 1000);
}
}
@@ -464,6 +395,361 @@ class ImportManager {
resultsDiv.classList.remove('hidden');
}
handleBulkFileSelection(event) {
const files = Array.from(event.target.files);
console.log('Selected files:', files);
this.bulkFiles = files.map(file => {
const suggestion = this.suggestTableType(file.name);
return {
file: file,
name: file.name,
size: file.size,
suggested_table: suggestion || 'skip' // Default to skip for unknown files
};
});
this.renderBulkFiles();
this.updateBulkControls();
}
suggestTableType(filename) {
const name = filename.toUpperCase().replace('.CSV', '');
const mapping = {
// Core supported tables
'ROLODEX': 'rolodex',
'ROLEX_V': 'rolodex', // ROLEX_V variant
'PHONE': 'phone',
'FILES': 'files',
'FILES_R': 'files',
'FILES_V': 'files',
'LEDGER': 'ledger',
'QDROS': 'qdros',
// Lookup and reference tables
'GRUPLKUP': 'gruplkup',
'PLANINFO': 'planinfo',
'RVARLKUP': 'rvarlkup',
'FVARLKUP': 'fvarlkup',
'TRNSLKUP': 'trnslkup',
'EMPLOYEE': 'employee',
'FILETYPE': 'filetype',
'TRNSTYPE': 'trnstype',
'TRNSACTN': 'trnsactn',
'FILENOTS': 'filenots',
'SETUP': 'setup',
'PENSIONS': 'pensions',
'PAYMENTS': 'payments',
'DEPOSITS': 'deposits',
// Form tables
'NUMBERAL': 'numberal',
'INX_LKUP': 'inx_lkup',
'FORM_LST': 'form_lst',
'FORM_INX': 'form_inx',
'LIFETABL': 'lifetabl',
// Pension tables
'MARRIAGE': 'marriage',
'DEATH': 'death',
'SEPARATE': 'separate',
'SCHEDULE': 'schedule'
};
return mapping[name] || '';
}
renderBulkFiles() {
const filesDisplay = document.getElementById('bulkFilesDisplay');
const filesList = document.getElementById('bulkFilesList');
const fileCount = document.getElementById('bulkFileCount');
fileCount.textContent = this.bulkFiles.length;
if (this.bulkFiles.length === 0) {
filesDisplay.classList.add('hidden');
return;
}
filesDisplay.classList.remove('hidden');
let html = '';
this.bulkFiles.forEach((fileObj, index) => {
const sizeKB = Math.round(fileObj.size / 1024);
html += `
<div class="flex items-center justify-between p-3 bg-white dark:bg-neutral-700 rounded-lg border border-neutral-200 dark:border-neutral-600">
<div class="flex items-center space-x-3">
<i class="fa-solid fa-file-csv text-primary-600"></i>
<div>
<div class="font-medium text-neutral-900 dark:text-neutral-100">
${fileObj.name}
</div>
<p class="text-sm text-neutral-500 dark:text-neutral-400">${sizeKB} KB</p>
</div>
</div>
<div class="flex items-center space-x-2">
<select onchange="importManager.updateBulkFileTableMapping(${index}, this.value)"
class="px-3 py-1 border border-neutral-300 dark:border-neutral-600 rounded-md text-sm bg-white dark:bg-neutral-700 text-neutral-900 dark:text-neutral-100">
<option value="">-- Select Table --</option>
<optgroup label="Core Tables">
<option value="rolodex" ${fileObj.suggested_table === 'rolodex' ? 'selected' : ''}>ROLODEX (Contacts)</option>
<option value="phone" ${fileObj.suggested_table === 'phone' ? 'selected' : ''}>PHONE (Phone Numbers)</option>
<option value="files" ${fileObj.suggested_table === 'files' ? 'selected' : ''}>FILES (Case Files)</option>
<option value="ledger" ${fileObj.suggested_table === 'ledger' ? 'selected' : ''}>LEDGER (Financial)</option>
<option value="qdros" ${fileObj.suggested_table === 'qdros' ? 'selected' : ''}>QDROS (Documents)</option>
</optgroup>
<optgroup label="Lookup Tables">
<option value="gruplkup" ${fileObj.suggested_table === 'gruplkup' ? 'selected' : ''}>GRUPLKUP</option>
<option value="employee" ${fileObj.suggested_table === 'employee' ? 'selected' : ''}>EMPLOYEE</option>
<option value="filetype" ${fileObj.suggested_table === 'filetype' ? 'selected' : ''}>FILETYPE</option>
<option value="trnstype" ${fileObj.suggested_table === 'trnstype' ? 'selected' : ''}>TRNSTYPE</option>
<option value="trnslkup" ${fileObj.suggested_table === 'trnslkup' ? 'selected' : ''}>TRNSLKUP</option>
<option value="rvarlkup" ${fileObj.suggested_table === 'rvarlkup' ? 'selected' : ''}>RVARLKUP</option>
<option value="fvarlkup" ${fileObj.suggested_table === 'fvarlkup' ? 'selected' : ''}>FVARLKUP</option>
</optgroup>
<optgroup label="Financial Tables">
<option value="payments" ${fileObj.suggested_table === 'payments' ? 'selected' : ''}>PAYMENTS</option>
<option value="deposits" ${fileObj.suggested_table === 'deposits' ? 'selected' : ''}>DEPOSITS</option>
<option value="trnsactn" ${fileObj.suggested_table === 'trnsactn' ? 'selected' : ''}>TRNSACTN</option>
</optgroup>
<optgroup label="Forms & Documents">
<option value="numberal" ${fileObj.suggested_table === 'numberal' ? 'selected' : ''}>NUMBERAL</option>
<option value="inx_lkup" ${fileObj.suggested_table === 'inx_lkup' ? 'selected' : ''}>INX_LKUP</option>
<option value="form_lst" ${fileObj.suggested_table === 'form_lst' ? 'selected' : ''}>FORM_LST</option>
<option value="form_inx" ${fileObj.suggested_table === 'form_inx' ? 'selected' : ''}>FORM_INX</option>
<option value="lifetabl" ${fileObj.suggested_table === 'lifetabl' ? 'selected' : ''}>LIFETABL</option>
</optgroup>
<optgroup label="Pension Tables">
<option value="pensions" ${fileObj.suggested_table === 'pensions' ? 'selected' : ''}>PENSIONS</option>
<option value="marriage" ${fileObj.suggested_table === 'marriage' ? 'selected' : ''}>MARRIAGE</option>
<option value="death" ${fileObj.suggested_table === 'death' ? 'selected' : ''}>DEATH</option>
<option value="separate" ${fileObj.suggested_table === 'separate' ? 'selected' : ''}>SEPARATE</option>
<option value="schedule" ${fileObj.suggested_table === 'schedule' ? 'selected' : ''}>SCHEDULE</option>
</optgroup>
<optgroup label="Configuration">
<option value="setup" ${fileObj.suggested_table === 'setup' ? 'selected' : ''}>SETUP</option>
<option value="planinfo" ${fileObj.suggested_table === 'planinfo' ? 'selected' : ''}>PLANINFO</option>
<option value="filenots" ${fileObj.suggested_table === 'filenots' ? 'selected' : ''}>FILENOTS</option>
</optgroup>
<optgroup label="Other">
<option value="skip" ${fileObj.suggested_table === 'skip' ? 'selected' : ''}>⚠️ SKIP (Don't Import)</option>
</optgroup>
</select>
<button onclick="importManager.removeBulkFile(${index})"
class="px-2 py-1 bg-red-600 text-white rounded-md hover:bg-red-700 text-sm">
<i class="fa-solid fa-times"></i>
</button>
</div>
</div>
`;
});
filesList.innerHTML = html;
}
updateBulkFileTableMapping(index, tableType) {
if (this.bulkFiles[index]) {
this.bulkFiles[index].suggested_table = tableType;
}
this.updateBulkControls();
}
removeBulkFile(index) {
this.bulkFiles.splice(index, 1);
this.renderBulkFiles();
this.updateBulkControls();
}
autoMapTables() {
this.bulkFiles.forEach(fileObj => {
if (!fileObj.suggested_table || fileObj.suggested_table === '') {
const suggestion = this.suggestTableType(fileObj.name);
// If no mapping found, default to 'skip' for unknown files
fileObj.suggested_table = suggestion || 'skip';
}
});
this.renderBulkFiles();
this.updateBulkControls();
}
clearAllFiles() {
this.bulkFiles = [];
document.getElementById('bulkFiles').value = '';
this.renderBulkFiles();
this.updateBulkControls();
}
updateBulkControls() {
const autoMapBtn = document.getElementById('autoMapBtn');
const clearAllBtn = document.getElementById('clearAllBtn');
const uploadBtn = document.getElementById('bulkUploadBtn');
const hasFiles = this.bulkFiles.length > 0;
const allMapped = this.bulkFiles.every(f => f.suggested_table && f.suggested_table !== '');
autoMapBtn.disabled = !hasFiles;
clearAllBtn.disabled = !hasFiles;
uploadBtn.disabled = !hasFiles || !allMapped;
if (hasFiles) {
const mappedCount = this.bulkFiles.filter(f => f.suggested_table && f.suggested_table !== '').length;
const importCount = this.bulkFiles.filter(f => f.suggested_table && f.suggested_table !== 'skip').length;
uploadBtn.innerHTML = `
<i class="fa-solid fa-cloud-upload"></i>
<span>Upload & Import ${importCount} Files (${mappedCount} mapped)</span>
`;
} else {
uploadBtn.innerHTML = `
<i class="fa-solid fa-cloud-upload"></i>
<span>Upload & Import All</span>
`;
}
}
async handleBulkUpload() {
if (this.bulkFiles.length === 0) {
window.alerts.error('Please select files to upload');
return;
}
// Validate all files have table mappings
const unmappedFiles = this.bulkFiles.filter(f => !f.suggested_table || f.suggested_table === '');
if (unmappedFiles.length > 0) {
window.alerts.error(`Please select table types for: ${unmappedFiles.map(f => f.name).join(', ')}`);
return;
}
// Filter out files marked as 'skip'
const filesToImport = this.bulkFiles.filter(f => f.suggested_table !== 'skip');
if (filesToImport.length === 0) {
window.alerts.error('No files selected for import (all marked as skip)');
return;
}
console.log('Starting bulk upload:', filesToImport);
// Show progress
this.showProgress();
try {
// Use the existing batch endpoint that handles FormData
const formData = new FormData();
// Add only files that are not marked as 'skip'
filesToImport.forEach(fileObj => {
formData.append('files', fileObj.file);
});
// Add corresponding table names
filesToImport.forEach(fileObj => {
formData.append('table_names', fileObj.suggested_table);
});
const response = await window.http.wrappedFetch('/api/admin/import/batch', {
method: 'POST',
body: formData
});
if (response.ok) {
const result = await response.json();
this.currentImportIds = result.import_ids;
this.updateProgress(`Bulk upload started for ${result.total_files} files`, 'info');
this.startBatchPolling();
} else {
const error = await response.json();
this.updateProgress(`Bulk upload failed: ${error.detail}`, 'error');
}
} catch (error) {
console.error('Bulk upload error:', error);
this.updateProgress('Bulk upload failed: Network error', 'error');
}
}
async loadImportStatus() {
try {
console.log('Loading import status...');
const response = await window.http.wrappedFetch('/api/admin/import/status');
if (response.ok) {
const data = await response.json();
this.displayImportStatus(data);
console.log('Import status loaded:', data);
} else {
console.error('Failed to load import status, status:', response.status);
window.alerts.error('Failed to load import status');
}
} catch (error) {
console.error('Failed to load import status:', error);
window.alerts.error('Failed to load import status');
}
}
displayImportStatus(data) {
const summary = data.summary;
const categories = data.categories;
// Update summary stats
document.getElementById('totalTables').textContent = summary.total_tables;
document.getElementById('importedTables').textContent = summary.imported_tables;
document.getElementById('emptyTables').textContent = summary.empty_tables;
document.getElementById('missingTables').textContent = summary.missing_tables;
document.getElementById('totalRows').textContent = summary.total_rows.toLocaleString();
document.getElementById('completionPercentage').textContent = `${summary.completion_percentage}%`;
// Update progress bar
const progressBar = document.getElementById('progressBar');
progressBar.style.width = `${summary.completion_percentage}%`;
// Display categories
const categoryContainer = document.getElementById('statusByCategory');
categoryContainer.innerHTML = '';
Object.keys(categories).forEach(categoryName => {
const tables = categories[categoryName];
const categoryDiv = document.createElement('div');
categoryDiv.className = 'bg-neutral-50 dark:bg-neutral-900 rounded-lg p-4';
const importedCount = tables.filter(t => t.imported).length;
const totalCount = tables.length;
categoryDiv.innerHTML = `
<h3 class="font-semibold text-neutral-900 dark:text-neutral-100 mb-3 flex items-center justify-between">
<span>${categoryName} Tables</span>
<span class="text-sm font-normal text-neutral-600 dark:text-neutral-400">${importedCount}/${totalCount} imported</span>
</h3>
<div class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-3">
${tables.map(table => `
<div class="flex items-center justify-between p-3 bg-white dark:bg-neutral-800 rounded-lg border border-neutral-200 dark:border-neutral-600">
<div class="flex items-center space-x-3">
<div class="flex-shrink-0">
${table.imported ?
'<i class="fa-solid fa-check-circle text-green-600"></i>' :
table.exists ?
'<i class="fa-solid fa-exclamation-circle text-yellow-600"></i>' :
'<i class="fa-solid fa-times-circle text-red-600"></i>'
}
</div>
<div>
<div class="font-medium text-neutral-900 dark:text-neutral-100">${table.display_name}</div>
<div class="text-sm text-neutral-500 dark:text-neutral-400">
${table.imported ?
`${table.row_count.toLocaleString()} rows` :
table.exists ?
'Empty table' :
'Not created'
}
</div>
<div class="text-xs text-neutral-400 dark:text-neutral-500">
Expected: ${table.expected_files.join(', ')}
</div>
</div>
</div>
</div>
`).join('')}
</div>
`;
categoryContainer.appendChild(categoryDiv);
});
}
escapeHtml(text) {
const div = document.createElement('div');
div.textContent = text;