1467 lines
70 KiB
HTML
1467 lines
70 KiB
HTML
{% extends "base.html" %}
|
|
|
|
{% block title %}Data Import - Delphi Database{% endblock %}
|
|
|
|
{% block content %}
|
|
<div class="space-y-6">
|
|
<!-- Page Header -->
|
|
<div class="flex flex-col sm:flex-row sm:items-center sm:justify-between gap-4">
|
|
<div class="flex items-center gap-3">
|
|
<div class="flex items-center justify-center w-10 h-10 bg-primary-100 dark:bg-primary-900/30 text-primary-600 dark:text-primary-400 rounded-xl">
|
|
<i class="fa-solid fa-upload text-lg"></i>
|
|
</div>
|
|
<h1 class="text-2xl font-bold text-neutral-900 dark:text-neutral-100">Data Import</h1>
|
|
</div>
|
|
<div class="flex items-center gap-3">
|
|
<button id="refreshStatusBtn" class="bg-info-600 text-white hover:bg-info-700 px-3 py-1.5 rounded-lg text-xs font-medium transition-colors duration-200 flex items-center gap-2">
|
|
<i class="fa-solid fa-rotate-right"></i>
|
|
<span>Refresh Status</span>
|
|
</button>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Import Status Panel -->
|
|
<div class="bg-white dark:bg-neutral-800 border border-neutral-200 dark:border-neutral-700 rounded-xl shadow-soft">
|
|
<div class="px-6 py-4 border-b border-neutral-200 dark:border-neutral-700">
|
|
<h5 class="text-lg font-semibold text-neutral-900 dark:text-neutral-100 flex items-center gap-2">
|
|
<i class="fa-solid fa-circle-info"></i>
|
|
<span>Current Database Status</span>
|
|
</h5>
|
|
</div>
|
|
<div class="p-6">
|
|
<div id="importStatus">
|
|
<div class="flex items-center justify-center py-4 text-neutral-500 dark:text-neutral-400">
|
|
<i class="fa-solid fa-rotate-right animate-spin text-xl mr-2"></i>
|
|
<p>Loading import status...</p>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- CSV File Upload Panel -->
|
|
<div class="bg-white dark:bg-neutral-800 border border-neutral-200 dark:border-neutral-700 rounded-xl shadow-soft">
|
|
<div class="px-6 py-4 border-b border-neutral-200 dark:border-neutral-700">
|
|
<div class="flex items-center justify-between">
|
|
<h5 class="text-lg font-semibold text-neutral-900 dark:text-neutral-100 flex items-center gap-2">
|
|
<i class="fa-regular fa-file-arrow-up"></i>
|
|
<span>Upload CSV Files</span>
|
|
</h5>
|
|
<div class="flex items-center gap-2">
|
|
<label class="text-sm font-medium text-neutral-700 dark:text-neutral-300">Mode:</label>
|
|
<select id="uploadMode" class="px-2 py-1 bg-white dark:bg-neutral-800 border border-neutral-300 dark:border-neutral-600 rounded text-sm">
|
|
<option value="batch">Batch Upload (Recommended)</option>
|
|
<option value="single">Single File</option>
|
|
</select>
|
|
<button type="button" id="importHelpBtn" class="ml-2 px-2 py-1 text-xs border border-neutral-300 dark:border-neutral-600 rounded hover:bg-neutral-100 dark:hover:bg-neutral-700" title="How to select and order files">
|
|
<i class="fa-solid fa-circle-question"></i>
|
|
Help
|
|
</button>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="p-6">
|
|
<!-- Single File Upload Form -->
|
|
<form id="importForm" enctype="multipart/form-data" class="single-upload hidden">
|
|
<div class="grid grid-cols-1 md:grid-cols-12 gap-4">
|
|
<div class="md:col-span-4" id="fileTypeContainer">
|
|
<label for="fileType" class="block text-sm font-medium text-neutral-700 dark:text-neutral-300 mb-2">Data Type *</label>
|
|
<select class="w-full px-3 py-3 bg-white dark:bg-neutral-800 border border-neutral-300 dark:border-neutral-600 rounded-lg text-neutral-900 dark:text-neutral-100 focus:ring-2 focus:ring-primary-500 focus:border-transparent transition-all duration-200" id="fileType" name="fileType" required>
|
|
<option value="">Select data type...</option>
|
|
</select>
|
|
<div class="text-sm text-neutral-500 dark:text-neutral-400 mt-1" id="fileTypeDescription"></div>
|
|
</div>
|
|
<div class="md:col-span-6">
|
|
<label for="csvFile" class="block text-sm font-medium text-neutral-700 dark:text-neutral-300 mb-2">CSV File *</label>
|
|
<input type="file" class="w-full px-3 py-3 bg-white dark:bg-neutral-800 border border-neutral-300 dark:border-neutral-600 rounded-lg text-neutral-900 dark:text-neutral-100 file:mr-4 file:py-2 file:px-4 file:rounded file:border-0 file:text-sm file:font-semibold file:bg-primary-100 file:text-primary-700 hover:file:bg-primary-200 transition-all duration-200" id="csvFile" name="csvFile" accept=".csv" required>
|
|
<div class="text-sm text-neutral-500 dark:text-neutral-400 mt-1">Select the CSV file to import</div>
|
|
</div>
|
|
<div class="md:col-span-2 flex items-end">
|
|
<div class="flex flex-col gap-2">
|
|
<label class="inline-flex items-center gap-2">
|
|
<input class="h-4 w-4 text-primary-600 border-neutral-300 rounded" type="checkbox" id="replaceExisting" name="replaceExisting">
|
|
<span class="text-sm">Replace existing</span>
|
|
</label>
|
|
<label class="inline-flex items-center gap-2">
|
|
<input class="h-4 w-4 text-primary-600 border-neutral-300 rounded" type="checkbox" id="flexibleOnly" name="flexibleOnly">
|
|
<span class="text-sm">Flexible-only</span>
|
|
</label>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="mt-4">
|
|
<div class="flex items-center gap-3">
|
|
<button type="button" class="px-4 py-2 bg-neutral-100 dark:bg-neutral-700 text-neutral-700 dark:text-neutral-300 hover:bg-neutral-200 dark:hover:bg-neutral-600 rounded-lg transition-colors duration-200 flex items-center gap-2" id="validateBtn">
|
|
<i class="fa-regular fa-circle-check"></i>
|
|
<span>Validate File</span>
|
|
</button>
|
|
<button type="submit" class="px-4 py-2 bg-primary-600 text-white hover:bg-primary-700 rounded-lg transition-colors duration-200 flex items-center gap-2" id="importBtn">
|
|
<i class="fa-solid fa-upload"></i>
|
|
<span>Import Data</span>
|
|
</button>
|
|
</div>
|
|
<p class="mt-2 text-xs text-neutral-500 dark:text-neutral-400" id="flexibleHint" style="display:none;">
|
|
Flexible-only: Upload any CSV. All columns will be stored as flexible JSON and visible in Flexible Imports.
|
|
</p>
|
|
</div>
|
|
</form>
|
|
|
|
<!-- Batch Upload Form -->
|
|
<form id="batchImportForm" enctype="multipart/form-data" class="batch-upload hidden">
|
|
<div class="space-y-4">
|
|
<div>
|
|
<label for="batchFiles" class="block text-sm font-medium text-neutral-700 dark:text-neutral-300 mb-2">Select Multiple CSV Files *</label>
|
|
<div class="relative">
|
|
<input type="file" class="w-full px-3 py-3 bg-white dark:bg-neutral-800 border border-neutral-300 dark:border-neutral-600 rounded-lg text-neutral-900 dark:text-neutral-100 file:mr-4 file:py-2 file:px-4 file:rounded file:border-0 file:text-sm file:font-semibold file:bg-primary-100 file:text-primary-700 hover:file:bg-primary-200 transition-all duration-200" id="batchFiles" name="batchFiles" accept=".csv" multiple required>
|
|
</div>
|
|
<div class="text-sm text-neutral-500 dark:text-neutral-400 mt-1">
|
|
Select all your CSV files at once (max 25). Files will be validated and imported in dependency order automatically.
|
|
<br><strong>Tip:</strong> Use Ctrl+A in the file dialog to select all CSV files from your export folder.
|
|
</div>
|
|
</div>
|
|
|
|
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
|
|
<div class="space-y-2">
|
|
<label class="inline-flex items-center gap-2">
|
|
<input class="h-4 w-4 text-primary-600 border-neutral-300 rounded" type="checkbox" id="batchReplaceExisting" name="batchReplaceExisting">
|
|
<span class="text-sm font-medium">Replace existing data</span>
|
|
</label>
|
|
<label class="inline-flex items-center gap-2">
|
|
<input class="h-4 w-4 text-success-600 border-neutral-300 rounded" type="checkbox" id="validateAllFirst" name="validateAllFirst" checked>
|
|
<span class="text-sm font-medium text-success-700 dark:text-success-400">Validate all files before import</span>
|
|
</label>
|
|
</div>
|
|
<div class="text-right">
|
|
<span class="text-sm text-neutral-600 dark:text-neutral-400" id="selectedFilesCount">0 files selected</span>
|
|
</div>
|
|
</div>
|
|
|
|
<div id="selectedFilesList" class="hidden">
|
|
<h6 class="text-sm font-medium text-neutral-700 dark:text-neutral-300 mb-2">Selected Files (Import Order):</h6>
|
|
<div class="bg-neutral-50 dark:bg-neutral-900 rounded-lg p-3 max-h-40 overflow-y-auto" id="filesList"></div>
|
|
</div>
|
|
|
|
<div class="flex items-center gap-3">
|
|
<button type="button" class="px-4 py-2 bg-info-600 text-white hover:bg-info-700 rounded-lg transition-colors duration-200 flex items-center gap-2" id="batchValidateBtn">
|
|
<i class="fa-solid fa-clipboard-check"></i>
|
|
<span>Validate All Files</span>
|
|
</button>
|
|
<button type="submit" class="px-4 py-2 bg-success-600 text-white hover:bg-success-700 rounded-lg transition-colors duration-200 flex items-center gap-2" id="batchImportBtn">
|
|
<i class="fa-solid fa-layer-group"></i>
|
|
<span>Import All Files</span>
|
|
</button>
|
|
<button type="button" class="px-4 py-2 bg-neutral-100 dark:bg-neutral-700 text-neutral-700 dark:text-neutral-300 hover:bg-neutral-200 dark:hover:bg-neutral-600 rounded-lg transition-colors duration-200 flex items-center gap-2" id="clearBatchBtn">
|
|
<i class="fa-solid fa-xmark"></i>
|
|
<span>Clear Selection</span>
|
|
</button>
|
|
</div>
|
|
</div>
|
|
</form>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Validation Results Panel -->
|
|
<div class="bg-white dark:bg-neutral-800 border border-neutral-200 dark:border-neutral-700 rounded-xl shadow-soft hidden" id="validationPanel">
|
|
<div class="px-6 py-4 border-b border-neutral-200 dark:border-neutral-700">
|
|
<h5 class="text-lg font-semibold text-neutral-900 dark:text-neutral-100 flex items-center gap-2">
|
|
<i class="fa-solid fa-clipboard-check"></i>
|
|
<span>File Validation Results</span>
|
|
</h5>
|
|
</div>
|
|
<div class="p-6" id="validationResults">
|
|
<!-- Validation results will be shown here -->
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Import Progress Panel -->
|
|
<div class="bg-white dark:bg-neutral-800 border border-neutral-200 dark:border-neutral-700 rounded-xl shadow-soft hidden" id="progressPanel">
|
|
<div class="px-6 py-4 border-b border-neutral-200 dark:border-neutral-700">
|
|
<h5 class="text-lg font-semibold text-neutral-900 dark:text-neutral-100 flex items-center gap-2">
|
|
<i class="fa-solid fa-hourglass-half"></i>
|
|
<span>Import Progress</span>
|
|
</h5>
|
|
</div>
|
|
<div class="p-6">
|
|
<div class="bg-neutral-200 dark:bg-neutral-700 rounded-full h-2 mb-3 overflow-hidden">
|
|
<div class="bg-primary-600 h-full rounded-full transition-all duration-300" style="width: 0%" id="progressBar"></div>
|
|
</div>
|
|
<div id="progressStatus" class="text-sm text-neutral-600 dark:text-neutral-400">Ready to import...</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Import Results Panel -->
|
|
<div class="bg-white dark:bg-neutral-800 border border-neutral-200 dark:border-neutral-700 rounded-xl shadow-soft hidden" id="resultsPanel">
|
|
<div class="px-6 py-4 border-b border-neutral-200 dark:border-neutral-700">
|
|
<h5 class="text-lg font-semibold text-neutral-900 dark:text-neutral-100 flex items-center gap-2">
|
|
<i class="fa-solid fa-circle-check"></i>
|
|
<span>Import Results</span>
|
|
</h5>
|
|
</div>
|
|
<div class="p-6" id="importResults">
|
|
<!-- Import results will be shown here -->
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Batch Results Panel -->
|
|
<div class="bg-white dark:bg-neutral-800 border border-neutral-200 dark:border-neutral-700 rounded-xl shadow-soft hidden" id="batchResultsPanel">
|
|
<div class="px-6 py-4 border-b border-neutral-200 dark:border-neutral-700">
|
|
<h5 class="text-lg font-semibold text-neutral-900 dark:text-neutral-100 flex items-center gap-2">
|
|
<i class="fa-solid fa-layer-group"></i>
|
|
<span>Batch Import Results</span>
|
|
</h5>
|
|
</div>
|
|
<div class="p-6" id="batchResults">
|
|
<!-- Batch import results will be shown here -->
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Recent Batch Uploads Panel -->
|
|
<div class="bg-white dark:bg-neutral-800 border border-neutral-200 dark:border-neutral-700 rounded-xl shadow-soft" id="recentBatchesPanel">
|
|
<div class="px-6 py-4 border-b border-neutral-200 dark:border-neutral-700">
|
|
<h5 class="text-lg font-semibold text-neutral-900 dark:text-neutral-100 flex items-center gap-2">
|
|
<i class="fa-solid fa-clock-rotate-left"></i>
|
|
<span>Recent Batch Uploads</span>
|
|
</h5>
|
|
</div>
|
|
<div class="p-6" id="recentBatches">
|
|
<div class="flex flex-col items-center justify-center py-4 text-neutral-500 dark:text-neutral-400">
|
|
<i class="fa-solid fa-file-arrow-up text-2xl mb-2"></i>
|
|
<p>Loading recent batches...</p>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Data Management Panel -->
|
|
<div class="bg-white dark:bg-neutral-800 border border-neutral-200 dark:border-neutral-700 rounded-xl shadow-soft">
|
|
<div class="px-6 py-4 border-b border-neutral-200 dark:border-neutral-700">
|
|
<h5 class="text-lg font-semibold text-neutral-900 dark:text-neutral-100 flex items-center gap-2">
|
|
<i class="fa-solid fa-database"></i>
|
|
<span>Data Management</span>
|
|
</h5>
|
|
</div>
|
|
<div class="p-6">
|
|
<div class="grid grid-cols-1 md:grid-cols-2 gap-6">
|
|
<div>
|
|
<h6 class="text-base font-semibold mb-2">Clear Table Data</h6>
|
|
<p class="text-sm text-neutral-500 dark:text-neutral-400 mb-3">Remove all records from a specific table (cannot be undone)</p>
|
|
<div class="flex gap-3">
|
|
<select class="flex-grow px-3 py-3 bg-white dark:bg-neutral-800 border border-neutral-300 dark:border-neutral-600 rounded-lg text-neutral-900 dark:text-neutral-100 focus:ring-2 focus:ring-primary-500 focus:border-transparent transition-all duration-200" id="clearTableType">
|
|
<option value="">Select table to clear...</option>
|
|
</select>
|
|
<button class="px-4 py-3 bg-danger-600 text-white hover:bg-danger-700 rounded-lg transition-colors duration-200 flex items-center gap-2 whitespace-nowrap" id="clearTableBtn">
|
|
<i class="fa-solid fa-trash"></i>
|
|
<span>Clear Table</span>
|
|
</button>
|
|
</div>
|
|
</div>
|
|
<div>
|
|
<h6 class="text-base font-semibold mb-2">Quick Actions</h6>
|
|
<div class="space-y-3">
|
|
<button class="w-full px-4 py-3 bg-neutral-100 dark:bg-neutral-700 text-neutral-700 dark:text-neutral-300 hover:bg-neutral-200 dark:hover:bg-neutral-600 rounded-lg transition-colors duration-200 flex items-center justify-center gap-2" id="backupBtn">
|
|
<i class="fa-solid fa-download"></i>
|
|
<span>Download Backup</span>
|
|
</button>
|
|
<button class="w-full px-4 py-3 bg-neutral-100 dark:bg-neutral-700 text-neutral-700 dark:text-neutral-300 hover:bg-neutral-200 dark:hover:bg-neutral-600 rounded-lg transition-colors duration-200 flex items-center justify-center gap-2" id="viewLogsBtn">
|
|
<i class="fa-regular fa-file-lines"></i>
|
|
<span>View Import Logs</span>
|
|
</button>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<script>
|
|
// Import functionality
|
|
let availableFiles = {};
|
|
let importInProgress = false;
|
|
let recentState = { limit: 5, offset: 0, status: 'all', start: '', end: '' };
|
|
|
|
// Authorization is injected by window.http.wrappedFetch
|
|
|
|
// Initialize on page load
|
|
document.addEventListener('DOMContentLoaded', function() {
|
|
// Check authentication first
|
|
const token = localStorage.getItem('auth_token');
|
|
if (!token) {
|
|
window.location.href = '/login';
|
|
return;
|
|
}
|
|
// Ensure admin only access for this page
|
|
(async () => {
|
|
try {
|
|
const resp = await window.http.wrappedFetch('/api/auth/me');
|
|
if (!resp.ok) {
|
|
window.location.href = '/login';
|
|
return;
|
|
}
|
|
const me = await resp.json();
|
|
if (!me || !me.is_admin) {
|
|
window.location.href = '/';
|
|
return;
|
|
}
|
|
} catch (_) {
|
|
window.location.href = '/login';
|
|
return;
|
|
}
|
|
})();
|
|
|
|
loadAvailableFiles();
|
|
loadImportStatus();
|
|
loadRecentBatches(false);
|
|
setupEventListeners();
|
|
|
|
// Set batch mode as default after a short delay to ensure DOM is ready
|
|
setTimeout(() => {
|
|
document.getElementById('uploadMode').value = 'batch';
|
|
switchUploadMode();
|
|
}, 100);
|
|
});
|
|
|
|
function setupEventListeners() {
|
|
// Form submission
|
|
document.getElementById('importForm').addEventListener('submit', handleImport);
|
|
document.getElementById('batchImportForm').addEventListener('submit', handleBatchImport);
|
|
|
|
// Upload mode switching
|
|
document.getElementById('uploadMode').addEventListener('change', switchUploadMode);
|
|
const helpBtn = document.getElementById('importHelpBtn');
|
|
if (helpBtn) helpBtn.addEventListener('click', showImportHelp);
|
|
|
|
// Validation buttons
|
|
document.getElementById('validateBtn').addEventListener('click', validateFile);
|
|
document.getElementById('batchValidateBtn').addEventListener('click', validateAllFiles);
|
|
|
|
// File type selection
|
|
document.getElementById('fileType').addEventListener('change', updateFileTypeDescription);
|
|
|
|
// Batch file selection
|
|
document.getElementById('batchFiles').addEventListener('change', updateSelectedFiles);
|
|
document.getElementById('clearBatchBtn').addEventListener('click', clearBatchSelection);
|
|
|
|
// Refresh status
|
|
document.getElementById('refreshStatusBtn').addEventListener('click', loadImportStatus);
|
|
|
|
// Clear table
|
|
document.getElementById('clearTableBtn').addEventListener('click', clearTable);
|
|
|
|
// Other buttons
|
|
document.getElementById('backupBtn').addEventListener('click', downloadBackup);
|
|
document.getElementById('viewLogsBtn').addEventListener('click', viewLogs);
|
|
const flexibleOnly = document.getElementById('flexibleOnly');
|
|
if (flexibleOnly) {
|
|
flexibleOnly.addEventListener('change', () => {
|
|
const isFlex = flexibleOnly.checked;
|
|
const fileTypeContainer = document.getElementById('fileTypeContainer');
|
|
const fileType = document.getElementById('fileType');
|
|
const hint = document.getElementById('flexibleHint');
|
|
if (isFlex) {
|
|
if (fileTypeContainer) fileTypeContainer.classList.add('opacity-50');
|
|
if (fileType) fileType.required = false;
|
|
if (hint) hint.style.display = '';
|
|
} else {
|
|
if (fileTypeContainer) fileTypeContainer.classList.remove('opacity-50');
|
|
if (fileType) fileType.required = true;
|
|
if (hint) hint.style.display = 'none';
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
async function loadAvailableFiles() {
|
|
try {
|
|
const response = await window.http.wrappedFetch('/api/import/available-files');
|
|
|
|
if (response.status === 401 || response.status === 403) {
|
|
console.error('Authentication error - redirecting to login');
|
|
window.location.href = '/login';
|
|
return;
|
|
}
|
|
|
|
if (!response.ok) {
|
|
const errorData = await response.json().catch(() => ({}));
|
|
throw new Error(errorData.error?.message || `HTTP ${response.status}: ${response.statusText}`);
|
|
}
|
|
|
|
const data = await response.json();
|
|
availableFiles = data;
|
|
|
|
// Populate file type dropdown
|
|
const fileTypeSelect = document.getElementById('fileType');
|
|
const clearTableSelect = document.getElementById('clearTableType');
|
|
|
|
fileTypeSelect.innerHTML = '<option value="">Select data type...</option>';
|
|
clearTableSelect.innerHTML = '<option value="">Select table to clear...</option>';
|
|
|
|
data.available_files.forEach(fileType => {
|
|
const description = data.descriptions[fileType] || fileType;
|
|
|
|
const option1 = document.createElement('option');
|
|
option1.value = fileType;
|
|
option1.textContent = `${fileType} - ${description}`;
|
|
fileTypeSelect.appendChild(option1);
|
|
|
|
const option2 = document.createElement('option');
|
|
option2.value = fileType;
|
|
option2.textContent = `${fileType} - ${description}`;
|
|
clearTableSelect.appendChild(option2);
|
|
});
|
|
|
|
} catch (error) {
|
|
console.error('Error loading available files:', error);
|
|
showAlert('Error loading available file types: ' + error.message, 'danger');
|
|
}
|
|
}
|
|
|
|
async function loadImportStatus() {
|
|
try {
|
|
const response = await window.http.wrappedFetch('/api/import/status');
|
|
|
|
if (response.status === 401 || response.status === 403) {
|
|
console.error('Authentication error - redirecting to login');
|
|
window.location.href = '/login';
|
|
return;
|
|
}
|
|
|
|
if (!response.ok) {
|
|
const errorData = await response.json().catch(() => ({}));
|
|
throw new Error(errorData.error?.message || `HTTP ${response.status}: ${response.statusText}`);
|
|
}
|
|
|
|
const status = await response.json();
|
|
displayImportStatus(status);
|
|
|
|
} catch (error) {
|
|
console.error('Error loading import status:', error);
|
|
document.getElementById('importStatus').innerHTML =
|
|
'<div class="p-4 bg-danger-100 dark:bg-danger-900/30 text-danger-700 dark:text-danger-300 rounded-lg">Error loading import status: ' + error.message + '</div>';
|
|
}
|
|
}
|
|
|
|
function displayImportStatus(status) {
|
|
const container = document.getElementById('importStatus');
|
|
|
|
let html = '<div class="grid grid-cols-1 md:grid-cols-3 gap-4">';
|
|
let totalRecords = 0;
|
|
|
|
Object.entries(status).forEach(([fileType, info]) => {
|
|
totalRecords += info.record_count || 0;
|
|
|
|
const statusClass = info.error ? 'danger' : (info.record_count > 0 ? 'success' : 'neutral');
|
|
const statusBg = info.error ? 'bg-danger-100 dark:bg-danger-900/30 border-danger-200 dark:border-danger-800' :
|
|
(info.record_count > 0 ? 'bg-success-100 dark:bg-success-900/30 border-success-200 dark:border-success-800' :
|
|
'bg-neutral-100 dark:bg-neutral-900/30 border-neutral-200 dark:border-neutral-800');
|
|
const statusIcon = info.error ? 'triangle-exclamation text-danger-600' :
|
|
(info.record_count > 0 ? 'circle-check text-success-600' : 'circle text-neutral-600');
|
|
|
|
html += `
|
|
<div class="bg-white dark:bg-neutral-800 border ${statusBg} rounded-lg p-4">
|
|
<div class="flex justify-between items-start">
|
|
<div>
|
|
<h6 class="font-semibold text-sm text-neutral-900 dark:text-neutral-100">${fileType}</h6>
|
|
<p class="text-xs text-neutral-600 dark:text-neutral-400">${info.table_name}</p>
|
|
</div>
|
|
<div class="text-right">
|
|
<i class="fa-solid fa-${statusIcon} text-lg"></i>
|
|
<p class="font-bold text-sm text-neutral-900 dark:text-neutral-100 mt-1">${info.record_count || 0}</p>
|
|
</div>
|
|
</div>
|
|
${info.error ? `<p class="text-xs text-danger-600 dark:text-danger-400 mt-2">${info.error}</p>` : ''}
|
|
</div>
|
|
`;
|
|
});
|
|
|
|
html += '</div>';
|
|
html += `<div class="mt-4 text-center">
|
|
<span class="font-medium text-neutral-900 dark:text-neutral-100">Total Records: ${totalRecords.toLocaleString()}</span>
|
|
</div>`;
|
|
|
|
container.innerHTML = html;
|
|
}
|
|
|
|
function updateFileTypeDescription() {
|
|
const fileType = document.getElementById('fileType').value;
|
|
const description = availableFiles.descriptions && availableFiles.descriptions[fileType];
|
|
document.getElementById('fileTypeDescription').textContent = description || '';
|
|
}
|
|
|
|
async function validateFile() {
|
|
const flexibleOnly = document.getElementById('flexibleOnly').checked;
|
|
const fileType = document.getElementById('fileType').value;
|
|
const fileInput = document.getElementById('csvFile');
|
|
|
|
if ((!flexibleOnly && !fileType) || !fileInput.files[0]) {
|
|
showAlert('Please select both data type and CSV file', 'warning');
|
|
return;
|
|
}
|
|
|
|
const formData = new FormData();
|
|
formData.append('file', fileInput.files[0]);
|
|
|
|
try {
|
|
showProgress(true, 'Validating file...');
|
|
|
|
const endpoint = flexibleOnly ? '/api/import/upload-flexible' : `/api/import/validate/${fileType}`;
|
|
const method = flexibleOnly ? 'POST' : 'POST';
|
|
const response = await window.http.wrappedFetch(endpoint, {
|
|
method: 'POST',
|
|
body: formData
|
|
});
|
|
|
|
if (!response.ok) {
|
|
const error = await response.json();
|
|
throw new Error(error.detail || 'Validation failed');
|
|
}
|
|
|
|
const result = await response.json();
|
|
if (flexibleOnly) {
|
|
// Synthesize a validation-like display for flexible-only
|
|
displayValidationResults({
|
|
valid: true,
|
|
headers: {
|
|
found: result.auto_mapping?.unmapped_headers || [],
|
|
mapped: {},
|
|
unmapped: result.auto_mapping?.unmapped_headers || [],
|
|
},
|
|
sample_data: [],
|
|
validation_errors: [],
|
|
total_errors: 0,
|
|
auto_mapping: { suggestions: {} },
|
|
});
|
|
} else {
|
|
displayValidationResults(result);
|
|
}
|
|
|
|
} catch (error) {
|
|
console.error('Validation error:', error);
|
|
showAlert('Validation failed: ' + error.message, 'danger');
|
|
} finally {
|
|
showProgress(false);
|
|
}
|
|
}
|
|
|
|
function displayValidationResults(result) {
|
|
const panel = document.getElementById('validationPanel');
|
|
const container = document.getElementById('validationResults');
|
|
|
|
let html = '';
|
|
|
|
// Overall status
|
|
const statusClass = result.valid ? 'success' : 'danger';
|
|
const statusIcon = result.valid ? 'circle-check text-success-600' : 'triangle-exclamation text-danger-600';
|
|
|
|
html += `
|
|
<div class="p-4 bg-${statusClass}-100 dark:bg-${statusClass}-900/30 rounded-lg mb-4">
|
|
<i class="fa-solid fa-${statusIcon} mr-2"></i>
|
|
<span class="font-medium">File validation ${result.valid ? 'passed' : 'completed with issues'}</span>
|
|
</div>
|
|
`;
|
|
|
|
// Auto-discovery mapping summary
|
|
html += '<h6 class="text-sm font-semibold mb-2">Auto-Discovery Mapping</h6>';
|
|
const mapped = (result.headers && result.headers.mapped) || {};
|
|
const unmapped = (result.headers && result.headers.unmapped) || [];
|
|
const suggestions = (result.auto_mapping && result.auto_mapping.suggestions) || {};
|
|
const mappedCount = Object.keys(mapped).length;
|
|
const unmappedCount = unmapped.length;
|
|
html += `<div class="p-3 bg-neutral-100 dark:bg-neutral-900/30 rounded-lg mb-3 text-sm">
|
|
<div>Mapped columns: <strong>${mappedCount}</strong> | Unmapped columns: <strong>${unmappedCount}</strong></div>
|
|
</div>`;
|
|
|
|
if (mappedCount > 0) {
|
|
html += '<div class="overflow-x-auto mb-3"><table class="w-full text-sm"><thead><tr class="bg-neutral-100 dark:bg-neutral-700"><th class="px-3 py-2 text-left font-medium">CSV Column</th><th class="px-3 py-2 text-left font-medium">Mapped To</th></tr></thead><tbody class="divide-y divide-neutral-200 dark:divide-neutral-700">';
|
|
Object.entries(mapped).forEach(([csvCol, dbField]) => {
|
|
html += `<tr><td class="px-3 py-2">${csvCol}</td><td class="px-3 py-2 font-mono">${dbField}</td></tr>`;
|
|
});
|
|
html += '</tbody></table></div>';
|
|
}
|
|
if (unmappedCount > 0) {
|
|
html += '<div class="p-3 bg-warning-100 dark:bg-warning-900/30 rounded-lg mb-2">Some columns were not recognized and will be stored as flexible JSON data:</div>';
|
|
html += '<div class="overflow-x-auto"><table class="w-full text-sm"><thead><tr class="bg-neutral-100 dark:bg-neutral-700"><th class="px-3 py-2 text-left font-medium">Unmapped CSV Column</th><th class="px-3 py-2 text-left font-medium">Top Suggestions</th></tr></thead><tbody class="divide-y divide-neutral-200 dark:divide-neutral-700">';
|
|
unmapped.forEach(col => {
|
|
const sug = suggestions[col] || [];
|
|
const sugText = sug.map(([name, score]) => `${name} (${(score*100).toFixed(0)}%)`).join(', ');
|
|
html += `<tr><td class="px-3 py-2">${col}</td><td class="px-3 py-2 text-neutral-600 dark:text-neutral-400">${sugText || '—'}</td></tr>`;
|
|
});
|
|
html += '</tbody></table></div>';
|
|
}
|
|
|
|
// Sample data
|
|
if (result.sample_data && result.sample_data.length > 0) {
|
|
html += '<h6 class="text-sm font-semibold mb-2 mt-4">Sample Data (First 10 rows)</h6>';
|
|
html += '<div class="overflow-x-auto"><table class="w-full text-sm text-neutral-900 dark:text-neutral-100">';
|
|
html += '<thead><tr class="bg-neutral-100 dark:bg-neutral-700">';
|
|
Object.keys(result.sample_data[0]).forEach(header => {
|
|
html += `<th class="px-3 py-2 text-left font-medium">${header}</th>`;
|
|
});
|
|
html += '</tr></thead><tbody class="divide-y divide-neutral-200 dark:divide-neutral-700">';
|
|
|
|
result.sample_data.forEach(row => {
|
|
html += '<tr>';
|
|
Object.values(row).forEach(value => {
|
|
html += `<td class="px-3 py-2">${value || ''}</td>`;
|
|
});
|
|
html += '</tr>';
|
|
});
|
|
html += '</tbody></table></div>';
|
|
}
|
|
|
|
// Validation errors
|
|
if (result.validation_errors && result.validation_errors.length > 0) {
|
|
html += '<h6 class="text-sm font-semibold mb-2 mt-4">Data Issues Found</h6>';
|
|
html += '<div class="p-3 bg-warning-100 dark:bg-warning-900/30 rounded-lg">';
|
|
result.validation_errors.forEach(error => {
|
|
html += `<div class="text-sm"><strong>Row ${error.row}, Field "${error.field}":</strong> ${error.error}</div>`;
|
|
});
|
|
if (result.total_errors > result.validation_errors.length) {
|
|
html += `<div class="mt-2 text-sm font-medium">... and ${result.total_errors - result.validation_errors.length} more errors</div>`;
|
|
}
|
|
html += '</div>';
|
|
}
|
|
|
|
container.innerHTML = html;
|
|
panel.classList.remove('hidden');
|
|
}
|
|
|
|
async function handleImport(event) {
|
|
event.preventDefault();
|
|
|
|
if (importInProgress) {
|
|
showAlert('Import already in progress', 'warning');
|
|
return;
|
|
}
|
|
|
|
const flexibleOnly = document.getElementById('flexibleOnly').checked;
|
|
const fileType = document.getElementById('fileType').value;
|
|
const fileInput = document.getElementById('csvFile');
|
|
const replaceExisting = document.getElementById('replaceExisting').checked;
|
|
|
|
if ((!flexibleOnly && !fileType) || !fileInput.files[0]) {
|
|
showAlert('Please select both data type and CSV file', 'warning');
|
|
return;
|
|
}
|
|
|
|
importInProgress = true;
|
|
|
|
const formData = new FormData();
|
|
formData.append('file', fileInput.files[0]);
|
|
formData.append('replace_existing', replaceExisting);
|
|
|
|
try {
|
|
showProgress(true, 'Importing data...');
|
|
|
|
let response;
|
|
if (flexibleOnly) {
|
|
response = await window.http.wrappedFetch('/api/import/upload-flexible', { method: 'POST', body: formData });
|
|
} else {
|
|
response = await window.http.wrappedFetch(`/api/import/upload/${fileType}`, { method: 'POST', body: formData });
|
|
}
|
|
|
|
if (!response.ok) {
|
|
const error = await response.json();
|
|
throw new Error(error.detail || 'Import failed');
|
|
}
|
|
|
|
const result = await response.json();
|
|
displayImportResults(result);
|
|
|
|
// Refresh status after successful import
|
|
await loadImportStatus();
|
|
|
|
// Reset form
|
|
document.getElementById('importForm').reset();
|
|
|
|
} catch (error) {
|
|
console.error('Import error:', error);
|
|
showAlert('Import failed: ' + error.message, 'danger');
|
|
} finally {
|
|
importInProgress = false;
|
|
showProgress(false);
|
|
}
|
|
}
|
|
|
|
function displayImportResults(result) {
|
|
const panel = document.getElementById('resultsPanel');
|
|
const container = document.getElementById('importResults');
|
|
|
|
const successClass = result.errors && result.errors.length > 0 ? 'warning' : 'success';
|
|
|
|
let html = `
|
|
<div class="p-4 bg-${successClass}-100 dark:bg-${successClass}-900/30 rounded-lg mb-4">
|
|
<h6 class="font-semibold flex items-center gap-2"><i class="fa-regular fa-circle-check"></i> Import Completed</h6>
|
|
<div class="text-sm mt-2 space-y-1">
|
|
<p><strong>File Type:</strong> ${result.file_type}</p>
|
|
<p><strong>Records Imported:</strong> ${result.imported_count}</p>
|
|
<p><strong>Errors:</strong> ${result.total_errors || 0}</p>
|
|
</div>
|
|
</div>
|
|
`;
|
|
|
|
if (result.auto_mapping) {
|
|
const mappedCount = Object.keys(result.auto_mapping.mapped_headers || {}).length;
|
|
const unmappedCount = (result.auto_mapping.unmapped_headers || []).length;
|
|
const flexSaved = result.auto_mapping.flexible_saved_rows || 0;
|
|
html += `
|
|
<div class="p-3 bg-neutral-100 dark:bg-neutral-900/30 rounded-lg mb-4 text-sm">
|
|
<strong>Auto-Discovery Summary</strong>
|
|
<div class="mt-1">Mapped columns: ${mappedCount} | Unmapped stored as flexible JSON: ${unmappedCount}</div>
|
|
<div>Rows with flexible data saved: ${flexSaved}</div>
|
|
</div>
|
|
`;
|
|
}
|
|
|
|
if (result.errors && result.errors.length > 0) {
|
|
html += '<h6 class="text-sm font-semibold mb-2">Import Errors</h6>';
|
|
html += '<div class="p-3 bg-danger-100 dark:bg-danger-900/30 rounded-lg">';
|
|
result.errors.forEach(error => {
|
|
html += `<div class="text-sm"><strong>Row ${error.row}:</strong> ${error.error}</div>`;
|
|
});
|
|
if (result.total_errors > result.errors.length) {
|
|
html += `<div class="mt-2 text-sm font-medium">... and ${result.total_errors - result.errors.length} more errors</div>`;
|
|
}
|
|
html += '</div>';
|
|
}
|
|
|
|
container.innerHTML = html;
|
|
panel.classList.remove('hidden');
|
|
}
|
|
|
|
function showProgress(show, message = '') {
|
|
const panel = document.getElementById('progressPanel');
|
|
const status = document.getElementById('progressStatus');
|
|
const bar = document.getElementById('progressBar');
|
|
|
|
if (show) {
|
|
status.textContent = message;
|
|
bar.style.width = '100%';
|
|
panel.classList.remove('hidden');
|
|
} else {
|
|
panel.classList.add('hidden');
|
|
}
|
|
}
|
|
|
|
async function clearTable() {
|
|
const fileType = document.getElementById('clearTableType').value;
|
|
|
|
if (!fileType) {
|
|
showAlert('Please select a table to clear', 'warning');
|
|
return;
|
|
}
|
|
|
|
if (!confirm(`Are you sure you want to clear all data from ${fileType}? This action cannot be undone.`)) {
|
|
return;
|
|
}
|
|
|
|
try {
|
|
const response = await window.http.wrappedFetch(`/api/import/clear/${fileType}`, {
|
|
method: 'DELETE'
|
|
});
|
|
|
|
if (!response.ok) {
|
|
const error = await response.json();
|
|
throw new Error(error.detail || 'Clear operation failed');
|
|
}
|
|
|
|
const result = await response.json();
|
|
showAlert(`Successfully cleared ${result.deleted_count} records from ${result.table_name}`, 'success');
|
|
|
|
// Refresh status
|
|
await loadImportStatus();
|
|
|
|
} catch (error) {
|
|
console.error('Clear table error:', error);
|
|
showAlert('Clear operation failed: ' + error.message, 'danger');
|
|
}
|
|
}
|
|
|
|
function downloadBackup() {
|
|
showAlert('Backup functionality coming soon', 'info');
|
|
}
|
|
|
|
function viewLogs() {
|
|
showAlert('Import logs functionality coming soon', 'info');
|
|
}
|
|
|
|
function switchUploadMode() {
|
|
const mode = document.getElementById('uploadMode').value;
|
|
const singleForm = document.querySelector('.single-upload');
|
|
const batchForm = document.querySelector('.batch-upload');
|
|
|
|
console.log('Switch mode to:', mode);
|
|
console.log('Single form found:', !!singleForm);
|
|
console.log('Batch form found:', !!batchForm);
|
|
|
|
if (mode === 'batch') {
|
|
if (singleForm) singleForm.classList.add('hidden');
|
|
if (batchForm) batchForm.classList.remove('hidden');
|
|
console.log('Switched to batch mode');
|
|
} else {
|
|
if (singleForm) singleForm.classList.remove('hidden');
|
|
if (batchForm) batchForm.classList.add('hidden');
|
|
console.log('Switched to single mode');
|
|
}
|
|
}
|
|
|
|
function showImportHelp() {
|
|
const tips = `
|
|
<div class="space-y-2 text-sm">
|
|
<div class="p-2 bg-neutral-100 dark:bg-neutral-900/40 rounded">
|
|
<strong>Recommended flow:</strong>
|
|
<ol class="list-decimal list-inside mt-1 space-y-1">
|
|
<li>Choose Batch Upload mode.</li>
|
|
<li>Select all exported CSVs at once (Cmd+A/ Ctrl+A).</li>
|
|
<li>Keep file names exactly as exported (e.g., STATES.csv, GRUPLKUP.csv, …).</li>
|
|
<li>Optionally Validate All first to catch header/format issues.</li>
|
|
<li>Click Import All Files.</li>
|
|
</ol>
|
|
</div>
|
|
<div>
|
|
Files will be imported in dependency order automatically:
|
|
<code class="block mt-1">STATES.csv → GRUPLKUP.csv → EMPLOYEE.csv → FILETYPE.csv → FILESTAT.csv → TRNSTYPE.csv → TRNSLKUP.csv → FOOTERS.csv → SETUP.csv → PRINTERS.csv → ROLODEX.csv → PHONE.csv → FILES.csv → LEDGER.csv → TRNSACTN.csv → QDROS.csv → PENSIONS.csv → PLANINFO.csv → PAYMENTS.csv → DEPOSITS.csv → FILENOTS.csv → FORM_INX.csv → FORM_LST.csv → FVARLKUP.csv → RVARLKUP.csv</code>
|
|
</div>
|
|
<div>
|
|
Unrecognized columns are saved as flexible JSON automatically. Unknown CSVs fall back to flexible-only storage.
|
|
</div>
|
|
<div class="text-xs text-neutral-500 mt-1">
|
|
Tip: Use Replace Existing to clear a table before importing its file.
|
|
</div>
|
|
</div>`;
|
|
if (window.alerts && window.alerts.show) {
|
|
window.alerts.show(tips, 'info', { html: true, duration: 0, title: 'Import Help' });
|
|
} else if (window.showNotification) {
|
|
window.showNotification('See import tips in console', 'info');
|
|
console.log('[Import Help]', tips);
|
|
} else {
|
|
alert('Batch mode → select all CSVs → Validate (optional) → Import. Files auto-ordered. Unknown columns saved as flexible.');
|
|
}
|
|
}
|
|
|
|
async function validateAllFiles() {
|
|
const fileInput = document.getElementById('batchFiles');
|
|
|
|
if (!fileInput.files || fileInput.files.length === 0) {
|
|
showAlert('Please select at least one CSV file', 'warning');
|
|
return;
|
|
}
|
|
|
|
if (fileInput.files.length > 25) {
|
|
showAlert('Maximum 25 files allowed per batch', 'warning');
|
|
return;
|
|
}
|
|
|
|
showProgress(true, 'Validating all selected files...');
|
|
|
|
try {
|
|
const formData = new FormData();
|
|
for (let file of fileInput.files) {
|
|
formData.append('files', file);
|
|
}
|
|
|
|
const response = await window.http.wrappedFetch('/api/import/batch-validate', {
|
|
method: 'POST',
|
|
body: formData
|
|
});
|
|
|
|
if (!response.ok) {
|
|
const error = await response.json();
|
|
throw new Error(error.detail || 'Batch validation failed');
|
|
}
|
|
|
|
const result = await response.json();
|
|
displayBatchValidationResults(result.batch_validation_results, result.summary.all_valid);
|
|
|
|
} catch (error) {
|
|
console.error('Batch validation error:', error);
|
|
showAlert('Batch validation failed: ' + error.message, 'danger');
|
|
} finally {
|
|
showProgress(false);
|
|
}
|
|
}
|
|
|
|
function displayBatchValidationResults(results, allValid) {
|
|
const panel = document.getElementById('validationPanel');
|
|
const container = document.getElementById('validationResults');
|
|
|
|
const statusClass = allValid ? 'success' : 'warning';
|
|
const statusIcon = allValid ? 'circle-check text-success-600' : 'triangle-exclamation text-warning-600';
|
|
|
|
const validCount = results.filter(r => r.valid).length;
|
|
const invalidCount = results.filter(r => !r.valid && r.error !== 'Unsupported file type').length;
|
|
const errorCount = results.filter(r => r.error && !r.valid).length;
|
|
const unsupportedCount = results.filter(r => r.error === 'Unsupported file type').length;
|
|
|
|
let html = `
|
|
<div class="p-4 bg-${statusClass}-100 dark:bg-${statusClass}-900/30 rounded-lg mb-4">
|
|
<div class="flex items-center gap-2">
|
|
<i class="fa-solid fa-${statusIcon}"></i>
|
|
<span class="font-medium">Batch validation ${allValid ? 'passed' : 'completed with issues'}</span>
|
|
</div>
|
|
<div class="text-sm mt-2">
|
|
Validated ${results.length} files:
|
|
<span class="text-success-600 dark:text-success-400">${validCount} valid</span>,
|
|
<span class="text-warning-600 dark:text-warning-400">${invalidCount} invalid</span>,
|
|
<span class="text-danger-600 dark:text-danger-400">${errorCount} errors</span>,
|
|
<span class="text-neutral-600 dark:text-neutral-400">${unsupportedCount} unsupported</span>
|
|
</div>
|
|
</div>
|
|
`;
|
|
|
|
html += '<h6 class="text-sm font-semibold mb-3">File Validation Details</h6>';
|
|
html += '<div class="space-y-2">';
|
|
|
|
results.forEach(result => {
|
|
let resultClass, resultIcon, status;
|
|
|
|
if (result.valid) {
|
|
resultClass = 'success';
|
|
resultIcon = 'circle-check';
|
|
status = 'valid';
|
|
} else if (result.error === 'Unsupported file type') {
|
|
resultClass = 'neutral';
|
|
resultIcon = 'circle-info';
|
|
status = 'unsupported';
|
|
} else if (result.error && result.error.includes('failed')) {
|
|
resultClass = 'danger';
|
|
resultIcon = 'circle-xmark';
|
|
status = 'error';
|
|
} else {
|
|
resultClass = 'warning';
|
|
resultIcon = 'triangle-exclamation';
|
|
status = 'invalid';
|
|
}
|
|
|
|
html += `
|
|
<div class="p-3 bg-${resultClass}-100 dark:bg-${resultClass}-900/30 rounded-lg">
|
|
<div class="flex items-center justify-between">
|
|
<div class="flex items-center gap-2">
|
|
<i class="fa-solid fa-${resultIcon} text-${resultClass}-600 dark:text-${resultClass}-400"></i>
|
|
<strong class="text-sm">${result.file_type}</strong>
|
|
</div>
|
|
<div class="text-sm text-${resultClass}-600 dark:text-${resultClass}-400 capitalize">${status}</div>
|
|
</div>
|
|
`;
|
|
|
|
if (result.headers && result.headers.mapped) {
|
|
const mappedCount = Object.keys(result.headers.mapped).length;
|
|
const unmappedCount = (result.headers.unmapped || []).length;
|
|
html += `<p class="text-xs text-neutral-600 dark:text-neutral-400 mt-1">${mappedCount} mapped, ${unmappedCount} unmapped</p>`;
|
|
}
|
|
|
|
if (result.total_errors > 0) {
|
|
html += `<p class="text-xs text-neutral-600 dark:text-neutral-400 mt-1">${result.total_errors} data validation errors found</p>`;
|
|
}
|
|
|
|
if (result.error) {
|
|
html += `<p class="text-xs text-${resultClass}-600 dark:text-${resultClass}-400 mt-1">${result.error}</p>`;
|
|
}
|
|
|
|
html += `</div>`;
|
|
});
|
|
|
|
html += '</div>';
|
|
|
|
if (allValid) {
|
|
html += `
|
|
<div class="mt-4 p-3 bg-success-100 dark:bg-success-900/30 rounded-lg">
|
|
<div class="flex items-center gap-2">
|
|
<i class="fa-solid fa-thumbs-up text-success-600"></i>
|
|
<span class="text-sm font-medium text-success-700 dark:text-success-300">All files passed validation! Ready for import.</span>
|
|
</div>
|
|
</div>
|
|
`;
|
|
} else {
|
|
html += `
|
|
<div class="mt-4 p-3 bg-warning-100 dark:bg-warning-900/30 rounded-lg">
|
|
<div class="flex items-center gap-2">
|
|
<i class="fa-solid fa-exclamation-triangle text-warning-600"></i>
|
|
<span class="text-sm font-medium text-warning-700 dark:text-warning-300">Some files have issues. Review the details above before importing.</span>
|
|
</div>
|
|
</div>
|
|
`;
|
|
}
|
|
|
|
container.innerHTML = html;
|
|
panel.classList.remove('hidden');
|
|
|
|
// Scroll to validation panel
|
|
panel.scrollIntoView({ behavior: 'smooth', block: 'start' });
|
|
}
|
|
|
|
function updateSelectedFiles() {
|
|
const fileInput = document.getElementById('batchFiles');
|
|
const countSpan = document.getElementById('selectedFilesCount');
|
|
const filesList = document.getElementById('selectedFilesList');
|
|
const filesListContent = document.getElementById('filesList');
|
|
|
|
const files = Array.from(fileInput.files);
|
|
countSpan.textContent = `${files.length} files selected`;
|
|
|
|
if (files.length > 0) {
|
|
// Define import order
|
|
const importOrder = [
|
|
"STATES.csv", "GRUPLKUP.csv", "EMPLOYEE.csv", "FILETYPE.csv", "FILESTAT.csv",
|
|
"TRNSTYPE.csv", "TRNSLKUP.csv", "FOOTERS.csv", "SETUP.csv", "PRINTERS.csv",
|
|
"ROLODEX.csv", "PHONE.csv", "FILES.csv", "LEDGER.csv", "TRNSACTN.csv",
|
|
"QDROS.csv", "PENSIONS.csv", "PLANINFO.csv", "PAYMENTS.csv", "DEPOSITS.csv",
|
|
"FILENOTS.csv", "FORM_INX.csv", "FORM_LST.csv", "FVARLKUP.csv", "RVARLKUP.csv"
|
|
];
|
|
|
|
// Sort files by import order
|
|
const orderedFiles = [];
|
|
const fileMap = {};
|
|
files.forEach(file => fileMap[file.name] = file);
|
|
|
|
importOrder.forEach(fileName => {
|
|
if (fileMap[fileName]) {
|
|
orderedFiles.push(fileMap[fileName]);
|
|
delete fileMap[fileName];
|
|
}
|
|
});
|
|
|
|
// Add remaining files
|
|
Object.values(fileMap).forEach(file => orderedFiles.push(file));
|
|
|
|
// Display ordered list
|
|
let html = '<div class="space-y-1">';
|
|
orderedFiles.forEach((file, index) => {
|
|
const isSupported = availableFiles.available_files && availableFiles.available_files.includes(file.name);
|
|
const statusClass = isSupported ? 'text-success-600 dark:text-success-400' : 'text-warning-600 dark:text-warning-400';
|
|
const statusIcon = isSupported ? 'circle-check' : 'triangle-exclamation';
|
|
|
|
html += `
|
|
<div class="flex items-center gap-2 text-sm">
|
|
<span class="text-neutral-500 dark:text-neutral-400 font-mono">${(index + 1).toString().padStart(2, '0')}.</span>
|
|
<i class="fa-solid fa-${statusIcon} ${statusClass}"></i>
|
|
<span class="text-neutral-900 dark:text-neutral-100">${file.name}</span>
|
|
<span class="text-xs text-neutral-500 dark:text-neutral-400">(${(file.size / 1024).toFixed(1)}KB)</span>
|
|
</div>
|
|
`;
|
|
});
|
|
html += '</div>';
|
|
|
|
filesListContent.innerHTML = html;
|
|
filesList.classList.remove('hidden');
|
|
} else {
|
|
filesList.classList.add('hidden');
|
|
}
|
|
}
|
|
|
|
function clearBatchSelection() {
|
|
document.getElementById('batchFiles').value = '';
|
|
updateSelectedFiles();
|
|
}
|
|
|
|
async function handleBatchImport(event) {
|
|
event.preventDefault();
|
|
|
|
if (importInProgress) {
|
|
showAlert('Import already in progress', 'warning');
|
|
return;
|
|
}
|
|
|
|
const fileInput = document.getElementById('batchFiles');
|
|
const replaceExisting = document.getElementById('batchReplaceExisting').checked;
|
|
const validateFirst = document.getElementById('validateAllFirst').checked;
|
|
|
|
if (!fileInput.files || fileInput.files.length === 0) {
|
|
showAlert('Please select at least one CSV file', 'warning');
|
|
return;
|
|
}
|
|
|
|
if (fileInput.files.length > 25) {
|
|
showAlert('Maximum 25 files allowed per batch', 'warning');
|
|
return;
|
|
}
|
|
|
|
importInProgress = true;
|
|
|
|
// Validate all files first if option is selected
|
|
if (validateFirst) {
|
|
try {
|
|
showProgress(true, 'Pre-validating all files before import...');
|
|
|
|
const validationResults = [];
|
|
let hasErrors = false;
|
|
|
|
for (let i = 0; i < fileInput.files.length; i++) {
|
|
const file = fileInput.files[i];
|
|
const fileName = file.name;
|
|
|
|
showProgress(true, `Pre-validating ${fileName} (${i + 1}/${fileInput.files.length})...`);
|
|
|
|
if (availableFiles.available_files && availableFiles.available_files.includes(fileName)) {
|
|
try {
|
|
const formData = new FormData();
|
|
formData.append('file', file);
|
|
|
|
const response = await window.http.wrappedFetch(`/api/import/validate/${fileName}`, {
|
|
method: 'POST',
|
|
body: formData
|
|
});
|
|
|
|
if (response.ok) {
|
|
const result = await response.json();
|
|
if (!result.valid) {
|
|
hasErrors = true;
|
|
validationResults.push({ fileName, valid: false, errors: result.validation_errors || [] });
|
|
}
|
|
} else {
|
|
hasErrors = true;
|
|
validationResults.push({ fileName, valid: false, errors: ['Validation request failed'] });
|
|
}
|
|
} catch (error) {
|
|
hasErrors = true;
|
|
validationResults.push({ fileName, valid: false, errors: [error.message] });
|
|
}
|
|
} else {
|
|
hasErrors = true;
|
|
validationResults.push({ fileName, valid: false, errors: ['Unsupported file type'] });
|
|
}
|
|
}
|
|
|
|
if (hasErrors) {
|
|
importInProgress = false;
|
|
showProgress(false);
|
|
|
|
let errorMessage = 'Pre-validation found issues in the following files:\n\n';
|
|
validationResults.forEach(result => {
|
|
if (!result.valid) {
|
|
errorMessage += `• ${result.fileName}: ${result.errors.join(', ')}\n`;
|
|
}
|
|
});
|
|
errorMessage += '\nPlease fix these issues before importing, or disable "Validate all files before import" to proceed anyway.';
|
|
|
|
showAlert(errorMessage, 'danger');
|
|
return;
|
|
} else {
|
|
showAlert('All files passed pre-validation. Proceeding with import...', 'success');
|
|
}
|
|
} catch (error) {
|
|
importInProgress = false;
|
|
showProgress(false);
|
|
showAlert('Pre-validation failed: ' + error.message, 'danger');
|
|
return;
|
|
}
|
|
}
|
|
|
|
// Continue with import if validation passed or was skipped
|
|
|
|
const formData = new FormData();
|
|
for (let file of fileInput.files) {
|
|
formData.append('files', file);
|
|
}
|
|
formData.append('replace_existing', replaceExisting);
|
|
|
|
try {
|
|
showProgress(true, 'Processing batch import...');
|
|
|
|
const response = await window.http.wrappedFetch('/api/import/batch-upload', {
|
|
method: 'POST',
|
|
body: formData
|
|
});
|
|
|
|
if (!response.ok) {
|
|
const error = await response.json();
|
|
throw new Error(error.detail || 'Batch import failed');
|
|
}
|
|
|
|
const result = await response.json();
|
|
displayBatchResults(result);
|
|
|
|
// Refresh status after successful import
|
|
await loadImportStatus();
|
|
|
|
// Reset form
|
|
document.getElementById('batchImportForm').reset();
|
|
updateSelectedFiles();
|
|
|
|
} catch (error) {
|
|
console.error('Batch import error:', error);
|
|
showAlert('Batch import failed: ' + error.message, 'danger');
|
|
} finally {
|
|
importInProgress = false;
|
|
showProgress(false);
|
|
}
|
|
}
|
|
|
|
function displayBatchResults(result) {
|
|
const panel = document.getElementById('batchResultsPanel');
|
|
const container = document.getElementById('batchResults');
|
|
|
|
const summary = result.summary;
|
|
const successRate = ((summary.total_imported / (summary.total_imported + summary.total_errors)) * 100) || 0;
|
|
|
|
let html = `
|
|
<div class="p-4 bg-info-100 dark:bg-info-900/30 rounded-lg mb-4">
|
|
<h6 class="font-semibold flex items-center gap-2"><i class="fa-solid fa-layer-group"></i> Batch Import Completed</h6>
|
|
<div class="grid grid-cols-2 md:grid-cols-4 gap-4 text-sm mt-3">
|
|
<div>
|
|
<strong>Total Files:</strong> ${summary.total_files}
|
|
</div>
|
|
<div>
|
|
<strong>Successful:</strong> <span class="text-success-600 dark:text-success-400">${summary.successful_files}</span>
|
|
</div>
|
|
<div>
|
|
<strong>Failed:</strong> <span class="text-danger-600 dark:text-danger-400">${summary.failed_files}</span>
|
|
</div>
|
|
<div>
|
|
<strong>Total Records:</strong> ${summary.total_imported.toLocaleString()}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
`;
|
|
|
|
// Individual file results
|
|
html += '<h6 class="text-sm font-semibold mb-3">File Import Details</h6>';
|
|
html += '<div class="space-y-2">';
|
|
|
|
result.batch_results.forEach(fileResult => {
|
|
let statusClass, statusIcon;
|
|
|
|
if (fileResult.status === 'success') {
|
|
statusClass = 'success';
|
|
statusIcon = 'circle-check';
|
|
} else if (fileResult.status === 'completed_with_errors') {
|
|
statusClass = 'warning';
|
|
statusIcon = 'triangle-exclamation';
|
|
} else if (fileResult.status === 'skipped') {
|
|
statusClass = 'info';
|
|
statusIcon = 'circle-info';
|
|
} else {
|
|
statusClass = 'danger';
|
|
statusIcon = 'circle-xmark';
|
|
}
|
|
|
|
html += `
|
|
<div class="p-3 bg-${statusClass}-100 dark:bg-${statusClass}-900/30 rounded-lg">
|
|
<div class="flex items-center justify-between">
|
|
<div class="flex items-center gap-2">
|
|
<i class="fa-solid fa-${statusIcon} text-${statusClass}-600 dark:text-${statusClass}-400"></i>
|
|
<strong class="text-sm">${fileResult.file_type}</strong>
|
|
</div>
|
|
<div class="text-right text-sm">
|
|
${fileResult.imported_count ? `<span class="text-success-600 dark:text-success-400">${fileResult.imported_count} imported</span>` : ''}
|
|
${fileResult.errors ? `<span class="text-danger-600 dark:text-danger-400 ml-2">${fileResult.errors} errors</span>` : ''}
|
|
</div>
|
|
</div>
|
|
<p class="text-sm text-neutral-600 dark:text-neutral-400 mt-1">${fileResult.message}</p>
|
|
${fileResult.auto_mapping ? `
|
|
<div class=\"mt-2 text-xs text-neutral-600 dark:text-neutral-400\">
|
|
<span>${Object.keys(fileResult.auto_mapping.mapped_headers || {}).length} mapped</span>
|
|
<span class=\"ml-2\">${(fileResult.auto_mapping.unmapped_headers || []).length} unmapped (stored as flexible)</span>
|
|
</div>
|
|
` : ''}
|
|
</div>
|
|
`;
|
|
});
|
|
|
|
html += '</div>';
|
|
|
|
container.innerHTML = html;
|
|
panel.classList.remove('hidden');
|
|
}
|
|
|
|
async function loadRecentBatches(append) {
|
|
try {
|
|
const params = new URLSearchParams();
|
|
params.set('limit', String(recentState.limit));
|
|
params.set('offset', String(recentState.offset));
|
|
if (recentState.status && recentState.status !== 'all') params.set('status', recentState.status);
|
|
if (recentState.start) params.set('start', recentState.start);
|
|
if (recentState.end) params.set('end', recentState.end);
|
|
const resp = await window.http.wrappedFetch(`/api/import/recent-batches?${params.toString()}`);
|
|
if (!resp.ok) return;
|
|
const data = await resp.json();
|
|
const rows = (data.recent || []).map(r => `
|
|
<tr class="hover:bg-neutral-50 dark:hover:bg-neutral-800 cursor-pointer" onclick="viewAuditDetails(${r.id})">
|
|
<td class="px-3 py-2 text-sm"><span class="inline-block px-2 py-0.5 rounded ${r.status === 'success' ? 'bg-green-100 text-green-700' : (r.status === 'completed_with_errors' ? 'bg-yellow-100 text-yellow-700' : (r.status === 'running' ? 'bg-blue-100 text-blue-700' : 'bg-red-100 text-red-700'))}">${r.status}</span></td>
|
|
<td class="px-3 py-2 text-sm">${r.started_at ? new Date(r.started_at).toLocaleString() : ''}</td>
|
|
<td class="px-3 py-2 text-sm">${r.finished_at ? new Date(r.finished_at).toLocaleString() : ''}</td>
|
|
<td class="px-3 py-2 text-sm">${r.successful_files}/${r.total_files}</td>
|
|
<td class="px-3 py-2 text-sm">${Number(r.total_imported || 0).toLocaleString()}</td>
|
|
<td class="px-3 py-2 text-right text-sm"><button class="px-2 py-1 border rounded" onclick="event.stopPropagation();downloadAuditJson(${r.id})">JSON</button></td>
|
|
</tr>
|
|
`).join('');
|
|
|
|
if (!append) {
|
|
const html = `
|
|
<div class="mb-3 flex flex-wrap items-end gap-2">
|
|
<div>
|
|
<label class="block text-xs text-neutral-600 dark:text-neutral-400 mb-1">Status</label>
|
|
<select id="recentStatusFilter" class="px-2 py-1 bg-white dark:bg-neutral-800 border border-neutral-300 dark:border-neutral-600 rounded text-sm">
|
|
<option value="all" ${recentState.status==='all'?'selected':''}>All</option>
|
|
<option value="running" ${recentState.status==='running'?'selected':''}>Running</option>
|
|
<option value="success" ${recentState.status==='success'?'selected':''}>Success</option>
|
|
<option value="completed_with_errors" ${recentState.status==='completed_with_errors'?'selected':''}>Completed with errors</option>
|
|
<option value="failed" ${recentState.status==='failed'?'selected':''}>Failed</option>
|
|
</select>
|
|
</div>
|
|
<div>
|
|
<label class="block text-xs text-neutral-600 dark:text-neutral-400 mb-1">Start</label>
|
|
<input id="recentStartFilter" type="datetime-local" class="px-2 py-1 bg-white dark:bg-neutral-800 border border-neutral-300 dark:border-neutral-600 rounded text-sm" value="${recentState.start || ''}">
|
|
</div>
|
|
<div>
|
|
<label class="block text-xs text-neutral-600 dark:text-neutral-400 mb-1">End</label>
|
|
<input id="recentEndFilter" type="datetime-local" class="px-2 py-1 bg-white dark:bg-neutral-800 border border-neutral-300 dark:border-neutral-600 rounded text-sm" value="${recentState.end || ''}">
|
|
</div>
|
|
<div class="ml-auto">
|
|
<button id="recentApplyBtn" class="px-3 py-1.5 bg-neutral-100 dark:bg-neutral-700 hover:bg-neutral-200 dark:hover:bg-neutral-600 rounded">Apply</button>
|
|
</div>
|
|
</div>
|
|
<div class="overflow-x-auto">
|
|
<table class="w-full text-sm border border-neutral-200 dark:border-neutral-700 rounded">
|
|
<thead class="bg-neutral-50 dark:bg-neutral-800">
|
|
<tr>
|
|
<th class="px-3 py-2 text-left">Status</th>
|
|
<th class="px-3 py-2 text-left">Started</th>
|
|
<th class="px-3 py-2 text-left">Finished</th>
|
|
<th class="px-3 py-2 text-left">Files</th>
|
|
<th class="px-3 py-2 text-left">Imported</th>
|
|
<th class="px-3 py-2 text-right">Actions</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody id="recentBatchesTableBody">
|
|
${rows || '<tr><td class="px-3 py-3 text-neutral-500" colspan="6">No batch uploads</td></tr>'}
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
<div class="mt-3 text-center">
|
|
<button id="recentLoadMoreBtn" class="px-3 py-1.5 border rounded ${((data.offset||0)+(data.recent?.length||0)) >= (data.total||0) ? 'opacity-50 cursor-not-allowed' : ''}">Load more</button>
|
|
<span class="ml-2 text-xs text-neutral-500">Showing ${(data.offset||0)+(data.recent?.length||0)} of ${data.total||0}</span>
|
|
</div>
|
|
`;
|
|
document.getElementById('recentBatches').innerHTML = html;
|
|
|
|
const statusEl = document.getElementById('recentStatusFilter');
|
|
const startEl = document.getElementById('recentStartFilter');
|
|
const endEl = document.getElementById('recentEndFilter');
|
|
const applyBtn = document.getElementById('recentApplyBtn');
|
|
const loadMoreBtn = document.getElementById('recentLoadMoreBtn');
|
|
if (applyBtn) applyBtn.addEventListener('click', (e) => {
|
|
e.preventDefault();
|
|
recentState.status = statusEl.value || 'all';
|
|
recentState.start = startEl.value || '';
|
|
recentState.end = endEl.value || '';
|
|
recentState.offset = 0;
|
|
loadRecentBatches(false);
|
|
});
|
|
if (loadMoreBtn) loadMoreBtn.addEventListener('click', (e) => {
|
|
e.preventDefault();
|
|
if (((data.offset||0)+(data.recent?.length||0)) >= (data.total||0)) return;
|
|
recentState.offset = (data.offset||0) + (data.recent?.length||0);
|
|
loadRecentBatches(true);
|
|
});
|
|
} else {
|
|
const tbody = document.getElementById('recentBatchesTableBody');
|
|
if (tbody) tbody.insertAdjacentHTML('beforeend', rows);
|
|
const showing = recentState.offset + (data.recent?.length || 0);
|
|
const loadMoreBtn = document.getElementById('recentLoadMoreBtn');
|
|
if (loadMoreBtn && showing >= (data.total||0)) {
|
|
loadMoreBtn.classList.add('opacity-50', 'cursor-not-allowed');
|
|
}
|
|
}
|
|
} catch (_) {}
|
|
}
|
|
|
|
async function viewAuditDetails(auditId) {
|
|
try {
|
|
const resp = await window.http.wrappedFetch(`/api/import/recent-batches/${auditId}`);
|
|
if (!resp.ok) return;
|
|
const data = await resp.json();
|
|
const files = (data.files || []).map(f => `
|
|
<tr>
|
|
<td class="px-3 py-2 text-sm font-mono">${f.file_type}</td>
|
|
<td class="px-3 py-2 text-sm"><span class="inline-block px-2 py-0.5 rounded ${f.status === 'success' ? 'bg-green-100 text-green-700' : (f.status === 'completed_with_errors' ? 'bg-yellow-100 text-yellow-700' : 'bg-red-100 text-red-700')}">${f.status}</span></td>
|
|
<td class="px-3 py-2 text-sm">${f.imported_count}</td>
|
|
<td class="px-3 py-2 text-sm">${f.errors}</td>
|
|
<td class="px-3 py-2 text-sm">${f.message || ''}</td>
|
|
</tr>
|
|
`).join('');
|
|
const hasFailed = Number(data.audit.failed_files || 0) > 0;
|
|
const content = `
|
|
<div class="space-y-2 text-sm">
|
|
<div><strong>Status:</strong> ${data.audit.status}</div>
|
|
<div><strong>Started:</strong> ${data.audit.started_at ? new Date(data.audit.started_at).toLocaleString() : ''}</div>
|
|
<div><strong>Finished:</strong> ${data.audit.finished_at ? new Date(data.audit.finished_at).toLocaleString() : ''}</div>
|
|
<div><strong>Files:</strong> ${data.audit.successful_files}/${data.audit.total_files}
|
|
<button class="ml-2 px-2 py-1 border rounded" onclick="downloadAuditJson(${data.audit.id})">Download JSON</button>
|
|
${hasFailed ? `<button class="ml-2 px-2 py-1 border rounded" onclick="rerunFailedFiles(${data.audit.id})">Rerun failed files</button>` : ''}
|
|
</div>
|
|
<div class="overflow-x-auto mt-2">
|
|
<table class="w-full text-sm border border-neutral-200 dark:border-neutral-700 rounded">
|
|
<thead class="bg-neutral-50 dark:bg-neutral-800">
|
|
<tr>
|
|
<th class="px-3 py-2 text-left">File</th>
|
|
<th class="px-3 py-2 text-left">Status</th>
|
|
<th class="px-3 py-2 text-left">Imported</th>
|
|
<th class="px-3 py-2 text-left">Errors</th>
|
|
<th class="px-3 py-2 text-left">Message</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
${files || '<tr><td class="px-3 py-3 text-neutral-500" colspan="5">No file records</td></tr>'}
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</div>
|
|
`;
|
|
if (window.alerts && window.alerts.show) {
|
|
window.alerts.show(content, 'info', { html: true, duration: 0, title: `Batch #${data.audit.id}` });
|
|
} else {
|
|
alert(`Batch ${data.audit.id}: ${data.audit.status}`);
|
|
}
|
|
} catch (_) {}
|
|
}
|
|
|
|
async function downloadAuditJson(auditId) {
|
|
try {
|
|
const resp = await window.http.wrappedFetch(`/api/import/recent-batches/${auditId}`);
|
|
if (!resp.ok) return;
|
|
const data = await resp.json();
|
|
const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' });
|
|
const url = URL.createObjectURL(blob);
|
|
const a = document.createElement('a');
|
|
a.href = url;
|
|
a.download = `import_audit_${auditId}.json`;
|
|
document.body.appendChild(a);
|
|
a.click();
|
|
URL.revokeObjectURL(url);
|
|
a.remove();
|
|
} catch (_) {}
|
|
}
|
|
|
|
async function rerunFailedFiles(auditId) {
|
|
try {
|
|
const confirmReplace = confirm('Replace existing records for these file types before rerun? Click OK to replace, Cancel to append.');
|
|
const formData = new FormData();
|
|
if (confirmReplace) formData.append('replace_existing', 'true');
|
|
showProgress(true, 'Re-running failed files...');
|
|
const resp = await window.http.wrappedFetch(`/api/import/recent-batches/${auditId}/rerun-failed`, { method: 'POST', body: formData });
|
|
if (!resp.ok) {
|
|
const err = await resp.json().catch(() => ({}));
|
|
throw new Error(err.detail || 'Rerun failed');
|
|
}
|
|
const result = await resp.json();
|
|
displayBatchResults(result);
|
|
await loadRecentBatches(false);
|
|
showAlert('Rerun completed', 'success');
|
|
} catch (e) {
|
|
console.error('Rerun failed', e);
|
|
showAlert('Rerun failed: ' + (e?.message || 'Unknown error'), 'danger');
|
|
} finally {
|
|
showProgress(false);
|
|
}
|
|
}
|
|
|
|
function showAlert(message, type = 'info') {
|
|
if (window.alerts && typeof window.alerts.show === 'function') {
|
|
window.alerts.show(message, type);
|
|
} else if (window.showNotification) {
|
|
window.showNotification(message, type);
|
|
} else {
|
|
alert(String(message));
|
|
}
|
|
}
|
|
</script>
|
|
{% endblock %} |