584 lines
22 KiB
HTML
584 lines
22 KiB
HTML
{% extends "base.html" %}
|
|
|
|
{% block title %}Data Import - Delphi Database{% endblock %}
|
|
|
|
{% block content %}
|
|
<div class="container-fluid">
|
|
<div class="row">
|
|
<div class="col-12">
|
|
<div class="d-flex justify-content-between align-items-center mb-3">
|
|
<h2><i class="bi bi-upload"></i> Data Import</h2>
|
|
<div>
|
|
<button class="btn btn-info" id="refreshStatusBtn">
|
|
<i class="bi bi-arrow-clockwise"></i> Refresh Status
|
|
</button>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Import Status Panel -->
|
|
<div class="card mb-4">
|
|
<div class="card-header">
|
|
<h5 class="mb-0"><i class="bi bi-info-circle"></i> Current Database Status</h5>
|
|
</div>
|
|
<div class="card-body">
|
|
<div id="importStatus">
|
|
<div class="text-center">
|
|
<div class="spinner-border text-primary" role="status">
|
|
<span class="visually-hidden">Loading...</span>
|
|
</div>
|
|
<p class="mt-2">Loading import status...</p>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- CSV File Upload Panel -->
|
|
<div class="card mb-4">
|
|
<div class="card-header">
|
|
<h5 class="mb-0"><i class="bi bi-file-earmark-arrow-up"></i> Upload CSV Files</h5>
|
|
</div>
|
|
<div class="card-body">
|
|
<form id="importForm" enctype="multipart/form-data">
|
|
<div class="row g-3">
|
|
<div class="col-md-4">
|
|
<label for="fileType" class="form-label">Data Type *</label>
|
|
<select class="form-select" id="fileType" name="fileType" required>
|
|
<option value="">Select data type...</option>
|
|
</select>
|
|
<div class="form-text" id="fileTypeDescription"></div>
|
|
</div>
|
|
<div class="col-md-6">
|
|
<label for="csvFile" class="form-label">CSV File *</label>
|
|
<input type="file" class="form-control" id="csvFile" name="csvFile" accept=".csv" required>
|
|
<div class="form-text">Select the CSV file to import</div>
|
|
</div>
|
|
<div class="col-md-2">
|
|
<label class="form-label"> </label>
|
|
<div class="form-check">
|
|
<input class="form-check-input" type="checkbox" id="replaceExisting" name="replaceExisting">
|
|
<label class="form-check-label" for="replaceExisting">
|
|
Replace existing data
|
|
</label>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="row mt-3">
|
|
<div class="col-12">
|
|
<div class="btn-group" role="group">
|
|
<button type="button" class="btn btn-secondary" id="validateBtn">
|
|
<i class="bi bi-check-circle"></i> Validate File
|
|
</button>
|
|
<button type="submit" class="btn btn-primary" id="importBtn">
|
|
<i class="bi bi-upload"></i> Import Data
|
|
</button>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</form>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Validation Results Panel -->
|
|
<div class="card mb-4" id="validationPanel" style="display: none;">
|
|
<div class="card-header">
|
|
<h5 class="mb-0"><i class="bi bi-clipboard-check"></i> File Validation Results</h5>
|
|
</div>
|
|
<div class="card-body" id="validationResults">
|
|
<!-- Validation results will be shown here -->
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Import Progress Panel -->
|
|
<div class="card mb-4" id="progressPanel" style="display: none;">
|
|
<div class="card-header">
|
|
<h5 class="mb-0"><i class="bi bi-hourglass-split"></i> Import Progress</h5>
|
|
</div>
|
|
<div class="card-body">
|
|
<div class="progress mb-3">
|
|
<div class="progress-bar progress-bar-striped progress-bar-animated"
|
|
role="progressbar" style="width: 0%" id="progressBar">0%</div>
|
|
</div>
|
|
<div id="progressStatus">Ready to import...</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Import Results Panel -->
|
|
<div class="card mb-4" id="resultsPanel" style="display: none;">
|
|
<div class="card-header">
|
|
<h5 class="mb-0"><i class="bi bi-check-circle-fill"></i> Import Results</h5>
|
|
</div>
|
|
<div class="card-body" id="importResults">
|
|
<!-- Import results will be shown here -->
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Data Management Panel -->
|
|
<div class="card">
|
|
<div class="card-header">
|
|
<h5 class="mb-0"><i class="bi bi-database"></i> Data Management</h5>
|
|
</div>
|
|
<div class="card-body">
|
|
<div class="row">
|
|
<div class="col-md-6">
|
|
<h6>Clear Table Data</h6>
|
|
<p class="text-muted small">Remove all records from a specific table (cannot be undone)</p>
|
|
<div class="input-group">
|
|
<select class="form-select" id="clearTableType">
|
|
<option value="">Select table to clear...</option>
|
|
</select>
|
|
<button class="btn btn-danger" id="clearTableBtn">
|
|
<i class="bi bi-trash"></i> Clear Table
|
|
</button>
|
|
</div>
|
|
</div>
|
|
<div class="col-md-6">
|
|
<h6>Quick Actions</h6>
|
|
<div class="d-grid gap-2">
|
|
<button class="btn btn-outline-warning" id="backupBtn">
|
|
<i class="bi bi-download"></i> Download Backup
|
|
</button>
|
|
<button class="btn btn-outline-info" id="viewLogsBtn">
|
|
<i class="bi bi-journal-text"></i> View Import Logs
|
|
</button>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<script>
|
|
// Import functionality
|
|
let availableFiles = {};
|
|
let importInProgress = false;
|
|
|
|
// Helper function for authenticated API calls
|
|
function getAuthHeaders() {
|
|
const token = localStorage.getItem('auth_token');
|
|
return {
|
|
'Authorization': `Bearer ${token}`
|
|
};
|
|
}
|
|
|
|
// Initialize on page load
|
|
document.addEventListener('DOMContentLoaded', function() {
|
|
// Check authentication first
|
|
const token = localStorage.getItem('auth_token');
|
|
if (!token) {
|
|
window.location.href = '/login';
|
|
return;
|
|
}
|
|
|
|
loadAvailableFiles();
|
|
loadImportStatus();
|
|
setupEventListeners();
|
|
});
|
|
|
|
function setupEventListeners() {
|
|
// Form submission
|
|
document.getElementById('importForm').addEventListener('submit', handleImport);
|
|
|
|
// Validation button
|
|
document.getElementById('validateBtn').addEventListener('click', validateFile);
|
|
|
|
// File type selection
|
|
document.getElementById('fileType').addEventListener('change', updateFileTypeDescription);
|
|
|
|
// Refresh status
|
|
document.getElementById('refreshStatusBtn').addEventListener('click', loadImportStatus);
|
|
|
|
// Clear table
|
|
document.getElementById('clearTableBtn').addEventListener('click', clearTable);
|
|
|
|
// Other buttons
|
|
document.getElementById('backupBtn').addEventListener('click', downloadBackup);
|
|
document.getElementById('viewLogsBtn').addEventListener('click', viewLogs);
|
|
}
|
|
|
|
async function loadAvailableFiles() {
|
|
try {
|
|
const response = await fetch('/api/import/available-files', {
|
|
headers: getAuthHeaders()
|
|
});
|
|
|
|
if (!response.ok) throw new Error('Failed to load available files');
|
|
|
|
const data = await response.json();
|
|
availableFiles = data;
|
|
|
|
// Populate file type dropdown
|
|
const fileTypeSelect = document.getElementById('fileType');
|
|
const clearTableSelect = document.getElementById('clearTableType');
|
|
|
|
fileTypeSelect.innerHTML = '<option value="">Select data type...</option>';
|
|
clearTableSelect.innerHTML = '<option value="">Select table to clear...</option>';
|
|
|
|
data.available_files.forEach(fileType => {
|
|
const description = data.descriptions[fileType] || fileType;
|
|
|
|
const option1 = document.createElement('option');
|
|
option1.value = fileType;
|
|
option1.textContent = `${fileType} - ${description}`;
|
|
fileTypeSelect.appendChild(option1);
|
|
|
|
const option2 = document.createElement('option');
|
|
option2.value = fileType;
|
|
option2.textContent = `${fileType} - ${description}`;
|
|
clearTableSelect.appendChild(option2);
|
|
});
|
|
|
|
} catch (error) {
|
|
console.error('Error loading available files:', error);
|
|
showAlert('Error loading available file types: ' + error.message, 'danger');
|
|
}
|
|
}
|
|
|
|
async function loadImportStatus() {
|
|
try {
|
|
const response = await fetch('/api/import/status', {
|
|
headers: getAuthHeaders()
|
|
});
|
|
|
|
if (!response.ok) throw new Error('Failed to load import status');
|
|
|
|
const status = await response.json();
|
|
displayImportStatus(status);
|
|
|
|
} catch (error) {
|
|
console.error('Error loading import status:', error);
|
|
document.getElementById('importStatus').innerHTML =
|
|
'<div class="alert alert-danger">Error loading import status: ' + error.message + '</div>';
|
|
}
|
|
}
|
|
|
|
function displayImportStatus(status) {
|
|
const container = document.getElementById('importStatus');
|
|
|
|
let html = '<div class="row">';
|
|
let totalRecords = 0;
|
|
|
|
Object.entries(status).forEach(([fileType, info], index) => {
|
|
totalRecords += info.record_count || 0;
|
|
|
|
const statusClass = info.error ? 'danger' : (info.record_count > 0 ? 'success' : 'secondary');
|
|
const statusIcon = info.error ? 'exclamation-triangle' : (info.record_count > 0 ? 'check-circle' : 'circle');
|
|
|
|
if (index % 3 === 0 && index > 0) {
|
|
html += '</div><div class="row mt-2">';
|
|
}
|
|
|
|
html += `
|
|
<div class="col-md-4">
|
|
<div class="card border-${statusClass}">
|
|
<div class="card-body p-2">
|
|
<div class="d-flex justify-content-between align-items-center">
|
|
<div>
|
|
<small class="fw-bold">${fileType}</small><br>
|
|
<small class="text-muted">${info.table_name}</small>
|
|
</div>
|
|
<div class="text-end">
|
|
<i class="bi bi-${statusIcon} text-${statusClass}"></i><br>
|
|
<small class="fw-bold">${info.record_count || 0}</small>
|
|
</div>
|
|
</div>
|
|
${info.error ? `<div class="text-danger small mt-1">${info.error}</div>` : ''}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
`;
|
|
});
|
|
|
|
html += '</div>';
|
|
html += `<div class="mt-3 text-center">
|
|
<strong>Total Records: ${totalRecords.toLocaleString()}</strong>
|
|
</div>`;
|
|
|
|
container.innerHTML = html;
|
|
}
|
|
|
|
function updateFileTypeDescription() {
|
|
const fileType = document.getElementById('fileType').value;
|
|
const description = availableFiles.descriptions && availableFiles.descriptions[fileType];
|
|
document.getElementById('fileTypeDescription').textContent = description || '';
|
|
}
|
|
|
|
async function validateFile() {
|
|
const fileType = document.getElementById('fileType').value;
|
|
const fileInput = document.getElementById('csvFile');
|
|
|
|
if (!fileType || !fileInput.files[0]) {
|
|
showAlert('Please select both file type and CSV file', 'warning');
|
|
return;
|
|
}
|
|
|
|
const formData = new FormData();
|
|
formData.append('file', fileInput.files[0]);
|
|
|
|
try {
|
|
showProgress(true, 'Validating file...');
|
|
|
|
const response = await fetch(`/api/import/validate/${fileType}`, {
|
|
method: 'POST',
|
|
headers: getAuthHeaders(),
|
|
body: formData
|
|
});
|
|
|
|
if (!response.ok) {
|
|
const error = await response.json();
|
|
throw new Error(error.detail || 'Validation failed');
|
|
}
|
|
|
|
const result = await response.json();
|
|
displayValidationResults(result);
|
|
|
|
} catch (error) {
|
|
console.error('Validation error:', error);
|
|
showAlert('Validation failed: ' + error.message, 'danger');
|
|
} finally {
|
|
showProgress(false);
|
|
}
|
|
}
|
|
|
|
function displayValidationResults(result) {
|
|
const panel = document.getElementById('validationPanel');
|
|
const container = document.getElementById('validationResults');
|
|
|
|
let html = '';
|
|
|
|
// Overall status
|
|
const statusClass = result.valid ? 'success' : 'danger';
|
|
const statusIcon = result.valid ? 'check-circle-fill' : 'exclamation-triangle-fill';
|
|
|
|
html += `
|
|
<div class="alert alert-${statusClass}">
|
|
<i class="bi bi-${statusIcon}"></i>
|
|
File validation ${result.valid ? 'passed' : 'failed'}
|
|
</div>
|
|
`;
|
|
|
|
// Headers validation
|
|
html += '<h6>Column Headers</h6>';
|
|
if (result.headers.missing.length > 0) {
|
|
html += `<div class="alert alert-warning">
|
|
<strong>Missing columns:</strong> ${result.headers.missing.join(', ')}
|
|
</div>`;
|
|
}
|
|
if (result.headers.extra.length > 0) {
|
|
html += `<div class="alert alert-info">
|
|
<strong>Extra columns:</strong> ${result.headers.extra.join(', ')}
|
|
</div>`;
|
|
}
|
|
if (result.headers.missing.length === 0 && result.headers.extra.length === 0) {
|
|
html += '<div class="alert alert-success">All expected columns found</div>';
|
|
}
|
|
|
|
// Sample data
|
|
if (result.sample_data && result.sample_data.length > 0) {
|
|
html += '<h6>Sample Data (First 10 rows)</h6>';
|
|
html += '<div class="table-responsive">';
|
|
html += '<table class="table table-sm table-striped">';
|
|
html += '<thead><tr>';
|
|
Object.keys(result.sample_data[0]).forEach(header => {
|
|
html += `<th>${header}</th>`;
|
|
});
|
|
html += '</tr></thead><tbody>';
|
|
|
|
result.sample_data.forEach(row => {
|
|
html += '<tr>';
|
|
Object.values(row).forEach(value => {
|
|
html += `<td class="small">${value || ''}</td>`;
|
|
});
|
|
html += '</tr>';
|
|
});
|
|
html += '</tbody></table></div>';
|
|
}
|
|
|
|
// Validation errors
|
|
if (result.validation_errors && result.validation_errors.length > 0) {
|
|
html += '<h6>Data Issues Found</h6>';
|
|
html += '<div class="alert alert-warning">';
|
|
result.validation_errors.forEach(error => {
|
|
html += `<div>Row ${error.row}, Field "${error.field}": ${error.error}</div>`;
|
|
});
|
|
if (result.total_errors > result.validation_errors.length) {
|
|
html += `<div class="mt-2"><strong>... and ${result.total_errors - result.validation_errors.length} more errors</strong></div>`;
|
|
}
|
|
html += '</div>';
|
|
}
|
|
|
|
container.innerHTML = html;
|
|
panel.style.display = 'block';
|
|
}
|
|
|
|
async function handleImport(event) {
|
|
event.preventDefault();
|
|
|
|
if (importInProgress) {
|
|
showAlert('Import already in progress', 'warning');
|
|
return;
|
|
}
|
|
|
|
const fileType = document.getElementById('fileType').value;
|
|
const fileInput = document.getElementById('csvFile');
|
|
const replaceExisting = document.getElementById('replaceExisting').checked;
|
|
|
|
if (!fileType || !fileInput.files[0]) {
|
|
showAlert('Please select both file type and CSV file', 'warning');
|
|
return;
|
|
}
|
|
|
|
importInProgress = true;
|
|
|
|
const formData = new FormData();
|
|
formData.append('file', fileInput.files[0]);
|
|
formData.append('replace_existing', replaceExisting);
|
|
|
|
try {
|
|
showProgress(true, 'Importing data...');
|
|
|
|
const response = await fetch(`/api/import/upload/${fileType}`, {
|
|
method: 'POST',
|
|
headers: getAuthHeaders(),
|
|
body: formData
|
|
});
|
|
|
|
if (!response.ok) {
|
|
const error = await response.json();
|
|
throw new Error(error.detail || 'Import failed');
|
|
}
|
|
|
|
const result = await response.json();
|
|
displayImportResults(result);
|
|
|
|
// Refresh status after successful import
|
|
await loadImportStatus();
|
|
|
|
// Reset form
|
|
document.getElementById('importForm').reset();
|
|
|
|
} catch (error) {
|
|
console.error('Import error:', error);
|
|
showAlert('Import failed: ' + error.message, 'danger');
|
|
} finally {
|
|
importInProgress = false;
|
|
showProgress(false);
|
|
}
|
|
}
|
|
|
|
function displayImportResults(result) {
|
|
const panel = document.getElementById('resultsPanel');
|
|
const container = document.getElementById('importResults');
|
|
|
|
const successClass = result.errors && result.errors.length > 0 ? 'warning' : 'success';
|
|
|
|
let html = `
|
|
<div class="alert alert-${successClass}">
|
|
<h6><i class="bi bi-check-circle"></i> Import Completed</h6>
|
|
<p class="mb-0">
|
|
<strong>File Type:</strong> ${result.file_type}<br>
|
|
<strong>Records Imported:</strong> ${result.imported_count}<br>
|
|
<strong>Errors:</strong> ${result.total_errors || 0}
|
|
</p>
|
|
</div>
|
|
`;
|
|
|
|
if (result.errors && result.errors.length > 0) {
|
|
html += '<h6>Import Errors</h6>';
|
|
html += '<div class="alert alert-danger">';
|
|
result.errors.forEach(error => {
|
|
html += `<div><strong>Row ${error.row}:</strong> ${error.error}</div>`;
|
|
});
|
|
if (result.total_errors > result.errors.length) {
|
|
html += `<div class="mt-2"><strong>... and ${result.total_errors - result.errors.length} more errors</strong></div>`;
|
|
}
|
|
html += '</div>';
|
|
}
|
|
|
|
container.innerHTML = html;
|
|
panel.style.display = 'block';
|
|
}
|
|
|
|
function showProgress(show, message = '') {
|
|
const panel = document.getElementById('progressPanel');
|
|
const status = document.getElementById('progressStatus');
|
|
const bar = document.getElementById('progressBar');
|
|
|
|
if (show) {
|
|
status.textContent = message;
|
|
bar.style.width = '100%';
|
|
bar.textContent = 'Processing...';
|
|
panel.style.display = 'block';
|
|
} else {
|
|
panel.style.display = 'none';
|
|
}
|
|
}
|
|
|
|
async function clearTable() {
|
|
const fileType = document.getElementById('clearTableType').value;
|
|
|
|
if (!fileType) {
|
|
showAlert('Please select a table to clear', 'warning');
|
|
return;
|
|
}
|
|
|
|
if (!confirm(`Are you sure you want to clear all data from ${fileType}? This action cannot be undone.`)) {
|
|
return;
|
|
}
|
|
|
|
try {
|
|
const response = await fetch(`/api/import/clear/${fileType}`, {
|
|
method: 'DELETE',
|
|
headers: getAuthHeaders()
|
|
});
|
|
|
|
if (!response.ok) {
|
|
const error = await response.json();
|
|
throw new Error(error.detail || 'Clear operation failed');
|
|
}
|
|
|
|
const result = await response.json();
|
|
showAlert(`Successfully cleared ${result.deleted_count} records from ${result.table_name}`, 'success');
|
|
|
|
// Refresh status
|
|
await loadImportStatus();
|
|
|
|
} catch (error) {
|
|
console.error('Clear table error:', error);
|
|
showAlert('Clear operation failed: ' + error.message, 'danger');
|
|
}
|
|
}
|
|
|
|
function downloadBackup() {
|
|
showAlert('Backup functionality coming soon', 'info');
|
|
}
|
|
|
|
function viewLogs() {
|
|
showAlert('Import logs functionality coming soon', 'info');
|
|
}
|
|
|
|
function showAlert(message, type = 'info') {
|
|
// Create and show Bootstrap alert
|
|
const alertDiv = document.createElement('div');
|
|
alertDiv.className = `alert alert-${type} alert-dismissible fade show position-fixed`;
|
|
alertDiv.style.top = '20px';
|
|
alertDiv.style.right = '20px';
|
|
alertDiv.style.zIndex = '9999';
|
|
alertDiv.innerHTML = `
|
|
${message}
|
|
<button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"alert\"></button>
|
|
`;
|
|
|
|
document.body.appendChild(alertDiv);
|
|
|
|
// Auto-dismiss after 5 seconds
|
|
setTimeout(() => {
|
|
if (alertDiv.parentNode) {
|
|
alertDiv.remove();
|
|
}
|
|
}, 5000);
|
|
}
|
|
</script>
|
|
{% endblock %} |