Select Multiple CSV Files *
-
-
Select multiple CSV files (max 20). Files will be imported in optimal dependency order.
+
+
+
+ Select all your CSV files at once (max 25). Files will be validated and imported in dependency order automatically.
+ Tip: Use Ctrl+A in the file dialog to select all CSV files from your export folder.
+
-
+
Replace existing data
+ Replace existing data
+
+
+ Validate all files before import
@@ -116,13 +138,17 @@
Selected Files (Import Order):
-
+
+
+ Validate All Files
+
Batch Import
+ Import All Files
+
+
+
+
+ Recent Batch Uploads
+
+
+
+
+
+
Loading recent batches...
+
+
+
+
@@ -234,6 +276,7 @@
// Import functionality
let availableFiles = {};
let importInProgress = false;
+let recentState = { limit: 5, offset: 0, status: 'all', start: '', end: '' };
// Authorization is injected by window.http.wrappedFetch
@@ -245,10 +288,35 @@ document.addEventListener('DOMContentLoaded', function() {
window.location.href = '/login';
return;
}
+ // Ensure admin only access for this page
+ (async () => {
+ try {
+ const resp = await window.http.wrappedFetch('/api/auth/me');
+ if (!resp.ok) {
+ window.location.href = '/login';
+ return;
+ }
+ const me = await resp.json();
+ if (!me || !me.is_admin) {
+ window.location.href = '/';
+ return;
+ }
+ } catch (_) {
+ window.location.href = '/login';
+ return;
+ }
+ })();
loadAvailableFiles();
loadImportStatus();
+ loadRecentBatches(false);
setupEventListeners();
+
+ // Set batch mode as default after a short delay to ensure DOM is ready
+ setTimeout(() => {
+ document.getElementById('uploadMode').value = 'batch';
+ switchUploadMode();
+ }, 100);
});
function setupEventListeners() {
@@ -258,9 +326,12 @@ function setupEventListeners() {
// Upload mode switching
document.getElementById('uploadMode').addEventListener('change', switchUploadMode);
+ const helpBtn = document.getElementById('importHelpBtn');
+ if (helpBtn) helpBtn.addEventListener('click', showImportHelp);
- // Validation button
+ // Validation buttons
document.getElementById('validateBtn').addEventListener('click', validateFile);
+ document.getElementById('batchValidateBtn').addEventListener('click', validateAllFiles);
// File type selection
document.getElementById('fileType').addEventListener('change', updateFileTypeDescription);
@@ -278,6 +349,24 @@ function setupEventListeners() {
// Other buttons
document.getElementById('backupBtn').addEventListener('click', downloadBackup);
document.getElementById('viewLogsBtn').addEventListener('click', viewLogs);
+ const flexibleOnly = document.getElementById('flexibleOnly');
+ if (flexibleOnly) {
+ flexibleOnly.addEventListener('change', () => {
+ const isFlex = flexibleOnly.checked;
+ const fileTypeContainer = document.getElementById('fileTypeContainer');
+ const fileType = document.getElementById('fileType');
+ const hint = document.getElementById('flexibleHint');
+ if (isFlex) {
+ if (fileTypeContainer) fileTypeContainer.classList.add('opacity-50');
+ if (fileType) fileType.required = false;
+ if (hint) hint.style.display = '';
+ } else {
+ if (fileTypeContainer) fileTypeContainer.classList.remove('opacity-50');
+ if (fileType) fileType.required = true;
+ if (hint) hint.style.display = 'none';
+ }
+ });
+ }
}
async function loadAvailableFiles() {
@@ -398,10 +487,11 @@ function updateFileTypeDescription() {
}
async function validateFile() {
+ const flexibleOnly = document.getElementById('flexibleOnly').checked;
const fileType = document.getElementById('fileType').value;
const fileInput = document.getElementById('csvFile');
- if (!fileType || !fileInput.files[0]) {
+ if ((!flexibleOnly && !fileType) || !fileInput.files[0]) {
showAlert('Please select both data type and CSV file', 'warning');
return;
}
@@ -412,7 +502,9 @@ async function validateFile() {
try {
showProgress(true, 'Validating file...');
- const response = await window.http.wrappedFetch(`/api/import/validate/${fileType}`, {
+ const endpoint = flexibleOnly ? '/api/import/upload-flexible' : `/api/import/validate/${fileType}`;
+ const method = flexibleOnly ? 'POST' : 'POST';
+ const response = await window.http.wrappedFetch(endpoint, {
method: 'POST',
body: formData
});
@@ -423,7 +515,23 @@ async function validateFile() {
}
const result = await response.json();
- displayValidationResults(result);
+ if (flexibleOnly) {
+ // Synthesize a validation-like display for flexible-only
+ displayValidationResults({
+ valid: true,
+ headers: {
+ found: result.auto_mapping?.unmapped_headers || [],
+ mapped: {},
+ unmapped: result.auto_mapping?.unmapped_headers || [],
+ },
+ sample_data: [],
+ validation_errors: [],
+ total_errors: 0,
+ auto_mapping: { suggestions: {} },
+ });
+ } else {
+ displayValidationResults(result);
+ }
} catch (error) {
console.error('Validation error:', error);
@@ -446,24 +554,37 @@ function displayValidationResults(result) {
html += `
File validation ${result.valid ? 'passed' : 'failed'}
+ File validation ${result.valid ? 'passed' : 'completed with issues'}
`;
- // Headers validation
- html += '
Column Headers ';
- if (result.headers.missing.length > 0) {
- html += `
- Missing columns: ${result.headers.missing.join(', ')}
-
`;
+ // Auto-discovery mapping summary
+ html += '
Auto-Discovery Mapping ';
+ const mapped = (result.headers && result.headers.mapped) || {};
+ const unmapped = (result.headers && result.headers.unmapped) || [];
+ const suggestions = (result.auto_mapping && result.auto_mapping.suggestions) || {};
+ const mappedCount = Object.keys(mapped).length;
+ const unmappedCount = unmapped.length;
+ html += `
+
Mapped columns: ${mappedCount} | Unmapped columns: ${unmappedCount}
+
`;
+
+ if (mappedCount > 0) {
+ html += '
CSV Column Mapped To ';
+ Object.entries(mapped).forEach(([csvCol, dbField]) => {
+ html += `${csvCol} ${dbField}
';
}
- if (result.headers.extra.length > 0) {
- html += `
- Extra columns: ${result.headers.extra.join(', ')}
-
`;
- }
- if (result.headers.missing.length === 0 && result.headers.extra.length === 0) {
- html += '
All expected columns found
';
+ if (unmappedCount > 0) {
+ html += '
Some columns were not recognized and will be stored as flexible JSON data:
';
+ html += '
Unmapped CSV Column Top Suggestions ';
+ unmapped.forEach(col => {
+ const sug = suggestions[col] || [];
+ const sugText = sug.map(([name, score]) => `${name} (${(score*100).toFixed(0)}%)`).join(', ');
+ html += `${col} ${sugText || '—'}
';
}
// Sample data
@@ -511,11 +632,12 @@ async function handleImport(event) {
return;
}
+ const flexibleOnly = document.getElementById('flexibleOnly').checked;
const fileType = document.getElementById('fileType').value;
const fileInput = document.getElementById('csvFile');
const replaceExisting = document.getElementById('replaceExisting').checked;
- if (!fileType || !fileInput.files[0]) {
+ if ((!flexibleOnly && !fileType) || !fileInput.files[0]) {
showAlert('Please select both data type and CSV file', 'warning');
return;
}
@@ -529,10 +651,12 @@ async function handleImport(event) {
try {
showProgress(true, 'Importing data...');
- const response = await window.http.wrappedFetch(`/api/import/upload/${fileType}`, {
- method: 'POST',
- body: formData
- });
+ let response;
+ if (flexibleOnly) {
+ response = await window.http.wrappedFetch('/api/import/upload-flexible', { method: 'POST', body: formData });
+ } else {
+ response = await window.http.wrappedFetch(`/api/import/upload/${fileType}`, { method: 'POST', body: formData });
+ }
if (!response.ok) {
const error = await response.json();
@@ -574,6 +698,19 @@ function displayImportResults(result) {
`;
+ if (result.auto_mapping) {
+ const mappedCount = Object.keys(result.auto_mapping.mapped_headers || {}).length;
+ const unmappedCount = (result.auto_mapping.unmapped_headers || []).length;
+ const flexSaved = result.auto_mapping.flexible_saved_rows || 0;
+ html += `
+
+
Auto-Discovery Summary
+
Mapped columns: ${mappedCount} | Unmapped stored as flexible JSON: ${unmappedCount}
+
Rows with flexible data saved: ${flexSaved}
+
+ `;
+ }
+
if (result.errors && result.errors.length > 0) {
html += '
Import Errors ';
html += '
';
@@ -651,15 +788,206 @@ function switchUploadMode() {
const singleForm = document.querySelector('.single-upload');
const batchForm = document.querySelector('.batch-upload');
+ console.log('Switch mode to:', mode);
+ console.log('Single form found:', !!singleForm);
+ console.log('Batch form found:', !!batchForm);
+
if (mode === 'batch') {
- singleForm.classList.add('hidden');
- batchForm.classList.remove('hidden');
+ if (singleForm) singleForm.classList.add('hidden');
+ if (batchForm) batchForm.classList.remove('hidden');
+ console.log('Switched to batch mode');
} else {
- singleForm.classList.remove('hidden');
- batchForm.classList.add('hidden');
+ if (singleForm) singleForm.classList.remove('hidden');
+ if (batchForm) batchForm.classList.add('hidden');
+ console.log('Switched to single mode');
}
}
+function showImportHelp() {
+ const tips = `
+
+
+
Recommended flow:
+
+ Choose Batch Upload mode.
+ Select all exported CSVs at once (Cmd+A/ Ctrl+A).
+ Keep file names exactly as exported (e.g., STATES.csv, GRUPLKUP.csv, …).
+ Optionally Validate All first to catch header/format issues.
+ Click Import All Files.
+
+
+
+ Files will be imported in dependency order automatically:
+ STATES.csv → GRUPLKUP.csv → EMPLOYEE.csv → FILETYPE.csv → FILESTAT.csv → TRNSTYPE.csv → TRNSLKUP.csv → FOOTERS.csv → SETUP.csv → PRINTERS.csv → ROLODEX.csv → PHONE.csv → FILES.csv → LEDGER.csv → TRNSACTN.csv → QDROS.csv → PENSIONS.csv → PLANINFO.csv → PAYMENTS.csv → DEPOSITS.csv → FILENOTS.csv → FORM_INX.csv → FORM_LST.csv → FVARLKUP.csv → RVARLKUP.csv
+
+
+ Unrecognized columns are saved as flexible JSON automatically. Unknown CSVs fall back to flexible-only storage.
+
+
+ Tip: Use Replace Existing to clear a table before importing its file.
+
+
`;
+ if (window.alerts && window.alerts.show) {
+ window.alerts.show(tips, 'info', { html: true, duration: 0, title: 'Import Help' });
+ } else if (window.showNotification) {
+ window.showNotification('See import tips in console', 'info');
+ console.log('[Import Help]', tips);
+ } else {
+ alert('Batch mode → select all CSVs → Validate (optional) → Import. Files auto-ordered. Unknown columns saved as flexible.');
+ }
+}
+
+async function validateAllFiles() {
+ const fileInput = document.getElementById('batchFiles');
+
+ if (!fileInput.files || fileInput.files.length === 0) {
+ showAlert('Please select at least one CSV file', 'warning');
+ return;
+ }
+
+ if (fileInput.files.length > 25) {
+ showAlert('Maximum 25 files allowed per batch', 'warning');
+ return;
+ }
+
+ showProgress(true, 'Validating all selected files...');
+
+ try {
+ const formData = new FormData();
+ for (let file of fileInput.files) {
+ formData.append('files', file);
+ }
+
+ const response = await window.http.wrappedFetch('/api/import/batch-validate', {
+ method: 'POST',
+ body: formData
+ });
+
+ if (!response.ok) {
+ const error = await response.json();
+ throw new Error(error.detail || 'Batch validation failed');
+ }
+
+ const result = await response.json();
+ displayBatchValidationResults(result.batch_validation_results, result.summary.all_valid);
+
+ } catch (error) {
+ console.error('Batch validation error:', error);
+ showAlert('Batch validation failed: ' + error.message, 'danger');
+ } finally {
+ showProgress(false);
+ }
+}
+
+function displayBatchValidationResults(results, allValid) {
+ const panel = document.getElementById('validationPanel');
+ const container = document.getElementById('validationResults');
+
+ const statusClass = allValid ? 'success' : 'warning';
+ const statusIcon = allValid ? 'circle-check text-success-600' : 'triangle-exclamation text-warning-600';
+
+ const validCount = results.filter(r => r.valid).length;
+ const invalidCount = results.filter(r => !r.valid && r.error !== 'Unsupported file type').length;
+ const errorCount = results.filter(r => r.error && !r.valid).length;
+ const unsupportedCount = results.filter(r => r.error === 'Unsupported file type').length;
+
+ let html = `
+
+
+ Batch validation ${allValid ? 'passed' : 'completed with issues'}
+
+
+ Validated ${results.length} files:
+ ${validCount} valid ,
+ ${invalidCount} invalid ,
+ ${errorCount} errors ,
+ ${unsupportedCount} unsupported
+
+
+ `;
+
+ html += '
File Validation Details ';
+ html += '
';
+
+ results.forEach(result => {
+ let resultClass, resultIcon, status;
+
+ if (result.valid) {
+ resultClass = 'success';
+ resultIcon = 'circle-check';
+ status = 'valid';
+ } else if (result.error === 'Unsupported file type') {
+ resultClass = 'neutral';
+ resultIcon = 'circle-info';
+ status = 'unsupported';
+ } else if (result.error && result.error.includes('failed')) {
+ resultClass = 'danger';
+ resultIcon = 'circle-xmark';
+ status = 'error';
+ } else {
+ resultClass = 'warning';
+ resultIcon = 'triangle-exclamation';
+ status = 'invalid';
+ }
+
+ html += `
+
+
+
+ ${result.file_type}
+
+
${status}
+
+ `;
+
+ if (result.headers && result.headers.mapped) {
+ const mappedCount = Object.keys(result.headers.mapped).length;
+ const unmappedCount = (result.headers.unmapped || []).length;
+ html += `
${mappedCount} mapped, ${unmappedCount} unmapped
`;
+ }
+
+ if (result.total_errors > 0) {
+ html += `
${result.total_errors} data validation errors found
`;
+ }
+
+ if (result.error) {
+ html += `
${result.error}
`;
+ }
+
+ html += `
`;
+ });
+
+ html += '
';
+
+ if (allValid) {
+ html += `
+
+
+ All files passed validation! Ready for import.
+
+
+ `;
+ } else {
+ html += `
+
+
+ Some files have issues. Review the details above before importing.
+
+
+ `;
+ }
+
+ container.innerHTML = html;
+ panel.classList.remove('hidden');
+
+ // Scroll to validation panel
+ panel.scrollIntoView({ behavior: 'smooth', block: 'start' });
+}
+
function updateSelectedFiles() {
const fileInput = document.getElementById('batchFiles');
const countSpan = document.getElementById('selectedFilesCount');
@@ -734,19 +1062,91 @@ async function handleBatchImport(event) {
const fileInput = document.getElementById('batchFiles');
const replaceExisting = document.getElementById('batchReplaceExisting').checked;
+ const validateFirst = document.getElementById('validateAllFirst').checked;
if (!fileInput.files || fileInput.files.length === 0) {
showAlert('Please select at least one CSV file', 'warning');
return;
}
- if (fileInput.files.length > 20) {
- showAlert('Maximum 20 files allowed per batch', 'warning');
+ if (fileInput.files.length > 25) {
+ showAlert('Maximum 25 files allowed per batch', 'warning');
return;
}
importInProgress = true;
+ // Validate all files first if option is selected
+ if (validateFirst) {
+ try {
+ showProgress(true, 'Pre-validating all files before import...');
+
+ const validationResults = [];
+ let hasErrors = false;
+
+ for (let i = 0; i < fileInput.files.length; i++) {
+ const file = fileInput.files[i];
+ const fileName = file.name;
+
+ showProgress(true, `Pre-validating ${fileName} (${i + 1}/${fileInput.files.length})...`);
+
+ if (availableFiles.available_files && availableFiles.available_files.includes(fileName)) {
+ try {
+ const formData = new FormData();
+ formData.append('file', file);
+
+ const response = await window.http.wrappedFetch(`/api/import/validate/${fileName}`, {
+ method: 'POST',
+ body: formData
+ });
+
+ if (response.ok) {
+ const result = await response.json();
+ if (!result.valid) {
+ hasErrors = true;
+ validationResults.push({ fileName, valid: false, errors: result.validation_errors || [] });
+ }
+ } else {
+ hasErrors = true;
+ validationResults.push({ fileName, valid: false, errors: ['Validation request failed'] });
+ }
+ } catch (error) {
+ hasErrors = true;
+ validationResults.push({ fileName, valid: false, errors: [error.message] });
+ }
+ } else {
+ hasErrors = true;
+ validationResults.push({ fileName, valid: false, errors: ['Unsupported file type'] });
+ }
+ }
+
+ if (hasErrors) {
+ importInProgress = false;
+ showProgress(false);
+
+ let errorMessage = 'Pre-validation found issues in the following files:\n\n';
+ validationResults.forEach(result => {
+ if (!result.valid) {
+ errorMessage += `• ${result.fileName}: ${result.errors.join(', ')}\n`;
+ }
+ });
+ errorMessage += '\nPlease fix these issues before importing, or disable "Validate all files before import" to proceed anyway.';
+
+ showAlert(errorMessage, 'danger');
+ return;
+ } else {
+ showAlert('All files passed pre-validation. Proceeding with import...', 'success');
+ }
+ } catch (error) {
+ importInProgress = false;
+ showProgress(false);
+ showAlert('Pre-validation failed: ' + error.message, 'danger');
+ return;
+ }
+ }
+
+ // Continue with import if validation passed or was skipped
+
const formData = new FormData();
for (let file of fileInput.files) {
formData.append('files', file);
@@ -846,6 +1246,12 @@ function displayBatchResults(result) {
${fileResult.message}
+ ${fileResult.auto_mapping ? `
+
+ ${Object.keys(fileResult.auto_mapping.mapped_headers || {}).length} mapped
+ ${(fileResult.auto_mapping.unmapped_headers || []).length} unmapped (stored as flexible)
+
+ ` : ''}
`;
});
@@ -856,6 +1262,198 @@ function displayBatchResults(result) {
panel.classList.remove('hidden');
}
+async function loadRecentBatches(append) {
+ try {
+ const params = new URLSearchParams();
+ params.set('limit', String(recentState.limit));
+ params.set('offset', String(recentState.offset));
+ if (recentState.status && recentState.status !== 'all') params.set('status', recentState.status);
+ if (recentState.start) params.set('start', recentState.start);
+ if (recentState.end) params.set('end', recentState.end);
+ const resp = await window.http.wrappedFetch(`/api/import/recent-batches?${params.toString()}`);
+ if (!resp.ok) return;
+ const data = await resp.json();
+ const rows = (data.recent || []).map(r => `
+
+ ${r.status} ${r.started_at ? new Date(r.started_at).toLocaleString() : ''}
+ ${r.finished_at ? new Date(r.finished_at).toLocaleString() : ''}
+ ${r.successful_files}/${r.total_files}
+ ${Number(r.total_imported || 0).toLocaleString()}
+ JSON
+ `).join('');
+
+ if (!append) {
+ const html = `
+
+
+ Status
+
+ All
+ Running
+ Success
+ Completed with errors
+ Failed
+
+
+
+ Start
+
+
+ End
+
+
+ Apply
+
+
+
+
+
+
+ Status
+ Started
+ Finished
+ Files
+ Imported
+ Actions
+
+
+
+ ${rows || 'No batch uploads
+
+
+
+ Load more
+ Showing ${(data.offset||0)+(data.recent?.length||0)} of ${data.total||0}
+
+ `;
+ document.getElementById('recentBatches').innerHTML = html;
+
+ const statusEl = document.getElementById('recentStatusFilter');
+ const startEl = document.getElementById('recentStartFilter');
+ const endEl = document.getElementById('recentEndFilter');
+ const applyBtn = document.getElementById('recentApplyBtn');
+ const loadMoreBtn = document.getElementById('recentLoadMoreBtn');
+ if (applyBtn) applyBtn.addEventListener('click', (e) => {
+ e.preventDefault();
+ recentState.status = statusEl.value || 'all';
+ recentState.start = startEl.value || '';
+ recentState.end = endEl.value || '';
+ recentState.offset = 0;
+ loadRecentBatches(false);
+ });
+ if (loadMoreBtn) loadMoreBtn.addEventListener('click', (e) => {
+ e.preventDefault();
+ if (((data.offset||0)+(data.recent?.length||0)) >= (data.total||0)) return;
+ recentState.offset = (data.offset||0) + (data.recent?.length||0);
+ loadRecentBatches(true);
+ });
+ } else {
+ const tbody = document.getElementById('recentBatchesTableBody');
+ if (tbody) tbody.insertAdjacentHTML('beforeend', rows);
+ const showing = recentState.offset + (data.recent?.length || 0);
+ const loadMoreBtn = document.getElementById('recentLoadMoreBtn');
+ if (loadMoreBtn && showing >= (data.total||0)) {
+ loadMoreBtn.classList.add('opacity-50', 'cursor-not-allowed');
+ }
+ }
+ } catch (_) {}
+}
+
+async function viewAuditDetails(auditId) {
+ try {
+ const resp = await window.http.wrappedFetch(`/api/import/recent-batches/${auditId}`);
+ if (!resp.ok) return;
+ const data = await resp.json();
+ const files = (data.files || []).map(f => `
+
+ ${f.file_type}
+ ${f.status} ${f.imported_count}
+ ${f.errors}
+ ${f.message || ''}
+
+ `).join('');
+ const hasFailed = Number(data.audit.failed_files || 0) > 0;
+ const content = `
+
+
Status: ${data.audit.status}
+
Started: ${data.audit.started_at ? new Date(data.audit.started_at).toLocaleString() : ''}
+
Finished: ${data.audit.finished_at ? new Date(data.audit.finished_at).toLocaleString() : ''}
+
Files: ${data.audit.successful_files}/${data.audit.total_files}
+ Download JSON
+ ${hasFailed ? `Rerun failed files ` : ''}
+
+
+
+
+
+ File
+ Status
+ Imported
+ Errors
+ Message
+
+
+
+ ${files || 'No file records
+
+
+
+ `;
+ if (window.alerts && window.alerts.show) {
+ window.alerts.show(content, 'info', { html: true, duration: 0, title: `Batch #${data.audit.id}` });
+ } else {
+ alert(`Batch ${data.audit.id}: ${data.audit.status}`);
+ }
+ } catch (_) {}
+}
+
+async function downloadAuditJson(auditId) {
+ try {
+ const resp = await window.http.wrappedFetch(`/api/import/recent-batches/${auditId}`);
+ if (!resp.ok) return;
+ const data = await resp.json();
+ const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' });
+ const url = URL.createObjectURL(blob);
+ const a = document.createElement('a');
+ a.href = url;
+ a.download = `import_audit_${auditId}.json`;
+ document.body.appendChild(a);
+ a.click();
+ URL.revokeObjectURL(url);
+ a.remove();
+ } catch (_) {}
+}
+
+async function rerunFailedFiles(auditId) {
+ try {
+ const confirmReplace = confirm('Replace existing records for these file types before rerun? Click OK to replace, Cancel to append.');
+ const formData = new FormData();
+ if (confirmReplace) formData.append('replace_existing', 'true');
+ showProgress(true, 'Re-running failed files...');
+ const resp = await window.http.wrappedFetch(`/api/import/recent-batches/${auditId}/rerun-failed`, { method: 'POST', body: formData });
+ if (!resp.ok) {
+ const err = await resp.json().catch(() => ({}));
+ throw new Error(err.detail || 'Rerun failed');
+ }
+ const result = await resp.json();
+ displayBatchResults(result);
+ await loadRecentBatches(false);
+ showAlert('Rerun completed', 'success');
+ } catch (e) {
+ console.error('Rerun failed', e);
+ showAlert('Rerun failed: ' + (e?.message || 'Unknown error'), 'danger');
+ } finally {
+ showProgress(false);
+ }
+}
+
function showAlert(message, type = 'info') {
if (window.alerts && typeof window.alerts.show === 'function') {
window.alerts.show(message, type);
diff --git a/templates/search.html b/templates/search.html
index 6c60b83..03713b4 100644
--- a/templates/search.html
+++ b/templates/search.html
@@ -441,6 +441,8 @@ function initializeAdvancedSearch() {
});
}
+// Use shared highlight utilities
+
async function loadSearchFacets() {
try {
const response = await window.http.wrappedFetch('/api/search/facets');
@@ -744,6 +746,9 @@ function displaySearchResults(data) {
const resultsContainer = document.getElementById('searchResults');
const statusElement = document.getElementById('searchStatus');
const facetsCard = document.getElementById('facetsCard');
+ const tokens = (window.highlightUtils && typeof window.highlightUtils.buildTokens === 'function')
+ ? window.highlightUtils.buildTokens(currentSearchCriteria.query || '')
+ : [];
// Update status
const executionTime = data.stats?.search_execution_time || 0;
@@ -777,6 +782,9 @@ function displaySearchResults(data) {
data.results.forEach(result => {
const typeIcon = getTypeIcon(result.type);
const typeBadge = getTypeBadge(result.type);
+ const matchHtml = (window.highlightUtils && typeof window.highlightUtils.formatSnippet === 'function')
+ ? window.highlightUtils.formatSnippet(result.highlight, tokens)
+ : (result.highlight || '');
resultsHTML += `
@@ -787,7 +795,7 @@ function displaySearchResults(data) {
@@ -795,16 +803,20 @@ function displaySearchResults(data) {
${result.updated_at ? `${formatDate(result.updated_at)} ` : ''}
-
${result.description}
- ${result.highlight ? `
Match: ${result.highlight}
` : ''}
- ${displayResultMetadata(result.metadata)}
+
${window.highlightUtils ? window.highlightUtils.highlight(result.description || '', tokens) : (result.description || '')}
+ ${matchHtml ? `
Match: ${matchHtml}
` : ''}
+ ${displayResultMetadata(result.metadata, tokens)}
`;
});
- resultsContainer.innerHTML = resultsHTML;
+ if (window.setSafeHTML) {
+ window.setSafeHTML(resultsContainer, resultsHTML);
+ } else {
+ resultsContainer.innerHTML = resultsHTML;
+ }
// Display pagination
if (data.page_info.total_pages > 1) {
@@ -903,14 +915,23 @@ function getTypeBadge(type) {
return badges[type] || '';
}
-function displayResultMetadata(metadata) {
+function displayResultMetadata(metadata, tokens) {
if (!metadata) return '';
let metadataHTML = '
';
Object.entries(metadata).forEach(([key, value]) => {
if (value && key !== 'phones') { // Skip complex objects
- metadataHTML += `${key.replace('_', ' ')}: ${value}${label}: ${valueHtml} around matches, preserving original casing
+ if cust.get("highlight"):
+ assert "" in cust["highlight"]
+ # The word 'Search' prefix should remain with original case if present
+ assert any(tag in cust["highlight"] for tag in ["Name:", "City:", "Email:"])
+
+ # Also create a ledger entry with mixed-case note and ensure highlight
+ resp = client.post(
+ "/api/financial/ledger/",
+ json=LedgerCreate(
+ file_no=file_no,
+ date=date.today().isoformat(),
+ t_code="NOTE",
+ t_type="2",
+ empl_num="E01",
+ quantity=0.0,
+ rate=0.0,
+ amount=0.0,
+ billed="N",
+ note=f"MixedCase DoE note {token}"
+ ).model_dump(mode="json")
+ )
+ assert resp.status_code == 200
+ # Ledger search via global endpoints isn't exposed directly here, but query through legacy ledger search when available
+ # We can at least ensure files search returns highlight on regarding; ledger highlight is already unit-tested
+
+
+def _create_qdro_with_form_name(file_no: str, form_name: str) -> int:
+ db = SessionLocal()
+ try:
+ qdro = QDRO(file_no=file_no, form_name=form_name, status="DRAFT")
+ db.add(qdro)
+ db.commit()
+ db.refresh(qdro)
+ return qdro.id
+ finally:
+ db.close()
+
+
+def test_advanced_search_highlights_mixed_case_for_customer_file_qdro(client: TestClient):
+ token_mixed = f"MiXeD{uuid.uuid4().hex[:6]}"
+ token_lower = token_mixed.lower()
+
+ # Customer with mixed-case in name
+ cust_id = _create_customer(client, last_suffix=token_mixed)
+
+ # File with mixed-case in regarding
+ file_no = _create_file(client, cust_id, regarding_token=token_mixed)
+
+ # QDRO seeded directly with mixed-case in form_name
+ qdro_id = _create_qdro_with_form_name(file_no, form_name=f"Form {token_mixed} Plan")
+
+ # Advanced search across types
+ payload = {
+ "query": token_lower,
+ "search_types": ["customer", "file", "qdro"],
+ "limit": 50,
+ }
+ resp = client.post("/api/search/advanced", json=payload)
+ assert resp.status_code == 200
+ data = resp.json()
+ assert data.get("total_results", 0) >= 3
+
+ # Index by (type, id)
+ results = data["results"]
+ by_key = {(r["type"], r["id"]): r for r in results}
+
+ # Customer
+ cust_res = by_key.get(("customer", cust_id))
+ assert cust_res is not None and isinstance(cust_res.get("highlight"), str)
+ assert "" in cust_res["highlight"]
+ assert f"{token_mixed} " in cust_res["highlight"]
+
+ # File
+ file_res = by_key.get(("file", file_no))
+ assert file_res is not None and isinstance(file_res.get("highlight"), str)
+ assert "" in file_res["highlight"]
+ assert f"{token_mixed} " in file_res["highlight"]
+
+ # QDRO
+ qdro_res = by_key.get(("qdro", qdro_id))
+ assert qdro_res is not None and isinstance(qdro_res.get("highlight"), str)
+ assert "" in qdro_res["highlight"]
+ assert f"{token_mixed} " in qdro_res["highlight"]
+
+
+def test_global_search_highlights_mixed_case_for_customer_file_qdro(client: TestClient):
+ token_mixed = f"MiXeD{uuid.uuid4().hex[:6]}"
+ token_lower = token_mixed.lower()
+
+ # Seed data
+ cust_id = _create_customer(client, last_suffix=token_mixed)
+ file_no = _create_file(client, cust_id, regarding_token=token_mixed)
+ qdro_id = _create_qdro_with_form_name(file_no, form_name=f"QDRO {token_mixed} Case")
+
+ # Global search
+ resp = client.get("/api/search/global", params={"q": token_lower, "limit": 50})
+ assert resp.status_code == 200
+ data = resp.json()
+
+ # Customers
+ custs = data.get("customers", [])
+ cust = next((r for r in custs if r.get("id") == cust_id), None)
+ assert cust is not None and isinstance(cust.get("highlight"), str)
+ assert "" in cust["highlight"]
+ assert f"{token_mixed} " in cust["highlight"]
+
+ # Files
+ files = data.get("files", [])
+ fil = next((r for r in files if r.get("id") == file_no), None)
+ assert fil is not None and isinstance(fil.get("highlight"), str)
+ assert "" in fil["highlight"]
+ assert f"{token_mixed} " in fil["highlight"]
+
+ # QDROs
+ qdros = data.get("qdros", [])
+ q = next((r for r in qdros if r.get("id") == qdro_id), None)
+ assert q is not None and isinstance(q.get("highlight"), str)
+ assert "" in q["highlight"]
+ assert f"{token_mixed} " in q["highlight"]
diff --git a/tests/test_search_highlight_utils.py b/tests/test_search_highlight_utils.py
new file mode 100644
index 0000000..7b74e84
--- /dev/null
+++ b/tests/test_search_highlight_utils.py
@@ -0,0 +1,186 @@
+from app.api.search_highlight import (
+ build_query_tokens,
+ highlight_text,
+ create_customer_highlight,
+ create_file_highlight,
+ create_ledger_highlight,
+ create_qdro_highlight,
+)
+
+
+def test_build_query_tokens_dedup_and_trim():
+ tokens = build_query_tokens(' John, Smith; "Smith" (J.) ')
+ assert tokens == ['John', 'Smith', 'J']
+
+
+def test_highlight_text_case_insensitive_preserves_original():
+ out = highlight_text('John Smith', ['joHN', 'smiTH'])
+ assert out == 'John Smith '
+
+
+def test_highlight_text_overlapping_tokens():
+ out = highlight_text('Anna and Ann went', ['ann', 'anna'])
+ # Should highlight both; merged ranges will encompass 'Anna' first, then 'Ann'
+ assert 'Anna ' in out
+ assert ' and Ann went' in out
+
+
+def test_highlight_text_multiple_occurrences():
+ out = highlight_text('alpha beta alpha', ['alpha'])
+ assert out.count('alpha ') == 2
+
+
+def test_highlight_text_returns_original_when_token_absent():
+ out = highlight_text('Hello World', ['zzz'])
+ assert out == 'Hello World'
+
+
+def test_highlight_text_merges_overlapping_tokens_single_range():
+ out = highlight_text('banana', ['ana', 'nan'])
+ assert out == 'banana '
+
+def test_build_query_tokens_mixed_case_dedup_order_preserving():
+ tokens = build_query_tokens('ALPHA alpha Beta beta BETA')
+ assert tokens == ['ALPHA', 'Beta']
+
+
+def test_build_query_tokens_trims_wrapping_punctuation_and_ignores_empties():
+ tokens = build_query_tokens('...Alpha!!!, __Alpha__, (Beta); "beta";; gamma---')
+ assert tokens == ['Alpha', 'Beta', 'gamma']
+
+
+def test_build_query_tokens_empty_input():
+ assert build_query_tokens(' ') == []
+
+
+def _make_customer(**attrs):
+ obj = type("CustomerStub", (), {})()
+ for k, v in attrs.items():
+ setattr(obj, k, v)
+ return obj
+
+
+def test_create_customer_highlight_prefers_name_over_other_fields():
+ customer = _make_customer(first='John', last='Smith', email='john@example.com', city='Johnstown')
+ out = create_customer_highlight(customer, 'john')
+ assert out.startswith('Name: ')
+ assert 'Email:' not in out and 'City:' not in out
+
+
+def test_create_customer_highlight_uses_email_when_name_not_matching():
+ customer = _make_customer(first='Alice', last='Wonder', email='johnson@example.com', city='Paris')
+ out = create_customer_highlight(customer, 'john')
+ assert out.startswith('Email: ')
+
+
+def test_create_customer_highlight_uses_city_when_only_city_matches():
+ customer = _make_customer(first='Alice', last='Wonder', email='awonder@example.com', city='Ann Arbor')
+ out = create_customer_highlight(customer, 'arbor')
+ assert out.startswith('City: ')
+
+
+def test_create_customer_highlight_requires_full_query_in_single_field():
+ customer = _make_customer(first='John', last='Smith', email='js@example.com', city='Boston')
+ # 'john boston' does not occur in any single attribute; should return empty string
+ out = create_customer_highlight(customer, 'john boston')
+ assert out == ''
+
+
+def test_create_customer_highlight_highlights_both_tokens_in_full_name():
+ customer = _make_customer(first='John', last='Smith', email='js@example.com', city='Boston')
+ out = create_customer_highlight(customer, 'John Smith')
+ assert out == 'Name: John Smith '
+
+
+def _make_file(**attrs):
+ obj = type("FileStub", (), {})()
+ for k, v in attrs.items():
+ setattr(obj, k, v)
+ return obj
+
+
+def test_create_file_highlight_prefers_matter_over_type():
+ file_obj = _make_file(regarding='Divorce Matter - John Doe', file_type='QDRO')
+ out = create_file_highlight(file_obj, 'divorce')
+ assert out.startswith('Matter: ')
+ assert 'Divorce ' in out
+
+
+def test_create_file_highlight_uses_type_when_matter_not_matching():
+ file_obj = _make_file(regarding='Miscellaneous', file_type='Income Tax')
+ out = create_file_highlight(file_obj, 'tax')
+ assert out.startswith('Type: ')
+ # Preserve original casing from the source
+ assert 'Tax ' in out
+
+
+def test_create_file_highlight_returns_empty_when_no_match():
+ file_obj = _make_file(regarding='Misc', file_type='General')
+ out = create_file_highlight(file_obj, 'unrelated')
+ assert out == ''
+
+
+def _make_ledger(**attrs):
+ obj = type("LedgerStub", (), {})()
+ for k, v in attrs.items():
+ setattr(obj, k, v)
+ return obj
+
+
+def test_create_ledger_highlight_truncates_to_160_with_suffix_and_highlights():
+ prefix = 'x' * 50
+ match = 'AlphaBeta'
+ filler = 'y' * 200
+ marker_after = 'ZZZ_AFTER'
+ note_text = prefix + match + filler + marker_after
+ ledger = _make_ledger(note=note_text)
+ out = create_ledger_highlight(ledger, 'alpha')
+ assert out.startswith('Note: ')
+ # Should include highlight within the preview
+ assert 'Alpha Beta' in out
+ # Should be truncated with suffix because original length > 160
+ assert out.endswith('...')
+ # Ensure content after 160 chars (marker_after) is not present
+ assert 'ZZZ_AFTER' not in out
+
+
+def _make_qdro(**attrs):
+ obj = type("QdroStub", (), {})()
+ for k, v in attrs.items():
+ setattr(obj, k, v)
+ return obj
+
+
+def test_create_qdro_highlight_prefers_form_name_over_pet_and_case():
+ qdro = _make_qdro(form_name='Domestic Relations Form - QDRO', pet='Jane Doe', case_number='2024-XYZ')
+ out = create_qdro_highlight(qdro, 'qdro')
+ assert out.startswith('Form: ')
+ assert 'QDRO ' in out
+
+
+def test_create_qdro_highlight_uses_pet_when_form_not_matching():
+ qdro = _make_qdro(form_name='Child Support', pet='John Johnson', case_number='A-1')
+ out = create_qdro_highlight(qdro, 'john')
+ assert out.startswith('Petitioner: ')
+
+
+def test_create_qdro_highlight_uses_case_when_only_case_matches():
+ qdro = _make_qdro(form_name='Child Support', pet='Mary Jane', case_number='Case 12345')
+ out = create_qdro_highlight(qdro, 'case 123')
+ assert out.startswith('Case: ')
+ assert 'Case ' in out and '123 ' in out
+
+
+def test_create_qdro_highlight_none_or_empty_fields_return_empty():
+ qdro = _make_qdro(form_name=None, pet=None, case_number=None)
+ assert create_qdro_highlight(qdro, 'anything') == ''
+ populated = _make_qdro(form_name='Form A', pet='Pet B', case_number='C-1')
+ assert create_qdro_highlight(populated, '') == ''
+
+
+def test_create_qdro_highlight_requires_full_query_in_single_field():
+ # Tokens present across fields but not as a contiguous substring in any single field
+ qdro = _make_qdro(form_name='QDRO Plan', pet='Alpha', case_number='123')
+ out = create_qdro_highlight(qdro, 'plan 123')
+ assert out == ''
+
diff --git a/tests/test_settings_api.py b/tests/test_settings_api.py
new file mode 100644
index 0000000..92dcda7
--- /dev/null
+++ b/tests/test_settings_api.py
@@ -0,0 +1,85 @@
+import os
+
+import pytest
+from fastapi.testclient import TestClient
+
+# Ensure required env vars for app import/config
+os.environ.setdefault("SECRET_KEY", "x" * 32)
+os.environ.setdefault("DATABASE_URL", "sqlite:////tmp/delphi_test.sqlite")
+
+from app.main import app # noqa: E402
+from app.auth.security import get_current_user, get_admin_user # noqa: E402
+from tests.helpers import assert_validation_error, assert_http_error # noqa: E402
+
+
+class _User:
+ def __init__(self, is_admin: bool):
+ self.id = 1 if is_admin else 2
+ self.username = "admin" if is_admin else "user"
+ self.is_admin = is_admin
+ self.is_active = True
+ self.first_name = "Test"
+ self.last_name = "User"
+
+
+@pytest.fixture()
+def client_admin():
+ app.dependency_overrides[get_current_user] = lambda: _User(True)
+ app.dependency_overrides[get_admin_user] = lambda: _User(True)
+ try:
+ yield TestClient(app)
+ finally:
+ app.dependency_overrides.pop(get_current_user, None)
+ app.dependency_overrides.pop(get_admin_user, None)
+
+
+@pytest.fixture()
+def client_user():
+ app.dependency_overrides[get_current_user] = lambda: _User(False)
+ try:
+ yield TestClient(app)
+ finally:
+ app.dependency_overrides.pop(get_current_user, None)
+
+
+def test_get_inactivity_warning_minutes_requires_auth_and_returns_shape(client_user: TestClient):
+ # Unauthenticated should 401 envelope
+ app.dependency_overrides.pop(get_current_user, None)
+ c = TestClient(app)
+ resp = c.get("/api/settings/inactivity_warning_minutes")
+ assert_http_error(resp, 403, "Not authenticated")
+
+ # Authenticated returns minutes field
+ app.dependency_overrides[get_current_user] = lambda: _User(False)
+ resp = c.get("/api/settings/inactivity_warning_minutes")
+ assert resp.status_code == 200
+ data = resp.json()
+ assert set(data.keys()) == {"minutes"}
+ assert isinstance(data["minutes"], int)
+
+
+def test_update_theme_preference_validation_and_auth(client_user: TestClient):
+ # Invalid theme value
+ resp = client_user.post("/api/auth/theme-preference", json={"theme_preference": "blue"})
+ assert_http_error(resp, 400, "Theme preference must be 'light' or 'dark'")
+
+ # Valid update
+ resp = client_user.post("/api/auth/theme-preference", json={"theme_preference": "dark"})
+ assert resp.status_code == 200
+ body = resp.json()
+ assert body == {"message": "Theme preference updated successfully", "theme": "dark"}
+
+ # Unauthenticated should 401
+ app.dependency_overrides.pop(get_current_user, None)
+ c = TestClient(app)
+ resp = c.post("/api/auth/theme-preference", json={"theme_preference": "light"})
+ assert_http_error(resp, 403, "Not authenticated")
+
+
+# If there are admin-only settings updates later, assert 403 for non-admin.
+# Placeholder: demonstrate 403 behavior using a known admin-only endpoint (/api/auth/users)
+def test_non_admin_forbidden_on_admin_endpoints(client_user: TestClient):
+ resp = client_user.get("/api/auth/users")
+ assert_http_error(resp, 403, "Not enough permissions")
+
+
diff --git a/tests/test_support_api.py b/tests/test_support_api.py
new file mode 100644
index 0000000..6fda9f0
--- /dev/null
+++ b/tests/test_support_api.py
@@ -0,0 +1,122 @@
+import os
+import uuid
+
+import pytest
+from fastapi.testclient import TestClient
+
+# Ensure required env vars for app import/config
+os.environ.setdefault("SECRET_KEY", "x" * 32)
+os.environ.setdefault("DATABASE_URL", "sqlite:////tmp/delphi_test.sqlite")
+
+from app.main import app # noqa: E402
+from app.auth.security import get_current_user, get_admin_user # noqa: E402
+from tests.helpers import assert_validation_error, assert_http_error # noqa: E402
+
+
+@pytest.fixture(scope="module")
+def client():
+ class _Admin:
+ def __init__(self):
+ self.id = 1
+ self.username = "admin"
+ self.is_admin = True
+ self.is_active = True
+ self.first_name = "Admin"
+ self.last_name = "User"
+
+ # For public create, current_user is optional; override admin endpoints
+ app.dependency_overrides[get_admin_user] = lambda: _Admin()
+ app.dependency_overrides[get_current_user] = lambda: _Admin()
+
+ try:
+ yield TestClient(app)
+ finally:
+ app.dependency_overrides.pop(get_admin_user, None)
+ app.dependency_overrides.pop(get_current_user, None)
+
+
+def test_create_ticket_validation_errors(client: TestClient):
+ # Missing required fields
+ resp = client.post("/api/support/tickets", json={})
+ assert_validation_error(resp, "subject")
+
+ # Too short subject/description and invalid email
+ payload = {
+ "subject": "Hey",
+ "description": "short",
+ "contact_name": "",
+ "contact_email": "not-an-email",
+ }
+ resp = client.post("/api/support/tickets", json=payload)
+ assert_validation_error(resp, "subject")
+ assert_validation_error(resp, "description")
+ assert_validation_error(resp, "contact_name")
+ assert_validation_error(resp, "contact_email")
+
+
+def _valid_ticket_payload() -> dict:
+ token = uuid.uuid4().hex[:6]
+ return {
+ "subject": f"Support issue {token}",
+ "description": "A reproducible problem description long enough",
+ "category": "bug_report",
+ "priority": "medium",
+ "contact_name": "John Tester",
+ "contact_email": f"john.{token}@example.com",
+ "current_page": "/dashboard",
+ "browser_info": "pytest-agent",
+ }
+
+
+def test_ticket_lifecycle_and_404s_with_audit(client: TestClient):
+ # Create ticket (public)
+ payload = _valid_ticket_payload()
+ resp = client.post("/api/support/tickets", json=payload)
+ assert resp.status_code == 200
+ body = resp.json()
+ ticket_id = body["ticket_id"]
+ assert body["status"] == "created"
+
+ # Get ticket as admin
+ resp = client.get(f"/api/support/tickets/{ticket_id}")
+ assert resp.status_code == 200
+ detail = resp.json()
+ assert detail["status"] == "open"
+ assert isinstance(detail.get("responses"), list)
+
+ # 404 on missing ticket get/update/respond
+ resp = client.get("/api/support/tickets/999999")
+ assert_http_error(resp, 404, "Ticket not found")
+ resp = client.put("/api/support/tickets/999999", json={"status": "in_progress"})
+ assert_http_error(resp, 404, "Ticket not found")
+ resp = client.post("/api/support/tickets/999999/responses", json={"message": "x"})
+ assert_http_error(resp, 404, "Ticket not found")
+
+ # State transitions: open -> in_progress -> resolved
+ resp = client.put(f"/api/support/tickets/{ticket_id}", json={"status": "in_progress"})
+ assert resp.status_code == 200
+ resp = client.get(f"/api/support/tickets/{ticket_id}")
+ assert resp.status_code == 200
+ assert resp.json()["status"] == "in_progress"
+
+ # Add public response
+ resp = client.post(
+ f"/api/support/tickets/{ticket_id}/responses",
+ json={"message": "We are working on it", "is_internal": False},
+ )
+ assert resp.status_code == 200
+
+ # Resolve ticket
+ resp = client.put(f"/api/support/tickets/{ticket_id}", json={"status": "resolved"})
+ assert resp.status_code == 200
+ resp = client.get(f"/api/support/tickets/{ticket_id}")
+ data = resp.json()
+ assert data["status"] == "resolved"
+ assert data["resolved_at"] is not None
+
+ # Basic list with pagination params should 200
+ resp = client.get("/api/support/tickets", params={"skip": 0, "limit": 10})
+ assert resp.status_code == 200
+ assert isinstance(resp.json(), list)
+
+