ready to test the import

app/api/import_data.py

@@ -10,9 +10,15 @@ from sqlalchemy.orm import Session
 from app.database.base import get_db
 from app.auth.security import get_current_user
 from app.models.user import User
-from app.models import *
+from app.models.rolodex import Rolodex, Phone
+from app.models.files import File
+from app.models.ledger import Ledger
+from app.models.qdro import QDRO
+from app.models.pensions import Pension, PensionSchedule, MarriageHistory, DeathBenefit, SeparationAgreement, LifeTable, NumberTable
+from app.models.lookups import Employee, FileType, FileStatus, TransactionType, TransactionCode, State, GroupLookup, Footer, PlanInfo, FormIndex, FormList, PrinterSetup, SystemSetup
+from app.models.additional import Payment, Deposit, FileNote, FormVariable, ReportVariable
 
-router = APIRouter(prefix="/api/import", tags=["import"])
+router = APIRouter(tags=["import"])
 
 
 # CSV to Model mapping
@@ -659,3 +665,143 @@ async def validate_csv_file(
         
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Validation failed: {str(e)}")
+
+
+@router.get("/progress/{import_id}")
+async def get_import_progress(
+    import_id: str,
+    current_user: User = Depends(get_current_user)
+):
+    """Get import progress status (placeholder for future implementation)"""
+    # This would be used for long-running imports with background tasks
+    return {
+        "import_id": import_id,
+        "status": "not_implemented",
+        "message": "Real-time progress tracking not yet implemented"
+    }
+
+
+@router.post("/batch-upload")
+async def batch_import_csv_files(
+    files: List[UploadFile] = UploadFileForm(...),
+    replace_existing: bool = Form(False),
+    db: Session = Depends(get_db),
+    current_user: User = Depends(get_current_user)
+):
+    """Import multiple CSV files in optimal order"""
+    
+    if len(files) > 20:
+        raise HTTPException(status_code=400, detail="Maximum 20 files allowed per batch")
+    
+    # Define optimal import order based on dependencies
+    import_order = [
+        "STATES.csv", "GRUPLKUP.csv", "EMPLOYEE.csv", "FILETYPE.csv", "FILESTAT.csv",
+        "TRNSTYPE.csv", "TRNSLKUP.csv", "FOOTERS.csv", "SETUP.csv", "PRINTERS.csv",
+        "ROLODEX.csv", "PHONE.csv", "FILES.csv", "LEDGER.csv", "TRNSACTN.csv",
+        "QDROS.csv", "PENSIONS.csv", "PLANINFO.csv", "PAYMENTS.csv", "DEPOSITS.csv",
+        "FILENOTS.csv", "FORM_INX.csv", "FORM_LST.csv", "FVARLKUP.csv", "RVARLKUP.csv"
+    ]
+    
+    # Sort uploaded files by optimal import order
+    file_map = {f.filename: f for f in files}
+    ordered_files = []
+    
+    for file_type in import_order:
+        if file_type in file_map:
+            ordered_files.append((file_type, file_map[file_type]))
+            del file_map[file_type]
+    
+    # Add any remaining files not in the predefined order
+    for filename, file in file_map.items():
+        ordered_files.append((filename, file))
+    
+    results = []
+    total_imported = 0
+    total_errors = 0
+    
+    for file_type, file in ordered_files:
+        if file_type not in CSV_MODEL_MAPPING:
+            results.append({
+                "file_type": file_type,
+                "status": "skipped",
+                "message": f"Unsupported file type: {file_type}"
+            })
+            continue
+        
+        try:
+            # Reset file pointer
+            await file.seek(0)
+            
+            # Import this file using simplified logic
+            model_class = CSV_MODEL_MAPPING[file_type]
+            field_mapping = FIELD_MAPPINGS.get(file_type, {})
+            
+            content = await file.read()
+            csv_content = content.decode('utf-8-sig')
+            csv_reader = csv.DictReader(io.StringIO(csv_content))
+            
+            imported_count = 0
+            errors = []
+            
+            # If replace_existing is True and this is the first file of this type
+            if replace_existing:
+                db.query(model_class).delete()
+                db.commit()
+            
+            for row_num, row in enumerate(csv_reader, start=2):
+                try:
+                    model_data = {}
+                    for csv_field, db_field in field_mapping.items():
+                        if csv_field in row:
+                            converted_value = convert_value(row[csv_field], csv_field)
+                            if converted_value is not None:
+                                model_data[db_field] = converted_value
+                    
+                    if not any(model_data.values()):
+                        continue
+                    
+                    instance = model_class(**model_data)
+                    db.add(instance)
+                    imported_count += 1
+                    
+                    if imported_count % 100 == 0:
+                        db.commit()
+                
+                except Exception as e:
+                    errors.append({
+                        "row": row_num,
+                        "error": str(e)
+                    })
+                    continue
+            
+            db.commit()
+            
+            total_imported += imported_count
+            total_errors += len(errors)
+            
+            results.append({
+                "file_type": file_type,
+                "status": "success" if len(errors) == 0 else "completed_with_errors",
+                "imported_count": imported_count,
+                "errors": len(errors),
+                "message": f"Imported {imported_count} records" + (f" with {len(errors)} errors" if errors else "")
+            })
+            
+        except Exception as e:
+            db.rollback()
+            results.append({
+                "file_type": file_type,
+                "status": "failed",
+                "message": f"Import failed: {str(e)}"
+            })
+    
+    return {
+        "batch_results": results,
+        "summary": {
+            "total_files": len(files),
+            "successful_files": len([r for r in results if r["status"] in ["success", "completed_with_errors"]]),
+            "failed_files": len([r for r in results if r["status"] == "failed"]),
+            "total_imported": total_imported,
+            "total_errors": total_errors
+        }
+    }

app/main.py

@@ -158,6 +158,15 @@ async def admin_page(request: Request):
     )
 
 
+@app.get("/import", response_class=HTMLResponse)
+async def import_page(request: Request):
+    """Data import management page (admin only)"""
+    return templates.TemplateResponse(
+        "import.html",
+        {"request": request, "title": "Data Import - " + settings.app_name}
+    )
+
+
 @app.get("/health")
 async def health_check():
     """Health check endpoint"""

templates/import.html

@@ -41,13 +41,23 @@
     <!-- CSV File Upload Panel -->
     <div class="bg-white dark:bg-neutral-800 border border-neutral-200 dark:border-neutral-700 rounded-xl shadow-soft">
         <div class="px-6 py-4 border-b border-neutral-200 dark:border-neutral-700">
-            <h5 class="text-lg font-semibold text-neutral-900 dark:text-neutral-100 flex items-center gap-2">
-                <i class="fa-regular fa-file-arrow-up"></i>
-                <span>Upload CSV Files</span>
-            </h5>
+            <div class="flex items-center justify-between">
+                <h5 class="text-lg font-semibold text-neutral-900 dark:text-neutral-100 flex items-center gap-2">
+                    <i class="fa-regular fa-file-arrow-up"></i>
+                    <span>Upload CSV Files</span>
+                </h5>
+                <div class="flex items-center gap-2">
+                    <label class="text-sm font-medium text-neutral-700 dark:text-neutral-300">Mode:</label>
+                    <select id="uploadMode" class="px-2 py-1 bg-white dark:bg-neutral-800 border border-neutral-300 dark:border-neutral-600 rounded text-sm">
+                        <option value="single">Single File</option>
+                        <option value="batch">Batch Upload</option>
+                    </select>
+                </div>
+            </div>
         </div>
         <div class="p-6">
-            <form id="importForm" enctype="multipart/form-data">
+            <!-- Single File Upload Form -->
+            <form id="importForm" enctype="multipart/form-data" class="single-upload">
                 <div class="grid grid-cols-1 md:grid-cols-12 gap-4">
                     <div class="md:col-span-4">
                         <label for="fileType" class="block text-sm font-medium text-neutral-700 dark:text-neutral-300 mb-2">Data Type *</label>
@@ -64,7 +74,7 @@
                     <div class="md:col-span-2 flex items-end">
                         <label class="inline-flex items-center gap-2">
                             <input class="h-4 w-4 text-primary-600 border-neutral-300 rounded" type="checkbox" id="replaceExisting" name="replaceExisting">
-                            <span>Replace existing data</span>
+                            <span class="text-sm">Replace existing</span>
                         </label>
                     </div>
                 </div>
@@ -82,6 +92,45 @@
                     </div>
                 </div>
             </form>
+            
+            <!-- Batch Upload Form -->
+            <form id="batchImportForm" enctype="multipart/form-data" class="batch-upload hidden">
+                <div class="space-y-4">
+                    <div>
+                        <label for="batchFiles" class="block text-sm font-medium text-neutral-700 dark:text-neutral-300 mb-2">Select Multiple CSV Files *</label>
+                        <input type="file" class="w-full px-3 py-3 bg-white dark:bg-neutral-800 border border-neutral-300 dark:border-neutral-600 rounded-lg text-neutral-900 dark:text-neutral-100 file:mr-4 file:py-2 file:px-4 file:rounded file:border-0 file:text-sm file:font-semibold file:bg-primary-100 file:text-primary-700 hover:file:bg-primary-200 transition-all duration-200" id="batchFiles" name="batchFiles" accept=".csv" multiple required>
+                        <div class="text-sm text-neutral-500 dark:text-neutral-400 mt-1">Select multiple CSV files (max 20). Files will be imported in optimal dependency order.</div>
+                    </div>
+                    
+                    <div class="grid grid-cols-1 md:grid-cols-2 gap-4">
+                        <div>
+                            <label class="inline-flex items-center gap-2">
+                                <input class="h-4 w-4 text-primary-600 border-neutral-300 rounded" type="checkbox" id="batchReplaceExisting" name="batchReplaceExisting">
+                                <span class="text-sm">Replace existing data</span>
+                            </label>
+                        </div>
+                        <div class="text-right">
+                            <span class="text-sm text-neutral-600 dark:text-neutral-400" id="selectedFilesCount">0 files selected</span>
+                        </div>
+                    </div>
+                    
+                    <div id="selectedFilesList" class="hidden">
+                        <h6 class="text-sm font-medium text-neutral-700 dark:text-neutral-300 mb-2">Selected Files (Import Order):</h6>
+                        <div class="bg-neutral-50 dark:bg-neutral-900 rounded-lg p-3 max-h-32 overflow-y-auto" id="filesList"></div>
+                    </div>
+                    
+                    <div class="flex items-center gap-3">
+                        <button type="submit" class="px-4 py-2 bg-success-600 text-white hover:bg-success-700 rounded-lg transition-colors duration-200 flex items-center gap-2" id="batchImportBtn">
+                            <i class="fa-solid fa-layer-group"></i>
+                            <span>Batch Import</span>
+                        </button>
+                        <button type="button" class="px-4 py-2 bg-neutral-100 dark:bg-neutral-700 text-neutral-700 dark:text-neutral-300 hover:bg-neutral-200 dark:hover:bg-neutral-600 rounded-lg transition-colors duration-200 flex items-center gap-2" id="clearBatchBtn">
+                            <i class="fa-solid fa-xmark"></i>
+                            <span>Clear Selection</span>
+                        </button>
+                    </div>
+                </div>
+            </form>
         </div>
     </div>
 
@@ -127,6 +176,19 @@
         </div>
     </div>
     
+    <!-- Batch Results Panel -->
+    <div class="bg-white dark:bg-neutral-800 border border-neutral-200 dark:border-neutral-700 rounded-xl shadow-soft hidden" id="batchResultsPanel">
+        <div class="px-6 py-4 border-b border-neutral-200 dark:border-neutral-700">
+            <h5 class="text-lg font-semibold text-neutral-900 dark:text-neutral-100 flex items-center gap-2">
+                <i class="fa-solid fa-layer-group"></i>
+                <span>Batch Import Results</span>
+            </h5>
+        </div>
+        <div class="p-6" id="batchResults">
+            <!-- Batch import results will be shown here -->
+        </div>
+    </div>
+
     <!-- Data Management Panel -->
     <div class="bg-white dark:bg-neutral-800 border border-neutral-200 dark:border-neutral-700 rounded-xl shadow-soft">
         <div class="px-6 py-4 border-b border-neutral-200 dark:border-neutral-700">
@@ -198,6 +260,10 @@ document.addEventListener('DOMContentLoaded', function() {
 function setupEventListeners() {
     // Form submission
     document.getElementById('importForm').addEventListener('submit', handleImport);
+    document.getElementById('batchImportForm').addEventListener('submit', handleBatchImport);
+    
+    // Upload mode switching
+    document.getElementById('uploadMode').addEventListener('change', switchUploadMode);
     
     // Validation button
     document.getElementById('validateBtn').addEventListener('click', validateFile);
@@ -205,6 +271,10 @@ function setupEventListeners() {
     // File type selection
     document.getElementById('fileType').addEventListener('change', updateFileTypeDescription);
     
+    // Batch file selection
+    document.getElementById('batchFiles').addEventListener('change', updateSelectedFiles);
+    document.getElementById('clearBatchBtn').addEventListener('click', clearBatchSelection);
+    
     // Refresh status
     document.getElementById('refreshStatusBtn').addEventListener('click', loadImportStatus);
     
@@ -222,7 +292,16 @@ async function loadAvailableFiles() {
             headers: getAuthHeaders()
         });
         
-        if (!response.ok) throw new Error('Failed to load available files');
+        if (response.status === 401 || response.status === 403) {
+            console.error('Authentication error - redirecting to login');
+            window.location.href = '/login';
+            return;
+        }
+        
+        if (!response.ok) {
+            const errorData = await response.json().catch(() => ({}));
+            throw new Error(errorData.error?.message || `HTTP ${response.status}: ${response.statusText}`);
+        }
         
         const data = await response.json();
         availableFiles = data;
@@ -260,7 +339,16 @@ async function loadImportStatus() {
             headers: getAuthHeaders()
         });
         
-        if (!response.ok) throw new Error('Failed to load import status');
+        if (response.status === 401 || response.status === 403) {
+            console.error('Authentication error - redirecting to login');
+            window.location.href = '/login';
+            return;
+        }
+        
+        if (!response.ok) {
+            const errorData = await response.json().catch(() => ({}));
+            throw new Error(errorData.error?.message || `HTTP ${response.status}: ${response.statusText}`);
+        }
         
         const status = await response.json();
         displayImportStatus(status);
@@ -571,6 +659,217 @@ function viewLogs() {
     showAlert('Import logs functionality coming soon', 'info');
 }
 
+function switchUploadMode() {
+    const mode = document.getElementById('uploadMode').value;
+    const singleForm = document.querySelector('.single-upload');
+    const batchForm = document.querySelector('.batch-upload');
+    
+    if (mode === 'batch') {
+        singleForm.classList.add('hidden');
+        batchForm.classList.remove('hidden');
+    } else {
+        singleForm.classList.remove('hidden');
+        batchForm.classList.add('hidden');
+    }
+}
+
+function updateSelectedFiles() {
+    const fileInput = document.getElementById('batchFiles');
+    const countSpan = document.getElementById('selectedFilesCount');
+    const filesList = document.getElementById('selectedFilesList');
+    const filesListContent = document.getElementById('filesList');
+    
+    const files = Array.from(fileInput.files);
+    countSpan.textContent = `${files.length} files selected`;
+    
+    if (files.length > 0) {
+        // Define import order
+        const importOrder = [
+            "STATES.csv", "GRUPLKUP.csv", "EMPLOYEE.csv", "FILETYPE.csv", "FILESTAT.csv",
+            "TRNSTYPE.csv", "TRNSLKUP.csv", "FOOTERS.csv", "SETUP.csv", "PRINTERS.csv",
+            "ROLODEX.csv", "PHONE.csv", "FILES.csv", "LEDGER.csv", "TRNSACTN.csv",
+            "QDROS.csv", "PENSIONS.csv", "PLANINFO.csv", "PAYMENTS.csv", "DEPOSITS.csv",
+            "FILENOTS.csv", "FORM_INX.csv", "FORM_LST.csv", "FVARLKUP.csv", "RVARLKUP.csv"
+        ];
+        
+        // Sort files by import order
+        const orderedFiles = [];
+        const fileMap = {};
+        files.forEach(file => fileMap[file.name] = file);
+        
+        importOrder.forEach(fileName => {
+            if (fileMap[fileName]) {
+                orderedFiles.push(fileMap[fileName]);
+                delete fileMap[fileName];
+            }
+        });
+        
+        // Add remaining files
+        Object.values(fileMap).forEach(file => orderedFiles.push(file));
+        
+        // Display ordered list
+        let html = '<div class="space-y-1">';
+        orderedFiles.forEach((file, index) => {
+            const isSupported = availableFiles.available_files && availableFiles.available_files.includes(file.name);
+            const statusClass = isSupported ? 'text-success-600 dark:text-success-400' : 'text-warning-600 dark:text-warning-400';
+            const statusIcon = isSupported ? 'circle-check' : 'triangle-exclamation';
+            
+            html += `
+                <div class="flex items-center gap-2 text-sm">
+                    <span class="text-neutral-500 dark:text-neutral-400 font-mono">${(index + 1).toString().padStart(2, '0')}.</span>
+                    <i class="fa-solid fa-${statusIcon} ${statusClass}"></i>
+                    <span class="text-neutral-900 dark:text-neutral-100">${file.name}</span>
+                    <span class="text-xs text-neutral-500 dark:text-neutral-400">(${(file.size / 1024).toFixed(1)}KB)</span>
+                </div>
+            `;
+        });
+        html += '</div>';
+        
+        filesListContent.innerHTML = html;
+        filesList.classList.remove('hidden');
+    } else {
+        filesList.classList.add('hidden');
+    }
+}
+
+function clearBatchSelection() {
+    document.getElementById('batchFiles').value = '';
+    updateSelectedFiles();
+}
+
+async function handleBatchImport(event) {
+    event.preventDefault();
+    
+    if (importInProgress) {
+        showAlert('Import already in progress', 'warning');
+        return;
+    }
+    
+    const fileInput = document.getElementById('batchFiles');
+    const replaceExisting = document.getElementById('batchReplaceExisting').checked;
+    
+    if (!fileInput.files || fileInput.files.length === 0) {
+        showAlert('Please select at least one CSV file', 'warning');
+        return;
+    }
+    
+    if (fileInput.files.length > 20) {
+        showAlert('Maximum 20 files allowed per batch', 'warning');
+        return;
+    }
+    
+    importInProgress = true;
+    
+    const formData = new FormData();
+    for (let file of fileInput.files) {
+        formData.append('files', file);
+    }
+    formData.append('replace_existing', replaceExisting);
+    
+    try {
+        showProgress(true, 'Processing batch import...');
+        
+        const response = await fetch('/api/import/batch-upload', {
+            method: 'POST',
+            headers: getAuthHeaders(),
+            body: formData
+        });
+        
+        if (!response.ok) {
+            const error = await response.json();
+            throw new Error(error.detail || 'Batch import failed');
+        }
+        
+        const result = await response.json();
+        displayBatchResults(result);
+        
+        // Refresh status after successful import
+        await loadImportStatus();
+        
+        // Reset form
+        document.getElementById('batchImportForm').reset();
+        updateSelectedFiles();
+        
+    } catch (error) {
+        console.error('Batch import error:', error);
+        showAlert('Batch import failed: ' + error.message, 'danger');
+    } finally {
+        importInProgress = false;
+        showProgress(false);
+    }
+}
+
+function displayBatchResults(result) {
+    const panel = document.getElementById('batchResultsPanel');
+    const container = document.getElementById('batchResults');
+    
+    const summary = result.summary;
+    const successRate = ((summary.total_imported / (summary.total_imported + summary.total_errors)) * 100) || 0;
+    
+    let html = `
+        <div class="p-4 bg-info-100 dark:bg-info-900/30 rounded-lg mb-4">
+            <h6 class="font-semibold flex items-center gap-2"><i class="fa-solid fa-layer-group"></i> Batch Import Completed</h6>
+            <div class="grid grid-cols-2 md:grid-cols-4 gap-4 text-sm mt-3">
+                <div>
+                    <strong>Total Files:</strong> ${summary.total_files}
+                </div>
+                <div>
+                    <strong>Successful:</strong> <span class="text-success-600 dark:text-success-400">${summary.successful_files}</span>
+                </div>
+                <div>
+                    <strong>Failed:</strong> <span class="text-danger-600 dark:text-danger-400">${summary.failed_files}</span>
+                </div>
+                <div>
+                    <strong>Total Records:</strong> ${summary.total_imported.toLocaleString()}
+                </div>
+            </div>
+        </div>
+    `;
+    
+    // Individual file results
+    html += '<h6 class="text-sm font-semibold mb-3">File Import Details</h6>';
+    html += '<div class="space-y-2">';
+    
+    result.batch_results.forEach(fileResult => {
+        let statusClass, statusIcon;
+        
+        if (fileResult.status === 'success') {
+            statusClass = 'success';
+            statusIcon = 'circle-check';
+        } else if (fileResult.status === 'completed_with_errors') {
+            statusClass = 'warning';
+            statusIcon = 'triangle-exclamation';
+        } else if (fileResult.status === 'skipped') {
+            statusClass = 'info';
+            statusIcon = 'circle-info';
+        } else {
+            statusClass = 'danger';
+            statusIcon = 'circle-xmark';
+        }
+        
+        html += `
+            <div class="p-3 bg-${statusClass}-100 dark:bg-${statusClass}-900/30 rounded-lg">
+                <div class="flex items-center justify-between">
+                    <div class="flex items-center gap-2">
+                        <i class="fa-solid fa-${statusIcon} text-${statusClass}-600 dark:text-${statusClass}-400"></i>
+                        <strong class="text-sm">${fileResult.file_type}</strong>
+                    </div>
+                    <div class="text-right text-sm">
+                        ${fileResult.imported_count ? `<span class="text-success-600 dark:text-success-400">${fileResult.imported_count} imported</span>` : ''}
+                        ${fileResult.errors ? `<span class="text-danger-600 dark:text-danger-400 ml-2">${fileResult.errors} errors</span>` : ''}
+                    </div>
+                </div>
+                <p class="text-sm text-neutral-600 dark:text-neutral-400 mt-1">${fileResult.message}</p>
+            </div>
+        `;
+    });
+    
+    html += '</div>';
+    
+    container.innerHTML = html;
+    panel.classList.remove('hidden');
+}
+
 function showAlert(message, type = 'info') {
     if (window.alerts && typeof window.alerts.show === 'function') {
         window.alerts.show(message, type);