From c76b68d009c8b6051fcf5ab4476584e69f8e12bc Mon Sep 17 00:00:00 2001 From: HotSwapp <47397945+HotSwapp@users.noreply.github.com> Date: Mon, 11 Aug 2025 21:58:25 -0500 Subject: [PATCH] fixing rolodex and search --- ADDRESS_VALIDATION_SERVICE.md | 304 +++++++ README.md | 10 + app/api/customers.py | 42 +- app/api/import_data.py | 175 +++- docker-compose.dev.yml | 2 +- package.json | 9 +- static/js/__tests__/alerts.test.js | 58 ++ static/js/__tests__/sanitizer.test.js | 56 ++ static/js/alerts.js | 53 +- static/js/customers-tailwind.js | 1107 ++++++++++++++----------- static/js/fetch-wrapper.js | 36 +- static/js/main.js | 61 +- static/js/sanitizer.js | 91 ++ templates/admin.html | 155 ++-- templates/base.html | 3 +- templates/customers.html | 146 ++-- templates/dashboard.html | 6 +- templates/documents.html | 72 +- templates/files.html | 75 +- templates/financial.html | 41 +- templates/import.html | 38 +- templates/login.html | 9 +- templates/search.html | 9 +- templates/support_modal.html | 5 +- test_deposits.csv | 3 + 25 files changed, 1651 insertions(+), 915 deletions(-) create mode 100644 ADDRESS_VALIDATION_SERVICE.md create mode 100644 static/js/__tests__/alerts.test.js create mode 100644 static/js/__tests__/sanitizer.test.js create mode 100644 static/js/sanitizer.js create mode 100644 test_deposits.csv diff --git a/ADDRESS_VALIDATION_SERVICE.md b/ADDRESS_VALIDATION_SERVICE.md new file mode 100644 index 0000000..fabfe3d --- /dev/null +++ b/ADDRESS_VALIDATION_SERVICE.md @@ -0,0 +1,304 @@ +# Address Validation Service Design + +## Overview +A separate Docker service for validating and standardizing addresses using a hybrid approach that prioritizes privacy and minimizes external API calls. + +## Architecture + +### Service Design +- **Standalone FastAPI service** running on port 8001 +- **SQLite database** containing USPS ZIP+4 data (~500MB) +- **USPS API integration** for street-level validation when needed +- **Redis cache** for validated addresses +- **Internal HTTP API** for communication with main legal application + +### Data Flow +``` +1. Legal App → Address Service (POST /validate) +2. Address Service checks local ZIP database +3. If ZIP/city/state valid → return immediately +4. If street validation needed → call USPS API +5. Cache result in Redis +6. Return standardized address to Legal App +``` + +## Technical Requirements + +### Dependencies +- FastAPI framework +- SQLAlchemy for database operations +- SQLite for ZIP+4 database storage +- Redis for caching validated addresses +- httpx for USPS API calls +- Pydantic for request/response validation + +### Database Schema +```sql +-- ZIP+4 Database (from USPS monthly files) +CREATE TABLE zip_codes ( + zip_code TEXT, + plus4 TEXT, + city TEXT, + state TEXT, + county TEXT, + delivery_point TEXT, + PRIMARY KEY (zip_code, plus4) +); + +CREATE INDEX idx_zip_city ON zip_codes(zip_code, city); +CREATE INDEX idx_city_state ON zip_codes(city, state); +``` + +### API Endpoints + +#### POST /validate +Validate and standardize an address. + +**Request:** +```json +{ + "street": "123 Main St", + "city": "Anytown", + "state": "CA", + "zip": "90210", + "strict": false // Optional: require exact match +} +``` + +**Response:** +```json +{ + "valid": true, + "confidence": 0.95, + "source": "local", // "local", "usps_api", "cached" + "standardized": { + "street": "123 MAIN ST", + "city": "ANYTOWN", + "state": "CA", + "zip": "90210", + "plus4": "1234", + "delivery_point": "12" + }, + "corrections": [ + "Standardized street abbreviation ST" + ] +} +``` + +#### GET /health +Health check endpoint. + +#### POST /batch-validate +Batch validation for multiple addresses (up to 50). + +#### GET /stats +Service statistics (cache hit rate, API usage, etc.). + +## Privacy & Security Features + +### Data Minimization +- Only street numbers/names sent to USPS API when necessary +- ZIP/city/state validation happens offline first +- Validated addresses cached to avoid repeat API calls +- No logging of personal addresses + +### Rate Limiting +- USPS API limited to 5 requests/second +- Internal queue system for burst requests +- Fallback to local-only validation when rate limited + +### Caching Strategy +- Redis cache with 30-day TTL for validated addresses +- Cache key: SHA256 hash of normalized address +- Cache hit ratio target: >80% after initial warmup + +## Data Sources + +### USPS ZIP+4 Database +- **Source:** USPS Address Management System +- **Update frequency:** Monthly +- **Size:** ~500MB compressed, ~2GB uncompressed +- **Format:** Fixed-width text files (legacy format) +- **Download:** Automated monthly sync via USPS FTP + +### USPS Address Validation API +- **Endpoint:** https://secure.shippingapis.com/ShippingAPI.dll +- **Rate limit:** 5 requests/second, 10,000/day free +- **Authentication:** USPS Web Tools User ID required +- **Response format:** XML (convert to JSON internally) + +## Implementation Phases + +### Phase 1: Basic Service (1-2 days) +- FastAPI service setup +- Basic ZIP code validation using downloaded USPS data +- Docker containerization +- Simple /validate endpoint + +### Phase 2: USPS Integration (1 day) +- USPS API client implementation +- Street-level validation +- Error handling and fallbacks + +### Phase 3: Caching & Optimization (1 day) +- Redis integration +- Performance optimization +- Batch validation endpoint + +### Phase 4: Data Management (1 day) +- Automated USPS data downloads +- Database update procedures +- Monitoring and alerting + +### Phase 5: Integration (0.5 day) +- Update legal app to use address service +- Form validation integration +- Error handling in UI + +## Docker Configuration + +### Dockerfile +```dockerfile +FROM python:3.11-slim + +WORKDIR /app +COPY requirements.txt . +RUN pip install -r requirements.txt + +COPY . . +EXPOSE 8001 + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8001"] +``` + +### Docker Compose Addition +```yaml +services: + address-service: + build: ./address-service + ports: + - "8001:8001" + environment: + - USPS_USER_ID=${USPS_USER_ID} + - REDIS_URL=redis://redis:6379 + depends_on: + - redis + volumes: + - ./address-service/data:/app/data +``` + +## Configuration + +### Environment Variables +- `USPS_USER_ID`: USPS Web Tools user ID +- `REDIS_URL`: Redis connection string +- `ZIP_DB_PATH`: Path to SQLite ZIP database +- `UPDATE_SCHEDULE`: Cron schedule for data updates +- `API_RATE_LIMIT`: USPS API rate limit (default: 5/second) +- `CACHE_TTL`: Cache time-to-live in seconds (default: 2592000 = 30 days) + +## Monitoring & Metrics + +### Key Metrics +- Cache hit ratio +- USPS API usage/limits +- Response times (local vs API) +- Validation success rates +- Database update status + +### Health Checks +- Service availability +- Database connectivity +- Redis connectivity +- USPS API connectivity +- Disk space for ZIP database + +## Error Handling + +### Graceful Degradation +1. USPS API down → Fall back to local ZIP validation only +2. Redis down → Skip caching, direct validation +3. ZIP database corrupt → Use USPS API only +4. All systems down → Return input address with warning + +### Error Responses +```json +{ + "valid": false, + "error": "USPS_API_UNAVAILABLE", + "message": "Street validation temporarily unavailable", + "fallback_used": "local_zip_only" +} +``` + +## Testing Strategy + +### Unit Tests +- Address normalization functions +- ZIP database queries +- USPS API client +- Caching logic + +### Integration Tests +- Full validation workflow +- Error handling scenarios +- Performance benchmarks +- Data update procedures + +### Load Testing +- Concurrent validation requests +- Cache performance under load +- USPS API rate limiting behavior + +## Security Considerations + +### Input Validation +- Sanitize all address inputs +- Prevent SQL injection in ZIP queries +- Validate against malicious payloads + +### Network Security +- Internal service communication only +- No direct external access to service +- HTTPS for USPS API calls +- Redis authentication if exposed + +### Data Protection +- No persistent logging of addresses +- Secure cache key generation +- Regular security updates for dependencies + +## Future Enhancements + +### Phase 2 Features +- International address validation (Google/SmartyStreets) +- Address autocomplete suggestions +- Geocoding integration +- Delivery route optimization + +### Performance Optimizations +- Database partitioning by state +- Compressed cache storage +- Async batch processing +- CDN for static ZIP data + +## Cost Analysis + +### Infrastructure Costs +- Additional container resources: ~$10/month +- Redis cache: ~$5/month +- USPS ZIP data storage: Minimal +- USPS API: Free tier (10K requests/day) + +### Development Time +- Initial implementation: 3-5 days +- Testing and refinement: 1-2 days +- Documentation and deployment: 0.5 day +- **Total: 4.5-7.5 days** + +### ROI +- Improved data quality +- Reduced shipping errors +- Better client communication +- Compliance with data standards +- Foundation for future location-based features \ No newline at end of file diff --git a/README.md b/README.md index 2125576..a403963 100644 --- a/README.md +++ b/README.md @@ -279,6 +279,16 @@ docker-compose --profile production up -d docker-compose ps ``` +## HTTP client usage in the frontend + +- Prefer calling `window.http.wrappedFetch(url, options)` instead of `fetch` directly. +- The wrapper automatically adds: + - `Authorization: Bearer ` when available + - `X-Correlation-ID` on every request + - `Content-Type: application/json` when you pass a string body (e.g., from `JSON.stringify`) +- It also exposes helpers: `window.http.parseErrorEnvelope`, `window.http.toError`, `window.http.formatAlert`. +- The global `fetch` remains wrapped for compatibility, but will log a one-time deprecation warning. New code should use `window.http.wrappedFetch`. + ### Traditional Deployment ```bash # With gunicorn for production diff --git a/app/api/customers.py b/app/api/customers.py index 662ba5d..4a2cc24 100644 --- a/app/api/customers.py +++ b/app/api/customers.py @@ -96,7 +96,7 @@ async def search_by_phone( """Search customers by phone number (legacy phone search feature)""" phones = db.query(Phone).join(Rolodex).filter( Phone.phone.contains(phone) - ).options(joinedload(Phone.rolodex)).all() + ).options(joinedload(Phone.rolodex_entry)).all() results = [] for phone_record in phones: @@ -104,10 +104,10 @@ async def search_by_phone( "phone": phone_record.phone, "location": phone_record.location, "customer": { - "id": phone_record.rolodex.id, - "name": f"{phone_record.rolodex.first or ''} {phone_record.rolodex.last}".strip(), - "city": phone_record.rolodex.city, - "state": phone_record.rolodex.abrev + "id": phone_record.rolodex_entry.id, + "name": f"{phone_record.rolodex_entry.first or ''} {phone_record.rolodex_entry.last}".strip(), + "city": phone_record.rolodex_entry.city, + "state": phone_record.rolodex_entry.abrev } }) @@ -178,21 +178,25 @@ async def list_customers( current_user: User = Depends(get_current_user) ): """List customers with pagination and search""" - query = db.query(Rolodex).options(joinedload(Rolodex.phone_numbers)) - - if search: - query = query.filter( - or_( - Rolodex.id.contains(search), - Rolodex.last.contains(search), - Rolodex.first.contains(search), - Rolodex.city.contains(search), - Rolodex.email.contains(search) + try: + query = db.query(Rolodex).options(joinedload(Rolodex.phone_numbers)) + + if search: + query = query.filter( + or_( + Rolodex.id.contains(search), + Rolodex.last.contains(search), + Rolodex.first.contains(search), + Rolodex.city.contains(search), + Rolodex.email.contains(search) + ) ) - ) - - customers = query.offset(skip).limit(limit).all() - return customers + + customers = query.offset(skip).limit(limit).all() + return customers + + except Exception as e: + raise HTTPException(status_code=500, detail=f"Error loading customers: {str(e)}") @router.get("/{customer_id}", response_model=CustomerResponse) diff --git a/app/api/import_data.py b/app/api/import_data.py index 36b81a3..600c096 100644 --- a/app/api/import_data.py +++ b/app/api/import_data.py @@ -72,7 +72,7 @@ FIELD_MAPPINGS = { "A3": "a3", "City": "city", "Abrev": "abrev", - # "St": "st", # Full state name - not mapped (model only has abrev) + "St": None, # Full state name - skip this field as model only has abrev "Zip": "zip", "Email": "email", "DOB": "dob", @@ -366,6 +366,7 @@ def parse_date(date_str: str) -> Optional[datetime]: return None + def convert_value(value: str, field_name: str) -> Any: """Convert string value to appropriate type based on field name""" if not value or value.strip() == "" or value.strip().lower() in ["null", "none", "n/a"]: @@ -483,8 +484,116 @@ async def import_csv_data( try: # Read CSV content content = await file.read() - csv_content = content.decode('utf-8') - csv_reader = csv.DictReader(io.StringIO(csv_content)) + + # Try multiple encodings for legacy CSV files + encodings = ['utf-8', 'windows-1252', 'iso-8859-1', 'cp1252'] + csv_content = None + for encoding in encodings: + try: + csv_content = content.decode(encoding) + break + except UnicodeDecodeError: + continue + + if csv_content is None: + raise HTTPException(status_code=400, detail="Could not decode CSV file. Please ensure it's saved in UTF-8, Windows-1252, or ISO-8859-1 encoding.") + + # Preprocess CSV content to fix common legacy issues + def preprocess_csv(content): + lines = content.split('\n') + cleaned_lines = [] + i = 0 + + while i < len(lines): + line = lines[i] + # If line doesn't have the expected number of commas, it might be a broken multi-line field + if i == 0: # Header line + cleaned_lines.append(line) + expected_comma_count = line.count(',') + i += 1 + continue + + # Check if this line has the expected number of commas + if line.count(',') < expected_comma_count: + # This might be a continuation of the previous line + # Try to merge with previous line + if cleaned_lines: + cleaned_lines[-1] += " " + line.replace('\n', ' ').replace('\r', ' ') + else: + cleaned_lines.append(line) + else: + cleaned_lines.append(line) + i += 1 + + return '\n'.join(cleaned_lines) + + # Custom robust parser for problematic legacy CSV files + class MockCSVReader: + def __init__(self, data, fieldnames): + self.data = data + self.fieldnames = fieldnames + self.index = 0 + + def __iter__(self): + return self + + def __next__(self): + if self.index >= len(self.data): + raise StopIteration + row = self.data[self.index] + self.index += 1 + return row + + try: + lines = csv_content.strip().split('\n') + if not lines: + raise ValueError("Empty CSV file") + + # Parse header using proper CSV parsing + header_reader = csv.reader(io.StringIO(lines[0])) + headers = next(header_reader) + headers = [h.strip() for h in headers] + print(f"DEBUG: Found {len(headers)} headers: {headers}") + + # Parse data rows with proper CSV parsing + rows_data = [] + skipped_rows = 0 + + for line_num, line in enumerate(lines[1:], start=2): + # Skip empty lines + if not line.strip(): + continue + + try: + # Use proper CSV parsing to handle commas within quoted fields + line_reader = csv.reader(io.StringIO(line)) + fields = next(line_reader) + fields = [f.strip() for f in fields] + + # Skip rows that are clearly malformed (too few fields) + if len(fields) < len(headers) // 2: # Less than half the expected fields + skipped_rows += 1 + continue + + # Pad or truncate to match header length + while len(fields) < len(headers): + fields.append('') + fields = fields[:len(headers)] + + row_dict = dict(zip(headers, fields)) + rows_data.append(row_dict) + + except Exception as row_error: + print(f"Skipping malformed row {line_num}: {row_error}") + skipped_rows += 1 + continue + + csv_reader = MockCSVReader(rows_data, headers) + print(f"SUCCESS: Parsed {len(rows_data)} rows (skipped {skipped_rows} malformed rows)") + + except Exception as e: + print(f"Custom parsing failed: {e}") + raise HTTPException(status_code=400, detail=f"Could not parse CSV file. The file appears to have serious formatting issues. Error: {str(e)}") imported_count = 0 errors = [] @@ -500,7 +609,7 @@ async def import_csv_data( model_data = {} for csv_field, db_field in field_mapping.items(): - if csv_field in row: + if csv_field in row and db_field is not None: # Skip fields mapped to None converted_value = convert_value(row[csv_field], csv_field) if converted_value is not None: model_data[db_field] = converted_value @@ -509,6 +618,15 @@ async def import_csv_data( if not any(model_data.values()): continue + # Special validation for models with required fields + if model_class == Phone: + if 'phone' not in model_data or not model_data['phone']: + continue # Skip phone records without a phone number + + if model_class == Rolodex: + if 'last' not in model_data or not model_data['last']: + continue # Skip rolodex records without a last name/company name + # Create model instance instance = model_class(**model_data) db.add(instance) @@ -542,6 +660,9 @@ async def import_csv_data( return result except Exception as e: + print(f"IMPORT ERROR DEBUG: {type(e).__name__}: {str(e)}") + import traceback + print(f"TRACEBACK: {traceback.format_exc()}") db.rollback() raise HTTPException(status_code=500, detail=f"Import failed: {str(e)}") @@ -616,8 +737,22 @@ async def validate_csv_file( try: content = await file.read() - csv_content = content.decode('utf-8') - csv_reader = csv.DictReader(io.StringIO(csv_content)) + + # Try multiple encodings for legacy CSV files + encodings = ['utf-8', 'windows-1252', 'iso-8859-1', 'cp1252'] + csv_content = None + for encoding in encodings: + try: + csv_content = content.decode(encoding) + break + except UnicodeDecodeError: + continue + + if csv_content is None: + raise HTTPException(status_code=400, detail="Could not decode CSV file. Please ensure it's saved in UTF-8, Windows-1252, or ISO-8859-1 encoding.") + + # Handle CSV parsing issues with legacy files + csv_reader = csv.DictReader(io.StringIO(csv_content), delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) # Check headers csv_headers = csv_reader.fieldnames @@ -664,6 +799,9 @@ async def validate_csv_file( } except Exception as e: + print(f"VALIDATION ERROR DEBUG: {type(e).__name__}: {str(e)}") + import traceback + print(f"VALIDATION TRACEBACK: {traceback.format_exc()}") raise HTTPException(status_code=500, detail=f"Validation failed: {str(e)}") @@ -737,8 +875,27 @@ async def batch_import_csv_files( field_mapping = FIELD_MAPPINGS.get(file_type, {}) content = await file.read() - csv_content = content.decode('utf-8-sig') - csv_reader = csv.DictReader(io.StringIO(csv_content)) + + # Try multiple encodings for legacy CSV files + encodings = ['utf-8-sig', 'utf-8', 'windows-1252', 'iso-8859-1', 'cp1252'] + csv_content = None + for encoding in encodings: + try: + csv_content = content.decode(encoding) + break + except UnicodeDecodeError: + continue + + if csv_content is None: + results.append({ + "file_type": file_type, + "status": "failed", + "message": "Could not decode CSV file encoding" + }) + continue + + # Handle CSV parsing issues with legacy files + csv_reader = csv.DictReader(io.StringIO(csv_content), delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) imported_count = 0 errors = [] @@ -752,7 +909,7 @@ async def batch_import_csv_files( try: model_data = {} for csv_field, db_field in field_mapping.items(): - if csv_field in row: + if csv_field in row and db_field is not None: # Skip fields mapped to None converted_value = convert_value(row[csv_field], csv_field) if converted_value is not None: model_data[db_field] = converted_value diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index cabc6ad..f91544f 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -28,7 +28,7 @@ services: - delphi_dev_uploads:/app/uploads # Database backups - delphi_dev_backups:/app/backups - command: python -m uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload + command: python -m uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload --reload-dir app --reload-dir templates --reload-dir static restart: unless-stopped healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8000/health"] diff --git a/package.json b/package.json index c6f3dfb..03f08ca 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "A modern Python web application built with FastAPI to replace the legacy Pascal-based database system. This system maintains the familiar keyboard shortcuts and workflows while providing a robust, modular backend with a clean web interface.", "main": "tailwind.config.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "test": "jest" }, "repository": { "type": "git", @@ -17,8 +17,13 @@ "url": "https://github.com/HotSwapp/delphi-database/issues" }, "homepage": "https://github.com/HotSwapp/delphi-database#readme", + "jest": { + "testEnvironment": "jsdom" + }, "devDependencies": { "@tailwindcss/forms": "^0.5.10", - "tailwindcss": "^3.4.10" + "tailwindcss": "^3.4.10", + "jest": "^29.7.0", + "jsdom": "^22.1.0" } } diff --git a/static/js/__tests__/alerts.test.js b/static/js/__tests__/alerts.test.js new file mode 100644 index 0000000..bb84e5c --- /dev/null +++ b/static/js/__tests__/alerts.test.js @@ -0,0 +1,58 @@ +/** @jest-environment jsdom */ + +const path = require('path'); +// Load sanitizer utility first so alerts can delegate to it +require(path.join(__dirname, '..', 'sanitizer.js')); +// Load the alerts module (IIFE attaches itself to window) +require(path.join(__dirname, '..', 'alerts.js')); + +describe('alerts._sanitize', () => { + const sanitize = window.alerts && window.alerts._sanitize; + + it('should be a function', () => { + expect(typeof sanitize).toBe('function'); + }); + + it('removes

Hello

'; + const clean = sanitize(dirty); + expect(clean).toContain(''); + expect(clean).toContain('

Hello

'); + expect(clean).not.toMatch(/ - + + diff --git a/templates/customers.html b/templates/customers.html index 3df8b80..eb37e51 100644 --- a/templates/customers.html +++ b/templates/customers.html @@ -32,12 +32,12 @@
- +
@@ -58,7 +58,7 @@
- +
@@ -77,13 +77,13 @@ - - - - - - - + + + + + + + @@ -277,7 +277,7 @@ - + {% endblock %} \ No newline at end of file diff --git a/templates/dashboard.html b/templates/dashboard.html index 4b76fb4..925af1c 100644 --- a/templates/dashboard.html +++ b/templates/dashboard.html @@ -191,11 +191,7 @@ // Load dashboard data async function loadDashboardData() { try { - const response = await fetch('/api/admin/stats', { - headers: { - 'Authorization': `Bearer ${localStorage.getItem('auth_token')}` - } - }); + const response = await window.http.wrappedFetch('/api/admin/stats'); if (response.ok) { const data = await response.json(); diff --git a/templates/documents.html b/templates/documents.html index 4327bc2..39d5218 100644 --- a/templates/documents.html +++ b/templates/documents.html @@ -446,9 +446,7 @@ document.addEventListener('DOMContentLoaded', function() { function initializeDocuments() { if (typeof apiGet === 'function') { // Ensure API headers are set up with token - if (window.apiHeaders && token) { - window.apiHeaders['Authorization'] = `Bearer ${token}`; - } + // Authorization is injected by window.http.wrappedFetch // Initialize the first tab as active document.getElementById('templates-tab').click(); @@ -594,14 +592,7 @@ function setupEventHandlers() { document.getElementById('refreshQdrosBtn').addEventListener('click', loadQdros); } -// Helper function for authenticated API calls -function getAuthHeaders() { - const token = localStorage.getItem('auth_token'); - return { - 'Authorization': `Bearer ${token}`, - 'Content-Type': 'application/json' - }; -} +// Authorization and JSON headers are injected by window.http.wrappedFetch async function loadTemplates() { try { @@ -624,9 +615,7 @@ async function loadTemplates() { console.log('🔍 DEBUG: Making API call to:', url); - const response = await fetch(url, { - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch(url); console.log('🔍 DEBUG: Response status:', response.status); if (!response.ok) { @@ -658,7 +647,7 @@ function createTemplateRow(template) { const row = document.createElement('tr'); const variableCount = Object.keys(template.variables || {}).length; - row.innerHTML = ` + const rowHtml = ` @@ -677,6 +666,7 @@ function createTemplateRow(template) { `; + if (window.setSafeHTML) { window.setSafeHTML(row, rowHtml); } else { row.innerHTML = rowHtml; } return row; } @@ -696,9 +686,7 @@ async function loadQdros() { return; } - const response = await fetch(url, { - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch(url); if (!response.ok) { const errorData = await response.json().catch(() => ({ detail: 'Unknown error' })); @@ -727,7 +715,7 @@ function displayQdros(qdros) { function createQdroRow(qdro) { const row = document.createElement('tr'); - row.innerHTML = ` + const qdroRowHtml = ` @@ -747,6 +735,7 @@ function createQdroRow(qdro) { `; + if (window.setSafeHTML) { window.setSafeHTML(row, qdroRowHtml); } else { row.innerHTML = qdroRowHtml; } return row; } @@ -768,9 +757,7 @@ async function loadCategories() { return; } - const response = await fetch('/api/documents/categories/', { - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch('/api/documents/categories/'); if (!response.ok) { const errorData = await response.json().catch(() => ({ detail: 'Unknown error' })); @@ -819,9 +806,7 @@ function openTemplateModal(templateId = null) { async function loadTemplateForEditing(templateId) { try { - const response = await fetch(`/api/documents/templates/${templateId}`, { - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch(`/api/documents/templates/${templateId}`); if (!response.ok) throw new Error('Failed to load template'); const template = await response.json(); @@ -856,11 +841,8 @@ async function saveTemplate() { const url = isEdit ? `/api/documents/templates/${templateData.form_id}` : '/api/documents/templates/'; const method = isEdit ? 'PUT' : 'POST'; - const response = await fetch(url, { + const response = await window.http.wrappedFetch(url, { method: method, - headers: { - 'Content-Type': 'application/json' - }, body: JSON.stringify(templateData) }); @@ -901,9 +883,7 @@ function extractVariables(content) { async function loadDocumentStats() { try { - const response = await fetch('/api/documents/stats/summary', { - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch('/api/documents/stats/summary'); if (!response.ok) throw new Error('Failed to load statistics'); const stats = await response.json(); @@ -918,7 +898,8 @@ async function loadDocumentStats() { Object.entries(stats.templates_by_category).forEach(([category, count]) => { const div = document.createElement('div'); div.className = 'flex items-center justify-between mb-1'; - div.innerHTML = `${category}${count}`; + const html = `${category}${count}`; + if (window.setSafeHTML) { window.setSafeHTML(div, html); } else { div.innerHTML = html; } categoriesDiv.appendChild(div); }); @@ -932,11 +913,12 @@ async function loadDocumentStats() { stats.recent_activity.forEach(activity => { const div = document.createElement('div'); div.className = 'mb-2 p-2 border rounded'; - div.innerHTML = ` + const activityHtml = ` ${activity.type}
File: ${activity.file_no}
${activity.status} `; + if (window.setSafeHTML) { window.setSafeHTML(div, activityHtml); } else { div.innerHTML = activityHtml; } activityDiv.appendChild(div); }); } @@ -983,9 +965,8 @@ async function logClientError({ message, action = null, error = null, extra = nu user_agent: navigator.userAgent, extra }; - await fetch('/api/documents/client-error', { + await window.http.wrappedFetch('/api/documents/client-error', { method: 'POST', - headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) }); } catch (_) { @@ -1010,9 +991,8 @@ async function generateFromTemplate(templateId) { async function deleteTemplate(templateId) { if (confirm('Are you sure you want to delete this template?')) { try { - const response = await fetch(`/api/documents/templates/${templateId}`, { - method: 'DELETE', - headers: getAuthHeaders() + const response = await window.http.wrappedFetch(`/api/documents/templates/${templateId}`, { + method: 'DELETE' }); if (!response.ok) throw new Error('Failed to delete template'); @@ -1034,9 +1014,7 @@ function openGenerateModal() { async function loadTemplatesForGeneration() { try { - const response = await fetch('/api/documents/templates/', { - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch('/api/documents/templates/'); if (!response.ok) throw new Error('Failed to load templates'); const templates = await response.json(); @@ -1057,9 +1035,7 @@ async function loadTemplatesForGeneration() { async function loadTemplatePreview(templateId) { try { - const response = await fetch(`/api/documents/templates/${templateId}`, { - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch(`/api/documents/templates/${templateId}`); if (!response.ok) throw new Error('Failed to load template'); const template = await response.json(); @@ -1100,9 +1076,8 @@ async function generateDocument() { }); } - const response = await fetch(`/api/documents/generate/${templateId}`, { + const response = await window.http.wrappedFetch(`/api/documents/generate/${templateId}`, { method: 'POST', - headers: getAuthHeaders(), body: JSON.stringify(requestData) }); @@ -1131,7 +1106,7 @@ function addCustomVariableInput() { const container = document.getElementById('customVariables'); const div = document.createElement('div'); div.className = 'grid grid-cols-12 gap-2 mb-2 custom-var-input'; - div.innerHTML = ` + const customVarHtml = `
@@ -1144,6 +1119,7 @@ function addCustomVariableInput() { `; + if (window.setSafeHTML) { window.setSafeHTML(div, customVarHtml); } else { div.innerHTML = customVarHtml; } container.appendChild(div); } diff --git a/templates/files.html b/templates/files.html index 3706d7e..af5508e 100644 --- a/templates/files.html +++ b/templates/files.html @@ -381,14 +381,7 @@ let lookupData = { employees: [] }; -// Helper function for authenticated API calls -function getAuthHeaders() { - const token = localStorage.getItem('auth_token'); - return { - 'Authorization': `Bearer ${token}`, - 'Content-Type': 'application/json' - }; -} +// Authorization and JSON headers are injected by window.http.wrappedFetch // Initialize on page load document.addEventListener('DOMContentLoaded', function() { @@ -446,9 +439,9 @@ async function loadLookupData() { try { // Load all lookup data in parallel const [fileTypesRes, statusesRes, employeesRes] = await Promise.all([ - fetch('/api/files/lookups/file-types', { headers: getAuthHeaders() }), - fetch('/api/files/lookups/file-statuses', { headers: getAuthHeaders() }), - fetch('/api/files/lookups/employees', { headers: getAuthHeaders() }) + window.http.wrappedFetch('/api/files/lookups/file-types'), + window.http.wrappedFetch('/api/files/lookups/file-statuses'), + window.http.wrappedFetch('/api/files/lookups/employees') ]); if (fileTypesRes.ok) { @@ -505,9 +498,7 @@ async function loadFiles(page = 0, filters = {}) { ...filters }); - const response = await fetch(`/api/files/?${params}`, { - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch(`/api/files/?${params}`); if (!response.ok) throw new Error('Failed to load files'); @@ -628,9 +619,7 @@ function showAddFileModal() { async function editFile(fileNo) { try { - const response = await fetch(`/api/files/${fileNo}`, { - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch(`/api/files/${fileNo}`); if (!response.ok) throw new Error('Failed to load file'); @@ -687,9 +676,7 @@ function populateFileForm(file) { async function loadClientInfo(clientId) { try { - const response = await fetch(`/api/customers/${clientId}`, { - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch(`/api/customers/${clientId}`); if (response.ok) { const client = await response.json(); @@ -705,9 +692,7 @@ async function loadClientInfo(clientId) { async function loadFinancialSummary(fileNo) { try { - const response = await fetch(`/api/files/${fileNo}/financial-summary`, { - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch(`/api/files/${fileNo}/financial-summary`); if (response.ok) { const data = await response.json(); @@ -771,9 +756,8 @@ async function saveFile() { const url = isEditing ? `/api/files/${editingFileNo}` : '/api/files/'; const method = isEditing ? 'PUT' : 'POST'; - const response = await fetch(url, { + const response = await window.http.wrappedFetch(url, { method: method, - headers: getAuthHeaders(), body: JSON.stringify(fileData) }); @@ -798,9 +782,8 @@ async function deleteFile() { } try { - const response = await fetch(`/api/files/${editingFileNo}`, { - method: 'DELETE', - headers: getAuthHeaders() + const response = await window.http.wrappedFetch(`/api/files/${editingFileNo}`, { + method: 'DELETE' }); if (!response.ok) throw new Error('Failed to delete file'); @@ -820,9 +803,8 @@ async function closeFile() { try { const body = closeDate ? JSON.stringify({ close_date: closeDate }) : ''; - const response = await fetch(`/api/files/${editingFileNo}/close`, { + const response = await window.http.wrappedFetch(`/api/files/${editingFileNo}/close`, { method: 'POST', - headers: getAuthHeaders(), body: body }); @@ -841,9 +823,8 @@ async function closeFile() { async function reopenFile() { try { - const response = await fetch(`/api/files/${editingFileNo}/reopen`, { - method: 'POST', - headers: getAuthHeaders() + const response = await window.http.wrappedFetch(`/api/files/${editingFileNo}/reopen`, { + method: 'POST' }); if (!response.ok) throw new Error('Failed to reopen file'); @@ -871,9 +852,7 @@ async function searchClients() { const params = new URLSearchParams({ limit: 100 }); if (search) params.append('search', search); - const response = await fetch(`/api/customers/?${params}`, { - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch(`/api/customers/?${params}`); if (!response.ok) throw new Error('Failed to search clients'); @@ -916,9 +895,7 @@ async function validateFileNumber() { if (!fileNo || isEditing) return; try { - const response = await fetch(`/api/files/${fileNo}`, { - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch(`/api/files/${fileNo}`); if (response.ok) { showAlert('File number already exists', 'warning'); @@ -951,9 +928,7 @@ async function performAdvancedSearch() { } try { - const response = await fetch(`/api/files/search/advanced?${params}`, { - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch(`/api/files/search/advanced?${params}`); if (!response.ok) throw new Error('Advanced search failed'); @@ -975,9 +950,7 @@ function clearAdvancedSearch() { async function showStats() { try { - const response = await fetch('/api/files/stats/summary', { - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch('/api/files/stats/summary'); if (!response.ok) throw new Error('Failed to load statistics'); @@ -1057,7 +1030,7 @@ function showAlert(message, type = 'info') { async function loadDocuments(fileNo) { try { - const response = await fetch(`/api/documents/${fileNo}/uploaded`, { headers: getAuthHeaders() }); + const response = await window.http.wrappedFetch(`/api/documents/${fileNo}/uploaded`); if (!response.ok) throw new Error('Failed to load documents'); const docs = await response.json(); displayDocuments(docs); @@ -1126,10 +1099,10 @@ async function uploadDocument() { formData.append('description', description); try { - const response = await fetch(`/api/documents/upload/${editingFileNo}`, { + const response = await window.http.wrappedFetch(`/api/documents/upload/${editingFileNo}`, { method: 'POST', body: formData, - headers: getAuthHeaders() // Note: no Content-Type, browser sets it + // Note: no Content-Type; browser sets multipart boundaries }); if (!response.ok) { const error = await response.json(); @@ -1164,9 +1137,8 @@ function downloadDocument(path) { async function deleteDocument(docId) { if (!confirm('Are you sure you want to delete this document?')) return; try { - const response = await fetch(`/api/documents/uploaded/${docId}`, { + const response = await window.http.wrappedFetch(`/api/documents/uploaded/${docId}`, { method: 'DELETE', - headers: getAuthHeaders() }); if (!response.ok) throw new Error('Failed to delete'); showAlert('Document deleted', 'success'); @@ -1186,10 +1158,9 @@ async function updateDocumentDescription(docId, description) { const formData = new FormData(); formData.append('description', description); try { - const response = await fetch(`/api/documents/uploaded/${docId}`, { + const response = await window.http.wrappedFetch(`/api/documents/uploaded/${docId}`, { method: 'PUT', body: formData, - headers: getAuthHeaders() }); if (!response.ok) throw new Error('Failed to update'); showAlert('Description updated', 'success'); diff --git a/templates/financial.html b/templates/financial.html index e5bb308..57ce27d 100644 --- a/templates/financial.html +++ b/templates/financial.html @@ -483,14 +483,7 @@ let dashboardData = null; let recentEntries = []; let unbilledData = null; -// Helper function for authenticated API calls -function getAuthHeaders() { - const token = localStorage.getItem('auth_token'); - return { - 'Authorization': `Bearer ${token}`, - 'Content-Type': 'application/json' - }; -} +// Authorization and JSON headers are injected by window.http.wrappedFetch // Initialize on page load document.addEventListener('DOMContentLoaded', function() { @@ -555,9 +548,7 @@ function setupEventListeners() { async function loadDashboardData() { try { - const response = await fetch('/api/financial/financial-dashboard', { - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch('/api/financial/financial-dashboard'); if (!response.ok) throw new Error('Failed to load dashboard data'); @@ -599,9 +590,7 @@ async function loadRecentTimeEntries() { const params = new URLSearchParams({ days }); if (employee) params.append('employee', employee); - const response = await fetch(`/api/financial/time-entries/recent?${params}`, { - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch(`/api/financial/time-entries/recent?${params}`); if (!response.ok) throw new Error('Failed to load recent entries'); @@ -652,9 +641,7 @@ function displayRecentTimeEntries(entries) { async function loadEmployeeOptions() { try { - const response = await fetch('/api/files/lookups/employees', { - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch('/api/files/lookups/employees'); if (response.ok) { const employees = await response.json(); @@ -719,9 +706,8 @@ async function saveQuickTime() { }; try { - const response = await fetch('/api/financial/time-entry/quick', { + const response = await window.http.wrappedFetch('/api/financial/time-entry/quick', { method: 'POST', - headers: getAuthHeaders(), body: JSON.stringify(data) }); @@ -769,9 +755,8 @@ async function savePayment() { }; try { - const response = await fetch('/api/financial/payments/', { + const response = await window.http.wrappedFetch('/api/financial/payments/', { method: 'POST', - headers: getAuthHeaders(), body: JSON.stringify(data) }); @@ -818,9 +803,8 @@ async function saveExpense() { }; try { - const response = await fetch('/api/financial/expenses/', { + const response = await window.http.wrappedFetch('/api/financial/expenses/', { method: 'POST', - headers: getAuthHeaders(), body: JSON.stringify(data) }); @@ -850,9 +834,7 @@ async function saveExpense() { async function showUnbilledModal() { // Load unbilled data try { - const response = await fetch('/api/financial/unbilled-entries', { - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch('/api/financial/unbilled-entries'); if (!response.ok) throw new Error('Failed to load unbilled entries'); @@ -979,9 +961,8 @@ async function billSelectedEntries() { } try { - const response = await fetch('/api/financial/bill-entries', { + const response = await window.http.wrappedFetch('/api/financial/bill-entries', { method: 'POST', - headers: getAuthHeaders(), body: JSON.stringify({ entry_ids: entryIds }) }); @@ -1081,9 +1062,7 @@ async function validateQuickTimeFile() { if (!fileNo) return; try { - const response = await fetch(`/api/files/${fileNo}`, { - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch(`/api/files/${fileNo}`); if (response.ok) { const file = await response.json(); diff --git a/templates/import.html b/templates/import.html index 4e2ebae..7bba7cf 100644 --- a/templates/import.html +++ b/templates/import.html @@ -235,13 +235,7 @@ let availableFiles = {}; let importInProgress = false; -// Helper function for authenticated API calls -function getAuthHeaders() { - const token = localStorage.getItem('auth_token'); - return { - 'Authorization': `Bearer ${token}` - }; -} +// Authorization is injected by window.http.wrappedFetch // Initialize on page load document.addEventListener('DOMContentLoaded', function() { @@ -288,9 +282,7 @@ function setupEventListeners() { async function loadAvailableFiles() { try { - const response = await fetch('/api/import/available-files', { - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch('/api/import/available-files'); if (response.status === 401 || response.status === 403) { console.error('Authentication error - redirecting to login'); @@ -335,9 +327,7 @@ async function loadAvailableFiles() { async function loadImportStatus() { try { - const response = await fetch('/api/import/status', { - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch('/api/import/status'); if (response.status === 401 || response.status === 403) { console.error('Authentication error - redirecting to login'); @@ -422,9 +412,8 @@ async function validateFile() { try { showProgress(true, 'Validating file...'); - const response = await fetch(`/api/import/validate/${fileType}`, { + const response = await window.http.wrappedFetch(`/api/import/validate/${fileType}`, { method: 'POST', - headers: getAuthHeaders(), body: formData }); @@ -540,9 +529,8 @@ async function handleImport(event) { try { showProgress(true, 'Importing data...'); - const response = await fetch(`/api/import/upload/${fileType}`, { + const response = await window.http.wrappedFetch(`/api/import/upload/${fileType}`, { method: 'POST', - headers: getAuthHeaders(), body: formData }); @@ -629,10 +617,9 @@ async function clearTable() { } try { - const response = await fetch(`/api/import/clear/${fileType}`, { - method: 'DELETE', - headers: getAuthHeaders() - }); + const response = await window.http.wrappedFetch(`/api/import/clear/${fileType}`, { + method: 'DELETE' + }); if (!response.ok) { const error = await response.json(); @@ -769,11 +756,10 @@ async function handleBatchImport(event) { try { showProgress(true, 'Processing batch import...'); - const response = await fetch('/api/import/batch-upload', { - method: 'POST', - headers: getAuthHeaders(), - body: formData - }); + const response = await window.http.wrappedFetch('/api/import/batch-upload', { + method: 'POST', + body: formData + }); if (!response.ok) { const error = await response.json(); diff --git a/templates/login.html b/templates/login.html index 7c01111..de7d2d5 100644 --- a/templates/login.html +++ b/templates/login.html @@ -4,6 +4,7 @@ Login - Delphi Consulting Group Database System + @@ -96,7 +97,7 @@ try { console.log('Sending request to /api/auth/login'); - const response = await fetch('/api/auth/login', { + const response = await window.http.wrappedFetch('/api/auth/login', { method: 'POST', headers: { 'Content-Type': 'application/json' @@ -154,11 +155,7 @@ async function checkTokenAndRedirect(token) { try { - const response = await fetch('/api/auth/me', { - headers: { - 'Authorization': `Bearer ${token}` - } - }); + const response = await window.http.wrappedFetch('/api/auth/me'); if (response.ok) { // Token is valid, redirect to customers page diff --git a/templates/search.html b/templates/search.html index b29b959..6c60b83 100644 --- a/templates/search.html +++ b/templates/search.html @@ -443,7 +443,7 @@ function initializeAdvancedSearch() { async function loadSearchFacets() { try { - const response = await fetch('/api/search/facets'); + const response = await window.http.wrappedFetch('/api/search/facets'); if (!response.ok) throw new Error('Failed to load search facets'); facetsData = await response.json(); @@ -576,7 +576,7 @@ function setupKeyboardShortcuts() { async function loadSearchSuggestions(query) { try { - const response = await fetch(`/api/search/suggestions?q=${encodeURIComponent(query)}&limit=10`); + const response = await window.http.wrappedFetch(`/api/search/suggestions?q=${encodeURIComponent(query)}&limit=10`); if (!response.ok) throw new Error('Failed to load suggestions'); const data = await response.json(); @@ -631,11 +631,8 @@ async function performSearch(offset = 0) { currentSearchCriteria = criteria; try { - const response = await fetch('/api/search/advanced', { + const response = await window.http.wrappedFetch('/api/search/advanced', { method: 'POST', - headers: { - 'Content-Type': 'application/json' - }, body: JSON.stringify(criteria) }); diff --git a/templates/support_modal.html b/templates/support_modal.html index 0b006a6..d21f1a6 100644 --- a/templates/support_modal.html +++ b/templates/support_modal.html @@ -295,11 +295,8 @@ let supportSystem = { browser_info: this.browserInfo }; - const response = await fetch('/api/support/tickets', { + const response = await window.http.wrappedFetch('/api/support/tickets', { method: 'POST', - headers: { - 'Content-Type': 'application/json' - }, body: JSON.stringify(ticketData) }); diff --git a/test_deposits.csv b/test_deposits.csv new file mode 100644 index 0000000..88c75a4 --- /dev/null +++ b/test_deposits.csv @@ -0,0 +1,3 @@ +Deposit_Date,Total +2024-01-01,100.00 +2024-01-02,200.50
IDNameGroupLocationPhoneEmailActionsCustomerNameGroupLocationPhoneEmailActions
${template.form_id} ${template.form_name} ${template.category} ${qdro.file_no} ${qdro.version} ${qdro.participant_name || ''}