changes

app/services/adaptive_cache.py

@@ -0,0 +1,399 @@
+"""
+Adaptive Cache TTL Service
+
+Dynamically adjusts cache TTL based on data update frequency and patterns.
+Provides intelligent caching that adapts to system usage patterns.
+"""
+import asyncio
+import time
+from typing import Dict, Optional, Tuple, Any, List
+from datetime import datetime, timedelta
+from collections import defaultdict, deque
+from dataclasses import dataclass
+from sqlalchemy.orm import Session
+from sqlalchemy import text, func
+
+from app.utils.logging import get_logger
+from app.services.cache import cache_get_json, cache_set_json
+
+logger = get_logger("adaptive_cache")
+
+
+@dataclass
+class UpdateMetrics:
+    """Metrics for tracking data update frequency"""
+    table_name: str
+    updates_per_hour: float
+    last_update: datetime
+    avg_query_frequency: float
+    cache_hit_rate: float
+    
+    
+@dataclass
+class CacheConfig:
+    """Cache configuration with adaptive TTL"""
+    base_ttl: int
+    min_ttl: int
+    max_ttl: int
+    update_weight: float = 0.7  # How much update frequency affects TTL
+    query_weight: float = 0.3   # How much query frequency affects TTL
+
+
+class AdaptiveCacheManager:
+    """
+    Manages adaptive caching with TTL that adjusts based on:
+    - Data update frequency
+    - Query frequency 
+    - Cache hit rates
+    - Time of day patterns
+    """
+    
+    def __init__(self):
+        # Track update frequencies by table
+        self.update_history: Dict[str, deque] = defaultdict(lambda: deque(maxlen=100))
+        self.query_history: Dict[str, deque] = defaultdict(lambda: deque(maxlen=200))
+        self.cache_stats: Dict[str, Dict[str, float]] = defaultdict(lambda: {
+            "hits": 0, "misses": 0, "total_queries": 0
+        })
+        
+        # Cache configurations for different data types
+        self.cache_configs = {
+            "customers": CacheConfig(base_ttl=300, min_ttl=60, max_ttl=1800),  # 5min base, 1min-30min range
+            "files": CacheConfig(base_ttl=240, min_ttl=60, max_ttl=1200),      # 4min base, 1min-20min range
+            "ledger": CacheConfig(base_ttl=120, min_ttl=30, max_ttl=600),      # 2min base, 30sec-10min range
+            "documents": CacheConfig(base_ttl=600, min_ttl=120, max_ttl=3600), # 10min base, 2min-1hr range
+            "templates": CacheConfig(base_ttl=900, min_ttl=300, max_ttl=7200), # 15min base, 5min-2hr range
+            "global": CacheConfig(base_ttl=180, min_ttl=45, max_ttl=900),      # 3min base, 45sec-15min range
+            "advanced": CacheConfig(base_ttl=300, min_ttl=60, max_ttl=1800),   # 5min base, 1min-30min range
+        }
+        
+        # Background task for monitoring
+        self._monitoring_task: Optional[asyncio.Task] = None
+        self._last_metrics_update = time.time()
+        
+    async def start_monitoring(self, db: Session):
+        """Start background monitoring of data update patterns"""
+        if self._monitoring_task is None or self._monitoring_task.done():
+            self._monitoring_task = asyncio.create_task(self._monitor_update_patterns(db))
+    
+    async def stop_monitoring(self):
+        """Stop background monitoring"""
+        if self._monitoring_task and not self._monitoring_task.done():
+            self._monitoring_task.cancel()
+            try:
+                await self._monitoring_task
+            except asyncio.CancelledError:
+                pass
+    
+    def record_data_update(self, table_name: str):
+        """Record that data was updated in a table"""
+        now = time.time()
+        self.update_history[table_name].append(now)
+        logger.debug(f"Recorded update for table: {table_name}")
+    
+    def record_query(self, cache_type: str, cache_key: str, hit: bool):
+        """Record a cache query (hit or miss)"""
+        now = time.time()
+        self.query_history[cache_type].append(now)
+        
+        stats = self.cache_stats[cache_type]
+        stats["total_queries"] += 1
+        if hit:
+            stats["hits"] += 1
+        else:
+            stats["misses"] += 1
+    
+    def get_adaptive_ttl(self, cache_type: str, fallback_ttl: int = 300) -> int:
+        """
+        Calculate adaptive TTL based on update and query patterns
+        
+        Args:
+            cache_type: Type of cache (customers, files, etc.)
+            fallback_ttl: Default TTL if no config found
+            
+        Returns:
+            Adaptive TTL in seconds
+        """
+        config = self.cache_configs.get(cache_type)
+        if not config:
+            return fallback_ttl
+        
+        # Get recent update frequency (updates per hour)
+        updates_per_hour = self._calculate_update_frequency(cache_type)
+        
+        # Get recent query frequency (queries per minute)
+        queries_per_minute = self._calculate_query_frequency(cache_type)
+        
+        # Get cache hit rate
+        hit_rate = self._calculate_hit_rate(cache_type)
+        
+        # Calculate adaptive TTL
+        ttl = self._calculate_adaptive_ttl(
+            config, updates_per_hour, queries_per_minute, hit_rate
+        )
+        
+        logger.debug(
+            f"Adaptive TTL for {cache_type}: {ttl}s "
+            f"(updates/hr: {updates_per_hour:.1f}, queries/min: {queries_per_minute:.1f}, hit_rate: {hit_rate:.2f})"
+        )
+        
+        return ttl
+    
+    def _calculate_update_frequency(self, table_name: str) -> float:
+        """Calculate updates per hour for the last hour"""
+        now = time.time()
+        hour_ago = now - 3600
+        
+        recent_updates = [
+            update_time for update_time in self.update_history[table_name]
+            if update_time >= hour_ago
+        ]
+        
+        return len(recent_updates)
+    
+    def _calculate_query_frequency(self, cache_type: str) -> float:
+        """Calculate queries per minute for the last 10 minutes"""
+        now = time.time()
+        ten_minutes_ago = now - 600
+        
+        recent_queries = [
+            query_time for query_time in self.query_history[cache_type]
+            if query_time >= ten_minutes_ago
+        ]
+        
+        return len(recent_queries) / 10.0  # per minute
+    
+    def _calculate_hit_rate(self, cache_type: str) -> float:
+        """Calculate cache hit rate"""
+        stats = self.cache_stats[cache_type]
+        total = stats["total_queries"]
+        
+        if total == 0:
+            return 0.5  # Neutral assumption
+        
+        return stats["hits"] / total
+    
+    def _calculate_adaptive_ttl(
+        self, 
+        config: CacheConfig, 
+        updates_per_hour: float, 
+        queries_per_minute: float, 
+        hit_rate: float
+    ) -> int:
+        """
+        Calculate adaptive TTL using multiple factors
+        
+        Logic:
+        - Higher update frequency = lower TTL
+        - Higher query frequency = shorter TTL (fresher data needed)
+        - Higher hit rate = can use longer TTL
+        - Apply time-of-day adjustments
+        """
+        base_ttl = config.base_ttl
+        
+        # Update frequency factor (0.1 to 2.0)
+        # More updates = shorter TTL
+        if updates_per_hour == 0:
+            update_factor = 1.5  # No recent updates, can cache longer
+        else:
+            # Logarithmic scaling: 1 update/hr = 1.0, 10 updates/hr = 0.5
+            update_factor = max(0.1, 1.0 / (1 + updates_per_hour * 0.1))
+        
+        # Query frequency factor (0.5 to 1.5)
+        # More queries = need fresher data
+        if queries_per_minute == 0:
+            query_factor = 1.2  # No queries, can cache longer
+        else:
+            # More queries = shorter TTL, but with diminishing returns
+            query_factor = max(0.5, 1.0 / (1 + queries_per_minute * 0.05))
+        
+        # Hit rate factor (0.8 to 1.3)
+        # Higher hit rate = working well, can extend TTL slightly
+        hit_rate_factor = 0.8 + (hit_rate * 0.5)
+        
+        # Time-of-day factor
+        time_factor = self._get_time_of_day_factor()
+        
+        # Combine factors
+        adaptive_factor = (
+            update_factor * config.update_weight +
+            query_factor * config.query_weight +
+            hit_rate_factor * 0.2 +
+            time_factor * 0.1
+        )
+        
+        # Apply to base TTL
+        adaptive_ttl = int(base_ttl * adaptive_factor)
+        
+        # Clamp to min/max bounds
+        return max(config.min_ttl, min(config.max_ttl, adaptive_ttl))
+    
+    def _get_time_of_day_factor(self) -> float:
+        """
+        Adjust TTL based on time of day
+        Business hours = shorter TTL (more activity)
+        Off hours = longer TTL (less activity)
+        """
+        now = datetime.now()
+        hour = now.hour
+        
+        # Business hours (8 AM - 6 PM): shorter TTL
+        if 8 <= hour <= 18:
+            return 0.9  # 10% shorter TTL
+        # Evening (6 PM - 10 PM): normal TTL
+        elif 18 < hour <= 22:
+            return 1.0
+        # Night/early morning: longer TTL
+        else:
+            return 1.3  # 30% longer TTL
+    
+    async def _monitor_update_patterns(self, db: Session):
+        """Background task to monitor database update patterns"""
+        logger.info("Starting adaptive cache monitoring")
+        
+        try:
+            while True:
+                await asyncio.sleep(300)  # Check every 5 minutes
+                await self._update_metrics(db)
+        except asyncio.CancelledError:
+            logger.info("Stopping adaptive cache monitoring")
+            raise
+        except Exception as e:
+            logger.error(f"Error in cache monitoring: {str(e)}")
+    
+    async def _update_metrics(self, db: Session):
+        """Update metrics from database statistics"""
+        try:
+            # Query recent update activity from audit logs or timestamp fields
+            now = datetime.now()
+            hour_ago = now - timedelta(hours=1)
+            
+            # Check for recent updates in key tables
+            tables_to_monitor = ['files', 'ledger', 'rolodex', 'documents', 'templates']
+            
+            for table in tables_to_monitor:
+                try:
+                    # Try to get update count from updated_at fields
+                    query = text(f"""
+                        SELECT COUNT(*) as update_count 
+                        FROM {table} 
+                        WHERE updated_at >= :hour_ago
+                    """)
+                    
+                    result = db.execute(query, {"hour_ago": hour_ago}).scalar()
+                    
+                    if result and result > 0:
+                        # Record the updates
+                        for _ in range(int(result)):
+                            self.record_data_update(table)
+                            
+                except Exception as e:
+                    # Table might not have updated_at field, skip silently
+                    logger.debug(f"Could not check updates for table {table}: {str(e)}")
+                    continue
+            
+            # Clean old data
+            self._cleanup_old_data()
+            
+        except Exception as e:
+            logger.error(f"Error updating cache metrics: {str(e)}")
+    
+    def _cleanup_old_data(self):
+        """Clean up old tracking data to prevent memory leaks"""
+        cutoff_time = time.time() - 7200  # Keep last 2 hours
+        
+        for table_history in self.update_history.values():
+            while table_history and table_history[0] < cutoff_time:
+                table_history.popleft()
+        
+        for query_history in self.query_history.values():
+            while query_history and query_history[0] < cutoff_time:
+                query_history.popleft()
+        
+        # Reset cache stats periodically
+        if time.time() - self._last_metrics_update > 3600:  # Every hour
+            for stats in self.cache_stats.values():
+                # Decay the stats to prevent them from growing indefinitely
+                stats["hits"] = int(stats["hits"] * 0.8)
+                stats["misses"] = int(stats["misses"] * 0.8)
+                stats["total_queries"] = stats["hits"] + stats["misses"]
+            
+            self._last_metrics_update = time.time()
+    
+    def get_cache_statistics(self) -> Dict[str, Any]:
+        """Get current cache statistics for monitoring"""
+        stats = {}
+        
+        for cache_type, config in self.cache_configs.items():
+            current_ttl = self.get_adaptive_ttl(cache_type, config.base_ttl)
+            update_freq = self._calculate_update_frequency(cache_type)
+            query_freq = self._calculate_query_frequency(cache_type)
+            hit_rate = self._calculate_hit_rate(cache_type)
+            
+            stats[cache_type] = {
+                "current_ttl": current_ttl,
+                "base_ttl": config.base_ttl,
+                "min_ttl": config.min_ttl,
+                "max_ttl": config.max_ttl,
+                "updates_per_hour": update_freq,
+                "queries_per_minute": query_freq,
+                "hit_rate": hit_rate,
+                "total_queries": self.cache_stats[cache_type]["total_queries"]
+            }
+        
+        return stats
+
+
+# Global instance
+adaptive_cache_manager = AdaptiveCacheManager()
+
+
+# Enhanced cache functions that use adaptive TTL
+async def adaptive_cache_get(
+    cache_type: str, 
+    cache_key: str, 
+    user_id: Optional[str] = None, 
+    parts: Optional[Dict] = None
+) -> Optional[Any]:
+    """Get from cache and record metrics"""
+    parts = parts or {}
+    
+    try:
+        result = await cache_get_json(cache_type, user_id, parts)
+        adaptive_cache_manager.record_query(cache_type, cache_key, hit=result is not None)
+        return result
+    except Exception as e:
+        logger.error(f"Cache get error: {str(e)}")
+        adaptive_cache_manager.record_query(cache_type, cache_key, hit=False)
+        return None
+
+
+async def adaptive_cache_set(
+    cache_type: str,
+    cache_key: str,
+    value: Any,
+    user_id: Optional[str] = None,
+    parts: Optional[Dict] = None,
+    ttl_override: Optional[int] = None
+) -> None:
+    """Set cache with adaptive TTL"""
+    parts = parts or {}
+    
+    # Use adaptive TTL unless overridden
+    ttl = ttl_override or adaptive_cache_manager.get_adaptive_ttl(cache_type)
+    
+    try:
+        await cache_set_json(cache_type, user_id, parts, value, ttl)
+        logger.debug(f"Cached {cache_type} with adaptive TTL: {ttl}s")
+    except Exception as e:
+        logger.error(f"Cache set error: {str(e)}")
+
+
+def record_data_update(table_name: str):
+    """Record that data was updated (call from model save/update operations)"""
+    adaptive_cache_manager.record_data_update(table_name)
+
+
+def get_cache_stats() -> Dict[str, Any]:
+    """Get current cache statistics"""
+    return adaptive_cache_manager.get_cache_statistics()

app/services/advanced_variables.py

@@ -0,0 +1,571 @@
+"""
+Advanced Variable Resolution Service
+
+This service handles complex variable processing including:
+- Conditional logic evaluation
+- Mathematical calculations and formulas
+- Dynamic data source queries
+- Variable dependency resolution
+- Caching and performance optimization
+"""
+from __future__ import annotations
+
+import re
+import json
+import math
+import operator
+from datetime import datetime, date, timedelta
+from typing import Dict, Any, List, Optional, Tuple, Union
+from decimal import Decimal, InvalidOperation
+import logging
+
+from sqlalchemy.orm import Session
+from sqlalchemy import text
+
+from app.models.template_variables import (
+    TemplateVariable, VariableContext, VariableAuditLog, 
+    VariableType, VariableTemplate
+)
+from app.models.files import File
+from app.models.rolodex import Rolodex
+from app.core.logging import get_logger
+
+logger = get_logger("advanced_variables")
+
+
+class VariableProcessor:
+    """
+    Handles advanced variable processing with conditional logic, calculations, and data sources
+    """
+    
+    def __init__(self, db: Session):
+        self.db = db
+        self._cache: Dict[str, Any] = {}
+        
+        # Safe functions available in formula expressions
+        self.safe_functions = {
+            'abs': abs,
+            'round': round,
+            'min': min,
+            'max': max,
+            'sum': sum,
+            'len': len,
+            'str': str,
+            'int': int,
+            'float': float,
+            'math_ceil': math.ceil,
+            'math_floor': math.floor,
+            'math_sqrt': math.sqrt,
+            'today': lambda: date.today(),
+            'now': lambda: datetime.now(),
+            'days_between': lambda d1, d2: (d1 - d2).days if isinstance(d1, date) and isinstance(d2, date) else 0,
+            'format_currency': lambda x: f"${float(x):,.2f}" if x is not None else "$0.00",
+            'format_date': lambda d, fmt='%B %d, %Y': d.strftime(fmt) if isinstance(d, date) else str(d),
+        }
+        
+        # Safe operators for formula evaluation
+        self.operators = {
+            '+': operator.add,
+            '-': operator.sub,
+            '*': operator.mul,
+            '/': operator.truediv,
+            '//': operator.floordiv,
+            '%': operator.mod,
+            '**': operator.pow,
+            '==': operator.eq,
+            '!=': operator.ne,
+            '<': operator.lt,
+            '<=': operator.le,
+            '>': operator.gt,
+            '>=': operator.ge,
+            'and': operator.and_,
+            'or': operator.or_,
+            'not': operator.not_,
+        }
+
+    def resolve_variables(
+        self, 
+        variables: List[str], 
+        context_type: str = "global", 
+        context_id: str = "default",
+        base_context: Optional[Dict[str, Any]] = None
+    ) -> Tuple[Dict[str, Any], List[str]]:
+        """
+        Resolve a list of variables with their current values
+        
+        Args:
+            variables: List of variable names to resolve
+            context_type: Context type (file, client, global, etc.)
+            context_id: Specific context identifier
+            base_context: Additional context values to use
+            
+        Returns:
+            Tuple of (resolved_variables, unresolved_variables)
+        """
+        resolved = {}
+        unresolved = []
+        processing_order = self._determine_processing_order(variables)
+        
+        # Start with base context
+        if base_context:
+            resolved.update(base_context)
+        
+        for var_name in processing_order:
+            try:
+                value = self._resolve_single_variable(
+                    var_name, context_type, context_id, resolved
+                )
+                if value is not None:
+                    resolved[var_name] = value
+                else:
+                    unresolved.append(var_name)
+            except Exception as e:
+                logger.error(f"Error resolving variable {var_name}: {str(e)}")
+                unresolved.append(var_name)
+        
+        return resolved, unresolved
+
+    def _resolve_single_variable(
+        self, 
+        var_name: str, 
+        context_type: str, 
+        context_id: str,
+        current_context: Dict[str, Any]
+    ) -> Any:
+        """
+        Resolve a single variable based on its type and configuration
+        """
+        # Get variable definition
+        var_def = self.db.query(TemplateVariable).filter(
+            TemplateVariable.name == var_name,
+            TemplateVariable.active == True
+        ).first()
+        
+        if not var_def:
+            return None
+        
+        # Check for static value first
+        if var_def.static_value is not None:
+            return self._convert_value(var_def.static_value, var_def.variable_type)
+        
+        # Check cache if enabled
+        cache_key = f"{var_name}:{context_type}:{context_id}"
+        if var_def.cache_duration_minutes > 0:
+            cached_value = self._get_cached_value(var_def, cache_key)
+            if cached_value is not None:
+                return cached_value
+        
+        # Get context-specific value
+        context_value = self._get_context_value(var_def.id, context_type, context_id)
+        
+        # Process based on variable type
+        if var_def.variable_type == VariableType.CALCULATED:
+            value = self._process_calculated_variable(var_def, current_context)
+        elif var_def.variable_type == VariableType.CONDITIONAL:
+            value = self._process_conditional_variable(var_def, current_context)
+        elif var_def.variable_type == VariableType.QUERY:
+            value = self._process_query_variable(var_def, current_context, context_type, context_id)
+        elif var_def.variable_type == VariableType.LOOKUP:
+            value = self._process_lookup_variable(var_def, current_context, context_type, context_id)
+        else:
+            # Simple variable types (string, number, date, boolean)
+            value = context_value if context_value is not None else var_def.default_value
+            value = self._convert_value(value, var_def.variable_type)
+        
+        # Apply validation
+        if not self._validate_value(value, var_def):
+            logger.warning(f"Validation failed for variable {var_name}")
+            return var_def.default_value
+        
+        # Cache the result
+        if var_def.cache_duration_minutes > 0:
+            self._cache_value(var_def, cache_key, value)
+        
+        return value
+
+    def _process_calculated_variable(
+        self, 
+        var_def: TemplateVariable, 
+        context: Dict[str, Any]
+    ) -> Any:
+        """
+        Process a calculated variable using its formula
+        """
+        if not var_def.formula:
+            return var_def.default_value
+        
+        try:
+            # Create safe execution environment
+            safe_context = {
+                **self.safe_functions,
+                **context,
+                '__builtins__': {}  # Disable built-ins for security
+            }
+            
+            # Evaluate the formula
+            result = eval(var_def.formula, safe_context)
+            return result
+            
+        except Exception as e:
+            logger.error(f"Error evaluating formula for {var_def.name}: {str(e)}")
+            return var_def.default_value
+
+    def _process_conditional_variable(
+        self, 
+        var_def: TemplateVariable, 
+        context: Dict[str, Any]
+    ) -> Any:
+        """
+        Process a conditional variable using if/then/else logic
+        """
+        if not var_def.conditional_logic:
+            return var_def.default_value
+        
+        try:
+            logic = var_def.conditional_logic
+            if isinstance(logic, str):
+                logic = json.loads(logic)
+            
+            # Process conditional rules
+            for rule in logic.get('rules', []):
+                condition = rule.get('condition')
+                if self._evaluate_condition(condition, context):
+                    return self._convert_value(rule.get('value'), var_def.variable_type)
+            
+            # No conditions matched, return default
+            return logic.get('default', var_def.default_value)
+            
+        except Exception as e:
+            logger.error(f"Error processing conditional logic for {var_def.name}: {str(e)}")
+            return var_def.default_value
+
+    def _process_query_variable(
+        self, 
+        var_def: TemplateVariable, 
+        context: Dict[str, Any],
+        context_type: str,
+        context_id: str
+    ) -> Any:
+        """
+        Process a variable that gets its value from a database query
+        """
+        if not var_def.data_source_query:
+            return var_def.default_value
+        
+        try:
+            # Substitute context variables in the query
+            query = var_def.data_source_query
+            for key, value in context.items():
+                query = query.replace(f":{key}", str(value) if value is not None else "NULL")
+            
+            # Add context parameters
+            query = query.replace(":context_id", context_id)
+            query = query.replace(":context_type", context_type)
+            
+            # Execute query
+            result = self.db.execute(text(query)).first()
+            if result:
+                return result[0] if len(result) == 1 else dict(result)
+            return var_def.default_value
+            
+        except Exception as e:
+            logger.error(f"Error executing query for {var_def.name}: {str(e)}")
+            return var_def.default_value
+
+    def _process_lookup_variable(
+        self, 
+        var_def: TemplateVariable, 
+        context: Dict[str, Any],
+        context_type: str,
+        context_id: str
+    ) -> Any:
+        """
+        Process a variable that looks up values from a reference table
+        """
+        if not all([var_def.lookup_table, var_def.lookup_key_field, var_def.lookup_value_field]):
+            return var_def.default_value
+        
+        try:
+            # Get the lookup key from context
+            lookup_key = context.get(var_def.lookup_key_field)
+            if lookup_key is None and context_type == "file":
+                # Try to get from file context
+                file_obj = self.db.query(File).filter(File.file_no == context_id).first()
+                if file_obj:
+                    lookup_key = getattr(file_obj, var_def.lookup_key_field, None)
+            
+            if lookup_key is None:
+                return var_def.default_value
+            
+            # Build and execute lookup query
+            query = text(f"""
+                SELECT {var_def.lookup_value_field} 
+                FROM {var_def.lookup_table} 
+                WHERE {var_def.lookup_key_field} = :lookup_key
+                LIMIT 1
+            """)
+            
+            result = self.db.execute(query, {"lookup_key": lookup_key}).first()
+            return result[0] if result else var_def.default_value
+            
+        except Exception as e:
+            logger.error(f"Error processing lookup for {var_def.name}: {str(e)}")
+            return var_def.default_value
+
+    def _evaluate_condition(self, condition: Dict[str, Any], context: Dict[str, Any]) -> bool:
+        """
+        Evaluate a conditional expression
+        """
+        try:
+            field = condition.get('field')
+            operator_name = condition.get('operator', 'equals')
+            expected_value = condition.get('value')
+            
+            actual_value = context.get(field)
+            
+            # Convert values for comparison
+            if operator_name in ['equals', 'not_equals']:
+                return (actual_value == expected_value) if operator_name == 'equals' else (actual_value != expected_value)
+            elif operator_name in ['greater_than', 'less_than', 'greater_equal', 'less_equal']:
+                try:
+                    actual_num = float(actual_value) if actual_value is not None else 0
+                    expected_num = float(expected_value) if expected_value is not None else 0
+                    
+                    if operator_name == 'greater_than':
+                        return actual_num > expected_num
+                    elif operator_name == 'less_than':
+                        return actual_num < expected_num
+                    elif operator_name == 'greater_equal':
+                        return actual_num >= expected_num
+                    elif operator_name == 'less_equal':
+                        return actual_num <= expected_num
+                except (ValueError, TypeError):
+                    return False
+            elif operator_name == 'contains':
+                return str(expected_value) in str(actual_value) if actual_value else False
+            elif operator_name == 'is_empty':
+                return actual_value is None or str(actual_value).strip() == ''
+            elif operator_name == 'is_not_empty':
+                return actual_value is not None and str(actual_value).strip() != ''
+            
+            return False
+            
+        except Exception:
+            return False
+
+    def _determine_processing_order(self, variables: List[str]) -> List[str]:
+        """
+        Determine the order to process variables based on dependencies
+        """
+        # Get all variable definitions
+        var_defs = self.db.query(TemplateVariable).filter(
+            TemplateVariable.name.in_(variables),
+            TemplateVariable.active == True
+        ).all()
+        
+        var_deps = {}
+        for var_def in var_defs:
+            deps = var_def.depends_on or []
+            if isinstance(deps, str):
+                deps = json.loads(deps)
+            var_deps[var_def.name] = [dep for dep in deps if dep in variables]
+        
+        # Topological sort for dependency resolution
+        ordered = []
+        remaining = set(variables)
+        
+        while remaining:
+            # Find variables with no unresolved dependencies
+            ready = [var for var in remaining if not any(dep in remaining for dep in var_deps.get(var, []))]
+            
+            if not ready:
+                # Circular dependency or missing dependency, add remaining arbitrarily
+                ready = list(remaining)
+            
+            ordered.extend(ready)
+            remaining -= set(ready)
+        
+        return ordered
+
+    def _get_context_value(self, variable_id: int, context_type: str, context_id: str) -> Any:
+        """
+        Get the context-specific value for a variable
+        """
+        context = self.db.query(VariableContext).filter(
+            VariableContext.variable_id == variable_id,
+            VariableContext.context_type == context_type,
+            VariableContext.context_id == context_id
+        ).first()
+        
+        return context.computed_value if context and context.computed_value else (context.value if context else None)
+
+    def _convert_value(self, value: Any, var_type: VariableType) -> Any:
+        """
+        Convert a value to the appropriate type
+        """
+        if value is None:
+            return None
+        
+        try:
+            if var_type == VariableType.NUMBER:
+                return float(value) if '.' in str(value) else int(value)
+            elif var_type == VariableType.BOOLEAN:
+                if isinstance(value, bool):
+                    return value
+                return str(value).lower() in ('true', '1', 'yes', 'on')
+            elif var_type == VariableType.DATE:
+                if isinstance(value, date):
+                    return value
+                # Try to parse date string
+                from dateutil.parser import parse
+                return parse(str(value)).date()
+            else:
+                return str(value)
+        except (ValueError, TypeError):
+            return value
+
+    def _validate_value(self, value: Any, var_def: TemplateVariable) -> bool:
+        """
+        Validate a value against the variable's validation rules
+        """
+        if var_def.required and (value is None or str(value).strip() == ''):
+            return False
+        
+        if not var_def.validation_rules:
+            return True
+        
+        try:
+            rules = var_def.validation_rules
+            if isinstance(rules, str):
+                rules = json.loads(rules)
+            
+            # Apply validation rules
+            for rule_type, rule_value in rules.items():
+                if rule_type == 'min_length' and len(str(value)) < rule_value:
+                    return False
+                elif rule_type == 'max_length' and len(str(value)) > rule_value:
+                    return False
+                elif rule_type == 'pattern' and not re.match(rule_value, str(value)):
+                    return False
+                elif rule_type == 'min_value' and float(value) < rule_value:
+                    return False
+                elif rule_type == 'max_value' and float(value) > rule_value:
+                    return False
+            
+            return True
+            
+        except Exception:
+            return True  # Don't fail validation on rule processing errors
+
+    def _get_cached_value(self, var_def: TemplateVariable, cache_key: str) -> Any:
+        """
+        Get cached value if still valid
+        """
+        if not var_def.last_cached_at:
+            return None
+        
+        cache_age = datetime.now() - var_def.last_cached_at
+        if cache_age.total_seconds() > (var_def.cache_duration_minutes * 60):
+            return None
+        
+        return var_def.cached_value
+
+    def _cache_value(self, var_def: TemplateVariable, cache_key: str, value: Any):
+        """
+        Cache a computed value
+        """
+        var_def.cached_value = str(value) if value is not None else None
+        var_def.last_cached_at = datetime.now()
+        self.db.commit()
+
+    def set_variable_value(
+        self, 
+        variable_name: str, 
+        value: Any, 
+        context_type: str = "global", 
+        context_id: str = "default",
+        user_name: Optional[str] = None
+    ) -> bool:
+        """
+        Set a variable value in a specific context
+        """
+        try:
+            var_def = self.db.query(TemplateVariable).filter(
+                TemplateVariable.name == variable_name,
+                TemplateVariable.active == True
+            ).first()
+            
+            if not var_def:
+                return False
+            
+            # Get or create context
+            context = self.db.query(VariableContext).filter(
+                VariableContext.variable_id == var_def.id,
+                VariableContext.context_type == context_type,
+                VariableContext.context_id == context_id
+            ).first()
+            
+            old_value = context.value if context else None
+            
+            if not context:
+                context = VariableContext(
+                    variable_id=var_def.id,
+                    context_type=context_type,
+                    context_id=context_id,
+                    value=str(value) if value is not None else None,
+                    source="manual"
+                )
+                self.db.add(context)
+            else:
+                context.value = str(value) if value is not None else None
+            
+            # Validate the value
+            converted_value = self._convert_value(value, var_def.variable_type)
+            context.is_valid = self._validate_value(converted_value, var_def)
+            
+            # Log the change
+            audit_log = VariableAuditLog(
+                variable_id=var_def.id,
+                context_type=context_type,
+                context_id=context_id,
+                old_value=old_value,
+                new_value=context.value,
+                change_type="updated",
+                changed_by=user_name
+            )
+            self.db.add(audit_log)
+            
+            self.db.commit()
+            return True
+            
+        except Exception as e:
+            logger.error(f"Error setting variable {variable_name}: {str(e)}")
+            self.db.rollback()
+            return False
+
+    def get_variables_for_template(self, template_id: int) -> List[Dict[str, Any]]:
+        """
+        Get all variables associated with a template
+        """
+        variables = self.db.query(TemplateVariable, VariableTemplate).join(
+            VariableTemplate, VariableTemplate.variable_id == TemplateVariable.id
+        ).filter(
+            VariableTemplate.template_id == template_id,
+            TemplateVariable.active == True
+        ).order_by(VariableTemplate.display_order, TemplateVariable.name).all()
+        
+        result = []
+        for var_def, var_template in variables:
+            result.append({
+                'id': var_def.id,
+                'name': var_def.name,
+                'display_name': var_def.display_name or var_def.name,
+                'description': var_def.description,
+                'type': var_def.variable_type.value,
+                'required': var_template.override_required if var_template.override_required is not None else var_def.required,
+                'default_value': var_template.override_default or var_def.default_value,
+                'group_name': var_template.group_name,
+                'validation_rules': var_def.validation_rules
+            })
+        
+        return result

app/services/async_file_operations.py

@@ -0,0 +1,527 @@
+"""
+Async file operations service for handling large files efficiently.
+
+Provides streaming file operations, chunked processing, and progress tracking
+to improve performance with large files and prevent memory exhaustion.
+"""
+import asyncio
+import aiofiles
+import os
+import hashlib
+import uuid
+from pathlib import Path
+from typing import AsyncGenerator, Callable, Optional, Tuple, Dict, Any
+from fastapi import UploadFile, HTTPException
+from app.config import settings
+from app.utils.logging import get_logger
+
+logger = get_logger("async_file_ops")
+
+# Configuration constants
+CHUNK_SIZE = 64 * 1024  # 64KB chunks for streaming
+LARGE_FILE_THRESHOLD = 10 * 1024 * 1024  # 10MB - files larger than this use streaming
+MAX_MEMORY_BUFFER = 50 * 1024 * 1024  # 50MB - max memory buffer for file operations
+
+
+class AsyncFileOperations:
+    """
+    Service for handling large file operations asynchronously with streaming support.
+    
+    Features:
+    - Streaming file uploads/downloads
+    - Chunked processing for large files
+    - Progress tracking callbacks
+    - Memory-efficient operations
+    - Async file validation
+    """
+    
+    def __init__(self, base_upload_dir: Optional[str] = None):
+        self.base_upload_dir = Path(base_upload_dir or settings.upload_dir)
+        self.base_upload_dir.mkdir(parents=True, exist_ok=True)
+    
+    async def stream_upload_file(
+        self,
+        file: UploadFile,
+        destination_path: str,
+        progress_callback: Optional[Callable[[int, int], None]] = None,
+        validate_callback: Optional[Callable[[bytes], None]] = None
+    ) -> Tuple[str, int, str]:
+        """
+        Stream upload file to destination with progress tracking.
+        
+        Args:
+            file: The uploaded file
+            destination_path: Relative path where to save the file
+            progress_callback: Optional callback for progress tracking (bytes_read, total_size)
+            validate_callback: Optional callback for chunk validation
+            
+        Returns:
+            Tuple of (final_path, file_size, checksum)
+        """
+        final_path = self.base_upload_dir / destination_path
+        final_path.parent.mkdir(parents=True, exist_ok=True)
+        
+        file_size = 0
+        checksum = hashlib.sha256()
+        
+        try:
+            async with aiofiles.open(final_path, 'wb') as dest_file:
+                # Reset file pointer to beginning
+                await file.seek(0)
+                
+                while True:
+                    chunk = await file.read(CHUNK_SIZE)
+                    if not chunk:
+                        break
+                    
+                    # Update size and checksum
+                    file_size += len(chunk)
+                    checksum.update(chunk)
+                    
+                    # Optional chunk validation
+                    if validate_callback:
+                        try:
+                            validate_callback(chunk)
+                        except Exception as e:
+                            logger.warning(f"Chunk validation failed: {str(e)}")
+                            raise HTTPException(status_code=400, detail=f"File validation failed: {str(e)}")
+                    
+                    # Write chunk asynchronously
+                    await dest_file.write(chunk)
+                    
+                    # Progress callback
+                    if progress_callback:
+                        progress_callback(file_size, file_size)  # We don't know total size in advance
+                        
+                    # Yield control to prevent blocking
+                    await asyncio.sleep(0)
+                    
+        except Exception as e:
+            # Clean up partial file on error
+            if final_path.exists():
+                try:
+                    final_path.unlink()
+                except:
+                    pass
+            raise HTTPException(status_code=500, detail=f"File upload failed: {str(e)}")
+        
+        return str(final_path), file_size, checksum.hexdigest()
+    
+    async def stream_read_file(
+        self,
+        file_path: str,
+        chunk_size: int = CHUNK_SIZE
+    ) -> AsyncGenerator[bytes, None]:
+        """
+        Stream read file in chunks.
+        
+        Args:
+            file_path: Path to the file to read
+            chunk_size: Size of chunks to read
+            
+        Yields:
+            File content chunks
+        """
+        full_path = self.base_upload_dir / file_path
+        
+        if not full_path.exists():
+            raise HTTPException(status_code=404, detail="File not found")
+            
+        try:
+            async with aiofiles.open(full_path, 'rb') as file:
+                while True:
+                    chunk = await file.read(chunk_size)
+                    if not chunk:
+                        break
+                    yield chunk
+                    # Yield control
+                    await asyncio.sleep(0)
+        except Exception as e:
+            logger.error(f"Failed to stream read file {file_path}: {str(e)}")
+            raise HTTPException(status_code=500, detail=f"Failed to read file: {str(e)}")
+    
+    async def validate_file_streaming(
+        self,
+        file: UploadFile,
+        max_size: Optional[int] = None,
+        allowed_extensions: Optional[set] = None,
+        malware_patterns: Optional[list] = None
+    ) -> Tuple[bool, str, Dict[str, Any]]:
+        """
+        Validate file using streaming to handle large files efficiently.
+        
+        Args:
+            file: The uploaded file
+            max_size: Maximum allowed file size
+            allowed_extensions: Set of allowed file extensions
+            malware_patterns: List of malware patterns to check for
+            
+        Returns:
+            Tuple of (is_valid, error_message, file_metadata)
+        """
+        metadata = {
+            "filename": file.filename,
+            "size": 0,
+            "checksum": "",
+            "content_type": file.content_type
+        }
+        
+        # Check filename and extension
+        if not file.filename:
+            return False, "No filename provided", metadata
+            
+        file_ext = Path(file.filename).suffix.lower()
+        if allowed_extensions and file_ext not in allowed_extensions:
+            return False, f"File extension {file_ext} not allowed", metadata
+        
+        # Stream validation
+        checksum = hashlib.sha256()
+        file_size = 0
+        first_chunk = b""
+        
+        try:
+            await file.seek(0)
+            
+            # Read and validate in chunks
+            is_first_chunk = True
+            while True:
+                chunk = await file.read(CHUNK_SIZE)
+                if not chunk:
+                    break
+                
+                file_size += len(chunk)
+                checksum.update(chunk)
+                
+                # Store first chunk for content type detection
+                if is_first_chunk:
+                    first_chunk = chunk
+                    is_first_chunk = False
+                
+                # Check size limit
+                if max_size and file_size > max_size:
+                    # Standardized message to match envelope tests
+                    return False, "File too large", metadata
+                
+                # Check for malware patterns
+                if malware_patterns:
+                    chunk_str = chunk.decode('utf-8', errors='ignore').lower()
+                    for pattern in malware_patterns:
+                        if pattern in chunk_str:
+                            return False, f"Malicious content detected", metadata
+                
+                # Yield control
+                await asyncio.sleep(0)
+            
+            # Update metadata
+            metadata.update({
+                "size": file_size,
+                "checksum": checksum.hexdigest(),
+                "first_chunk": first_chunk[:512]  # First 512 bytes for content detection
+            })
+            
+            return True, "", metadata
+            
+        except Exception as e:
+            logger.error(f"File validation failed: {str(e)}")
+            return False, f"Validation error: {str(e)}", metadata
+        finally:
+            # Reset file pointer
+            await file.seek(0)
+    
+    async def process_csv_file_streaming(
+        self,
+        file: UploadFile,
+        row_processor: Callable[[str], Any],
+        progress_callback: Optional[Callable[[int], None]] = None,
+        batch_size: int = 1000
+    ) -> Tuple[int, int, list]:
+        """
+        Process CSV file in streaming fashion for large files.
+        
+        Args:
+            file: The CSV file to process
+            row_processor: Function to process each row
+            progress_callback: Optional callback for progress (rows_processed)
+            batch_size: Number of rows to process in each batch
+            
+        Returns:
+            Tuple of (total_rows, successful_rows, errors)
+        """
+        total_rows = 0
+        successful_rows = 0
+        errors = []
+        batch = []
+        
+        try:
+            await file.seek(0)
+            
+            # Read file in chunks and process line by line
+            buffer = ""
+            header_processed = False
+            
+            while True:
+                chunk = await file.read(CHUNK_SIZE)
+                if not chunk:
+                    # Process remaining buffer
+                    if buffer.strip():
+                        lines = buffer.split('\n')
+                        for line in lines:
+                            if line.strip():
+                                await self._process_csv_line(
+                                    line, row_processor, batch, batch_size,
+                                    total_rows, successful_rows, errors,
+                                    progress_callback, header_processed
+                                )
+                                total_rows += 1
+                                if not header_processed:
+                                    header_processed = True
+                    break
+                
+                # Decode chunk and add to buffer
+                try:
+                    chunk_text = chunk.decode('utf-8')
+                except UnicodeDecodeError:
+                    # Try with error handling
+                    chunk_text = chunk.decode('utf-8', errors='replace')
+                
+                buffer += chunk_text
+                
+                # Process complete lines
+                while '\n' in buffer:
+                    line, buffer = buffer.split('\n', 1)
+                    
+                    if line.strip():  # Skip empty lines
+                        success = await self._process_csv_line(
+                            line, row_processor, batch, batch_size,
+                            total_rows, successful_rows, errors,
+                            progress_callback, header_processed
+                        )
+                        
+                        total_rows += 1
+                        if success:
+                            successful_rows += 1
+                        
+                        if not header_processed:
+                            header_processed = True
+                
+                # Yield control
+                await asyncio.sleep(0)
+            
+            # Process any remaining batch
+            if batch:
+                await self._process_csv_batch(batch, errors)
+            
+        except Exception as e:
+            logger.error(f"CSV processing failed: {str(e)}")
+            errors.append(f"Processing error: {str(e)}")
+        
+        return total_rows, successful_rows, errors
+    
+    async def _process_csv_line(
+        self,
+        line: str,
+        row_processor: Callable,
+        batch: list,
+        batch_size: int,
+        total_rows: int,
+        successful_rows: int,
+        errors: list,
+        progress_callback: Optional[Callable],
+        header_processed: bool
+    ) -> bool:
+        """Process a single CSV line"""
+        try:
+            # Skip header row
+            if not header_processed:
+                return True
+                
+            # Add to batch
+            batch.append(line)
+            
+            # Process batch when full
+            if len(batch) >= batch_size:
+                await self._process_csv_batch(batch, errors)
+                batch.clear()
+            
+            # Progress callback
+            if progress_callback:
+                progress_callback(total_rows)
+            
+            return True
+            
+        except Exception as e:
+            errors.append(f"Row {total_rows}: {str(e)}")
+            return False
+    
+    async def _process_csv_batch(self, batch: list, errors: list):
+        """Process a batch of CSV rows"""
+        try:
+            # Process batch - this would be customized based on needs
+            for line in batch:
+                # Individual row processing would happen here
+                pass
+        except Exception as e:
+            errors.append(f"Batch processing error: {str(e)}")
+    
+    async def copy_file_async(
+        self,
+        source_path: str,
+        destination_path: str,
+        progress_callback: Optional[Callable[[int, int], None]] = None
+    ) -> bool:
+        """
+        Copy file asynchronously with progress tracking.
+        
+        Args:
+            source_path: Source file path
+            destination_path: Destination file path
+            progress_callback: Optional progress callback
+            
+        Returns:
+            True if successful, False otherwise
+        """
+        source = self.base_upload_dir / source_path
+        destination = self.base_upload_dir / destination_path
+        
+        if not source.exists():
+            logger.error(f"Source file does not exist: {source}")
+            return False
+        
+        try:
+            # Create destination directory
+            destination.parent.mkdir(parents=True, exist_ok=True)
+            
+            file_size = source.stat().st_size
+            bytes_copied = 0
+            
+            async with aiofiles.open(source, 'rb') as src_file:
+                async with aiofiles.open(destination, 'wb') as dest_file:
+                    while True:
+                        chunk = await src_file.read(CHUNK_SIZE)
+                        if not chunk:
+                            break
+                        
+                        await dest_file.write(chunk)
+                        bytes_copied += len(chunk)
+                        
+                        if progress_callback:
+                            progress_callback(bytes_copied, file_size)
+                        
+                        # Yield control
+                        await asyncio.sleep(0)
+            
+            return True
+            
+        except Exception as e:
+            logger.error(f"Failed to copy file {source} to {destination}: {str(e)}")
+            return False
+    
+    async def get_file_info_async(self, file_path: str) -> Optional[Dict[str, Any]]:
+        """
+        Get file information asynchronously.
+        
+        Args:
+            file_path: Path to the file
+            
+        Returns:
+            File information dictionary or None if file doesn't exist
+        """
+        full_path = self.base_upload_dir / file_path
+        
+        if not full_path.exists():
+            return None
+        
+        try:
+            stat = full_path.stat()
+            
+            # Calculate checksum for smaller files
+            checksum = None
+            if stat.st_size <= LARGE_FILE_THRESHOLD:
+                checksum = hashlib.sha256()
+                async with aiofiles.open(full_path, 'rb') as file:
+                    while True:
+                        chunk = await file.read(CHUNK_SIZE)
+                        if not chunk:
+                            break
+                        checksum.update(chunk)
+                        await asyncio.sleep(0)
+                checksum = checksum.hexdigest()
+            
+            return {
+                "path": file_path,
+                "size": stat.st_size,
+                "created": stat.st_ctime,
+                "modified": stat.st_mtime,
+                "checksum": checksum,
+                "is_large_file": stat.st_size > LARGE_FILE_THRESHOLD
+            }
+            
+        except Exception as e:
+            logger.error(f"Failed to get file info for {file_path}: {str(e)}")
+            return None
+
+
+# Global instance
+async_file_ops = AsyncFileOperations()
+
+
+# Utility functions for backward compatibility
+async def stream_save_upload(
+    file: UploadFile,
+    subdir: str,
+    filename_override: Optional[str] = None,
+    progress_callback: Optional[Callable[[int, int], None]] = None
+) -> Tuple[str, int]:
+    """
+    Save uploaded file using streaming operations.
+    
+    Returns:
+        Tuple of (relative_path, file_size)
+    """
+    # Generate safe filename
+    safe_filename = filename_override or file.filename
+    if not safe_filename:
+        safe_filename = f"upload_{uuid.uuid4().hex}"
+    
+    # Create unique filename to prevent conflicts
+    unique_filename = f"{uuid.uuid4().hex}_{safe_filename}"
+    relative_path = f"{subdir}/{unique_filename}"
+    
+    final_path, file_size, checksum = await async_file_ops.stream_upload_file(
+        file, relative_path, progress_callback
+    )
+    
+    return relative_path, file_size
+
+
+async def validate_large_upload(
+    file: UploadFile,
+    category: str = "document",
+    max_size: Optional[int] = None
+) -> Tuple[bool, str, Dict[str, Any]]:
+    """
+    Validate uploaded file using streaming for large files.
+    
+    Returns:
+        Tuple of (is_valid, error_message, metadata)
+    """
+    # Define allowed extensions by category
+    allowed_extensions = {
+        "document": {".pdf", ".doc", ".docx", ".txt", ".rtf"},
+        "image": {".jpg", ".jpeg", ".png", ".gif", ".bmp"},
+        "csv": {".csv", ".txt"},
+        "archive": {".zip", ".rar", ".7z", ".tar", ".gz"}
+    }
+    
+    # Define basic malware patterns
+    malware_patterns = [
+        "eval(", "exec(", "system(", "shell_exec(",
+        "<script", "javascript:", "vbscript:",
+        "cmd.exe", "powershell.exe"
+    ]
+    
+    extensions = allowed_extensions.get(category, set())
+    
+    return await async_file_ops.validate_file_streaming(
+        file, max_size, extensions, malware_patterns
+    )

app/services/async_storage.py

@@ -0,0 +1,346 @@
+"""
+Async storage abstraction for handling large files efficiently.
+
+Extends the existing storage abstraction with async capabilities
+for better performance with large files.
+"""
+import asyncio
+import aiofiles
+import os
+import uuid
+from pathlib import Path
+from typing import Optional, AsyncGenerator, Callable, Tuple
+from app.config import settings
+from app.utils.logging import get_logger
+
+logger = get_logger("async_storage")
+
+CHUNK_SIZE = 64 * 1024  # 64KB chunks
+
+
+class AsyncStorageAdapter:
+    """Abstract async storage adapter."""
+
+    async def save_bytes_async(
+        self, 
+        content: bytes, 
+        filename_hint: str, 
+        subdir: Optional[str] = None, 
+        content_type: Optional[str] = None,
+        progress_callback: Optional[Callable[[int, int], None]] = None
+    ) -> str:
+        raise NotImplementedError
+
+    async def save_stream_async(
+        self,
+        content_stream: AsyncGenerator[bytes, None],
+        filename_hint: str,
+        subdir: Optional[str] = None,
+        content_type: Optional[str] = None,
+        progress_callback: Optional[Callable[[int, int], None]] = None
+    ) -> str:
+        raise NotImplementedError
+
+    async def open_bytes_async(self, storage_path: str) -> bytes:
+        raise NotImplementedError
+
+    async def open_stream_async(self, storage_path: str) -> AsyncGenerator[bytes, None]:
+        raise NotImplementedError
+
+    async def delete_async(self, storage_path: str) -> bool:
+        raise NotImplementedError
+
+    async def exists_async(self, storage_path: str) -> bool:
+        raise NotImplementedError
+
+    async def get_size_async(self, storage_path: str) -> Optional[int]:
+        raise NotImplementedError
+
+    def public_url(self, storage_path: str) -> Optional[str]:
+        return None
+
+
+class AsyncLocalStorageAdapter(AsyncStorageAdapter):
+    """Async local storage adapter for handling large files efficiently."""
+
+    def __init__(self, base_dir: Optional[str] = None) -> None:
+        self.base_dir = Path(base_dir or settings.upload_dir).resolve()
+        self.base_dir.mkdir(parents=True, exist_ok=True)
+
+    async def _ensure_dir_async(self, directory: Path) -> None:
+        """Ensure directory exists asynchronously."""
+        if not directory.exists():
+            directory.mkdir(parents=True, exist_ok=True)
+
+    def _generate_unique_filename(self, filename_hint: str, subdir: Optional[str] = None) -> Tuple[Path, str]:
+        """Generate unique filename and return full path and relative path."""
+        safe_name = filename_hint.replace("/", "_").replace("\\", "_")
+        if not Path(safe_name).suffix:
+            safe_name = f"{safe_name}.bin"
+        
+        unique = uuid.uuid4().hex
+        final_name = f"{unique}_{safe_name}"
+        
+        if subdir:
+            directory = self.base_dir / subdir
+            full_path = directory / final_name
+            relative_path = f"{subdir}/{final_name}"
+        else:
+            directory = self.base_dir
+            full_path = directory / final_name
+            relative_path = final_name
+        
+        return full_path, relative_path
+
+    async def save_bytes_async(
+        self, 
+        content: bytes, 
+        filename_hint: str, 
+        subdir: Optional[str] = None, 
+        content_type: Optional[str] = None,
+        progress_callback: Optional[Callable[[int, int], None]] = None
+    ) -> str:
+        """Save bytes to storage asynchronously."""
+        full_path, relative_path = self._generate_unique_filename(filename_hint, subdir)
+        
+        # Ensure directory exists
+        await self._ensure_dir_async(full_path.parent)
+        
+        try:
+            async with aiofiles.open(full_path, "wb") as f:
+                if len(content) <= CHUNK_SIZE:
+                    # Small file - write directly
+                    await f.write(content)
+                    if progress_callback:
+                        progress_callback(len(content), len(content))
+                else:
+                    # Large file - write in chunks
+                    total_size = len(content)
+                    written = 0
+                    
+                    for i in range(0, len(content), CHUNK_SIZE):
+                        chunk = content[i:i + CHUNK_SIZE]
+                        await f.write(chunk)
+                        written += len(chunk)
+                        
+                        if progress_callback:
+                            progress_callback(written, total_size)
+                        
+                        # Yield control
+                        await asyncio.sleep(0)
+                        
+            return relative_path
+            
+        except Exception as e:
+            # Clean up on failure
+            if full_path.exists():
+                try:
+                    full_path.unlink()
+                except:
+                    pass
+            logger.error(f"Failed to save file {relative_path}: {str(e)}")
+            raise
+
+    async def save_stream_async(
+        self,
+        content_stream: AsyncGenerator[bytes, None],
+        filename_hint: str,
+        subdir: Optional[str] = None,
+        content_type: Optional[str] = None,
+        progress_callback: Optional[Callable[[int, int], None]] = None
+    ) -> str:
+        """Save streaming content to storage asynchronously."""
+        full_path, relative_path = self._generate_unique_filename(filename_hint, subdir)
+        
+        # Ensure directory exists
+        await self._ensure_dir_async(full_path.parent)
+        
+        try:
+            total_written = 0
+            async with aiofiles.open(full_path, "wb") as f:
+                async for chunk in content_stream:
+                    await f.write(chunk)
+                    total_written += len(chunk)
+                    
+                    if progress_callback:
+                        progress_callback(total_written, total_written)  # Unknown total for streams
+                    
+                    # Yield control
+                    await asyncio.sleep(0)
+                    
+            return relative_path
+            
+        except Exception as e:
+            # Clean up on failure
+            if full_path.exists():
+                try:
+                    full_path.unlink()
+                except:
+                    pass
+            logger.error(f"Failed to save stream {relative_path}: {str(e)}")
+            raise
+
+    async def open_bytes_async(self, storage_path: str) -> bytes:
+        """Read entire file as bytes asynchronously."""
+        full_path = self.base_dir / storage_path
+        
+        if not full_path.exists():
+            raise FileNotFoundError(f"File not found: {storage_path}")
+        
+        try:
+            async with aiofiles.open(full_path, "rb") as f:
+                return await f.read()
+        except Exception as e:
+            logger.error(f"Failed to read file {storage_path}: {str(e)}")
+            raise
+
+    async def open_stream_async(self, storage_path: str) -> AsyncGenerator[bytes, None]:
+        """Stream file content asynchronously."""
+        full_path = self.base_dir / storage_path
+        
+        if not full_path.exists():
+            raise FileNotFoundError(f"File not found: {storage_path}")
+        
+        try:
+            async with aiofiles.open(full_path, "rb") as f:
+                while True:
+                    chunk = await f.read(CHUNK_SIZE)
+                    if not chunk:
+                        break
+                    yield chunk
+                    # Yield control
+                    await asyncio.sleep(0)
+        except Exception as e:
+            logger.error(f"Failed to stream file {storage_path}: {str(e)}")
+            raise
+
+    async def delete_async(self, storage_path: str) -> bool:
+        """Delete file asynchronously."""
+        full_path = self.base_dir / storage_path
+        
+        try:
+            if full_path.exists():
+                full_path.unlink()
+                return True
+            return False
+        except Exception as e:
+            logger.error(f"Failed to delete file {storage_path}: {str(e)}")
+            return False
+
+    async def exists_async(self, storage_path: str) -> bool:
+        """Check if file exists asynchronously."""
+        full_path = self.base_dir / storage_path
+        return full_path.exists()
+
+    async def get_size_async(self, storage_path: str) -> Optional[int]:
+        """Get file size asynchronously."""
+        full_path = self.base_dir / storage_path
+        
+        try:
+            if full_path.exists():
+                return full_path.stat().st_size
+            return None
+        except Exception as e:
+            logger.error(f"Failed to get size for {storage_path}: {str(e)}")
+            return None
+
+    def public_url(self, storage_path: str) -> Optional[str]:
+        """Get public URL for file."""
+        return f"/uploads/{storage_path}".replace("\\", "/")
+
+
+class HybridStorageAdapter:
+    """
+    Hybrid storage adapter that provides both sync and async interfaces.
+    
+    Uses async operations internally but provides sync compatibility
+    for existing code.
+    """
+    
+    def __init__(self, base_dir: Optional[str] = None):
+        self.async_adapter = AsyncLocalStorageAdapter(base_dir)
+        self.base_dir = self.async_adapter.base_dir
+    
+    # Sync interface for backward compatibility
+    def save_bytes(
+        self, 
+        content: bytes, 
+        filename_hint: str, 
+        subdir: Optional[str] = None, 
+        content_type: Optional[str] = None
+    ) -> str:
+        """Sync wrapper for save_bytes_async."""
+        return asyncio.run(self.async_adapter.save_bytes_async(
+            content, filename_hint, subdir, content_type
+        ))
+    
+    def open_bytes(self, storage_path: str) -> bytes:
+        """Sync wrapper for open_bytes_async."""
+        return asyncio.run(self.async_adapter.open_bytes_async(storage_path))
+    
+    def delete(self, storage_path: str) -> bool:
+        """Sync wrapper for delete_async."""
+        return asyncio.run(self.async_adapter.delete_async(storage_path))
+    
+    def exists(self, storage_path: str) -> bool:
+        """Sync wrapper for exists_async."""
+        return asyncio.run(self.async_adapter.exists_async(storage_path))
+    
+    def public_url(self, storage_path: str) -> Optional[str]:
+        """Get public URL for file."""
+        return self.async_adapter.public_url(storage_path)
+    
+    # Async interface
+    async def save_bytes_async(
+        self, 
+        content: bytes, 
+        filename_hint: str, 
+        subdir: Optional[str] = None, 
+        content_type: Optional[str] = None,
+        progress_callback: Optional[Callable[[int, int], None]] = None
+    ) -> str:
+        """Save bytes asynchronously."""
+        return await self.async_adapter.save_bytes_async(
+            content, filename_hint, subdir, content_type, progress_callback
+        )
+    
+    async def save_stream_async(
+        self,
+        content_stream: AsyncGenerator[bytes, None],
+        filename_hint: str,
+        subdir: Optional[str] = None,
+        content_type: Optional[str] = None,
+        progress_callback: Optional[Callable[[int, int], None]] = None
+    ) -> str:
+        """Save stream asynchronously."""
+        return await self.async_adapter.save_stream_async(
+            content_stream, filename_hint, subdir, content_type, progress_callback
+        )
+    
+    async def open_bytes_async(self, storage_path: str) -> bytes:
+        """Read file as bytes asynchronously."""
+        return await self.async_adapter.open_bytes_async(storage_path)
+    
+    async def open_stream_async(self, storage_path: str) -> AsyncGenerator[bytes, None]:
+        """Stream file content asynchronously."""
+        async for chunk in self.async_adapter.open_stream_async(storage_path):
+            yield chunk
+    
+    async def get_size_async(self, storage_path: str) -> Optional[int]:
+        """Get file size asynchronously."""
+        return await self.async_adapter.get_size_async(storage_path)
+
+
+def get_async_storage() -> AsyncLocalStorageAdapter:
+    """Get async storage adapter instance."""
+    return AsyncLocalStorageAdapter()
+
+
+def get_hybrid_storage() -> HybridStorageAdapter:
+    """Get hybrid storage adapter with both sync and async interfaces."""
+    return HybridStorageAdapter()
+
+
+# Global instances
+async_storage = get_async_storage()
+hybrid_storage = get_hybrid_storage()

app/services/batch_generation.py

@@ -0,0 +1,203 @@
+"""
+Batch statement generation helpers.
+
+This module extracts request validation, batch ID construction, estimated completion
+calculation, and database persistence from the API layer.
+"""
+from __future__ import annotations
+
+from typing import List, Optional, Any, Dict, Tuple
+from datetime import datetime, timezone, timedelta
+from dataclasses import dataclass, field
+
+from fastapi import HTTPException, status
+from sqlalchemy.orm import Session
+
+from app.models.billing import BillingBatch, BillingBatchFile
+
+
+def prepare_batch_parameters(file_numbers: Optional[List[str]]) -> List[str]:
+    """Validate incoming file numbers and return de-duplicated list, preserving order."""
+    if not file_numbers:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="At least one file number must be provided",
+        )
+    if len(file_numbers) > 50:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Maximum 50 files allowed per batch operation",
+        )
+    # Remove duplicates while preserving order
+    return list(dict.fromkeys(file_numbers))
+
+
+def make_batch_id(unique_file_numbers: List[str], start_time: datetime) -> str:
+    """Create a stable batch ID matching the previous public behavior."""
+    return f"batch_{start_time.strftime('%Y%m%d_%H%M%S')}_{abs(hash(str(unique_file_numbers))) % 10000:04d}"
+
+
+def compute_estimated_completion(
+    *,
+    processed_files: int,
+    total_files: int,
+    started_at_iso: str,
+    now: datetime,
+) -> Optional[str]:
+    """Calculate estimated completion time as ISO string based on average rate."""
+    if processed_files <= 0:
+        return None
+    try:
+        start_time = datetime.fromisoformat(started_at_iso.replace("Z", "+00:00"))
+    except Exception:
+        return None
+
+    elapsed_seconds = (now - start_time).total_seconds()
+    if elapsed_seconds <= 0:
+        return None
+
+    remaining_files = max(total_files - processed_files, 0)
+    if remaining_files == 0:
+        return now.isoformat()
+
+    avg_time_per_file = elapsed_seconds / processed_files
+    estimated_remaining_seconds = avg_time_per_file * remaining_files
+    estimated_completion = now + timedelta(seconds=estimated_remaining_seconds)
+    return estimated_completion.isoformat()
+
+
+def persist_batch_results(
+    db: Session,
+    *,
+    batch_id: str,
+    progress: Any,
+    processing_time_seconds: float,
+    success_rate: float,
+) -> None:
+    """Persist batch summary and per-file results using the DB models.
+
+    The `progress` object is expected to expose attributes consistent with the API's
+    BatchProgress model:
+      - status, total_files, successful_files, failed_files
+      - started_at, updated_at, completed_at, error_message
+      - files: list with {file_no, status, error_message, statement_meta, started_at, completed_at}
+    """
+
+    def _parse_iso(dt: Optional[str]):
+        if not dt:
+            return None
+        try:
+            from datetime import datetime as _dt
+            return _dt.fromisoformat(str(dt).replace('Z', '+00:00'))
+        except Exception:
+            return None
+
+    batch_row = BillingBatch(
+        batch_id=batch_id,
+        status=str(getattr(progress, "status", "")),
+        total_files=int(getattr(progress, "total_files", 0)),
+        successful_files=int(getattr(progress, "successful_files", 0)),
+        failed_files=int(getattr(progress, "failed_files", 0)),
+        started_at=_parse_iso(getattr(progress, "started_at", None)),
+        updated_at=_parse_iso(getattr(progress, "updated_at", None)),
+        completed_at=_parse_iso(getattr(progress, "completed_at", None)),
+        processing_time_seconds=float(processing_time_seconds),
+        success_rate=float(success_rate),
+        error_message=getattr(progress, "error_message", None),
+    )
+    db.add(batch_row)
+
+    for f in list(getattr(progress, "files", []) or []):
+        meta = getattr(f, "statement_meta", None)
+        filename = None
+        size = None
+        if meta is not None:
+            try:
+                filename = getattr(meta, "filename", None)
+                size = getattr(meta, "size", None)
+            except Exception:
+                filename = None
+                size = None
+            if filename is None and isinstance(meta, dict):
+                filename = meta.get("filename")
+                size = meta.get("size")
+        db.add(
+            BillingBatchFile(
+                batch_id=batch_id,
+                file_no=getattr(f, "file_no", None),
+                status=str(getattr(f, "status", "")),
+                error_message=getattr(f, "error_message", None),
+                filename=filename,
+                size=size,
+                started_at=_parse_iso(getattr(f, "started_at", None)),
+                completed_at=_parse_iso(getattr(f, "completed_at", None)),
+            )
+        )
+
+    db.commit()
+
+
+
+@dataclass
+class BatchProgressEntry:
+    """Lightweight progress entry shape used in tests for compatibility."""
+    file_no: str
+    status: str
+    started_at: Optional[str] = None
+    completed_at: Optional[str] = None
+    error_message: Optional[str] = None
+    statement_meta: Optional[Dict[str, Any]] = None
+
+
+@dataclass
+class BatchProgress:
+    """Lightweight batch progress shape used in tests for topic formatting checks."""
+    batch_id: str
+    status: str
+    total_files: int
+    processed_files: int
+    successful_files: int
+    failed_files: int
+    current_file: Optional[str] = None
+    started_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    updated_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    completed_at: Optional[datetime] = None
+    estimated_completion: Optional[datetime] = None
+    processing_time_seconds: Optional[float] = None
+    success_rate: Optional[float] = None
+    files: List[BatchProgressEntry] = field(default_factory=list)
+    error_message: Optional[str] = None
+
+    def model_dump(self) -> Dict[str, Any]:
+        """Provide a dict representation similar to Pydantic for broadcasting."""
+        def _dt(v):
+            if isinstance(v, datetime):
+                return v.isoformat()
+            return v
+        return {
+            "batch_id": self.batch_id,
+            "status": self.status,
+            "total_files": self.total_files,
+            "processed_files": self.processed_files,
+            "successful_files": self.successful_files,
+            "failed_files": self.failed_files,
+            "current_file": self.current_file,
+            "started_at": _dt(self.started_at),
+            "updated_at": _dt(self.updated_at),
+            "completed_at": _dt(self.completed_at),
+            "estimated_completion": _dt(self.estimated_completion),
+            "processing_time_seconds": self.processing_time_seconds,
+            "success_rate": self.success_rate,
+            "files": [
+                {
+                    "file_no": f.file_no,
+                    "status": f.status,
+                    "started_at": f.started_at,
+                    "completed_at": f.completed_at,
+                    "error_message": f.error_message,
+                    "statement_meta": f.statement_meta,
+                }
+                for f in self.files
+            ],
+            "error_message": self.error_message,
+        }

app/services/customers_search.py

@@ -4,7 +4,15 @@ from sqlalchemy import or_, and_, func, asc, desc
 from app.models.rolodex import Rolodex
 
 
-def apply_customer_filters(base_query, search: Optional[str], group: Optional[str], state: Optional[str], groups: Optional[List[str]], states: Optional[List[str]]):
+def apply_customer_filters(
+    base_query,
+    search: Optional[str],
+    group: Optional[str],
+    state: Optional[str],
+    groups: Optional[List[str]],
+    states: Optional[List[str]],
+    name_prefix: Optional[str] = None,
+):
     """Apply shared search and group/state filters to the provided base_query.
 
     This helper is used by both list and export endpoints to keep logic in sync.
@@ -53,6 +61,16 @@ def apply_customer_filters(base_query, search: Optional[str], group: Optional[st
     if effective_states:
         base_query = base_query.filter(Rolodex.abrev.in_(effective_states))
 
+    # Optional: prefix filter on name (matches first OR last starting with the prefix, case-insensitive)
+    p = (name_prefix or "").strip().lower()
+    if p:
+        base_query = base_query.filter(
+            or_(
+                func.lower(Rolodex.last).like(f"{p}%"),
+                func.lower(Rolodex.first).like(f"{p}%"),
+            )
+        )
+
     return base_query
 
 

app/services/deadline_calendar.py

@@ -0,0 +1,698 @@
+"""
+Deadline calendar integration service
+Provides calendar views and scheduling utilities for deadlines
+"""
+from typing import List, Dict, Any, Optional, Tuple
+from datetime import datetime, date, timezone, timedelta
+from calendar import monthrange, weekday
+from sqlalchemy.orm import Session, joinedload
+from sqlalchemy import and_, func, or_, desc
+
+from app.models import (
+    Deadline, CourtCalendar, User, Employee,
+    DeadlineType, DeadlinePriority, DeadlineStatus
+)
+from app.utils.logging import app_logger
+
+logger = app_logger
+
+
+class DeadlineCalendarService:
+    """Service for deadline calendar views and scheduling"""
+    
+    def __init__(self, db: Session):
+        self.db = db
+    
+    def get_monthly_calendar(
+        self,
+        year: int,
+        month: int,
+        user_id: Optional[int] = None,
+        employee_id: Optional[str] = None,
+        show_completed: bool = False
+    ) -> Dict[str, Any]:
+        """Get monthly calendar view with deadlines"""
+        
+        # Get first and last day of month
+        first_day = date(year, month, 1)
+        last_day = date(year, month, monthrange(year, month)[1])
+        
+        # Get first Monday of calendar view (may be in previous month)
+        first_monday = first_day - timedelta(days=first_day.weekday())
+        
+        # Get last Sunday of calendar view (may be in next month)
+        last_sunday = last_day + timedelta(days=(6 - last_day.weekday()))
+        
+        # Build query for deadlines in the calendar period
+        query = self.db.query(Deadline).filter(
+            Deadline.deadline_date.between(first_monday, last_sunday)
+        )
+        
+        if not show_completed:
+            query = query.filter(Deadline.status != DeadlineStatus.COMPLETED)
+        
+        if user_id:
+            query = query.filter(Deadline.assigned_to_user_id == user_id)
+        
+        if employee_id:
+            query = query.filter(Deadline.assigned_to_employee_id == employee_id)
+        
+        deadlines = query.options(
+            joinedload(Deadline.file),
+            joinedload(Deadline.client),
+            joinedload(Deadline.assigned_to_user),
+            joinedload(Deadline.assigned_to_employee)
+        ).order_by(
+            Deadline.deadline_date.asc(),
+            Deadline.deadline_time.asc(),
+            Deadline.priority.desc()
+        ).all()
+        
+        # Build calendar grid (6 weeks x 7 days)
+        calendar_weeks = []
+        current_date = first_monday
+        
+        for week in range(6):
+            week_days = []
+            
+            for day in range(7):
+                day_date = current_date + timedelta(days=day)
+                
+                # Get deadlines for this day
+                day_deadlines = [
+                    d for d in deadlines if d.deadline_date == day_date
+                ]
+                
+                # Format deadline data
+                formatted_deadlines = []
+                for deadline in day_deadlines:
+                    formatted_deadlines.append({
+                        "id": deadline.id,
+                        "title": deadline.title,
+                        "deadline_time": deadline.deadline_time.strftime("%H:%M") if deadline.deadline_time else None,
+                        "priority": deadline.priority.value,
+                        "deadline_type": deadline.deadline_type.value,
+                        "status": deadline.status.value,
+                        "file_no": deadline.file_no,
+                        "client_name": self._get_client_name(deadline),
+                        "court_name": deadline.court_name,
+                        "is_overdue": deadline.is_overdue,
+                        "is_court_date": deadline.deadline_type == DeadlineType.COURT_HEARING
+                    })
+                
+                week_days.append({
+                    "date": day_date,
+                    "day_number": day_date.day,
+                    "is_current_month": day_date.month == month,
+                    "is_today": day_date == date.today(),
+                    "is_weekend": day_date.weekday() >= 5,
+                    "deadlines": formatted_deadlines,
+                    "deadline_count": len(formatted_deadlines),
+                    "has_overdue": any(d["is_overdue"] for d in formatted_deadlines),
+                    "has_court_date": any(d["is_court_date"] for d in formatted_deadlines),
+                    "max_priority": self._get_max_priority(day_deadlines)
+                })
+            
+            calendar_weeks.append({
+                "week_start": current_date,
+                "days": week_days
+            })
+            
+            current_date += timedelta(days=7)
+        
+        # Calculate summary statistics
+        month_deadlines = [d for d in deadlines if d.deadline_date.month == month]
+        
+        return {
+            "year": year,
+            "month": month,
+            "month_name": first_day.strftime("%B"),
+            "calendar_period": {
+                "start_date": first_monday,
+                "end_date": last_sunday
+            },
+            "summary": {
+                "total_deadlines": len(month_deadlines),
+                "overdue": len([d for d in month_deadlines if d.is_overdue]),
+                "pending": len([d for d in month_deadlines if d.status == DeadlineStatus.PENDING]),
+                "completed": len([d for d in month_deadlines if d.status == DeadlineStatus.COMPLETED]),
+                "court_dates": len([d for d in month_deadlines if d.deadline_type == DeadlineType.COURT_HEARING])
+            },
+            "weeks": calendar_weeks
+        }
+    
+    def get_weekly_calendar(
+        self,
+        year: int,
+        week: int,
+        user_id: Optional[int] = None,
+        employee_id: Optional[str] = None,
+        show_completed: bool = False
+    ) -> Dict[str, Any]:
+        """Get weekly calendar view with detailed scheduling"""
+        
+        # Calculate the Monday of the specified week
+        jan_1 = date(year, 1, 1)
+        jan_1_weekday = jan_1.weekday()
+        
+        # Find the Monday of week 1
+        days_to_monday = -jan_1_weekday if jan_1_weekday == 0 else 7 - jan_1_weekday
+        first_monday = jan_1 + timedelta(days=days_to_monday)
+        
+        # Calculate the target week's Monday
+        week_monday = first_monday + timedelta(weeks=week - 1)
+        week_sunday = week_monday + timedelta(days=6)
+        
+        # Build query for deadlines in the week
+        query = self.db.query(Deadline).filter(
+            Deadline.deadline_date.between(week_monday, week_sunday)
+        )
+        
+        if not show_completed:
+            query = query.filter(Deadline.status != DeadlineStatus.COMPLETED)
+        
+        if user_id:
+            query = query.filter(Deadline.assigned_to_user_id == user_id)
+        
+        if employee_id:
+            query = query.filter(Deadline.assigned_to_employee_id == employee_id)
+        
+        deadlines = query.options(
+            joinedload(Deadline.file),
+            joinedload(Deadline.client),
+            joinedload(Deadline.assigned_to_user),
+            joinedload(Deadline.assigned_to_employee)
+        ).order_by(
+            Deadline.deadline_date.asc(),
+            Deadline.deadline_time.asc(),
+            Deadline.priority.desc()
+        ).all()
+        
+        # Build daily schedule
+        week_days = []
+        
+        for day_offset in range(7):
+            day_date = week_monday + timedelta(days=day_offset)
+            
+            # Get deadlines for this day
+            day_deadlines = [d for d in deadlines if d.deadline_date == day_date]
+            
+            # Group deadlines by time
+            timed_deadlines = []
+            all_day_deadlines = []
+            
+            for deadline in day_deadlines:
+                deadline_data = {
+                    "id": deadline.id,
+                    "title": deadline.title,
+                    "deadline_time": deadline.deadline_time,
+                    "priority": deadline.priority.value,
+                    "deadline_type": deadline.deadline_type.value,
+                    "status": deadline.status.value,
+                    "file_no": deadline.file_no,
+                    "client_name": self._get_client_name(deadline),
+                    "assigned_to": self._get_assigned_to(deadline),
+                    "court_name": deadline.court_name,
+                    "case_number": deadline.case_number,
+                    "description": deadline.description,
+                    "is_overdue": deadline.is_overdue,
+                    "estimated_duration": self._get_estimated_duration(deadline)
+                }
+                
+                if deadline.deadline_time:
+                    timed_deadlines.append(deadline_data)
+                else:
+                    all_day_deadlines.append(deadline_data)
+            
+            # Sort timed deadlines by time
+            timed_deadlines.sort(key=lambda x: x["deadline_time"])
+            
+            week_days.append({
+                "date": day_date,
+                "day_name": day_date.strftime("%A"),
+                "day_short": day_date.strftime("%a"),
+                "is_today": day_date == date.today(),
+                "is_weekend": day_date.weekday() >= 5,
+                "timed_deadlines": timed_deadlines,
+                "all_day_deadlines": all_day_deadlines,
+                "total_deadlines": len(day_deadlines),
+                "has_court_dates": any(d.deadline_type == DeadlineType.COURT_HEARING for d in day_deadlines)
+            })
+        
+        return {
+            "year": year,
+            "week": week,
+            "week_period": {
+                "start_date": week_monday,
+                "end_date": week_sunday
+            },
+            "summary": {
+                "total_deadlines": len(deadlines),
+                "timed_deadlines": len([d for d in deadlines if d.deadline_time]),
+                "all_day_deadlines": len([d for d in deadlines if not d.deadline_time]),
+                "court_dates": len([d for d in deadlines if d.deadline_type == DeadlineType.COURT_HEARING])
+            },
+            "days": week_days
+        }
+    
+    def get_daily_schedule(
+        self,
+        target_date: date,
+        user_id: Optional[int] = None,
+        employee_id: Optional[str] = None,
+        show_completed: bool = False
+    ) -> Dict[str, Any]:
+        """Get detailed daily schedule with time slots"""
+        
+        # Build query for deadlines on the target date
+        query = self.db.query(Deadline).filter(
+            Deadline.deadline_date == target_date
+        )
+        
+        if not show_completed:
+            query = query.filter(Deadline.status != DeadlineStatus.COMPLETED)
+        
+        if user_id:
+            query = query.filter(Deadline.assigned_to_user_id == user_id)
+        
+        if employee_id:
+            query = query.filter(Deadline.assigned_to_employee_id == employee_id)
+        
+        deadlines = query.options(
+            joinedload(Deadline.file),
+            joinedload(Deadline.client),
+            joinedload(Deadline.assigned_to_user),
+            joinedload(Deadline.assigned_to_employee)
+        ).order_by(
+            Deadline.deadline_time.asc(),
+            Deadline.priority.desc()
+        ).all()
+        
+        # Create time slots (30-minute intervals from 8 AM to 6 PM)
+        time_slots = []
+        start_hour = 8
+        end_hour = 18
+        
+        for hour in range(start_hour, end_hour):
+            for minute in [0, 30]:
+                slot_time = datetime.combine(target_date, datetime.min.time().replace(hour=hour, minute=minute))
+                
+                # Find deadlines in this time slot
+                slot_deadlines = []
+                for deadline in deadlines:
+                    if deadline.deadline_time:
+                        deadline_time = deadline.deadline_time.replace(tzinfo=None)
+                        
+                        # Check if deadline falls within this 30-minute slot
+                        if (slot_time <= deadline_time < slot_time + timedelta(minutes=30)):
+                            slot_deadlines.append({
+                                "id": deadline.id,
+                                "title": deadline.title,
+                                "deadline_time": deadline.deadline_time,
+                                "priority": deadline.priority.value,
+                                "deadline_type": deadline.deadline_type.value,
+                                "status": deadline.status.value,
+                                "file_no": deadline.file_no,
+                                "client_name": self._get_client_name(deadline),
+                                "court_name": deadline.court_name,
+                                "case_number": deadline.case_number,
+                                "description": deadline.description,
+                                "estimated_duration": self._get_estimated_duration(deadline)
+                            })
+                
+                time_slots.append({
+                    "time": slot_time.strftime("%H:%M"),
+                    "datetime": slot_time,
+                    "deadlines": slot_deadlines,
+                    "is_busy": len(slot_deadlines) > 0
+                })
+        
+        # Get all-day deadlines
+        all_day_deadlines = []
+        for deadline in deadlines:
+            if not deadline.deadline_time:
+                all_day_deadlines.append({
+                    "id": deadline.id,
+                    "title": deadline.title,
+                    "priority": deadline.priority.value,
+                    "deadline_type": deadline.deadline_type.value,
+                    "status": deadline.status.value,
+                    "file_no": deadline.file_no,
+                    "client_name": self._get_client_name(deadline),
+                    "description": deadline.description
+                })
+        
+        return {
+            "date": target_date,
+            "day_name": target_date.strftime("%A, %B %d, %Y"),
+            "is_today": target_date == date.today(),
+            "summary": {
+                "total_deadlines": len(deadlines),
+                "timed_deadlines": len([d for d in deadlines if d.deadline_time]),
+                "all_day_deadlines": len(all_day_deadlines),
+                "court_dates": len([d for d in deadlines if d.deadline_type == DeadlineType.COURT_HEARING]),
+                "overdue": len([d for d in deadlines if d.is_overdue])
+            },
+            "all_day_deadlines": all_day_deadlines,
+            "time_slots": time_slots,
+            "business_hours": {
+                "start": f"{start_hour:02d}:00",
+                "end": f"{end_hour:02d}:00"
+            }
+        }
+    
+    def find_available_slots(
+        self,
+        start_date: date,
+        end_date: date,
+        duration_minutes: int = 60,
+        user_id: Optional[int] = None,
+        employee_id: Optional[str] = None,
+        business_hours_only: bool = True
+    ) -> List[Dict[str, Any]]:
+        """Find available time slots for scheduling new deadlines"""
+        
+        # Get existing deadlines in the period
+        query = self.db.query(Deadline).filter(
+            Deadline.deadline_date.between(start_date, end_date),
+            Deadline.status == DeadlineStatus.PENDING,
+            Deadline.deadline_time.isnot(None)
+        )
+        
+        if user_id:
+            query = query.filter(Deadline.assigned_to_user_id == user_id)
+        
+        if employee_id:
+            query = query.filter(Deadline.assigned_to_employee_id == employee_id)
+        
+        existing_deadlines = query.all()
+        
+        # Define business hours
+        if business_hours_only:
+            start_hour, end_hour = 8, 18
+        else:
+            start_hour, end_hour = 0, 24
+        
+        available_slots = []
+        current_date = start_date
+        
+        while current_date <= end_date:
+            # Skip weekends if business hours only
+            if business_hours_only and current_date.weekday() >= 5:
+                current_date += timedelta(days=1)
+                continue
+            
+            # Get deadlines for this day
+            day_deadlines = [
+                d for d in existing_deadlines 
+                if d.deadline_date == current_date
+            ]
+            
+            # Sort by time
+            day_deadlines.sort(key=lambda d: d.deadline_time)
+            
+            # Find gaps between deadlines
+            for hour in range(start_hour, end_hour):
+                for minute in range(0, 60, 30):  # 30-minute intervals
+                    slot_start = datetime.combine(
+                        current_date, 
+                        datetime.min.time().replace(hour=hour, minute=minute)
+                    )
+                    slot_end = slot_start + timedelta(minutes=duration_minutes)
+                    
+                    # Check if this slot conflicts with existing deadlines
+                    is_available = True
+                    for deadline in day_deadlines:
+                        deadline_start = deadline.deadline_time.replace(tzinfo=None)
+                        deadline_end = deadline_start + timedelta(
+                            minutes=self._get_estimated_duration(deadline)
+                        )
+                        
+                        # Check for overlap
+                        if not (slot_end <= deadline_start or slot_start >= deadline_end):
+                            is_available = False
+                            break
+                    
+                    if is_available:
+                        available_slots.append({
+                            "start_datetime": slot_start,
+                            "end_datetime": slot_end,
+                            "date": current_date,
+                            "start_time": slot_start.strftime("%H:%M"),
+                            "end_time": slot_end.strftime("%H:%M"),
+                            "duration_minutes": duration_minutes,
+                            "day_name": current_date.strftime("%A")
+                        })
+            
+            current_date += timedelta(days=1)
+        
+        return available_slots[:50]  # Limit to first 50 slots
+    
+    def get_conflict_analysis(
+        self,
+        proposed_datetime: datetime,
+        duration_minutes: int = 60,
+        user_id: Optional[int] = None,
+        employee_id: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """Analyze potential conflicts for a proposed deadline time"""
+        
+        proposed_date = proposed_datetime.date()
+        proposed_end = proposed_datetime + timedelta(minutes=duration_minutes)
+        
+        # Get existing deadlines on the same day
+        query = self.db.query(Deadline).filter(
+            Deadline.deadline_date == proposed_date,
+            Deadline.status == DeadlineStatus.PENDING,
+            Deadline.deadline_time.isnot(None)
+        )
+        
+        if user_id:
+            query = query.filter(Deadline.assigned_to_user_id == user_id)
+        
+        if employee_id:
+            query = query.filter(Deadline.assigned_to_employee_id == employee_id)
+        
+        existing_deadlines = query.options(
+            joinedload(Deadline.file),
+            joinedload(Deadline.client)
+        ).all()
+        
+        conflicts = []
+        nearby_deadlines = []
+        
+        for deadline in existing_deadlines:
+            deadline_start = deadline.deadline_time.replace(tzinfo=None)
+            deadline_end = deadline_start + timedelta(
+                minutes=self._get_estimated_duration(deadline)
+            )
+            
+            # Check for direct overlap
+            if not (proposed_end <= deadline_start or proposed_datetime >= deadline_end):
+                conflicts.append({
+                    "id": deadline.id,
+                    "title": deadline.title,
+                    "start_time": deadline_start,
+                    "end_time": deadline_end,
+                    "conflict_type": "overlap",
+                    "file_no": deadline.file_no,
+                    "client_name": self._get_client_name(deadline)
+                })
+            
+            # Check for nearby deadlines (within 30 minutes)
+            elif (abs((proposed_datetime - deadline_start).total_seconds()) <= 1800 or
+                  abs((proposed_end - deadline_end).total_seconds()) <= 1800):
+                nearby_deadlines.append({
+                    "id": deadline.id,
+                    "title": deadline.title,
+                    "start_time": deadline_start,
+                    "end_time": deadline_end,
+                    "file_no": deadline.file_no,
+                    "client_name": self._get_client_name(deadline),
+                    "minutes_gap": min(
+                        abs((proposed_datetime - deadline_end).total_seconds() / 60),
+                        abs((deadline_start - proposed_end).total_seconds() / 60)
+                    )
+                })
+        
+        return {
+            "proposed_datetime": proposed_datetime,
+            "proposed_end": proposed_end,
+            "duration_minutes": duration_minutes,
+            "has_conflicts": len(conflicts) > 0,
+            "conflicts": conflicts,
+            "nearby_deadlines": nearby_deadlines,
+            "recommendation": self._get_scheduling_recommendation(
+                conflicts, nearby_deadlines, proposed_datetime
+            )
+        }
+    
+    # Private helper methods
+    
+    def _get_client_name(self, deadline: Deadline) -> Optional[str]:
+        """Get formatted client name from deadline"""
+        
+        if deadline.client:
+            return f"{deadline.client.first or ''} {deadline.client.last or ''}".strip()
+        elif deadline.file and deadline.file.owner:
+            return f"{deadline.file.owner.first or ''} {deadline.file.owner.last or ''}".strip()
+        return None
+    
+    def _get_assigned_to(self, deadline: Deadline) -> Optional[str]:
+        """Get assigned person name from deadline"""
+        
+        if deadline.assigned_to_user:
+            return deadline.assigned_to_user.username
+        elif deadline.assigned_to_employee:
+            employee = deadline.assigned_to_employee
+            return f"{employee.first_name or ''} {employee.last_name or ''}".strip()
+        return None
+    
+    def _get_max_priority(self, deadlines: List[Deadline]) -> str:
+        """Get the highest priority from a list of deadlines"""
+        
+        if not deadlines:
+            return "none"
+        
+        priority_order = {
+            DeadlinePriority.CRITICAL: 4,
+            DeadlinePriority.HIGH: 3,
+            DeadlinePriority.MEDIUM: 2,
+            DeadlinePriority.LOW: 1
+        }
+        
+        max_priority = max(deadlines, key=lambda d: priority_order.get(d.priority, 0))
+        return max_priority.priority.value
+    
+    def _get_estimated_duration(self, deadline: Deadline) -> int:
+        """Get estimated duration in minutes for a deadline type"""
+        
+        # Default durations by deadline type
+        duration_map = {
+            DeadlineType.COURT_HEARING: 120,      # 2 hours
+            DeadlineType.COURT_FILING: 30,        # 30 minutes
+            DeadlineType.CLIENT_MEETING: 60,      # 1 hour
+            DeadlineType.DISCOVERY: 30,           # 30 minutes
+            DeadlineType.ADMINISTRATIVE: 30,      # 30 minutes
+            DeadlineType.INTERNAL: 60,            # 1 hour
+            DeadlineType.CONTRACT: 30,            # 30 minutes
+            DeadlineType.STATUTE_OF_LIMITATIONS: 30,  # 30 minutes
+            DeadlineType.OTHER: 60                # 1 hour default
+        }
+        
+        return duration_map.get(deadline.deadline_type, 60)
+    
+    def _get_scheduling_recommendation(
+        self,
+        conflicts: List[Dict],
+        nearby_deadlines: List[Dict],
+        proposed_datetime: datetime
+    ) -> str:
+        """Get scheduling recommendation based on conflicts"""
+        
+        if conflicts:
+            return "CONFLICT - Choose a different time slot"
+        
+        if nearby_deadlines:
+            min_gap = min(d["minutes_gap"] for d in nearby_deadlines)
+            if min_gap < 15:
+                return "CAUTION - Very tight schedule, consider more buffer time"
+            elif min_gap < 30:
+                return "ACCEPTABLE - Close to other deadlines but manageable"
+        
+        return "OPTIMAL - No conflicts detected"
+
+
+class CalendarExportService:
+    """Service for exporting deadlines to external calendar formats"""
+    
+    def __init__(self, db: Session):
+        self.db = db
+    
+    def export_to_ical(
+        self,
+        start_date: date,
+        end_date: date,
+        user_id: Optional[int] = None,
+        employee_id: Optional[str] = None,
+        deadline_types: Optional[List[DeadlineType]] = None
+    ) -> str:
+        """Export deadlines to iCalendar format"""
+        
+        # Get deadlines for export
+        query = self.db.query(Deadline).filter(
+            Deadline.deadline_date.between(start_date, end_date),
+            Deadline.status == DeadlineStatus.PENDING
+        )
+        
+        if user_id:
+            query = query.filter(Deadline.assigned_to_user_id == user_id)
+        
+        if employee_id:
+            query = query.filter(Deadline.assigned_to_employee_id == employee_id)
+        
+        if deadline_types:
+            query = query.filter(Deadline.deadline_type.in_(deadline_types))
+        
+        deadlines = query.options(
+            joinedload(Deadline.file),
+            joinedload(Deadline.client)
+        ).order_by(Deadline.deadline_date.asc()).all()
+        
+        # Build iCal content
+        ical_lines = [
+            "BEGIN:VCALENDAR",
+            "VERSION:2.0",
+            "PRODID:-//Delphi Consulting//Deadline Manager//EN",
+            "CALSCALE:GREGORIAN",
+            "METHOD:PUBLISH"
+        ]
+        
+        for deadline in deadlines:
+            # Format datetime for iCal
+            if deadline.deadline_time:
+                dtstart = deadline.deadline_time.strftime("%Y%m%dT%H%M%S")
+                dtend = (deadline.deadline_time + timedelta(hours=1)).strftime("%Y%m%dT%H%M%S")
+            else:
+                dtstart = deadline.deadline_date.strftime("%Y%m%d")
+                dtend = dtstart
+                ical_lines.extend([
+                    "BEGIN:VEVENT",
+                    f"DTSTART;VALUE=DATE:{dtstart}",
+                    f"DTEND;VALUE=DATE:{dtend}"
+                ])
+            
+            if deadline.deadline_time:
+                ical_lines.extend([
+                    "BEGIN:VEVENT",
+                    f"DTSTART:{dtstart}",
+                    f"DTEND:{dtend}"
+                ])
+            
+            # Add event details
+            ical_lines.extend([
+                f"UID:deadline-{deadline.id}@delphi-consulting.com",
+                f"SUMMARY:{deadline.title}",
+                f"DESCRIPTION:{deadline.description or ''}",
+                f"PRIORITY:{self._get_ical_priority(deadline.priority)}",
+                f"CATEGORIES:{deadline.deadline_type.value.upper()}",
+                f"STATUS:CONFIRMED",
+                f"DTSTAMP:{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}",
+                "END:VEVENT"
+            ])
+        
+        ical_lines.append("END:VCALENDAR")
+        
+        return "\r\n".join(ical_lines)
+    
+    def _get_ical_priority(self, priority: DeadlinePriority) -> str:
+        """Convert deadline priority to iCal priority"""
+        
+        priority_map = {
+            DeadlinePriority.CRITICAL: "1",  # High
+            DeadlinePriority.HIGH: "3",     # Medium-High
+            DeadlinePriority.MEDIUM: "5",   # Medium
+            DeadlinePriority.LOW: "7"       # Low
+        }
+        
+        return priority_map.get(priority, "5")

app/services/deadline_notifications.py

@@ -0,0 +1,536 @@
+"""
+Deadline notification and alert service
+Handles automated deadline reminders and notifications with workflow integration
+"""
+from typing import List, Dict, Any, Optional
+from datetime import datetime, date, timezone, timedelta
+from sqlalchemy.orm import Session, joinedload
+from sqlalchemy import and_, func, or_, desc
+
+from app.models import (
+    Deadline, DeadlineReminder, User, Employee,
+    DeadlineStatus, DeadlinePriority, NotificationFrequency
+)
+from app.services.deadlines import DeadlineService
+from app.services.workflow_integration import log_deadline_approaching_sync
+from app.utils.logging import app_logger
+
+logger = app_logger
+
+
+class DeadlineNotificationService:
+    """Service for managing deadline notifications and alerts"""
+    
+    def __init__(self, db: Session):
+        self.db = db
+        self.deadline_service = DeadlineService(db)
+    
+    def process_daily_reminders(self, notification_date: date = None) -> Dict[str, Any]:
+        """Process all reminders that should be sent today"""
+        
+        if notification_date is None:
+            notification_date = date.today()
+        
+        logger.info(f"Processing deadline reminders for {notification_date}")
+        
+        # First, check for approaching deadlines and trigger workflow events
+        workflow_events_triggered = self.check_approaching_deadlines_for_workflows(notification_date)
+        
+        # Get pending reminders for today
+        pending_reminders = self.deadline_service.get_pending_reminders(notification_date)
+        
+        results = {
+            "date": notification_date,
+            "total_reminders": len(pending_reminders),
+            "sent_successfully": 0,
+            "failed": 0,
+            "workflow_events_triggered": workflow_events_triggered,
+            "errors": []
+        }
+        
+        for reminder in pending_reminders:
+            try:
+                # Send the notification
+                success = self._send_reminder_notification(reminder)
+                
+                if success:
+                    # Mark as sent
+                    self.deadline_service.mark_reminder_sent(
+                        reminder.id, 
+                        delivery_status="sent"
+                    )
+                    results["sent_successfully"] += 1
+                    logger.info(f"Sent reminder {reminder.id} for deadline '{reminder.deadline.title}'")
+                else:
+                    # Mark as failed
+                    self.deadline_service.mark_reminder_sent(
+                        reminder.id, 
+                        delivery_status="failed",
+                        error_message="Failed to send notification"
+                    )
+                    results["failed"] += 1
+                    results["errors"].append(f"Failed to send reminder {reminder.id}")
+                    
+            except Exception as e:
+                # Mark as failed with error
+                self.deadline_service.mark_reminder_sent(
+                    reminder.id,
+                    delivery_status="failed", 
+                    error_message=str(e)
+                )
+                results["failed"] += 1
+                results["errors"].append(f"Error sending reminder {reminder.id}: {str(e)}")
+                logger.error(f"Error processing reminder {reminder.id}: {str(e)}")
+        
+        logger.info(f"Reminder processing complete: {results['sent_successfully']} sent, {results['failed']} failed, {workflow_events_triggered} workflow events triggered")
+        return results
+    
+    def check_approaching_deadlines_for_workflows(self, check_date: date = None) -> int:
+        """Check for approaching deadlines and trigger workflow events"""
+        
+        if check_date is None:
+            check_date = date.today()
+        
+        # Get deadlines approaching within the next 7 days
+        end_date = check_date + timedelta(days=7)
+        
+        approaching_deadlines = self.db.query(Deadline).filter(
+            Deadline.status == DeadlineStatus.PENDING,
+            Deadline.deadline_date.between(check_date, end_date)
+        ).options(
+            joinedload(Deadline.file),
+            joinedload(Deadline.client)
+        ).all()
+        
+        events_triggered = 0
+        
+        for deadline in approaching_deadlines:
+            try:
+                # Calculate days until deadline
+                days_until = (deadline.deadline_date - check_date).days
+                
+                # Determine deadline type for workflow context
+                deadline_type = getattr(deadline, 'deadline_type', None)
+                deadline_type_str = deadline_type.value if deadline_type else 'other'
+                
+                # Log workflow event for deadline approaching
+                log_deadline_approaching_sync(
+                    db=self.db,
+                    deadline_id=deadline.id,
+                    file_no=deadline.file_no,
+                    client_id=deadline.client_id,
+                    days_until_deadline=days_until,
+                    deadline_type=deadline_type_str
+                )
+                
+                events_triggered += 1
+                logger.debug(f"Triggered workflow event for deadline {deadline.id} '{deadline.title}' ({days_until} days away)")
+                
+            except Exception as e:
+                logger.error(f"Error triggering workflow event for deadline {deadline.id}: {str(e)}")
+        
+        if events_triggered > 0:
+            logger.info(f"Triggered {events_triggered} deadline approaching workflow events")
+        
+        return events_triggered
+    
+    def get_urgent_alerts(
+        self,
+        user_id: Optional[int] = None,
+        employee_id: Optional[str] = None
+    ) -> List[Dict[str, Any]]:
+        """Get urgent deadline alerts that need immediate attention"""
+        
+        today = date.today()
+        
+        # Build base query for urgent deadlines
+        query = self.db.query(Deadline).filter(
+            Deadline.status == DeadlineStatus.PENDING
+        )
+        
+        if user_id:
+            query = query.filter(Deadline.assigned_to_user_id == user_id)
+        
+        if employee_id:
+            query = query.filter(Deadline.assigned_to_employee_id == employee_id)
+        
+        # Get overdue and critical upcoming deadlines
+        urgent_deadlines = query.filter(
+            or_(
+                # Overdue deadlines
+                Deadline.deadline_date < today,
+                # Critical priority deadlines due within 3 days
+                and_(
+                    Deadline.priority == DeadlinePriority.CRITICAL,
+                    Deadline.deadline_date <= today + timedelta(days=3)
+                ),
+                # High priority deadlines due within 1 day
+                and_(
+                    Deadline.priority == DeadlinePriority.HIGH,
+                    Deadline.deadline_date <= today + timedelta(days=1)
+                )
+            )
+        ).options(
+            joinedload(Deadline.file),
+            joinedload(Deadline.client),
+            joinedload(Deadline.assigned_to_user),
+            joinedload(Deadline.assigned_to_employee)
+        ).order_by(
+            Deadline.deadline_date.asc(),
+            Deadline.priority.desc()
+        ).all()
+        
+        alerts = []
+        for deadline in urgent_deadlines:
+            alert_level = self._determine_alert_level(deadline, today)
+            
+            alerts.append({
+                "deadline_id": deadline.id,
+                "title": deadline.title,
+                "deadline_date": deadline.deadline_date,
+                "deadline_time": deadline.deadline_time,
+                "priority": deadline.priority.value,
+                "deadline_type": deadline.deadline_type.value,
+                "alert_level": alert_level,
+                "days_until_deadline": deadline.days_until_deadline,
+                "is_overdue": deadline.is_overdue,
+                "file_no": deadline.file_no,
+                "client_name": self._get_client_name(deadline),
+                "assigned_to": self._get_assigned_to(deadline),
+                "court_name": deadline.court_name,
+                "case_number": deadline.case_number
+            })
+        
+        return alerts
+    
+    def get_dashboard_summary(
+        self,
+        user_id: Optional[int] = None,
+        employee_id: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """Get deadline summary for dashboard display"""
+        
+        today = date.today()
+        
+        # Build base query
+        query = self.db.query(Deadline).filter(
+            Deadline.status == DeadlineStatus.PENDING
+        )
+        
+        if user_id:
+            query = query.filter(Deadline.assigned_to_user_id == user_id)
+        
+        if employee_id:
+            query = query.filter(Deadline.assigned_to_employee_id == employee_id)
+        
+        # Calculate counts
+        overdue_count = query.filter(Deadline.deadline_date < today).count()
+        
+        due_today_count = query.filter(Deadline.deadline_date == today).count()
+        
+        due_tomorrow_count = query.filter(
+            Deadline.deadline_date == today + timedelta(days=1)
+        ).count()
+        
+        due_this_week_count = query.filter(
+            Deadline.deadline_date.between(
+                today,
+                today + timedelta(days=7)
+            )
+        ).count()
+        
+        due_next_week_count = query.filter(
+            Deadline.deadline_date.between(
+                today + timedelta(days=8),
+                today + timedelta(days=14)
+            )
+        ).count()
+        
+        # Critical priority counts
+        critical_overdue = query.filter(
+            Deadline.priority == DeadlinePriority.CRITICAL,
+            Deadline.deadline_date < today
+        ).count()
+        
+        critical_upcoming = query.filter(
+            Deadline.priority == DeadlinePriority.CRITICAL,
+            Deadline.deadline_date.between(today, today + timedelta(days=7))
+        ).count()
+        
+        return {
+            "overdue": overdue_count,
+            "due_today": due_today_count,
+            "due_tomorrow": due_tomorrow_count,
+            "due_this_week": due_this_week_count,
+            "due_next_week": due_next_week_count,
+            "critical_overdue": critical_overdue,
+            "critical_upcoming": critical_upcoming,
+            "total_pending": query.count(),
+            "needs_attention": overdue_count + critical_overdue + critical_upcoming
+        }
+    
+    def create_adhoc_reminder(
+        self,
+        deadline_id: int,
+        recipient_user_id: int,
+        reminder_date: date,
+        custom_message: Optional[str] = None
+    ) -> DeadlineReminder:
+        """Create an ad-hoc reminder for a specific deadline"""
+        
+        deadline = self.db.query(Deadline).filter(Deadline.id == deadline_id).first()
+        if not deadline:
+            raise ValueError(f"Deadline {deadline_id} not found")
+        
+        recipient = self.db.query(User).filter(User.id == recipient_user_id).first()
+        if not recipient:
+            raise ValueError(f"User {recipient_user_id} not found")
+        
+        # Calculate days before deadline
+        days_before = (deadline.deadline_date - reminder_date).days
+        
+        reminder = DeadlineReminder(
+            deadline_id=deadline_id,
+            reminder_date=reminder_date,
+            days_before_deadline=days_before,
+            recipient_user_id=recipient_user_id,
+            recipient_email=recipient.email if hasattr(recipient, 'email') else None,
+            subject=f"Custom Reminder: {deadline.title}",
+            message=custom_message or f"Custom reminder for deadline '{deadline.title}' due on {deadline.deadline_date}",
+            notification_method="email"
+        )
+        
+        self.db.add(reminder)
+        self.db.commit()
+        self.db.refresh(reminder)
+        
+        logger.info(f"Created ad-hoc reminder {reminder.id} for deadline {deadline_id}")
+        return reminder
+    
+    def get_notification_preferences(self, user_id: int) -> Dict[str, Any]:
+        """Get user's notification preferences (placeholder for future implementation)"""
+        
+        # This would be expanded to include user-specific notification settings
+        # For now, return default preferences
+        return {
+            "email_enabled": True,
+            "in_app_enabled": True,
+            "sms_enabled": False,
+            "advance_notice_days": {
+                "critical": 7,
+                "high": 3,
+                "medium": 1,
+                "low": 1
+            },
+            "notification_times": ["09:00", "17:00"],  # When to send daily notifications
+            "quiet_hours": {
+                "start": "18:00",
+                "end": "08:00"
+            }
+        }
+    
+    def schedule_court_date_reminders(
+        self,
+        deadline_id: int,
+        court_date: date,
+        preparation_days: int = 7
+    ):
+        """Schedule special reminders for court dates with preparation milestones"""
+        
+        deadline = self.db.query(Deadline).filter(Deadline.id == deadline_id).first()
+        if not deadline:
+            raise ValueError(f"Deadline {deadline_id} not found")
+        
+        recipient_user_id = deadline.assigned_to_user_id or deadline.created_by_user_id
+        
+        # Schedule preparation milestone reminders
+        preparation_milestones = [
+            (preparation_days, "Begin case preparation"),
+            (3, "Final preparation and document review"),
+            (1, "Last-minute preparation and travel arrangements"),
+            (0, "Court appearance today")
+        ]
+        
+        for days_before, milestone_message in preparation_milestones:
+            reminder_date = court_date - timedelta(days=days_before)
+            
+            if reminder_date >= date.today():
+                reminder = DeadlineReminder(
+                    deadline_id=deadline_id,
+                    reminder_date=reminder_date,
+                    days_before_deadline=days_before,
+                    recipient_user_id=recipient_user_id,
+                    subject=f"Court Date Preparation: {deadline.title}",
+                    message=f"{milestone_message} - Court appearance on {court_date}",
+                    notification_method="email"
+                )
+                
+                self.db.add(reminder)
+        
+        self.db.commit()
+        logger.info(f"Scheduled court date reminders for deadline {deadline_id}")
+    
+    # Private helper methods
+    
+    def _send_reminder_notification(self, reminder: DeadlineReminder) -> bool:
+        """Send a reminder notification (placeholder for actual implementation)"""
+        
+        try:
+            # In a real implementation, this would:
+            # 1. Format the notification message
+            # 2. Send via email/SMS/push notification
+            # 3. Handle delivery confirmations
+            # 4. Retry failed deliveries
+            
+            # For now, just log the notification
+            logger.info(
+                f"NOTIFICATION: {reminder.subject} to user {reminder.recipient_user_id} "
+                f"for deadline '{reminder.deadline.title}' due {reminder.deadline.deadline_date}"
+            )
+            
+            # Simulate successful delivery
+            return True
+            
+        except Exception as e:
+            logger.error(f"Failed to send notification: {str(e)}")
+            return False
+    
+    def _determine_alert_level(self, deadline: Deadline, today: date) -> str:
+        """Determine the alert level for a deadline"""
+        
+        days_until = deadline.days_until_deadline
+        
+        if deadline.is_overdue:
+            return "critical"
+        
+        if deadline.priority == DeadlinePriority.CRITICAL:
+            if days_until <= 1:
+                return "critical"
+            elif days_until <= 3:
+                return "high"
+            else:
+                return "medium"
+        
+        elif deadline.priority == DeadlinePriority.HIGH:
+            if days_until <= 0:
+                return "critical"
+            elif days_until <= 1:
+                return "high"
+            else:
+                return "medium"
+        
+        else:
+            if days_until <= 0:
+                return "high"
+            else:
+                return "low"
+    
+    def _get_client_name(self, deadline: Deadline) -> Optional[str]:
+        """Get formatted client name from deadline"""
+        
+        if deadline.client:
+            return f"{deadline.client.first or ''} {deadline.client.last or ''}".strip()
+        elif deadline.file and deadline.file.owner:
+            return f"{deadline.file.owner.first or ''} {deadline.file.owner.last or ''}".strip()
+        return None
+    
+    def _get_assigned_to(self, deadline: Deadline) -> Optional[str]:
+        """Get assigned person name from deadline"""
+        
+        if deadline.assigned_to_user:
+            return deadline.assigned_to_user.username
+        elif deadline.assigned_to_employee:
+            employee = deadline.assigned_to_employee
+            return f"{employee.first_name or ''} {employee.last_name or ''}".strip()
+        return None
+
+
+class DeadlineAlertManager:
+    """Manager for deadline alert workflows and automation"""
+    
+    def __init__(self, db: Session):
+        self.db = db
+        self.notification_service = DeadlineNotificationService(db)
+    
+    def run_daily_alert_processing(self, process_date: date = None) -> Dict[str, Any]:
+        """Run the daily deadline alert processing workflow"""
+        
+        if process_date is None:
+            process_date = date.today()
+        
+        logger.info(f"Starting daily deadline alert processing for {process_date}")
+        
+        results = {
+            "process_date": process_date,
+            "reminders_processed": {},
+            "urgent_alerts_generated": 0,
+            "errors": []
+        }
+        
+        try:
+            # Process scheduled reminders
+            reminder_results = self.notification_service.process_daily_reminders(process_date)
+            results["reminders_processed"] = reminder_results
+            
+            # Generate urgent alerts for overdue items
+            urgent_alerts = self.notification_service.get_urgent_alerts()
+            results["urgent_alerts_generated"] = len(urgent_alerts)
+            
+            # Log summary
+            logger.info(
+                f"Daily processing complete: {reminder_results['sent_successfully']} reminders sent, "
+                f"{results['urgent_alerts_generated']} urgent alerts generated"
+            )
+            
+        except Exception as e:
+            error_msg = f"Error in daily alert processing: {str(e)}"
+            results["errors"].append(error_msg)
+            logger.error(error_msg)
+        
+        return results
+    
+    def escalate_overdue_deadlines(
+        self,
+        escalation_days: int = 1
+    ) -> List[Dict[str, Any]]:
+        """Escalate deadlines that have been overdue for specified days"""
+        
+        cutoff_date = date.today() - timedelta(days=escalation_days)
+        
+        overdue_deadlines = self.db.query(Deadline).filter(
+            Deadline.status == DeadlineStatus.PENDING,
+            Deadline.deadline_date <= cutoff_date
+        ).options(
+            joinedload(Deadline.file),
+            joinedload(Deadline.assigned_to_user),
+            joinedload(Deadline.assigned_to_employee)
+        ).all()
+        
+        escalations = []
+        
+        for deadline in overdue_deadlines:
+            # Create escalation record
+            escalation = {
+                "deadline_id": deadline.id,
+                "title": deadline.title,
+                "deadline_date": deadline.deadline_date,
+                "days_overdue": (date.today() - deadline.deadline_date).days,
+                "priority": deadline.priority.value,
+                "assigned_to": self.notification_service._get_assigned_to(deadline),
+                "file_no": deadline.file_no,
+                "escalation_date": date.today()
+            }
+            
+            escalations.append(escalation)
+            
+            # In a real system, this would:
+            # 1. Send escalation notifications to supervisors
+            # 2. Create escalation tasks
+            # 3. Update deadline status if needed
+            
+            logger.warning(
+                f"ESCALATION: Deadline '{deadline.title}' (ID: {deadline.id}) "
+                f"overdue by {escalation['days_overdue']} days"
+            )
+        
+        return escalations

app/services/deadline_reports.py

@@ -0,0 +1,838 @@
+"""
+Deadline reporting and dashboard services
+Provides comprehensive reporting and analytics for deadline management
+"""
+from typing import List, Dict, Any, Optional, Tuple
+from datetime import datetime, date, timezone, timedelta
+from sqlalchemy.orm import Session, joinedload
+from sqlalchemy import and_, func, or_, desc, case, extract
+from decimal import Decimal
+
+from app.models import (
+    Deadline, DeadlineHistory, User, Employee, File, Rolodex,
+    DeadlineType, DeadlinePriority, DeadlineStatus, NotificationFrequency
+)
+from app.utils.logging import app_logger
+
+logger = app_logger
+
+
+class DeadlineReportService:
+    """Service for deadline reporting and analytics"""
+    
+    def __init__(self, db: Session):
+        self.db = db
+    
+    def generate_upcoming_deadlines_report(
+        self,
+        start_date: date = None,
+        end_date: date = None,
+        employee_id: Optional[str] = None,
+        user_id: Optional[int] = None,
+        deadline_type: Optional[DeadlineType] = None,
+        priority: Optional[DeadlinePriority] = None
+    ) -> Dict[str, Any]:
+        """Generate comprehensive upcoming deadlines report"""
+        
+        if start_date is None:
+            start_date = date.today()
+        
+        if end_date is None:
+            end_date = start_date + timedelta(days=30)
+        
+        # Build query
+        query = self.db.query(Deadline).filter(
+            Deadline.status == DeadlineStatus.PENDING,
+            Deadline.deadline_date.between(start_date, end_date)
+        )
+        
+        if employee_id:
+            query = query.filter(Deadline.assigned_to_employee_id == employee_id)
+        
+        if user_id:
+            query = query.filter(Deadline.assigned_to_user_id == user_id)
+        
+        if deadline_type:
+            query = query.filter(Deadline.deadline_type == deadline_type)
+        
+        if priority:
+            query = query.filter(Deadline.priority == priority)
+        
+        deadlines = query.options(
+            joinedload(Deadline.file),
+            joinedload(Deadline.client),
+            joinedload(Deadline.assigned_to_user),
+            joinedload(Deadline.assigned_to_employee)
+        ).order_by(
+            Deadline.deadline_date.asc(),
+            Deadline.priority.desc()
+        ).all()
+        
+        # Group deadlines by week
+        weeks = {}
+        for deadline in deadlines:
+            # Calculate week start (Monday)
+            days_since_monday = deadline.deadline_date.weekday()
+            week_start = deadline.deadline_date - timedelta(days=days_since_monday)
+            week_key = week_start.strftime("%Y-%m-%d")
+            
+            if week_key not in weeks:
+                weeks[week_key] = {
+                    "week_start": week_start,
+                    "week_end": week_start + timedelta(days=6),
+                    "deadlines": [],
+                    "counts": {
+                        "total": 0,
+                        "critical": 0,
+                        "high": 0,
+                        "medium": 0,
+                        "low": 0
+                    }
+                }
+            
+            deadline_data = {
+                "id": deadline.id,
+                "title": deadline.title,
+                "deadline_date": deadline.deadline_date,
+                "deadline_time": deadline.deadline_time,
+                "priority": deadline.priority.value,
+                "deadline_type": deadline.deadline_type.value,
+                "file_no": deadline.file_no,
+                "client_name": self._get_client_name(deadline),
+                "assigned_to": self._get_assigned_to(deadline),
+                "court_name": deadline.court_name,
+                "case_number": deadline.case_number,
+                "days_until": (deadline.deadline_date - date.today()).days
+            }
+            
+            weeks[week_key]["deadlines"].append(deadline_data)
+            weeks[week_key]["counts"]["total"] += 1
+            weeks[week_key]["counts"][deadline.priority.value] += 1
+        
+        # Sort weeks by date
+        sorted_weeks = sorted(weeks.values(), key=lambda x: x["week_start"])
+        
+        # Calculate summary statistics
+        total_deadlines = len(deadlines)
+        priority_breakdown = {}
+        type_breakdown = {}
+        
+        for priority in DeadlinePriority:
+            count = sum(1 for d in deadlines if d.priority == priority)
+            priority_breakdown[priority.value] = count
+        
+        for deadline_type in DeadlineType:
+            count = sum(1 for d in deadlines if d.deadline_type == deadline_type)
+            type_breakdown[deadline_type.value] = count
+        
+        return {
+            "report_period": {
+                "start_date": start_date,
+                "end_date": end_date,
+                "days": (end_date - start_date).days + 1
+            },
+            "filters": {
+                "employee_id": employee_id,
+                "user_id": user_id,
+                "deadline_type": deadline_type.value if deadline_type else None,
+                "priority": priority.value if priority else None
+            },
+            "summary": {
+                "total_deadlines": total_deadlines,
+                "priority_breakdown": priority_breakdown,
+                "type_breakdown": type_breakdown
+            },
+            "weeks": sorted_weeks
+        }
+    
+    def generate_overdue_report(
+        self,
+        cutoff_date: date = None,
+        employee_id: Optional[str] = None,
+        user_id: Optional[int] = None
+    ) -> Dict[str, Any]:
+        """Generate report of overdue deadlines"""
+        
+        if cutoff_date is None:
+            cutoff_date = date.today()
+        
+        query = self.db.query(Deadline).filter(
+            Deadline.status == DeadlineStatus.PENDING,
+            Deadline.deadline_date < cutoff_date
+        )
+        
+        if employee_id:
+            query = query.filter(Deadline.assigned_to_employee_id == employee_id)
+        
+        if user_id:
+            query = query.filter(Deadline.assigned_to_user_id == user_id)
+        
+        overdue_deadlines = query.options(
+            joinedload(Deadline.file),
+            joinedload(Deadline.client),
+            joinedload(Deadline.assigned_to_user),
+            joinedload(Deadline.assigned_to_employee)
+        ).order_by(
+            Deadline.deadline_date.asc()
+        ).all()
+        
+        # Group by days overdue
+        overdue_groups = {
+            "1-3_days": [],
+            "4-7_days": [],
+            "8-30_days": [],
+            "over_30_days": []
+        }
+        
+        for deadline in overdue_deadlines:
+            days_overdue = (cutoff_date - deadline.deadline_date).days
+            
+            deadline_data = {
+                "id": deadline.id,
+                "title": deadline.title,
+                "deadline_date": deadline.deadline_date,
+                "priority": deadline.priority.value,
+                "deadline_type": deadline.deadline_type.value,
+                "file_no": deadline.file_no,
+                "client_name": self._get_client_name(deadline),
+                "assigned_to": self._get_assigned_to(deadline),
+                "days_overdue": days_overdue
+            }
+            
+            if days_overdue <= 3:
+                overdue_groups["1-3_days"].append(deadline_data)
+            elif days_overdue <= 7:
+                overdue_groups["4-7_days"].append(deadline_data)
+            elif days_overdue <= 30:
+                overdue_groups["8-30_days"].append(deadline_data)
+            else:
+                overdue_groups["over_30_days"].append(deadline_data)
+        
+        return {
+            "report_date": cutoff_date,
+            "filters": {
+                "employee_id": employee_id,
+                "user_id": user_id
+            },
+            "summary": {
+                "total_overdue": len(overdue_deadlines),
+                "by_timeframe": {
+                    "1-3_days": len(overdue_groups["1-3_days"]),
+                    "4-7_days": len(overdue_groups["4-7_days"]),
+                    "8-30_days": len(overdue_groups["8-30_days"]),
+                    "over_30_days": len(overdue_groups["over_30_days"])
+                }
+            },
+            "overdue_groups": overdue_groups
+        }
+    
+    def generate_completion_report(
+        self,
+        start_date: date,
+        end_date: date,
+        employee_id: Optional[str] = None,
+        user_id: Optional[int] = None
+    ) -> Dict[str, Any]:
+        """Generate deadline completion performance report"""
+        
+        # Get all deadlines that were due within the period
+        query = self.db.query(Deadline).filter(
+            Deadline.deadline_date.between(start_date, end_date)
+        )
+        
+        if employee_id:
+            query = query.filter(Deadline.assigned_to_employee_id == employee_id)
+        
+        if user_id:
+            query = query.filter(Deadline.assigned_to_user_id == user_id)
+        
+        deadlines = query.options(
+            joinedload(Deadline.assigned_to_user),
+            joinedload(Deadline.assigned_to_employee)
+        ).all()
+        
+        # Calculate completion statistics
+        total_deadlines = len(deadlines)
+        completed_on_time = 0
+        completed_late = 0
+        still_pending = 0
+        missed = 0
+        
+        completion_by_priority = {}
+        completion_by_type = {}
+        completion_by_assignee = {}
+        
+        for deadline in deadlines:
+            # Determine completion status
+            if deadline.status == DeadlineStatus.COMPLETED:
+                if deadline.completed_date and deadline.completed_date.date() <= deadline.deadline_date:
+                    completed_on_time += 1
+                    status = "on_time"
+                else:
+                    completed_late += 1
+                    status = "late"
+            elif deadline.status == DeadlineStatus.PENDING:
+                if deadline.deadline_date < date.today():
+                    missed += 1
+                    status = "missed"
+                else:
+                    still_pending += 1
+                    status = "pending"
+            elif deadline.status == DeadlineStatus.CANCELLED:
+                status = "cancelled"
+            else:
+                status = "other"
+            
+            # Track by priority
+            priority_key = deadline.priority.value
+            if priority_key not in completion_by_priority:
+                completion_by_priority[priority_key] = {
+                    "total": 0, "on_time": 0, "late": 0, "missed": 0, "pending": 0, "cancelled": 0
+                }
+            completion_by_priority[priority_key]["total"] += 1
+            completion_by_priority[priority_key][status] += 1
+            
+            # Track by type
+            type_key = deadline.deadline_type.value
+            if type_key not in completion_by_type:
+                completion_by_type[type_key] = {
+                    "total": 0, "on_time": 0, "late": 0, "missed": 0, "pending": 0, "cancelled": 0
+                }
+            completion_by_type[type_key]["total"] += 1
+            completion_by_type[type_key][status] += 1
+            
+            # Track by assignee
+            assignee = self._get_assigned_to(deadline) or "Unassigned"
+            if assignee not in completion_by_assignee:
+                completion_by_assignee[assignee] = {
+                    "total": 0, "on_time": 0, "late": 0, "missed": 0, "pending": 0, "cancelled": 0
+                }
+            completion_by_assignee[assignee]["total"] += 1
+            completion_by_assignee[assignee][status] += 1
+        
+        # Calculate completion rates
+        completed_total = completed_on_time + completed_late
+        on_time_rate = (completed_on_time / completed_total * 100) if completed_total > 0 else 0
+        completion_rate = (completed_total / total_deadlines * 100) if total_deadlines > 0 else 0
+        
+        return {
+            "report_period": {
+                "start_date": start_date,
+                "end_date": end_date
+            },
+            "filters": {
+                "employee_id": employee_id,
+                "user_id": user_id
+            },
+            "summary": {
+                "total_deadlines": total_deadlines,
+                "completed_on_time": completed_on_time,
+                "completed_late": completed_late,
+                "still_pending": still_pending,
+                "missed": missed,
+                "on_time_rate": round(on_time_rate, 2),
+                "completion_rate": round(completion_rate, 2)
+            },
+            "breakdown": {
+                "by_priority": completion_by_priority,
+                "by_type": completion_by_type,
+                "by_assignee": completion_by_assignee
+            }
+        }
+    
+    def generate_workload_report(
+        self,
+        target_date: date = None,
+        days_ahead: int = 30
+    ) -> Dict[str, Any]:
+        """Generate workload distribution report by assignee"""
+        
+        if target_date is None:
+            target_date = date.today()
+        
+        end_date = target_date + timedelta(days=days_ahead)
+        
+        # Get pending deadlines in the timeframe
+        deadlines = self.db.query(Deadline).filter(
+            Deadline.status == DeadlineStatus.PENDING,
+            Deadline.deadline_date.between(target_date, end_date)
+        ).options(
+            joinedload(Deadline.assigned_to_user),
+            joinedload(Deadline.assigned_to_employee),
+            joinedload(Deadline.file),
+            joinedload(Deadline.client)
+        ).all()
+        
+        # Group by assignee
+        workload_by_assignee = {}
+        unassigned_deadlines = []
+        
+        for deadline in deadlines:
+            assignee = self._get_assigned_to(deadline)
+            
+            if not assignee:
+                unassigned_deadlines.append({
+                    "id": deadline.id,
+                    "title": deadline.title,
+                    "deadline_date": deadline.deadline_date,
+                    "priority": deadline.priority.value,
+                    "deadline_type": deadline.deadline_type.value,
+                    "file_no": deadline.file_no
+                })
+                continue
+            
+            if assignee not in workload_by_assignee:
+                workload_by_assignee[assignee] = {
+                    "total_deadlines": 0,
+                    "critical": 0,
+                    "high": 0,
+                    "medium": 0,
+                    "low": 0,
+                    "overdue": 0,
+                    "due_this_week": 0,
+                    "due_next_week": 0,
+                    "deadlines": []
+                }
+            
+            # Count by priority
+            workload_by_assignee[assignee]["total_deadlines"] += 1
+            workload_by_assignee[assignee][deadline.priority.value] += 1
+            
+            # Count by timeframe
+            days_until = (deadline.deadline_date - target_date).days
+            if days_until < 0:
+                workload_by_assignee[assignee]["overdue"] += 1
+            elif days_until <= 7:
+                workload_by_assignee[assignee]["due_this_week"] += 1
+            elif days_until <= 14:
+                workload_by_assignee[assignee]["due_next_week"] += 1
+            
+            workload_by_assignee[assignee]["deadlines"].append({
+                "id": deadline.id,
+                "title": deadline.title,
+                "deadline_date": deadline.deadline_date,
+                "priority": deadline.priority.value,
+                "deadline_type": deadline.deadline_type.value,
+                "file_no": deadline.file_no,
+                "days_until": days_until
+            })
+        
+        # Sort assignees by workload
+        sorted_assignees = sorted(
+            workload_by_assignee.items(),
+            key=lambda x: (x[1]["critical"] + x[1]["high"], x[1]["total_deadlines"]),
+            reverse=True
+        )
+        
+        return {
+            "report_date": target_date,
+            "timeframe_days": days_ahead,
+            "summary": {
+                "total_assignees": len(workload_by_assignee),
+                "total_deadlines": len(deadlines),
+                "unassigned_deadlines": len(unassigned_deadlines)
+            },
+            "workload_by_assignee": dict(sorted_assignees),
+            "unassigned_deadlines": unassigned_deadlines
+        }
+    
+    def generate_trends_report(
+        self,
+        start_date: date,
+        end_date: date,
+        granularity: str = "month"  # "week", "month", "quarter"
+    ) -> Dict[str, Any]:
+        """Generate deadline trends and analytics over time"""
+        
+        # Get all deadlines created within the period
+        deadlines = self.db.query(Deadline).filter(
+            func.date(Deadline.created_at) >= start_date,
+            func.date(Deadline.created_at) <= end_date
+        ).all()
+        
+        # Group by time periods
+        periods = {}
+        
+        for deadline in deadlines:
+            created_date = deadline.created_at.date()
+            
+            if granularity == "week":
+                # Get Monday of the week
+                days_since_monday = created_date.weekday()
+                period_start = created_date - timedelta(days=days_since_monday)
+                period_key = period_start.strftime("%Y-W%U")
+            elif granularity == "month":
+                period_key = created_date.strftime("%Y-%m")
+            elif granularity == "quarter":
+                quarter = (created_date.month - 1) // 3 + 1
+                period_key = f"{created_date.year}-Q{quarter}"
+            else:
+                period_key = created_date.strftime("%Y-%m-%d")
+            
+            if period_key not in periods:
+                periods[period_key] = {
+                    "total_created": 0,
+                    "completed": 0,
+                    "missed": 0,
+                    "pending": 0,
+                    "by_type": {},
+                    "by_priority": {},
+                    "avg_completion_days": 0
+                }
+            
+            periods[period_key]["total_created"] += 1
+            
+            # Track completion status
+            if deadline.status == DeadlineStatus.COMPLETED:
+                periods[period_key]["completed"] += 1
+            elif deadline.status == DeadlineStatus.PENDING and deadline.deadline_date < date.today():
+                periods[period_key]["missed"] += 1
+            else:
+                periods[period_key]["pending"] += 1
+            
+            # Track by type and priority
+            type_key = deadline.deadline_type.value
+            priority_key = deadline.priority.value
+            
+            if type_key not in periods[period_key]["by_type"]:
+                periods[period_key]["by_type"][type_key] = 0
+            periods[period_key]["by_type"][type_key] += 1
+            
+            if priority_key not in periods[period_key]["by_priority"]:
+                periods[period_key]["by_priority"][priority_key] = 0
+            periods[period_key]["by_priority"][priority_key] += 1
+        
+        # Calculate trends
+        sorted_periods = sorted(periods.items())
+        
+        return {
+            "report_period": {
+                "start_date": start_date,
+                "end_date": end_date,
+                "granularity": granularity
+            },
+            "summary": {
+                "total_periods": len(periods),
+                "total_deadlines": len(deadlines)
+            },
+            "trends": {
+                "by_period": sorted_periods
+            }
+        }
+    
+    # Private helper methods
+    
+    def _get_client_name(self, deadline: Deadline) -> Optional[str]:
+        """Get formatted client name from deadline"""
+        
+        if deadline.client:
+            return f"{deadline.client.first or ''} {deadline.client.last or ''}".strip()
+        elif deadline.file and deadline.file.owner:
+            return f"{deadline.file.owner.first or ''} {deadline.file.owner.last or ''}".strip()
+        return None
+    
+    def _get_assigned_to(self, deadline: Deadline) -> Optional[str]:
+        """Get assigned person name from deadline"""
+        
+        if deadline.assigned_to_user:
+            return deadline.assigned_to_user.username
+        elif deadline.assigned_to_employee:
+            employee = deadline.assigned_to_employee
+            return f"{employee.first_name or ''} {employee.last_name or ''}".strip()
+        return None
+
+
+class DeadlineDashboardService:
+    """Service for deadline dashboard widgets and summaries"""
+    
+    def __init__(self, db: Session):
+        self.db = db
+        self.report_service = DeadlineReportService(db)
+    
+    def get_dashboard_widgets(
+        self,
+        user_id: Optional[int] = None,
+        employee_id: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """Get all dashboard widgets for deadline management"""
+        
+        today = date.today()
+        
+        return {
+            "summary_cards": self._get_summary_cards(user_id, employee_id),
+            "upcoming_deadlines": self._get_upcoming_deadlines_widget(user_id, employee_id),
+            "overdue_alerts": self._get_overdue_alerts_widget(user_id, employee_id),
+            "priority_breakdown": self._get_priority_breakdown_widget(user_id, employee_id),
+            "recent_completions": self._get_recent_completions_widget(user_id, employee_id),
+            "weekly_calendar": self._get_weekly_calendar_widget(today, user_id, employee_id)
+        }
+    
+    def _get_summary_cards(
+        self,
+        user_id: Optional[int] = None,
+        employee_id: Optional[str] = None
+    ) -> List[Dict[str, Any]]:
+        """Get summary cards for dashboard"""
+        
+        base_query = self.db.query(Deadline).filter(Deadline.status == DeadlineStatus.PENDING)
+        
+        if user_id:
+            base_query = base_query.filter(Deadline.assigned_to_user_id == user_id)
+        
+        if employee_id:
+            base_query = base_query.filter(Deadline.assigned_to_employee_id == employee_id)
+        
+        today = date.today()
+        
+        # Calculate counts
+        total_pending = base_query.count()
+        overdue = base_query.filter(Deadline.deadline_date < today).count()
+        due_today = base_query.filter(Deadline.deadline_date == today).count()
+        due_this_week = base_query.filter(
+            Deadline.deadline_date.between(today, today + timedelta(days=7))
+        ).count()
+        
+        return [
+            {
+                "title": "Total Pending",
+                "value": total_pending,
+                "icon": "calendar",
+                "color": "blue"
+            },
+            {
+                "title": "Overdue",
+                "value": overdue,
+                "icon": "exclamation-triangle",
+                "color": "red"
+            },
+            {
+                "title": "Due Today",
+                "value": due_today,
+                "icon": "clock",
+                "color": "orange"
+            },
+            {
+                "title": "Due This Week",
+                "value": due_this_week,
+                "icon": "calendar-week",
+                "color": "green"
+            }
+        ]
+    
+    def _get_upcoming_deadlines_widget(
+        self,
+        user_id: Optional[int] = None,
+        employee_id: Optional[str] = None,
+        limit: int = 5
+    ) -> Dict[str, Any]:
+        """Get upcoming deadlines widget"""
+        
+        query = self.db.query(Deadline).filter(
+            Deadline.status == DeadlineStatus.PENDING,
+            Deadline.deadline_date >= date.today()
+        )
+        
+        if user_id:
+            query = query.filter(Deadline.assigned_to_user_id == user_id)
+        
+        if employee_id:
+            query = query.filter(Deadline.assigned_to_employee_id == employee_id)
+        
+        deadlines = query.options(
+            joinedload(Deadline.file),
+            joinedload(Deadline.client)
+        ).order_by(
+            Deadline.deadline_date.asc(),
+            Deadline.priority.desc()
+        ).limit(limit).all()
+        
+        return {
+            "title": "Upcoming Deadlines",
+            "deadlines": [
+                {
+                    "id": d.id,
+                    "title": d.title,
+                    "deadline_date": d.deadline_date,
+                    "priority": d.priority.value,
+                    "deadline_type": d.deadline_type.value,
+                    "file_no": d.file_no,
+                    "client_name": self.report_service._get_client_name(d),
+                    "days_until": (d.deadline_date - date.today()).days
+                }
+                for d in deadlines
+            ]
+        }
+    
+    def _get_overdue_alerts_widget(
+        self,
+        user_id: Optional[int] = None,
+        employee_id: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """Get overdue alerts widget"""
+        
+        query = self.db.query(Deadline).filter(
+            Deadline.status == DeadlineStatus.PENDING,
+            Deadline.deadline_date < date.today()
+        )
+        
+        if user_id:
+            query = query.filter(Deadline.assigned_to_user_id == user_id)
+        
+        if employee_id:
+            query = query.filter(Deadline.assigned_to_employee_id == employee_id)
+        
+        overdue_deadlines = query.options(
+            joinedload(Deadline.file),
+            joinedload(Deadline.client)
+        ).order_by(
+            Deadline.deadline_date.asc()
+        ).limit(10).all()
+        
+        return {
+            "title": "Overdue Deadlines",
+            "count": len(overdue_deadlines),
+            "deadlines": [
+                {
+                    "id": d.id,
+                    "title": d.title,
+                    "deadline_date": d.deadline_date,
+                    "priority": d.priority.value,
+                    "file_no": d.file_no,
+                    "client_name": self.report_service._get_client_name(d),
+                    "days_overdue": (date.today() - d.deadline_date).days
+                }
+                for d in overdue_deadlines
+            ]
+        }
+    
+    def _get_priority_breakdown_widget(
+        self,
+        user_id: Optional[int] = None,
+        employee_id: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """Get priority breakdown widget"""
+        
+        base_query = self.db.query(Deadline).filter(Deadline.status == DeadlineStatus.PENDING)
+        
+        if user_id:
+            base_query = base_query.filter(Deadline.assigned_to_user_id == user_id)
+        
+        if employee_id:
+            base_query = base_query.filter(Deadline.assigned_to_employee_id == employee_id)
+        
+        breakdown = {}
+        for priority in DeadlinePriority:
+            count = base_query.filter(Deadline.priority == priority).count()
+            breakdown[priority.value] = count
+        
+        return {
+            "title": "Priority Breakdown",
+            "breakdown": breakdown
+        }
+    
+    def _get_recent_completions_widget(
+        self,
+        user_id: Optional[int] = None,
+        employee_id: Optional[str] = None,
+        days_back: int = 7
+    ) -> Dict[str, Any]:
+        """Get recent completions widget"""
+        
+        cutoff_date = date.today() - timedelta(days=days_back)
+        
+        query = self.db.query(Deadline).filter(
+            Deadline.status == DeadlineStatus.COMPLETED,
+            func.date(Deadline.completed_date) >= cutoff_date
+        )
+        
+        if user_id:
+            query = query.filter(Deadline.assigned_to_user_id == user_id)
+        
+        if employee_id:
+            query = query.filter(Deadline.assigned_to_employee_id == employee_id)
+        
+        completed = query.options(
+            joinedload(Deadline.file),
+            joinedload(Deadline.client)
+        ).order_by(
+            Deadline.completed_date.desc()
+        ).limit(5).all()
+        
+        return {
+            "title": "Recently Completed",
+            "count": len(completed),
+            "deadlines": [
+                {
+                    "id": d.id,
+                    "title": d.title,
+                    "deadline_date": d.deadline_date,
+                    "completed_date": d.completed_date.date() if d.completed_date else None,
+                    "priority": d.priority.value,
+                    "file_no": d.file_no,
+                    "client_name": self.report_service._get_client_name(d),
+                    "on_time": d.completed_date.date() <= d.deadline_date if d.completed_date else False
+                }
+                for d in completed
+            ]
+        }
+    
+    def _get_weekly_calendar_widget(
+        self,
+        week_start: date,
+        user_id: Optional[int] = None,
+        employee_id: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """Get weekly calendar widget"""
+        
+        # Adjust to Monday
+        days_since_monday = week_start.weekday()
+        monday = week_start - timedelta(days=days_since_monday)
+        sunday = monday + timedelta(days=6)
+        
+        query = self.db.query(Deadline).filter(
+            Deadline.status == DeadlineStatus.PENDING,
+            Deadline.deadline_date.between(monday, sunday)
+        )
+        
+        if user_id:
+            query = query.filter(Deadline.assigned_to_user_id == user_id)
+        
+        if employee_id:
+            query = query.filter(Deadline.assigned_to_employee_id == employee_id)
+        
+        deadlines = query.options(
+            joinedload(Deadline.file),
+            joinedload(Deadline.client)
+        ).order_by(
+            Deadline.deadline_date.asc(),
+            Deadline.deadline_time.asc()
+        ).all()
+        
+        # Group by day
+        calendar_days = {}
+        for i in range(7):
+            day = monday + timedelta(days=i)
+            calendar_days[day.strftime("%Y-%m-%d")] = {
+                "date": day,
+                "day_name": day.strftime("%A"),
+                "deadlines": []
+            }
+        
+        for deadline in deadlines:
+            day_key = deadline.deadline_date.strftime("%Y-%m-%d")
+            if day_key in calendar_days:
+                calendar_days[day_key]["deadlines"].append({
+                    "id": deadline.id,
+                    "title": deadline.title,
+                    "deadline_time": deadline.deadline_time,
+                    "priority": deadline.priority.value,
+                    "deadline_type": deadline.deadline_type.value,
+                    "file_no": deadline.file_no
+                })
+        
+        return {
+            "title": "This Week",
+            "week_start": monday,
+            "week_end": sunday,
+            "days": list(calendar_days.values())
+        }

app/services/deadlines.py

@@ -0,0 +1,684 @@
+"""
+Deadline management service
+Handles deadline creation, tracking, notifications, and reporting
+"""
+from typing import List, Dict, Any, Optional, Tuple
+from datetime import datetime, date, timedelta, timezone
+from sqlalchemy.orm import Session, joinedload
+from sqlalchemy import and_, func, or_, desc, asc
+from decimal import Decimal
+
+from app.models import (
+    Deadline, DeadlineReminder, DeadlineTemplate, DeadlineHistory, CourtCalendar,
+    DeadlineType, DeadlinePriority, DeadlineStatus, NotificationFrequency,
+    File, Rolodex, Employee, User
+)
+from app.utils.logging import app_logger
+
+logger = app_logger
+
+
+class DeadlineManagementError(Exception):
+    """Exception raised when deadline management operations fail"""
+    pass
+
+
+class DeadlineService:
+    """Service for deadline management operations"""
+    
+    def __init__(self, db: Session):
+        self.db = db
+    
+    def create_deadline(
+        self,
+        title: str,
+        deadline_date: date,
+        created_by_user_id: int,
+        deadline_type: DeadlineType = DeadlineType.OTHER,
+        priority: DeadlinePriority = DeadlinePriority.MEDIUM,
+        description: Optional[str] = None,
+        file_no: Optional[str] = None,
+        client_id: Optional[str] = None,
+        assigned_to_user_id: Optional[int] = None,
+        assigned_to_employee_id: Optional[str] = None,
+        deadline_time: Optional[datetime] = None,
+        court_name: Optional[str] = None,
+        case_number: Optional[str] = None,
+        advance_notice_days: int = 7,
+        notification_frequency: NotificationFrequency = NotificationFrequency.WEEKLY
+    ) -> Deadline:
+        """Create a new deadline"""
+        
+        # Validate file exists if provided
+        if file_no:
+            file_obj = self.db.query(File).filter(File.file_no == file_no).first()
+            if not file_obj:
+                raise DeadlineManagementError(f"File {file_no} not found")
+        
+        # Validate client exists if provided
+        if client_id:
+            client_obj = self.db.query(Rolodex).filter(Rolodex.id == client_id).first()
+            if not client_obj:
+                raise DeadlineManagementError(f"Client {client_id} not found")
+        
+        # Validate assigned employee if provided
+        if assigned_to_employee_id:
+            employee_obj = self.db.query(Employee).filter(Employee.empl_num == assigned_to_employee_id).first()
+            if not employee_obj:
+                raise DeadlineManagementError(f"Employee {assigned_to_employee_id} not found")
+        
+        # Create deadline
+        deadline = Deadline(
+            title=title,
+            description=description,
+            deadline_date=deadline_date,
+            deadline_time=deadline_time,
+            deadline_type=deadline_type,
+            priority=priority,
+            file_no=file_no,
+            client_id=client_id,
+            assigned_to_user_id=assigned_to_user_id,
+            assigned_to_employee_id=assigned_to_employee_id,
+            created_by_user_id=created_by_user_id,
+            court_name=court_name,
+            case_number=case_number,
+            advance_notice_days=advance_notice_days,
+            notification_frequency=notification_frequency
+        )
+        
+        self.db.add(deadline)
+        self.db.flush()  # Get the ID
+        
+        # Create history record
+        self._create_deadline_history(
+            deadline.id, "created", None, None, None, created_by_user_id, "Deadline created"
+        )
+        
+        # Schedule automatic reminders
+        if notification_frequency != NotificationFrequency.NONE:
+            self._schedule_reminders(deadline)
+        
+        self.db.commit()
+        self.db.refresh(deadline)
+        
+        logger.info(f"Created deadline {deadline.id}: '{title}' for {deadline_date}")
+        return deadline
+    
+    def update_deadline(
+        self,
+        deadline_id: int,
+        user_id: int,
+        **updates
+    ) -> Deadline:
+        """Update an existing deadline"""
+        
+        deadline = self.db.query(Deadline).filter(Deadline.id == deadline_id).first()
+        if not deadline:
+            raise DeadlineManagementError(f"Deadline {deadline_id} not found")
+        
+        # Track changes for history
+        changes = []
+        for field, new_value in updates.items():
+            if hasattr(deadline, field):
+                old_value = getattr(deadline, field)
+                if old_value != new_value:
+                    changes.append((field, old_value, new_value))
+                    setattr(deadline, field, new_value)
+        
+        # Update timestamp
+        deadline.updated_at = datetime.now(timezone.utc)
+        
+        # Create history records for changes
+        for field, old_value, new_value in changes:
+            self._create_deadline_history(
+                deadline_id, "updated", field, str(old_value), str(new_value), user_id
+            )
+        
+        # If deadline date changed, reschedule reminders
+        if any(field == 'deadline_date' for field, _, _ in changes):
+            self._reschedule_reminders(deadline)
+        
+        self.db.commit()
+        self.db.refresh(deadline)
+        
+        logger.info(f"Updated deadline {deadline_id} - {len(changes)} changes made")
+        return deadline
+    
+    def complete_deadline(
+        self,
+        deadline_id: int,
+        user_id: int,
+        completion_notes: Optional[str] = None
+    ) -> Deadline:
+        """Mark a deadline as completed"""
+        
+        deadline = self.db.query(Deadline).filter(Deadline.id == deadline_id).first()
+        if not deadline:
+            raise DeadlineManagementError(f"Deadline {deadline_id} not found")
+        
+        if deadline.status != DeadlineStatus.PENDING:
+            raise DeadlineManagementError(f"Only pending deadlines can be completed")
+        
+        # Update deadline
+        deadline.status = DeadlineStatus.COMPLETED
+        deadline.completed_date = datetime.now(timezone.utc)
+        deadline.completed_by_user_id = user_id
+        deadline.completion_notes = completion_notes
+        
+        # Create history record
+        self._create_deadline_history(
+            deadline_id, "completed", "status", "pending", "completed", user_id, completion_notes
+        )
+        
+        # Cancel pending reminders
+        self._cancel_pending_reminders(deadline_id)
+        
+        self.db.commit()
+        self.db.refresh(deadline)
+        
+        logger.info(f"Completed deadline {deadline_id}")
+        return deadline
+    
+    def extend_deadline(
+        self,
+        deadline_id: int,
+        new_deadline_date: date,
+        user_id: int,
+        extension_reason: Optional[str] = None,
+        extension_granted_by: Optional[str] = None
+    ) -> Deadline:
+        """Extend a deadline to a new date"""
+        
+        deadline = self.db.query(Deadline).filter(Deadline.id == deadline_id).first()
+        if not deadline:
+            raise DeadlineManagementError(f"Deadline {deadline_id} not found")
+        
+        if deadline.status not in [DeadlineStatus.PENDING, DeadlineStatus.EXTENDED]:
+            raise DeadlineManagementError("Only pending or previously extended deadlines can be extended")
+        
+        # Store original deadline if this is the first extension
+        if not deadline.original_deadline_date:
+            deadline.original_deadline_date = deadline.deadline_date
+        
+        old_date = deadline.deadline_date
+        deadline.deadline_date = new_deadline_date
+        deadline.status = DeadlineStatus.EXTENDED
+        deadline.extension_reason = extension_reason
+        deadline.extension_granted_by = extension_granted_by
+        
+        # Create history record
+        self._create_deadline_history(
+            deadline_id, "extended", "deadline_date", str(old_date), str(new_deadline_date), 
+            user_id, f"Extension reason: {extension_reason or 'Not specified'}"
+        )
+        
+        # Reschedule reminders for new date
+        self._reschedule_reminders(deadline)
+        
+        self.db.commit()
+        self.db.refresh(deadline)
+        
+        logger.info(f"Extended deadline {deadline_id} from {old_date} to {new_deadline_date}")
+        return deadline
+    
+    def cancel_deadline(
+        self,
+        deadline_id: int,
+        user_id: int,
+        cancellation_reason: Optional[str] = None
+    ) -> Deadline:
+        """Cancel a deadline"""
+        
+        deadline = self.db.query(Deadline).filter(Deadline.id == deadline_id).first()
+        if not deadline:
+            raise DeadlineManagementError(f"Deadline {deadline_id} not found")
+        
+        deadline.status = DeadlineStatus.CANCELLED
+        
+        # Create history record
+        self._create_deadline_history(
+            deadline_id, "cancelled", "status", deadline.status.value, "cancelled", 
+            user_id, cancellation_reason
+        )
+        
+        # Cancel pending reminders
+        self._cancel_pending_reminders(deadline_id)
+        
+        self.db.commit()
+        self.db.refresh(deadline)
+        
+        logger.info(f"Cancelled deadline {deadline_id}")
+        return deadline
+    
+    def get_deadlines_by_file(self, file_no: str) -> List[Deadline]:
+        """Get all deadlines for a specific file"""
+        
+        return self.db.query(Deadline).filter(
+            Deadline.file_no == file_no
+        ).options(
+            joinedload(Deadline.assigned_to_user),
+            joinedload(Deadline.assigned_to_employee),
+            joinedload(Deadline.created_by)
+        ).order_by(Deadline.deadline_date.asc()).all()
+    
+    def get_upcoming_deadlines(
+        self,
+        days_ahead: int = 30,
+        user_id: Optional[int] = None,
+        employee_id: Optional[str] = None,
+        priority: Optional[DeadlinePriority] = None,
+        deadline_type: Optional[DeadlineType] = None
+    ) -> List[Deadline]:
+        """Get upcoming deadlines within specified timeframe"""
+        
+        end_date = date.today() + timedelta(days=days_ahead)
+        
+        query = self.db.query(Deadline).filter(
+            Deadline.status == DeadlineStatus.PENDING,
+            Deadline.deadline_date <= end_date
+        )
+        
+        if user_id:
+            query = query.filter(Deadline.assigned_to_user_id == user_id)
+        
+        if employee_id:
+            query = query.filter(Deadline.assigned_to_employee_id == employee_id)
+        
+        if priority:
+            query = query.filter(Deadline.priority == priority)
+        
+        if deadline_type:
+            query = query.filter(Deadline.deadline_type == deadline_type)
+        
+        return query.options(
+            joinedload(Deadline.file),
+            joinedload(Deadline.client),
+            joinedload(Deadline.assigned_to_user),
+            joinedload(Deadline.assigned_to_employee)
+        ).order_by(Deadline.deadline_date.asc(), Deadline.priority.desc()).all()
+    
+    def get_overdue_deadlines(
+        self,
+        user_id: Optional[int] = None,
+        employee_id: Optional[str] = None
+    ) -> List[Deadline]:
+        """Get overdue deadlines"""
+        
+        query = self.db.query(Deadline).filter(
+            Deadline.status == DeadlineStatus.PENDING,
+            Deadline.deadline_date < date.today()
+        )
+        
+        if user_id:
+            query = query.filter(Deadline.assigned_to_user_id == user_id)
+        
+        if employee_id:
+            query = query.filter(Deadline.assigned_to_employee_id == employee_id)
+        
+        return query.options(
+            joinedload(Deadline.file),
+            joinedload(Deadline.client),
+            joinedload(Deadline.assigned_to_user),
+            joinedload(Deadline.assigned_to_employee)
+        ).order_by(Deadline.deadline_date.asc()).all()
+    
+    def get_deadline_statistics(
+        self,
+        user_id: Optional[int] = None,
+        employee_id: Optional[str] = None,
+        start_date: Optional[date] = None,
+        end_date: Optional[date] = None
+    ) -> Dict[str, Any]:
+        """Get deadline statistics for reporting"""
+        
+        base_query = self.db.query(Deadline)
+        
+        if user_id:
+            base_query = base_query.filter(Deadline.assigned_to_user_id == user_id)
+        
+        if employee_id:
+            base_query = base_query.filter(Deadline.assigned_to_employee_id == employee_id)
+        
+        if start_date:
+            base_query = base_query.filter(Deadline.deadline_date >= start_date)
+        
+        if end_date:
+            base_query = base_query.filter(Deadline.deadline_date <= end_date)
+        
+        # Calculate statistics
+        total_deadlines = base_query.count()
+        pending_deadlines = base_query.filter(Deadline.status == DeadlineStatus.PENDING).count()
+        completed_deadlines = base_query.filter(Deadline.status == DeadlineStatus.COMPLETED).count()
+        overdue_deadlines = base_query.filter(
+            Deadline.status == DeadlineStatus.PENDING,
+            Deadline.deadline_date < date.today()
+        ).count()
+        
+        # Deadlines by priority
+        priority_counts = {}
+        for priority in DeadlinePriority:
+            count = base_query.filter(Deadline.priority == priority).count()
+            priority_counts[priority.value] = count
+        
+        # Deadlines by type
+        type_counts = {}
+        for deadline_type in DeadlineType:
+            count = base_query.filter(Deadline.deadline_type == deadline_type).count()
+            type_counts[deadline_type.value] = count
+        
+        # Upcoming deadlines (next 7, 14, 30 days)
+        today = date.today()
+        upcoming_7_days = base_query.filter(
+            Deadline.status == DeadlineStatus.PENDING,
+            Deadline.deadline_date.between(today, today + timedelta(days=7))
+        ).count()
+        
+        upcoming_14_days = base_query.filter(
+            Deadline.status == DeadlineStatus.PENDING,
+            Deadline.deadline_date.between(today, today + timedelta(days=14))
+        ).count()
+        
+        upcoming_30_days = base_query.filter(
+            Deadline.status == DeadlineStatus.PENDING,
+            Deadline.deadline_date.between(today, today + timedelta(days=30))
+        ).count()
+        
+        return {
+            "total_deadlines": total_deadlines,
+            "pending_deadlines": pending_deadlines,
+            "completed_deadlines": completed_deadlines,
+            "overdue_deadlines": overdue_deadlines,
+            "completion_rate": (completed_deadlines / total_deadlines * 100) if total_deadlines > 0 else 0,
+            "priority_breakdown": priority_counts,
+            "type_breakdown": type_counts,
+            "upcoming": {
+                "next_7_days": upcoming_7_days,
+                "next_14_days": upcoming_14_days,
+                "next_30_days": upcoming_30_days
+            }
+        }
+    
+    def create_deadline_from_template(
+        self,
+        template_id: int,
+        user_id: int,
+        file_no: Optional[str] = None,
+        client_id: Optional[str] = None,
+        deadline_date: Optional[date] = None,
+        **overrides
+    ) -> Deadline:
+        """Create a deadline from a template"""
+        
+        template = self.db.query(DeadlineTemplate).filter(DeadlineTemplate.id == template_id).first()
+        if not template:
+            raise DeadlineManagementError(f"Deadline template {template_id} not found")
+        
+        if not template.active:
+            raise DeadlineManagementError("Template is not active")
+        
+        # Calculate deadline date if not provided
+        if not deadline_date:
+            if template.days_from_file_open and file_no:
+                file_obj = self.db.query(File).filter(File.file_no == file_no).first()
+                if file_obj:
+                    deadline_date = file_obj.opened + timedelta(days=template.days_from_file_open)
+            else:
+                deadline_date = date.today() + timedelta(days=template.days_from_event or 30)
+        
+        # Get file and client info for template substitution
+        file_obj = None
+        client_obj = None
+        
+        if file_no:
+            file_obj = self.db.query(File).filter(File.file_no == file_no).first()
+            if file_obj and file_obj.owner:
+                client_obj = file_obj.owner
+        elif client_id:
+            client_obj = self.db.query(Rolodex).filter(Rolodex.id == client_id).first()
+        
+        # Process template strings with substitutions
+        title = self._process_template_string(
+            template.default_title_template, file_obj, client_obj
+        )
+        
+        description = self._process_template_string(
+            template.default_description_template, file_obj, client_obj
+        ) if template.default_description_template else None
+        
+        # Create deadline with template defaults and overrides
+        deadline_data = {
+            "title": title,
+            "description": description,
+            "deadline_date": deadline_date,
+            "deadline_type": template.deadline_type,
+            "priority": template.priority,
+            "file_no": file_no,
+            "client_id": client_id,
+            "advance_notice_days": template.default_advance_notice_days,
+            "notification_frequency": template.default_notification_frequency,
+            "created_by_user_id": user_id
+        }
+        
+        # Apply any overrides
+        deadline_data.update(overrides)
+        
+        return self.create_deadline(**deadline_data)
+    
+    def get_pending_reminders(self, reminder_date: date = None) -> List[DeadlineReminder]:
+        """Get pending reminders that need to be sent"""
+        
+        if reminder_date is None:
+            reminder_date = date.today()
+        
+        return self.db.query(DeadlineReminder).join(Deadline).filter(
+            DeadlineReminder.reminder_date <= reminder_date,
+            DeadlineReminder.notification_sent == False,
+            Deadline.status == DeadlineStatus.PENDING
+        ).options(
+            joinedload(DeadlineReminder.deadline),
+            joinedload(DeadlineReminder.recipient)
+        ).all()
+    
+    def mark_reminder_sent(
+        self,
+        reminder_id: int,
+        delivery_status: str = "sent",
+        error_message: Optional[str] = None
+    ):
+        """Mark a reminder as sent"""
+        
+        reminder = self.db.query(DeadlineReminder).filter(DeadlineReminder.id == reminder_id).first()
+        if reminder:
+            reminder.notification_sent = True
+            reminder.sent_at = datetime.now(timezone.utc)
+            reminder.delivery_status = delivery_status
+            if error_message:
+                reminder.error_message = error_message
+            
+            self.db.commit()
+    
+    # Private helper methods
+    
+    def _create_deadline_history(
+        self,
+        deadline_id: int,
+        change_type: str,
+        field_changed: Optional[str],
+        old_value: Optional[str],
+        new_value: Optional[str],
+        user_id: int,
+        change_reason: Optional[str] = None
+    ):
+        """Create a deadline history record"""
+        
+        history_record = DeadlineHistory(
+            deadline_id=deadline_id,
+            change_type=change_type,
+            field_changed=field_changed,
+            old_value=old_value,
+            new_value=new_value,
+            user_id=user_id,
+            change_reason=change_reason
+        )
+        
+        self.db.add(history_record)
+    
+    def _schedule_reminders(self, deadline: Deadline):
+        """Schedule automatic reminders for a deadline"""
+        
+        if deadline.notification_frequency == NotificationFrequency.NONE:
+            return
+        
+        # Calculate reminder dates
+        reminder_dates = []
+        advance_days = deadline.advance_notice_days or 7
+        
+        if deadline.notification_frequency == NotificationFrequency.DAILY:
+            # Daily reminders starting from advance notice days
+            for i in range(advance_days, 0, -1):
+                reminder_date = deadline.deadline_date - timedelta(days=i)
+                if reminder_date >= date.today():
+                    reminder_dates.append((reminder_date, i))
+        
+        elif deadline.notification_frequency == NotificationFrequency.WEEKLY:
+            # Weekly reminders
+            weeks_ahead = max(1, advance_days // 7)
+            for week in range(weeks_ahead, 0, -1):
+                reminder_date = deadline.deadline_date - timedelta(weeks=week)
+                if reminder_date >= date.today():
+                    reminder_dates.append((reminder_date, week * 7))
+        
+        elif deadline.notification_frequency == NotificationFrequency.MONTHLY:
+            # Monthly reminder
+            reminder_date = deadline.deadline_date - timedelta(days=30)
+            if reminder_date >= date.today():
+                reminder_dates.append((reminder_date, 30))
+        
+        # Create reminder records
+        for reminder_date, days_before in reminder_dates:
+            recipient_user_id = deadline.assigned_to_user_id or deadline.created_by_user_id
+            
+            reminder = DeadlineReminder(
+                deadline_id=deadline.id,
+                reminder_date=reminder_date,
+                days_before_deadline=days_before,
+                recipient_user_id=recipient_user_id,
+                subject=f"Deadline Reminder: {deadline.title}",
+                message=f"Reminder: {deadline.title} is due on {deadline.deadline_date} ({days_before} days from now)"
+            )
+            
+            self.db.add(reminder)
+    
+    def _reschedule_reminders(self, deadline: Deadline):
+        """Reschedule reminders after deadline date change"""
+        
+        # Delete existing unsent reminders
+        self.db.query(DeadlineReminder).filter(
+            DeadlineReminder.deadline_id == deadline.id,
+            DeadlineReminder.notification_sent == False
+        ).delete()
+        
+        # Schedule new reminders
+        self._schedule_reminders(deadline)
+    
+    def _cancel_pending_reminders(self, deadline_id: int):
+        """Cancel all pending reminders for a deadline"""
+        
+        self.db.query(DeadlineReminder).filter(
+            DeadlineReminder.deadline_id == deadline_id,
+            DeadlineReminder.notification_sent == False
+        ).delete()
+    
+    def _process_template_string(
+        self,
+        template_string: Optional[str],
+        file_obj: Optional[File],
+        client_obj: Optional[Rolodex]
+    ) -> Optional[str]:
+        """Process template string with variable substitutions"""
+        
+        if not template_string:
+            return None
+        
+        result = template_string
+        
+        # File substitutions
+        if file_obj:
+            result = result.replace("{file_no}", file_obj.file_no or "")
+            result = result.replace("{regarding}", file_obj.regarding or "")
+            result = result.replace("{attorney}", file_obj.empl_num or "")
+        
+        # Client substitutions
+        if client_obj:
+            client_name = f"{client_obj.first or ''} {client_obj.last or ''}".strip()
+            result = result.replace("{client_name}", client_name)
+            result = result.replace("{client_id}", client_obj.id or "")
+        
+        # Date substitutions
+        today = date.today()
+        result = result.replace("{today}", today.strftime("%Y-%m-%d"))
+        result = result.replace("{today_formatted}", today.strftime("%B %d, %Y"))
+        
+        return result
+
+
+class DeadlineTemplateService:
+    """Service for managing deadline templates"""
+    
+    def __init__(self, db: Session):
+        self.db = db
+    
+    def create_template(
+        self,
+        name: str,
+        deadline_type: DeadlineType,
+        user_id: int,
+        description: Optional[str] = None,
+        priority: DeadlinePriority = DeadlinePriority.MEDIUM,
+        default_title_template: Optional[str] = None,
+        default_description_template: Optional[str] = None,
+        default_advance_notice_days: int = 7,
+        default_notification_frequency: NotificationFrequency = NotificationFrequency.WEEKLY,
+        days_from_file_open: Optional[int] = None,
+        days_from_event: Optional[int] = None
+    ) -> DeadlineTemplate:
+        """Create a new deadline template"""
+        
+        # Check for duplicate name
+        existing = self.db.query(DeadlineTemplate).filter(DeadlineTemplate.name == name).first()
+        if existing:
+            raise DeadlineManagementError(f"Template with name '{name}' already exists")
+        
+        template = DeadlineTemplate(
+            name=name,
+            description=description,
+            deadline_type=deadline_type,
+            priority=priority,
+            default_title_template=default_title_template,
+            default_description_template=default_description_template,
+            default_advance_notice_days=default_advance_notice_days,
+            default_notification_frequency=default_notification_frequency,
+            days_from_file_open=days_from_file_open,
+            days_from_event=days_from_event,
+            created_by_user_id=user_id
+        )
+        
+        self.db.add(template)
+        self.db.commit()
+        self.db.refresh(template)
+        
+        logger.info(f"Created deadline template: {name}")
+        return template
+    
+    def get_active_templates(
+        self,
+        deadline_type: Optional[DeadlineType] = None
+    ) -> List[DeadlineTemplate]:
+        """Get all active deadline templates"""
+        
+        query = self.db.query(DeadlineTemplate).filter(DeadlineTemplate.active == True)
+        
+        if deadline_type:
+            query = query.filter(DeadlineTemplate.deadline_type == deadline_type)
+        
+        return query.order_by(DeadlineTemplate.name).all()

app/services/document_notifications.py

@@ -0,0 +1,172 @@
+"""
+Document Notifications Service
+
+Provides convenience helpers to broadcast real-time document processing
+status updates over the centralized WebSocket pool. Targets both per-file
+topics for end users and an admin-wide topic for monitoring.
+"""
+from __future__ import annotations
+
+from typing import Any, Dict, Optional
+from datetime import datetime, timezone
+from uuid import uuid4
+
+from app.core.logging import get_logger
+from app.middleware.websocket_middleware import get_websocket_manager
+from app.database.base import SessionLocal
+from app.models.document_workflows import EventLog
+
+
+logger = get_logger("document_notifications")
+
+
+# Topic helpers
+def topic_for_file(file_no: str) -> str:
+    return f"documents_{file_no}"
+
+
+ADMIN_DOCUMENTS_TOPIC = "admin_documents"
+
+
+# ----------------------------------------------------------------------------
+# Lightweight in-memory status store for backfill
+# ----------------------------------------------------------------------------
+_last_status_by_file: Dict[str, Dict[str, Any]] = {}
+
+
+def _record_last_status(*, file_no: str, status: str, data: Optional[Dict[str, Any]] = None) -> None:
+    try:
+        _last_status_by_file[file_no] = {
+            "file_no": file_no,
+            "status": status,
+            "data": dict(data or {}),
+            "timestamp": datetime.now(timezone.utc),
+        }
+    except Exception:
+        # Avoid ever failing core path
+        pass
+
+
+def get_last_status(file_no: str) -> Optional[Dict[str, Any]]:
+    """Return the last known status record for a file, if any.
+
+    Record shape: { file_no, status, data, timestamp: datetime }
+    """
+    try:
+        return _last_status_by_file.get(file_no)
+    except Exception:
+        return None
+
+
+async def broadcast_status(
+    *,
+    file_no: str,
+    status: str,  # "processing" | "completed" | "failed"
+    data: Optional[Dict[str, Any]] = None,
+    user_id: Optional[int] = None,
+) -> int:
+    """
+    Broadcast a document status update to:
+    - The per-file topic for subscribers
+    - The admin monitoring topic
+    - Optionally to a specific user's active connections
+    Returns number of messages successfully sent to the per-file topic.
+    """
+    wm = get_websocket_manager()
+
+    event_data: Dict[str, Any] = {
+        "file_no": file_no,
+        "status": status,
+        **(data or {}),
+    }
+
+    # Update in-memory last-known status for backfill
+    _record_last_status(file_no=file_no, status=status, data=data)
+
+    # Best-effort persistence to event log for history/backfill
+    try:
+        db = SessionLocal()
+        try:
+            ev = EventLog(
+                event_id=str(uuid4()),
+                event_type=f"document_{status}",
+                event_source="document_management",
+                file_no=file_no,
+                user_id=user_id,
+                resource_type="document",
+                resource_id=str(event_data.get("document_id") or event_data.get("job_id") or ""),
+                event_data=event_data,
+                previous_state=None,
+                new_state={"status": status},
+                occurred_at=datetime.now(timezone.utc),
+            )
+            db.add(ev)
+            db.commit()
+        except Exception:
+            try: db.rollback()
+            except Exception: pass
+        finally:
+            try: db.close()
+            except Exception: pass
+    except Exception:
+        # Never fail core path
+        pass
+
+    # Per-file topic broadcast
+    topic = topic_for_file(file_no)
+    sent_to_file = await wm.broadcast_to_topic(
+        topic=topic,
+        message_type=f"document_{status}",
+        data=event_data,
+    )
+
+    # Admin monitoring broadcast (best-effort)
+    try:
+        await wm.broadcast_to_topic(
+            topic=ADMIN_DOCUMENTS_TOPIC,
+            message_type="admin_document_event",
+            data=event_data,
+        )
+    except Exception:
+        # Never fail core path if admin broadcast fails
+        pass
+
+    # Optional direct-to-user notification
+    if user_id is not None:
+        try:
+            await wm.send_to_user(
+                user_id=user_id,
+                message_type=f"document_{status}",
+                data=event_data,
+            )
+        except Exception:
+            # Ignore failures to keep UX resilient
+            pass
+
+    logger.info(
+        "Document notification broadcast",
+        file_no=file_no,
+        status=status,
+        sent_to_file_topic=sent_to_file,
+    )
+    return sent_to_file
+
+
+async def notify_processing(
+    *, file_no: str, user_id: Optional[int] = None, data: Optional[Dict[str, Any]] = None
+) -> int:
+    return await broadcast_status(file_no=file_no, status="processing", data=data, user_id=user_id)
+
+
+async def notify_completed(
+    *, file_no: str, user_id: Optional[int] = None, data: Optional[Dict[str, Any]] = None
+) -> int:
+    return await broadcast_status(file_no=file_no, status="completed", data=data, user_id=user_id)
+
+
+async def notify_failed(
+    *, file_no: str, user_id: Optional[int] = None, data: Optional[Dict[str, Any]] = None
+) -> int:
+    return await broadcast_status(file_no=file_no, status="failed", data=data, user_id=user_id)
+
+

app/services/file_management.py

@@ -9,9 +9,10 @@ from sqlalchemy.orm import Session, joinedload
 from sqlalchemy import and_, func, or_, desc
 
 from app.models import (
-    File, Ledger, FileStatus, FileType, Rolodex, Employee, 
-    BillingStatement, Timer, TimeEntry, User, FileStatusHistory, 
-    FileTransferHistory, FileArchiveInfo
+    File, Ledger, FileStatus, FileType, Rolodex, Employee,
+    BillingStatement, Timer, TimeEntry, User, FileStatusHistory,
+    FileTransferHistory, FileArchiveInfo, FileClosureChecklist, FileAlert,
+    FileRelationship
 )
 from app.utils.logging import app_logger
 
@@ -432,6 +433,284 @@ class FileManagementService:
         
         logger.info(f"Bulk status update: {len(results['successful'])} successful, {len(results['failed'])} failed")
         return results
+
+    # Checklist management
+
+    def get_closure_checklist(self, file_no: str) -> List[Dict[str, Any]]:
+        """Return the closure checklist items for a file."""
+        items = self.db.query(FileClosureChecklist).filter(
+            FileClosureChecklist.file_no == file_no
+        ).order_by(FileClosureChecklist.sort_order.asc(), FileClosureChecklist.id.asc()).all()
+
+        return [
+            {
+                "id": i.id,
+                "file_no": i.file_no,
+                "item_name": i.item_name,
+                "item_description": i.item_description,
+                "is_required": bool(i.is_required),
+                "is_completed": bool(i.is_completed),
+                "completed_date": i.completed_date,
+                "completed_by_name": i.completed_by_name,
+                "notes": i.notes,
+                "sort_order": i.sort_order,
+            }
+            for i in items
+        ]
+
+    def add_checklist_item(
+        self,
+        *,
+        file_no: str,
+        item_name: str,
+        item_description: Optional[str] = None,
+        is_required: bool = True,
+        sort_order: int = 0,
+    ) -> FileClosureChecklist:
+        """Add a checklist item to a file."""
+        # Ensure file exists
+        if not self.db.query(File).filter(File.file_no == file_no).first():
+            raise FileManagementError(f"File {file_no} not found")
+
+        item = FileClosureChecklist(
+            file_no=file_no,
+            item_name=item_name,
+            item_description=item_description,
+            is_required=is_required,
+            sort_order=sort_order,
+        )
+        self.db.add(item)
+        self.db.commit()
+        self.db.refresh(item)
+        logger.info(f"Added checklist item '{item_name}' to file {file_no}")
+        return item
+
+    def update_checklist_item(
+        self,
+        *,
+        item_id: int,
+        item_name: Optional[str] = None,
+        item_description: Optional[str] = None,
+        is_required: Optional[bool] = None,
+        is_completed: Optional[bool] = None,
+        sort_order: Optional[int] = None,
+        user_id: Optional[int] = None,
+        notes: Optional[str] = None,
+    ) -> FileClosureChecklist:
+        """Update attributes of a checklist item; optionally mark complete/incomplete."""
+        item = self.db.query(FileClosureChecklist).filter(FileClosureChecklist.id == item_id).first()
+        if not item:
+            raise FileManagementError("Checklist item not found")
+
+        if item_name is not None:
+            item.item_name = item_name
+        if item_description is not None:
+            item.item_description = item_description
+        if is_required is not None:
+            item.is_required = bool(is_required)
+        if sort_order is not None:
+            item.sort_order = int(sort_order)
+        if is_completed is not None:
+            item.is_completed = bool(is_completed)
+            if item.is_completed:
+                item.completed_date = datetime.now(timezone.utc)
+                if user_id:
+                    user = self.db.query(User).filter(User.id == user_id).first()
+                    item.completed_by_user_id = user_id
+                    item.completed_by_name = user.username if user else f"user_{user_id}"
+            else:
+                item.completed_date = None
+                item.completed_by_user_id = None
+                item.completed_by_name = None
+        if notes is not None:
+            item.notes = notes
+
+        self.db.commit()
+        self.db.refresh(item)
+        logger.info(f"Updated checklist item {item_id}")
+        return item
+
+    def delete_checklist_item(self, *, item_id: int) -> None:
+        item = self.db.query(FileClosureChecklist).filter(FileClosureChecklist.id == item_id).first()
+        if not item:
+            raise FileManagementError("Checklist item not found")
+        self.db.delete(item)
+        self.db.commit()
+        logger.info(f"Deleted checklist item {item_id}")
+
+    # Alerts management
+
+    def create_alert(
+        self,
+        *,
+        file_no: str,
+        alert_type: str,
+        title: str,
+        message: str,
+        alert_date: date,
+        notify_attorney: bool = True,
+        notify_admin: bool = False,
+        notification_days_advance: int = 7,
+    ) -> FileAlert:
+        if not self.db.query(File).filter(File.file_no == file_no).first():
+            raise FileManagementError(f"File {file_no} not found")
+        alert = FileAlert(
+            file_no=file_no,
+            alert_type=alert_type,
+            title=title,
+            message=message,
+            alert_date=alert_date,
+            notify_attorney=notify_attorney,
+            notify_admin=notify_admin,
+            notification_days_advance=notification_days_advance,
+        )
+        self.db.add(alert)
+        self.db.commit()
+        self.db.refresh(alert)
+        logger.info(f"Created alert {alert.id} for file {file_no} on {alert_date}")
+        return alert
+
+    def get_alerts(
+        self,
+        *,
+        file_no: str,
+        active_only: bool = True,
+        upcoming_only: bool = False,
+        limit: int = 100,
+    ) -> List[FileAlert]:
+        query = self.db.query(FileAlert).filter(FileAlert.file_no == file_no)
+        if active_only:
+            query = query.filter(FileAlert.is_active == True)
+        if upcoming_only:
+            today = datetime.now(timezone.utc).date()
+            query = query.filter(FileAlert.alert_date >= today)
+        return query.order_by(FileAlert.alert_date.asc(), FileAlert.id.asc()).limit(limit).all()
+
+    def acknowledge_alert(self, *, alert_id: int, user_id: int) -> FileAlert:
+        alert = self.db.query(FileAlert).filter(FileAlert.id == alert_id).first()
+        if not alert:
+            raise FileManagementError("Alert not found")
+        if not alert.is_active:
+            return alert
+        alert.is_acknowledged = True
+        alert.acknowledged_at = datetime.now(timezone.utc)
+        alert.acknowledged_by_user_id = user_id
+        self.db.commit()
+        self.db.refresh(alert)
+        logger.info(f"Acknowledged alert {alert_id} by user {user_id}")
+        return alert
+
+    def update_alert(
+        self,
+        *,
+        alert_id: int,
+        title: Optional[str] = None,
+        message: Optional[str] = None,
+        alert_date: Optional[date] = None,
+        is_active: Optional[bool] = None,
+    ) -> FileAlert:
+        alert = self.db.query(FileAlert).filter(FileAlert.id == alert_id).first()
+        if not alert:
+            raise FileManagementError("Alert not found")
+        if title is not None:
+            alert.title = title
+        if message is not None:
+            alert.message = message
+        if alert_date is not None:
+            alert.alert_date = alert_date
+        if is_active is not None:
+            alert.is_active = bool(is_active)
+        self.db.commit()
+        self.db.refresh(alert)
+        logger.info(f"Updated alert {alert_id}")
+        return alert
+
+    def delete_alert(self, *, alert_id: int) -> None:
+        alert = self.db.query(FileAlert).filter(FileAlert.id == alert_id).first()
+        if not alert:
+            raise FileManagementError("Alert not found")
+        self.db.delete(alert)
+        self.db.commit()
+        logger.info(f"Deleted alert {alert_id}")
+
+    # Relationship management
+
+    def create_relationship(
+        self,
+        *,
+        source_file_no: str,
+        target_file_no: str,
+        relationship_type: str,
+        user_id: Optional[int] = None,
+        notes: Optional[str] = None,
+    ) -> FileRelationship:
+        if source_file_no == target_file_no:
+            raise FileManagementError("Source and target file cannot be the same")
+        source = self.db.query(File).filter(File.file_no == source_file_no).first()
+        target = self.db.query(File).filter(File.file_no == target_file_no).first()
+        if not source:
+            raise FileManagementError(f"File {source_file_no} not found")
+        if not target:
+            raise FileManagementError(f"File {target_file_no} not found")
+        user_name: Optional[str] = None
+        if user_id is not None:
+            user = self.db.query(User).filter(User.id == user_id).first()
+            user_name = user.username if user else f"user_{user_id}"
+        # Prevent duplicate exact relationship
+        existing = self.db.query(FileRelationship).filter(
+            FileRelationship.source_file_no == source_file_no,
+            FileRelationship.target_file_no == target_file_no,
+            FileRelationship.relationship_type == relationship_type,
+        ).first()
+        if existing:
+            return existing
+        rel = FileRelationship(
+            source_file_no=source_file_no,
+            target_file_no=target_file_no,
+            relationship_type=relationship_type,
+            notes=notes,
+            created_by_user_id=user_id,
+            created_by_name=user_name,
+        )
+        self.db.add(rel)
+        self.db.commit()
+        self.db.refresh(rel)
+        logger.info(
+            f"Created relationship {relationship_type}: {source_file_no} -> {target_file_no}"
+        )
+        return rel
+
+    def get_relationships(self, *, file_no: str) -> List[Dict[str, Any]]:
+        """Return relationships where the given file is source or target."""
+        rels = self.db.query(FileRelationship).filter(
+            (FileRelationship.source_file_no == file_no) | (FileRelationship.target_file_no == file_no)
+        ).order_by(FileRelationship.id.desc()).all()
+        results: List[Dict[str, Any]] = []
+        for r in rels:
+            direction = "outbound" if r.source_file_no == file_no else "inbound"
+            other_file_no = r.target_file_no if direction == "outbound" else r.source_file_no
+            results.append(
+                {
+                    "id": r.id,
+                    "direction": direction,
+                    "relationship_type": r.relationship_type,
+                    "notes": r.notes,
+                    "source_file_no": r.source_file_no,
+                    "target_file_no": r.target_file_no,
+                    "other_file_no": other_file_no,
+                    "created_by_name": r.created_by_name,
+                    "created_at": getattr(r, "created_at", None),
+                }
+            )
+        return results
+
+    def delete_relationship(self, *, relationship_id: int) -> None:
+        rel = self.db.query(FileRelationship).filter(FileRelationship.id == relationship_id).first()
+        if not rel:
+            raise FileManagementError("Relationship not found")
+        self.db.delete(rel)
+        self.db.commit()
+        logger.info(f"Deleted relationship {relationship_id}")
     
     # Private helper methods
     

app/services/mailing.py

@@ -0,0 +1,229 @@
+"""
+Mailing utilities for generating printable labels and envelopes.
+
+MVP scope:
+- Build address blocks from `Rolodex` entries
+- Generate printable HTML for Avery 5160 labels (3 x 10)
+- Generate simple envelope HTML (No. 10) with optional return address
+- Save bytes via storage adapter for easy download at /uploads
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Iterable, List, Optional, Sequence
+
+from sqlalchemy.orm import Session
+
+from app.models.rolodex import Rolodex
+from app.models.files import File
+from app.services.storage import get_default_storage
+
+
+@dataclass
+class Address:
+    display_name: str
+    line1: Optional[str] = None
+    line2: Optional[str] = None
+    line3: Optional[str] = None
+    city: Optional[str] = None
+    state: Optional[str] = None
+    postal_code: Optional[str] = None
+
+    def compact_lines(self, include_name: bool = True) -> List[str]:
+        lines: List[str] = []
+        if include_name and self.display_name:
+            lines.append(self.display_name)
+        for part in [self.line1, self.line2, self.line3]:
+            if part:
+                lines.append(part)
+        city_state_zip: List[str] = []
+        if self.city:
+            city_state_zip.append(self.city)
+        if self.state:
+            city_state_zip.append(self.state)
+        if self.postal_code:
+            city_state_zip.append(self.postal_code)
+        if city_state_zip:
+            # Join as "City, ST ZIP" when state and city present, otherwise simple join
+            if self.city and self.state:
+                last = " ".join([p for p in [self.state, self.postal_code] if p])
+                lines.append(f"{self.city}, {last}".strip())
+            else:
+                lines.append(" ".join(city_state_zip))
+        return lines
+
+
+def build_address_from_rolodex(entry: Rolodex) -> Address:
+    name_parts: List[str] = []
+    if getattr(entry, "prefix", None):
+        name_parts.append(entry.prefix)
+    if getattr(entry, "first", None):
+        name_parts.append(entry.first)
+    if getattr(entry, "middle", None):
+        name_parts.append(entry.middle)
+    # Always include last/company
+    if getattr(entry, "last", None):
+        name_parts.append(entry.last)
+    if getattr(entry, "suffix", None):
+        name_parts.append(entry.suffix)
+    display_name = " ".join([p for p in name_parts if p]).strip()
+    return Address(
+        display_name=display_name or (entry.last or ""),
+        line1=getattr(entry, "a1", None),
+        line2=getattr(entry, "a2", None),
+        line3=getattr(entry, "a3", None),
+        city=getattr(entry, "city", None),
+        state=getattr(entry, "abrev", None),
+        postal_code=getattr(entry, "zip", None),
+    )
+
+
+def build_addresses_from_files(db: Session, file_nos: Sequence[str]) -> List[Address]:
+    if not file_nos:
+        return []
+    files = (
+        db.query(File)
+        .filter(File.file_no.in_([fn for fn in file_nos if fn]))
+        .all()
+    )
+    addresses: List[Address] = []
+    # Resolve owners in one extra query across unique owner ids
+    owner_ids = list({f.id for f in files if getattr(f, "id", None)})
+    if owner_ids:
+        owners_by_id = {
+            r.id: r for r in db.query(Rolodex).filter(Rolodex.id.in_(owner_ids)).all()
+        }
+    else:
+        owners_by_id = {}
+    for f in files:
+        owner = owners_by_id.get(getattr(f, "id", None))
+        if owner:
+            addresses.append(build_address_from_rolodex(owner))
+    return addresses
+
+
+def build_addresses_from_rolodex(db: Session, rolodex_ids: Sequence[str]) -> List[Address]:
+    if not rolodex_ids:
+        return []
+    entries = (
+        db.query(Rolodex)
+        .filter(Rolodex.id.in_([rid for rid in rolodex_ids if rid]))
+        .all()
+    )
+    return [build_address_from_rolodex(r) for r in entries]
+
+
+def _labels_5160_css() -> str:
+    # 3 columns x 10 rows; label size 2.625" x 1.0"; sheet Letter 8.5"x11"
+    # Basic approximated layout suitable for quick printing.
+    return """
+    @page { size: letter; margin: 0.5in; }
+    body { font-family: Arial, sans-serif; margin: 0; }
+    .sheet { display: grid; grid-template-columns: repeat(3, 2.625in); grid-auto-rows: 1in; column-gap: 0.125in; row-gap: 0.0in; }
+    .label { box-sizing: border-box; padding: 0.1in 0.15in; overflow: hidden; }
+    .label p { margin: 0; line-height: 1.1; font-size: 11pt; }
+    .hint { margin: 12px 0; color: #666; font-size: 10pt; }
+    """
+
+
+def render_labels_html(addresses: Sequence[Address], *, start_position: int = 1, include_name: bool = True) -> bytes:
+    # Fill with empty slots up to start_position - 1 to allow partial sheets
+    blocks: List[str] = []
+    empty_slots = max(0, min(29, (start_position - 1)))
+    for _ in range(empty_slots):
+        blocks.append('<div class="label"></div>')
+    for addr in addresses:
+        lines = addr.compact_lines(include_name=include_name)
+        inner = "".join([f"<p>{line}</p>" for line in lines if line])
+        blocks.append(f'<div class="label">{inner}</div>')
+    css = _labels_5160_css()
+    html = f"""
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset=\"utf-8\" />
+  <title>Mailing Labels (Avery 5160)</title>
+  <style>{css}</style>
+  <meta name=\"generator\" content=\"delphi\" />
+</head>
+<body>
+  <div class=\"hint\">Avery 5160 — 30 labels per sheet. Print at 100% scale. Do not fit to page.</div>
+  <div class=\"sheet\">{''.join(blocks)}</div>
+</body>
+</html>
+"""
+    return html.encode("utf-8")
+
+
+def _envelope_css() -> str:
+    # Simple layout: place return address top-left, recipient in center-right area.
+    return """
+    @page { size: letter; margin: 0.5in; }
+    body { font-family: Arial, sans-serif; margin: 0; }
+    .envelope { position: relative; width: 9.5in; height: 4.125in; border: 1px dashed #ddd; margin: 0 auto; }
+    .return { position: absolute; top: 0.5in; left: 0.6in; font-size: 10pt; line-height: 1.2; }
+    .recipient { position: absolute; top: 1.6in; left: 3.7in; font-size: 12pt; line-height: 1.25; }
+    .envelope p { margin: 0; }
+    .page { page-break-after: always; margin: 0 0 12px 0; }
+    .hint { margin: 12px 0; color: #666; font-size: 10pt; }
+    """
+
+
+def render_envelopes_html(
+    addresses: Sequence[Address],
+    *,
+    return_address_lines: Optional[Sequence[str]] = None,
+    include_name: bool = True,
+) -> bytes:
+    css = _envelope_css()
+    pages: List[str] = []
+    return_html = "".join([f"<p>{line}</p>" for line in (return_address_lines or []) if line])
+    for addr in addresses:
+        to_lines = addr.compact_lines(include_name=include_name)
+        to_html = "".join([f"<p>{line}</p>" for line in to_lines if line])
+        page = f"""
+  <div class=\"page\">
+    <div class=\"envelope\">
+      {'<div class=\"return\">' + return_html + '</div>' if return_html else ''}
+      <div class=\"recipient\">{to_html}</div>
+    </div>
+  </div>
+        """
+        pages.append(page)
+    html = f"""
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset=\"utf-8\" />
+  <title>Envelopes (No. 10)</title>
+  <style>{css}</style>
+  <meta name=\"generator\" content=\"delphi\" />
+</head>
+<body>
+  <div class=\"hint\">No. 10 envelope layout. Print at 100% scale.</div>
+  {''.join(pages)}
+</body>
+</html>
+"""
+    return html.encode("utf-8")
+
+
+def save_html_bytes(content: bytes, *, filename_hint: str, subdir: str) -> dict:
+    storage = get_default_storage()
+    storage_path = storage.save_bytes(
+        content=content,
+        filename_hint=filename_hint if filename_hint.endswith(".html") else f"{filename_hint}.html",
+        subdir=subdir,
+        content_type="text/html",
+    )
+    url = storage.public_url(storage_path)
+    return {
+        "storage_path": storage_path,
+        "url": url,
+        "created_at": datetime.now(timezone.utc).isoformat(),
+        "mime_type": "text/html",
+        "size": len(content),
+    }
+
+

app/services/pension_valuation.py

@@ -0,0 +1,502 @@
+"""
+Pension valuation (annuity evaluator) service.
+
+Computes present value for:
+- Single-life level annuity with optional COLA and discounting
+- Joint-survivor annuity with survivor continuation percentage
+
+Survival probabilities are sourced from `number_tables` if available
+for the requested month range, using the ratio NA_t / NA_0 for the
+specified sex and race. If monthly entries are missing and life table
+values are available, a simple exponential survival curve is derived
+from life expectancy (LE) to approximate monthly survival.
+
+Rates are provided as percentages (e.g., 3.0 = 3%).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import math
+from typing import Dict, List, Optional, Tuple
+
+from sqlalchemy.orm import Session
+
+from app.models.pensions import LifeTable, NumberTable
+
+
+class InvalidCodeError(ValueError):
+    pass
+
+
+_RACE_MAP: Dict[str, str] = {
+    "W": "w",  # White
+    "B": "b",  # Black
+    "H": "h",  # Hispanic
+    "A": "a",  # All races
+}
+
+_SEX_MAP: Dict[str, str] = {
+    "M": "m",
+    "F": "f",
+    "A": "a",  # All sexes
+}
+
+
+def _normalize_codes(sex: str, race: str) -> Tuple[str, str, str]:
+    sex_u = (sex or "A").strip().upper()
+    race_u = (race or "A").strip().upper()
+    if sex_u not in _SEX_MAP:
+        raise InvalidCodeError("Invalid sex code; expected one of M, F, A")
+    if race_u not in _RACE_MAP:
+        raise InvalidCodeError("Invalid race code; expected one of W, B, H, A")
+    return _RACE_MAP[race_u] + _SEX_MAP[sex_u], sex_u, race_u
+
+
+def _to_monthly_rate(annual_percent: float) -> float:
+    """Convert an annual percentage (e.g. 6.0) to monthly effective rate."""
+    annual_rate = float(annual_percent or 0.0) / 100.0
+    if annual_rate <= -1.0:
+        # Avoid invalid negative base
+        raise ValueError("Annual rate too negative")
+    return (1.0 + annual_rate) ** (1.0 / 12.0) - 1.0
+
+
+def _load_monthly_na_series(
+    db: Session,
+    *,
+    sex: str,
+    race: str,
+    start_month: int,
+    months: int,
+    interpolate_missing: bool = False,
+    interpolation_method: str = "linear",  # "linear" or "step"
+) -> Optional[List[float]]:
+    """Return NA series for months [start_month, start_month + months - 1].
+
+    Values are floats for the column `na_{suffix}`. If any month in the
+    requested range is missing, returns None to indicate fallback.
+    """
+    if months <= 0:
+        return []
+
+    suffix, _, _ = _normalize_codes(sex, race)
+    na_col = f"na_{suffix}"
+
+    month_values: Dict[int, float] = {}
+    rows: List[NumberTable] = (
+        db.query(NumberTable)
+        .filter(NumberTable.month >= start_month, NumberTable.month < start_month + months)
+        .all()
+    )
+    for row in rows:
+        value = getattr(row, na_col, None)
+        if value is not None:
+            month_values[int(row.month)] = float(value)
+
+    # Build initial series with possible gaps
+    series_vals: List[Optional[float]] = []
+    for m in range(start_month, start_month + months):
+        series_vals.append(month_values.get(m))
+
+    if any(v is None for v in series_vals) and interpolate_missing:
+        # Linear interpolation for internal gaps
+        if (interpolation_method or "linear").lower() == "step":
+            # Step-wise: carry forward previous known; if leading gaps, use next known
+            # Fill leading gaps
+            first_known = None
+            for idx, val in enumerate(series_vals):
+                if val is not None:
+                    first_known = float(val)
+                    break
+            if first_known is None:
+                return None
+            for i in range(len(series_vals)):
+                if series_vals[i] is None:
+                    # find prev known
+                    prev_val = None
+                    for k in range(i - 1, -1, -1):
+                        if series_vals[k] is not None:
+                            prev_val = float(series_vals[k])
+                            break
+                    if prev_val is not None:
+                        series_vals[i] = prev_val
+                    else:
+                        # Use first known for leading gap
+                        series_vals[i] = first_known
+        else:
+            for i in range(len(series_vals)):
+                if series_vals[i] is None:
+                    # find prev
+                    prev_idx = None
+                    for k in range(i - 1, -1, -1):
+                        if series_vals[k] is not None:
+                            prev_idx = k
+                            break
+                    # find next
+                    next_idx = None
+                    for k in range(i + 1, len(series_vals)):
+                        if series_vals[k] is not None:
+                            next_idx = k
+                            break
+                    if prev_idx is None or next_idx is None:
+                        return None
+                    v0 = float(series_vals[prev_idx])
+                    v1 = float(series_vals[next_idx])
+                    frac = (i - prev_idx) / (next_idx - prev_idx)
+                    series_vals[i] = v0 + (v1 - v0) * frac
+
+    if any(v is None for v in series_vals):
+        return None
+
+    return [float(v) for v in series_vals]  # type: ignore
+
+
+def _approximate_survival_from_le(le_years: float, months: int) -> List[float]:
+    """Approximate monthly survival probabilities using an exponential model.
+
+    Given life expectancy in years (LE), approximate a constant hazard rate
+    such that expected remaining life equals LE. For a memoryless exponential
+    distribution, E[T] = 1/lambda. We discretize monthly: p_survive(t) = exp(-lambda * t_years).
+    """
+    if le_years is None or le_years <= 0:
+        # No survival; return zero beyond t=0
+        return [1.0] + [0.0] * (max(0, months - 1))
+
+    lam = 1.0 / float(le_years)
+    series: List[float] = []
+    for idx in range(months):
+        t_years = idx / 12.0
+        series.append(float(pow(2.718281828459045, -lam * t_years)))
+    return series
+
+
+def _load_life_expectancy(db: Session, *, age: int, sex: str, race: str) -> Optional[float]:
+    suffix, _, _ = _normalize_codes(sex, race)
+    le_col = f"le_{suffix}"
+    row: Optional[LifeTable] = db.query(LifeTable).filter(LifeTable.age == age).first()
+    if not row:
+        return None
+    val = getattr(row, le_col, None)
+    return float(val) if val is not None else None
+
+
+def _to_survival_probabilities(
+    db: Session,
+    *,
+    start_age: Optional[int],
+    sex: str,
+    race: str,
+    term_months: int,
+    interpolation_method: str = "linear",
+) -> List[float]:
+    """Build per-month survival probabilities p(t) for t in [0, term_months-1].
+
+    Prefer monthly NumberTable NA series if contiguous; otherwise approximate
+    from LifeTable life expectancy at `start_age`.
+    """
+    if term_months <= 0:
+        return []
+
+    # Try exact monthly NA series first
+    na_series = _load_monthly_na_series(
+        db,
+        sex=sex,
+        race=race,
+        start_month=0,
+        months=term_months,
+        interpolate_missing=True,
+        interpolation_method=interpolation_method,
+    )
+    if na_series is not None and len(na_series) > 0:
+        base = na_series[0]
+        if base is None or base <= 0:
+            # Degenerate base; fall back
+            na_series = None
+        else:
+            probs = [float(v) / float(base) for v in na_series]
+            # Clamp to [0,1]
+            return [0.0 if p < 0.0 else (1.0 if p > 1.0 else p) for p in probs]
+
+    # Fallback to LE approximation
+    le_years = _load_life_expectancy(db, age=int(start_age or 0), sex=sex, race=race)
+    return _approximate_survival_from_le(le_years if le_years is not None else 0.0, term_months)
+
+
+def _present_value_from_stream(
+    payments: List[float],
+    *,
+    discount_monthly: float,
+    cola_monthly: float,
+) -> float:
+    """PV of a cash-flow stream with monthly discount and monthly COLA growth applied."""
+    pv = 0.0
+    growth_factor = 1.0
+    discount_factor = 1.0
+    for idx, base_payment in enumerate(payments):
+        if idx == 0:
+            growth_factor = 1.0
+            discount_factor = 1.0
+        else:
+            growth_factor *= (1.0 + cola_monthly)
+            discount_factor *= (1.0 + discount_monthly)
+        pv += (base_payment * growth_factor) / discount_factor
+    return float(pv)
+
+
+def _compute_growth_factor_at_month(
+    month_index: int,
+    *,
+    cola_annual_percent: float,
+    cola_mode: str,
+    cola_cap_percent: Optional[float] = None,
+) -> float:
+    """Compute nominal COLA growth factor at month t relative to t=0.
+
+    cola_mode:
+      - "monthly": compound monthly using effective monthly rate derived from annual percent
+      - "annual_prorated": step annually, prorate linearly within the year
+    """
+    annual_pct = float(cola_annual_percent or 0.0)
+    if cola_cap_percent is not None:
+        try:
+            annual_pct = min(annual_pct, float(cola_cap_percent))
+        except Exception:
+            pass
+
+    if month_index <= 0 or annual_pct == 0.0:
+        return 1.0
+
+    if (cola_mode or "monthly").lower() == "annual_prorated":
+        years_completed = month_index // 12
+        remainder_months = month_index % 12
+        a = annual_pct / 100.0
+        step = (1.0 + a) ** years_completed
+        prorata = 1.0 + a * (remainder_months / 12.0)
+        return float(step * prorata)
+    else:
+        # monthly compounding from annual percent
+        m = _to_monthly_rate(annual_pct)
+        return float((1.0 + m) ** month_index)
+
+
+@dataclass
+class SingleLifeInputs:
+    monthly_benefit: float
+    term_months: int
+    start_age: Optional[int]
+    sex: str
+    race: str
+    discount_rate: float = 0.0  # annual percent
+    cola_rate: float = 0.0  # annual percent
+    defer_months: float = 0.0  # months to delay first payment (supports fractional)
+    payment_period_months: int = 1  # months per payment (1=monthly, 3=quarterly, etc.)
+    certain_months: int = 0  # months guaranteed from commencement regardless of mortality
+    cola_mode: str = "monthly"  # "monthly" or "annual_prorated"
+    cola_cap_percent: Optional[float] = None
+    interpolation_method: str = "linear"
+    max_age: Optional[int] = None
+
+
+def present_value_single_life(db: Session, inputs: SingleLifeInputs) -> float:
+    """Compute PV of a single-life level annuity under mortality and economic assumptions."""
+    if inputs.monthly_benefit < 0:
+        raise ValueError("monthly_benefit must be non-negative")
+    if inputs.term_months < 0:
+        raise ValueError("term_months must be non-negative")
+
+    if inputs.payment_period_months <= 0:
+        raise ValueError("payment_period_months must be >= 1")
+    if inputs.defer_months < 0:
+        raise ValueError("defer_months must be >= 0")
+    if inputs.certain_months < 0:
+        raise ValueError("certain_months must be >= 0")
+
+    # Survival probabilities for participant
+    # Adjust term if max_age is provided and start_age known
+    term_months = inputs.term_months
+    if inputs.max_age is not None and inputs.start_age is not None:
+        max_months = max(0, (int(inputs.max_age) - int(inputs.start_age)) * 12)
+        term_months = min(term_months, max_months)
+
+    p_survive = _to_survival_probabilities(
+        db,
+        start_age=inputs.start_age,
+        sex=inputs.sex,
+        race=inputs.race,
+        term_months=term_months,
+        interpolation_method=inputs.interpolation_method,
+    )
+
+    i_m = _to_monthly_rate(inputs.discount_rate)
+    period = int(inputs.payment_period_months)
+    t0 = int(math.ceil(inputs.defer_months))
+    t = t0
+    guarantee_end = float(inputs.defer_months) + float(inputs.certain_months)
+
+    pv = 0.0
+    first = True
+    while t < term_months:
+        p_t = p_survive[t] if t < len(p_survive) else 0.0
+        base_amt = inputs.monthly_benefit * float(period)
+        # Pro-rata first payment if deferral is fractional
+        if first:
+            frac_defer = float(inputs.defer_months) - math.floor(float(inputs.defer_months))
+            pro_rata = 1.0 - (frac_defer / float(period)) if frac_defer > 0 else 1.0
+        else:
+            pro_rata = 1.0
+        eff_base = base_amt * pro_rata
+        amount = eff_base if t < guarantee_end else eff_base * p_t
+        growth = _compute_growth_factor_at_month(
+            t,
+            cola_annual_percent=inputs.cola_rate,
+            cola_mode=inputs.cola_mode,
+            cola_cap_percent=inputs.cola_cap_percent,
+        )
+        discount = (1.0 + i_m) ** t
+        pv += (amount * growth) / discount
+        t += period
+        first = False
+    return float(pv)
+
+
+@dataclass
+class JointSurvivorInputs:
+    monthly_benefit: float
+    term_months: int
+    participant_age: Optional[int]
+    participant_sex: str
+    participant_race: str
+    spouse_age: Optional[int]
+    spouse_sex: str
+    spouse_race: str
+    survivor_percent: float  # as percent (0-100)
+    discount_rate: float = 0.0  # annual percent
+    cola_rate: float = 0.0  # annual percent
+    defer_months: float = 0.0
+    payment_period_months: int = 1
+    certain_months: int = 0
+    cola_mode: str = "monthly"
+    cola_cap_percent: Optional[float] = None
+    survivor_basis: str = "contingent"  # "contingent" or "last_survivor"
+    survivor_commence_participant_only: bool = False
+    interpolation_method: str = "linear"
+    max_age: Optional[int] = None
+
+
+def present_value_joint_survivor(db: Session, inputs: JointSurvivorInputs) -> Dict[str, float]:
+    """Compute PV for a joint-survivor annuity.
+
+    Expected monthly payment at time t:
+      E[Payment_t] = B * P(both alive at t) + B * s * P(spouse alive only at t)
+                   = B * [ (1 - s) * P(both alive) + s * P(spouse alive) ]
+    where s = survivor_percent (0..1)
+    """
+    if inputs.monthly_benefit < 0:
+        raise ValueError("monthly_benefit must be non-negative")
+    if inputs.term_months < 0:
+        raise ValueError("term_months must be non-negative")
+    if inputs.survivor_percent < 0 or inputs.survivor_percent > 100:
+        raise ValueError("survivor_percent must be between 0 and 100")
+
+    if inputs.payment_period_months <= 0:
+        raise ValueError("payment_period_months must be >= 1")
+    if inputs.defer_months < 0:
+        raise ValueError("defer_months must be >= 0")
+    if inputs.certain_months < 0:
+        raise ValueError("certain_months must be >= 0")
+
+    # Adjust term if max_age is provided and participant_age known
+    term_months = inputs.term_months
+    if inputs.max_age is not None and inputs.participant_age is not None:
+        max_months = max(0, (int(inputs.max_age) - int(inputs.participant_age)) * 12)
+        term_months = min(term_months, max_months)
+
+    p_part = _to_survival_probabilities(
+        db,
+        start_age=inputs.participant_age,
+        sex=inputs.participant_sex,
+        race=inputs.participant_race,
+        term_months=term_months,
+        interpolation_method=inputs.interpolation_method,
+    )
+    p_sp = _to_survival_probabilities(
+        db,
+        start_age=inputs.spouse_age,
+        sex=inputs.spouse_sex,
+        race=inputs.spouse_race,
+        term_months=term_months,
+        interpolation_method=inputs.interpolation_method,
+    )
+
+    s_frac = float(inputs.survivor_percent) / 100.0
+
+    i_m = _to_monthly_rate(inputs.discount_rate)
+    period = int(inputs.payment_period_months)
+    t0 = int(math.ceil(inputs.defer_months))
+    t = t0
+    guarantee_end = float(inputs.defer_months) + float(inputs.certain_months)
+
+    pv_total = 0.0
+    pv_both = 0.0
+    pv_surv = 0.0
+    first = True
+    while t < term_months:
+        p_part_t = p_part[t] if t < len(p_part) else 0.0
+        p_sp_t = p_sp[t] if t < len(p_sp) else 0.0
+        p_both = p_part_t * p_sp_t
+        p_sp_only = p_sp_t - p_both
+        base_amt = inputs.monthly_benefit * float(period)
+        # Pro-rata first payment if deferral is fractional
+        if first:
+            frac_defer = float(inputs.defer_months) - math.floor(float(inputs.defer_months))
+            pro_rata = 1.0 - (frac_defer / float(period)) if frac_defer > 0 else 1.0
+        else:
+            pro_rata = 1.0
+        both_amt = base_amt * pro_rata * p_both
+        if inputs.survivor_commence_participant_only:
+            surv_basis_prob = p_part_t
+        else:
+            surv_basis_prob = p_sp_only
+        surv_amt = base_amt * pro_rata * s_frac * surv_basis_prob
+        if (inputs.survivor_basis or "contingent").lower() == "last_survivor":
+            # Last-survivor: pay full while either is alive, then 0
+            # E[Payment_t] = base_amt * P(participant alive OR spouse alive)
+            p_either = p_part_t + p_sp_t - p_both
+            total_amt = base_amt * pro_rata * p_either
+            # Components are less meaningful; keep mortality-only decomposition
+        else:
+            # Contingent: full while both alive, survivor_percent to spouse when only spouse alive
+            total_amt = base_amt * pro_rata if t < guarantee_end else (both_amt + surv_amt)
+
+        growth = _compute_growth_factor_at_month(
+            t,
+            cola_annual_percent=inputs.cola_rate,
+            cola_mode=inputs.cola_mode,
+            cola_cap_percent=inputs.cola_cap_percent,
+        )
+        discount = (1.0 + i_m) ** t
+
+        pv_total += (total_amt * growth) / discount
+        # Components exclude guarantee to reflect mortality-only decomposition
+        pv_both += (both_amt * growth) / discount
+        pv_surv += (surv_amt * growth) / discount
+        t += period
+        first = False
+
+    return {
+        "pv_total": float(pv_total),
+        "pv_participant_component": float(pv_both),
+        "pv_survivor_component": float(pv_surv),
+    }
+
+
+__all__ = [
+    "SingleLifeInputs",
+    "JointSurvivorInputs",
+    "present_value_single_life",
+    "present_value_joint_survivor",
+    "InvalidCodeError",
+]
+
+

app/services/statement_generation.py

@@ -0,0 +1,237 @@
+"""
+Statement generation helpers extracted from API layer.
+
+These functions encapsulate database access, validation, and file generation
+for billing statements so API endpoints can remain thin controllers.
+"""
+from __future__ import annotations
+
+from typing import Optional, Tuple, List, Dict, Any
+from pathlib import Path
+from datetime import datetime, timezone, date
+
+from fastapi import HTTPException, status
+from sqlalchemy.orm import Session, joinedload
+
+from app.models.files import File
+from app.models.ledger import Ledger
+
+
+def _safe_round(value: Optional[float]) -> float:
+    try:
+        return round(float(value or 0.0), 2)
+    except Exception:
+        return 0.0
+
+
+def parse_period_month(period: Optional[str]) -> Optional[Tuple[date, date]]:
+    """Parse period in the form YYYY-MM and return (start_date, end_date) inclusive.
+    Returns None when period is not provided or invalid.
+    """
+    if not period:
+        return None
+    import re as _re
+    m = _re.fullmatch(r"(\d{4})-(\d{2})", str(period).strip())
+    if not m:
+        return None
+    year = int(m.group(1))
+    month = int(m.group(2))
+    if month < 1 or month > 12:
+        return None
+    from calendar import monthrange
+    last_day = monthrange(year, month)[1]
+    return date(year, month, 1), date(year, month, last_day)
+
+
+def render_statement_html(
+    *,
+    file_no: str,
+    client_name: Optional[str],
+    matter: Optional[str],
+    as_of_iso: str,
+    period: Optional[str],
+    totals: Dict[str, float],
+    unbilled_entries: List[Dict[str, Any]],
+) -> str:
+    """Create a simple, self-contained HTML statement string.
+
+    The API constructs pydantic models for totals and entries; this helper accepts
+    primitive dicts to avoid coupling to API types.
+    """
+
+    def _fmt(val: Optional[float]) -> str:
+        try:
+            return f"{float(val or 0):.2f}"
+        except Exception:
+            return "0.00"
+
+    rows: List[str] = []
+    for e in unbilled_entries:
+        date_val = e.get("date")
+        date_str = date_val.isoformat() if hasattr(date_val, "isoformat") else (date_val or "")
+        rows.append(
+            f"<tr><td>{date_str}</td><td>{e.get('t_code','')}</td><td>{str(e.get('description','')).replace('<','&lt;').replace('>','&gt;')}</td>"
+            f"<td style='text-align:right'>{_fmt(e.get('quantity'))}</td><td style='text-align:right'>{_fmt(e.get('rate'))}</td><td style='text-align:right'>{_fmt(e.get('amount'))}</td></tr>"
+        )
+    rows_html = "\n".join(rows) if rows else "<tr><td colspan='6' style='text-align:center;color:#666'>No unbilled entries</td></tr>"
+
+    period_html = f"<div><strong>Period:</strong> {period}</div>" if period else ""
+
+    html = f"""
+<!DOCTYPE html>
+<html lang=\"en\">
+  <head>
+    <meta charset=\"utf-8\" />
+    <title>Statement {file_no}</title>
+    <style>
+      body {{ font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Helvetica, Arial, sans-serif; margin: 24px; }}
+      h1 {{ margin: 0 0 8px 0; }}
+      .meta {{ color: #444; margin-bottom: 16px; }}
+      table {{ border-collapse: collapse; width: 100%; }}
+      th, td {{ border: 1px solid #ddd; padding: 8px; font-size: 14px; }}
+      th {{ background: #f6f6f6; text-align: left; }}
+      .totals {{ margin: 16px 0; display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 8px; }}
+      .totals div {{ background: #fafafa; border: 1px solid #eee; padding: 8px; }}
+    </style>
+  </head>
+  <body>
+    <h1>Statement</h1>
+    <div class=\"meta\">\n      <div><strong>File:</strong> {file_no}</div>\n      <div><strong>Client:</strong> {client_name or ''}</div>\n      <div><strong>Matter:</strong> {matter or ''}</div>\n      <div><strong>As of:</strong> {as_of_iso}</div>\n      {period_html}
+    </div>
+
+    <div class=\"totals\">\n      <div><strong>Charges (billed)</strong><br/>${_fmt(totals.get('charges_billed'))}</div>\n      <div><strong>Charges (unbilled)</strong><br/>${_fmt(totals.get('charges_unbilled'))}</div>\n      <div><strong>Charges (total)</strong><br/>${_fmt(totals.get('charges_total'))}</div>\n      <div><strong>Payments</strong><br/>${_fmt(totals.get('payments'))}</div>\n      <div><strong>Trust balance</strong><br/>${_fmt(totals.get('trust_balance'))}</div>\n      <div><strong>Current balance</strong><br/>${_fmt(totals.get('current_balance'))}</div>
+    </div>
+
+    <h2>Unbilled Entries</h2>
+    <table>
+      <thead>
+        <tr>
+          <th>Date</th>
+          <th>Code</th>
+          <th>Description</th>
+          <th style=\"text-align:right\">Qty</th>
+          <th style=\"text-align:right\">Rate</th>
+          <th style=\"text-align:right\">Amount</th>
+        </tr>
+      </thead>
+      <tbody>
+        {rows_html}
+      </tbody>
+    </table>
+  </body>
+ </html>
+"""
+    return html
+
+
+def generate_single_statement(
+    file_no: str,
+    period: Optional[str],
+    db: Session,
+) -> Dict[str, Any]:
+    """Generate a statement for a single file and write an HTML artifact to exports/.
+
+    Returns a dict matching the "GeneratedStatementMeta" schema expected by the API layer.
+    Raises HTTPException on not found or internal errors.
+    """
+    file_obj = (
+        db.query(File)
+        .options(joinedload(File.owner))
+        .filter(File.file_no == file_no)
+        .first()
+    )
+
+    if not file_obj:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"File {file_no} not found",
+        )
+
+    # Optional period filtering (YYYY-MM)
+    date_range = parse_period_month(period)
+    q = db.query(Ledger).filter(Ledger.file_no == file_no)
+    if date_range:
+        start_date, end_date = date_range
+        q = q.filter(Ledger.date >= start_date).filter(Ledger.date <= end_date)
+    entries: List[Ledger] = q.all()
+
+    CHARGE_TYPES = {"2", "3", "4"}
+    charges_billed = sum(e.amount for e in entries if e.t_type in CHARGE_TYPES and e.billed == "Y")
+    charges_unbilled = sum(e.amount for e in entries if e.t_type in CHARGE_TYPES and e.billed != "Y")
+    charges_total = charges_billed + charges_unbilled
+    payments_total = sum(e.amount for e in entries if e.t_type == "5")
+    trust_balance = file_obj.trust_bal or 0.0
+    current_balance = charges_total - payments_total
+
+    unbilled_entries: List[Dict[str, Any]] = [
+        {
+            "id": e.id,
+            "date": e.date,
+            "t_code": e.t_code,
+            "t_type": e.t_type,
+            "description": e.note,
+            "quantity": e.quantity or 0.0,
+            "rate": e.rate or 0.0,
+            "amount": e.amount,
+        }
+        for e in entries
+        if e.t_type in CHARGE_TYPES and e.billed != "Y"
+    ]
+
+    client_name: Optional[str] = None
+    if file_obj.owner:
+        client_name = f"{file_obj.owner.first or ''} {file_obj.owner.last}".strip()
+
+    as_of_iso = datetime.now(timezone.utc).isoformat()
+    totals_dict: Dict[str, float] = {
+        "charges_billed": _safe_round(charges_billed),
+        "charges_unbilled": _safe_round(charges_unbilled),
+        "charges_total": _safe_round(charges_total),
+        "payments": _safe_round(payments_total),
+        "trust_balance": _safe_round(trust_balance),
+        "current_balance": _safe_round(current_balance),
+    }
+
+    # Render HTML
+    html = render_statement_html(
+        file_no=file_no,
+        client_name=client_name or None,
+        matter=file_obj.regarding,
+        as_of_iso=as_of_iso,
+        period=period,
+        totals=totals_dict,
+        unbilled_entries=unbilled_entries,
+    )
+
+    # Ensure exports directory and write file
+    exports_dir = Path("exports")
+    try:
+        exports_dir.mkdir(exist_ok=True)
+    except Exception:
+        # Best-effort: if cannot create, bubble up internal error
+        raise HTTPException(status_code=500, detail="Unable to create exports directory")
+
+    timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S_%f")
+    safe_file_no = str(file_no).replace("/", "_").replace("\\", "_")
+    filename = f"statement_{safe_file_no}_{timestamp}.html"
+    export_path = exports_dir / filename
+    html_bytes = html.encode("utf-8")
+    with open(export_path, "wb") as f:
+        f.write(html_bytes)
+
+    size = export_path.stat().st_size
+
+    return {
+        "file_no": file_no,
+        "client_name": client_name or None,
+        "as_of": as_of_iso,
+        "period": period,
+        "totals": totals_dict,
+        "unbilled_count": len(unbilled_entries),
+        "export_path": str(export_path),
+        "filename": filename,
+        "size": size,
+        "content_type": "text/html",
+    }
+
+

app/services/template_merge.py

@@ -1,7 +1,13 @@
 """
-Template variable resolution and DOCX preview using docxtpl.
+Advanced Template Processing Engine
 
-MVP features:
+Enhanced features:
+- Rich variable resolution with formatting options
+- Conditional content blocks (IF/ENDIF sections)
+- Loop functionality for data tables (FOR/ENDFOR sections)  
+- Advanced variable substitution with built-in functions
+- PDF generation support
+- Template function library
 - Resolve variables from explicit context, FormVariable, ReportVariable
 - Built-in variables (dates)
 - Render DOCX using docxtpl when mime_type is docx; otherwise return bytes as-is
@@ -11,21 +17,39 @@ from __future__ import annotations
 
 import io
 import re
+import warnings
+import subprocess
+import tempfile
+import os
 from datetime import date, datetime
-from typing import Any, Dict, List, Tuple
+from typing import Any, Dict, List, Tuple, Optional, Union
+from decimal import Decimal, InvalidOperation
 
 from sqlalchemy.orm import Session
 
 from app.models.additional import FormVariable, ReportVariable
+from app.core.logging import get_logger
+
+logger = get_logger("template_merge")
 
 try:
-    from docxtpl import DocxTemplate
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", category=UserWarning)
+        from docxtpl import DocxTemplate
     DOCXTPL_AVAILABLE = True
 except Exception:
     DOCXTPL_AVAILABLE = False
 
 
+# Enhanced token patterns for different template features
 TOKEN_PATTERN = re.compile(r"\{\{\s*([a-zA-Z0-9_\.]+)\s*\}\}")
+FORMATTED_TOKEN_PATTERN = re.compile(r"\{\{\s*([a-zA-Z0-9_\.]+)\s*\|\s*([^}]+)\s*\}\}")
+CONDITIONAL_START_PATTERN = re.compile(r"\{\%\s*if\s+([^%]+)\s*\%\}")
+CONDITIONAL_ELSE_PATTERN = re.compile(r"\{\%\s*else\s*\%\}")
+CONDITIONAL_END_PATTERN = re.compile(r"\{\%\s*endif\s*\%\}")
+LOOP_START_PATTERN = re.compile(r"\{\%\s*for\s+(\w+)\s+in\s+([^%]+)\s*\%\}")
+LOOP_END_PATTERN = re.compile(r"\{\%\s*endfor\s*\%\}")
+FUNCTION_PATTERN = re.compile(r"\{\{\s*(\w+)\s*\(\s*([^)]*)\s*\)\s*\}\}")
 
 
 def extract_tokens_from_bytes(content: bytes) -> List[str]:
@@ -47,20 +71,281 @@ def extract_tokens_from_bytes(content: bytes) -> List[str]:
     return sorted({m.group(1) for m in TOKEN_PATTERN.finditer(text)})
 
 
-def build_context(payload_context: Dict[str, Any]) -> Dict[str, Any]:
-    # Built-ins
+class TemplateFunctions:
+    """
+    Built-in template functions available in document templates
+    """
+    
+    @staticmethod
+    def format_currency(value: Any, symbol: str = "$", decimal_places: int = 2) -> str:
+        """Format a number as currency"""
+        try:
+            num_value = float(value) if value is not None else 0.0
+            return f"{symbol}{num_value:,.{decimal_places}f}"
+        except (ValueError, TypeError):
+            return f"{symbol}0.00"
+    
+    @staticmethod
+    def format_date(value: Any, format_str: str = "%B %d, %Y") -> str:
+        """Format a date with a custom format string"""
+        if value is None:
+            return ""
+        try:
+            if isinstance(value, str):
+                from dateutil.parser import parse
+                value = parse(value).date()
+            elif isinstance(value, datetime):
+                value = value.date()
+            
+            if isinstance(value, date):
+                return value.strftime(format_str)
+            return str(value)
+        except Exception:
+            return str(value)
+    
+    @staticmethod
+    def format_number(value: Any, decimal_places: int = 2, thousands_sep: str = ",") -> str:
+        """Format a number with specified decimal places and thousands separator"""
+        try:
+            num_value = float(value) if value is not None else 0.0
+            if thousands_sep == ",":
+                return f"{num_value:,.{decimal_places}f}"
+            else:
+                formatted = f"{num_value:.{decimal_places}f}"
+                if thousands_sep:
+                    # Simple thousands separator replacement
+                    parts = formatted.split(".")
+                    parts[0] = parts[0][::-1]  # Reverse
+                    parts[0] = thousands_sep.join([parts[0][i:i+3] for i in range(0, len(parts[0]), 3)])
+                    parts[0] = parts[0][::-1]  # Reverse back
+                    formatted = ".".join(parts)
+                return formatted
+        except (ValueError, TypeError):
+            return "0.00"
+    
+    @staticmethod
+    def format_percentage(value: Any, decimal_places: int = 1) -> str:
+        """Format a number as a percentage"""
+        try:
+            num_value = float(value) if value is not None else 0.0
+            return f"{num_value:.{decimal_places}f}%"
+        except (ValueError, TypeError):
+            return "0.0%"
+    
+    @staticmethod
+    def format_phone(value: Any, format_type: str = "us") -> str:
+        """Format a phone number"""
+        if not value:
+            return ""
+        
+        # Remove all non-digit characters
+        digits = re.sub(r'\D', '', str(value))
+        
+        if format_type.lower() == "us" and len(digits) == 10:
+            return f"({digits[:3]}) {digits[3:6]}-{digits[6:]}"
+        elif format_type.lower() == "us" and len(digits) == 11 and digits[0] == "1":
+            return f"1-({digits[1:4]}) {digits[4:7]}-{digits[7:]}"
+        
+        return str(value)
+    
+    @staticmethod
+    def uppercase(value: Any) -> str:
+        """Convert text to uppercase"""
+        return str(value).upper() if value is not None else ""
+    
+    @staticmethod
+    def lowercase(value: Any) -> str:
+        """Convert text to lowercase"""
+        return str(value).lower() if value is not None else ""
+    
+    @staticmethod
+    def titlecase(value: Any) -> str:
+        """Convert text to title case"""
+        return str(value).title() if value is not None else ""
+    
+    @staticmethod
+    def truncate(value: Any, length: int = 50, suffix: str = "...") -> str:
+        """Truncate text to a specified length"""
+        text = str(value) if value is not None else ""
+        if len(text) <= length:
+            return text
+        return text[:length - len(suffix)] + suffix
+    
+    @staticmethod
+    def default(value: Any, default_value: str = "") -> str:
+        """Return default value if the input is empty/null"""
+        if value is None or str(value).strip() == "":
+            return default_value
+        return str(value)
+    
+    @staticmethod
+    def join(items: List[Any], separator: str = ", ") -> str:
+        """Join a list of items with a separator"""
+        if not isinstance(items, (list, tuple)):
+            return str(items) if items is not None else ""
+        return separator.join(str(item) for item in items if item is not None)
+    
+    @staticmethod
+    def length(value: Any) -> int:
+        """Get the length of a string or list"""
+        if value is None:
+            return 0
+        if isinstance(value, (list, tuple, dict)):
+            return len(value)
+        return len(str(value))
+    
+    @staticmethod
+    def math_add(a: Any, b: Any) -> float:
+        """Add two numbers"""
+        try:
+            return float(a or 0) + float(b or 0)
+        except (ValueError, TypeError):
+            return 0.0
+    
+    @staticmethod
+    def math_subtract(a: Any, b: Any) -> float:
+        """Subtract two numbers"""
+        try:
+            return float(a or 0) - float(b or 0)
+        except (ValueError, TypeError):
+            return 0.0
+    
+    @staticmethod
+    def math_multiply(a: Any, b: Any) -> float:
+        """Multiply two numbers"""
+        try:
+            return float(a or 0) * float(b or 0)
+        except (ValueError, TypeError):
+            return 0.0
+    
+    @staticmethod
+    def math_divide(a: Any, b: Any) -> float:
+        """Divide two numbers"""
+        try:
+            divisor = float(b or 0)
+            if divisor == 0:
+                return 0.0
+            return float(a or 0) / divisor
+        except (ValueError, TypeError):
+            return 0.0
+
+
+def apply_variable_formatting(value: Any, format_spec: str) -> str:
+    """
+    Apply formatting to a variable value based on format specification
+    
+    Format specifications:
+    - currency[:symbol][:decimal_places] - Format as currency
+    - date[:format_string] - Format as date
+    - number[:decimal_places][:thousands_sep] - Format as number
+    - percentage[:decimal_places] - Format as percentage
+    - phone[:format_type] - Format as phone number
+    - upper - Convert to uppercase
+    - lower - Convert to lowercase
+    - title - Convert to title case
+    - truncate[:length][:suffix] - Truncate text
+    - default[:default_value] - Use default if empty
+    """
+    if not format_spec:
+        return str(value) if value is not None else ""
+    
+    parts = format_spec.split(":")
+    format_type = parts[0].lower()
+    
+    try:
+        if format_type == "currency":
+            symbol = parts[1] if len(parts) > 1 else "$"
+            decimal_places = int(parts[2]) if len(parts) > 2 else 2
+            return TemplateFunctions.format_currency(value, symbol, decimal_places)
+        
+        elif format_type == "date":
+            format_str = parts[1] if len(parts) > 1 else "%B %d, %Y"
+            return TemplateFunctions.format_date(value, format_str)
+        
+        elif format_type == "number":
+            decimal_places = int(parts[1]) if len(parts) > 1 else 2
+            thousands_sep = parts[2] if len(parts) > 2 else ","
+            return TemplateFunctions.format_number(value, decimal_places, thousands_sep)
+        
+        elif format_type == "percentage":
+            decimal_places = int(parts[1]) if len(parts) > 1 else 1
+            return TemplateFunctions.format_percentage(value, decimal_places)
+        
+        elif format_type == "phone":
+            format_type_spec = parts[1] if len(parts) > 1 else "us"
+            return TemplateFunctions.format_phone(value, format_type_spec)
+        
+        elif format_type == "upper":
+            return TemplateFunctions.uppercase(value)
+        
+        elif format_type == "lower":
+            return TemplateFunctions.lowercase(value)
+        
+        elif format_type == "title":
+            return TemplateFunctions.titlecase(value)
+        
+        elif format_type == "truncate":
+            length = int(parts[1]) if len(parts) > 1 else 50
+            suffix = parts[2] if len(parts) > 2 else "..."
+            return TemplateFunctions.truncate(value, length, suffix)
+        
+        elif format_type == "default":
+            default_value = parts[1] if len(parts) > 1 else ""
+            return TemplateFunctions.default(value, default_value)
+        
+        else:
+            logger.warning(f"Unknown format type: {format_type}")
+            return str(value) if value is not None else ""
+    
+    except Exception as e:
+        logger.error(f"Error applying format '{format_spec}' to value '{value}': {e}")
+        return str(value) if value is not None else ""
+
+
+def build_context(payload_context: Dict[str, Any], context_type: str = "global", context_id: str = "default") -> Dict[str, Any]:
+    # Built-ins with enhanced date/time functions
     today = date.today()
+    now = datetime.utcnow()
     builtins = {
         "TODAY": today.strftime("%B %d, %Y"),
         "TODAY_ISO": today.isoformat(),
-        "NOW": datetime.utcnow().isoformat() + "Z",
+        "TODAY_SHORT": today.strftime("%m/%d/%Y"),
+        "TODAY_YEAR": str(today.year),
+        "TODAY_MONTH": str(today.month),
+        "TODAY_DAY": str(today.day),
+        "NOW": now.isoformat() + "Z",
+        "NOW_TIME": now.strftime("%I:%M %p"),
+        "NOW_TIMESTAMP": str(int(now.timestamp())),
+        # Context identifiers for enhanced variable processing
+        "_context_type": context_type,
+        "_context_id": context_id,
+        
+        # Template functions
+        "format_currency": TemplateFunctions.format_currency,
+        "format_date": TemplateFunctions.format_date,
+        "format_number": TemplateFunctions.format_number,
+        "format_percentage": TemplateFunctions.format_percentage,
+        "format_phone": TemplateFunctions.format_phone,
+        "uppercase": TemplateFunctions.uppercase,
+        "lowercase": TemplateFunctions.lowercase,
+        "titlecase": TemplateFunctions.titlecase,
+        "truncate": TemplateFunctions.truncate,
+        "default": TemplateFunctions.default,
+        "join": TemplateFunctions.join,
+        "length": TemplateFunctions.length,
+        "math_add": TemplateFunctions.math_add,
+        "math_subtract": TemplateFunctions.math_subtract,
+        "math_multiply": TemplateFunctions.math_multiply,
+        "math_divide": TemplateFunctions.math_divide,
     }
     merged = {**builtins}
+    
     # Normalize keys to support both FOO and foo
     for k, v in payload_context.items():
         merged[k] = v
         if isinstance(k, str):
             merged.setdefault(k.upper(), v)
+    
     return merged
 
 
@@ -83,6 +368,41 @@ def _safe_lookup_variable(db: Session, identifier: str) -> Any:
 def resolve_tokens(db: Session, tokens: List[str], context: Dict[str, Any]) -> Tuple[Dict[str, Any], List[str]]:
     resolved: Dict[str, Any] = {}
     unresolved: List[str] = []
+    
+    # Try enhanced variable processor first for advanced features
+    try:
+        from app.services.advanced_variables import VariableProcessor
+        processor = VariableProcessor(db)
+        
+        # Extract context information for enhanced processing
+        context_type = context.get('_context_type', 'global')
+        context_id = context.get('_context_id', 'default')
+        
+        # Remove internal context markers from the context
+        clean_context = {k: v for k, v in context.items() if not k.startswith('_')}
+        
+        enhanced_resolved, enhanced_unresolved = processor.resolve_variables(
+            variables=tokens,
+            context_type=context_type,
+            context_id=context_id,
+            base_context=clean_context
+        )
+        
+        resolved.update(enhanced_resolved)
+        unresolved.extend(enhanced_unresolved)
+        
+        # Remove successfully resolved tokens from further processing
+        tokens = [tok for tok in tokens if tok not in enhanced_resolved]
+        
+    except ImportError:
+        # Enhanced variables not available, fall back to legacy processing
+        pass
+    except Exception as e:
+        # Log error but continue with legacy processing
+        import logging
+        logging.warning(f"Enhanced variable processing failed: {e}")
+    
+    # Fallback to legacy variable resolution for remaining tokens
     for tok in tokens:
         # Order: payload context (case-insensitive via upper) -> FormVariable -> ReportVariable
         value = context.get(tok)
@@ -91,22 +411,338 @@ def resolve_tokens(db: Session, tokens: List[str], context: Dict[str, Any]) -> T
         if value is None:
             value = _safe_lookup_variable(db, tok)
         if value is None:
-            unresolved.append(tok)
+            if tok not in unresolved:  # Avoid duplicates from enhanced processing
+                unresolved.append(tok)
         else:
             resolved[tok] = value
+    
     return resolved, unresolved
 
 
+def process_conditional_sections(content: str, context: Dict[str, Any]) -> str:
+    """
+    Process conditional sections in template content
+    
+    Syntax:
+    {% if condition %}
+    content to include if condition is true
+    {% else %}
+    content to include if condition is false (optional)
+    {% endif %}
+    """
+    result = content
+    
+    # Find all conditional blocks
+    while True:
+        start_match = CONDITIONAL_START_PATTERN.search(result)
+        if not start_match:
+            break
+        
+        # Find corresponding endif
+        start_pos = start_match.end()
+        endif_match = CONDITIONAL_END_PATTERN.search(result, start_pos)
+        if not endif_match:
+            logger.warning("Found {% if %} without matching {% endif %}")
+            break
+        
+        # Find optional else clause
+        else_match = CONDITIONAL_ELSE_PATTERN.search(result, start_pos, endif_match.start())
+        
+        condition = start_match.group(1).strip()
+        
+        # Extract content blocks
+        if else_match:
+            if_content = result[start_pos:else_match.start()]
+            else_content = result[else_match.end():endif_match.start()]
+        else:
+            if_content = result[start_pos:endif_match.start()]
+            else_content = ""
+        
+        # Evaluate condition
+        try:
+            condition_result = evaluate_condition(condition, context)
+            selected_content = if_content if condition_result else else_content
+        except Exception as e:
+            logger.error(f"Error evaluating condition '{condition}': {e}")
+            selected_content = else_content  # Default to else content on error
+        
+        # Replace the entire conditional block with the selected content
+        result = result[:start_match.start()] + selected_content + result[endif_match.end():]
+    
+    return result
+
+
+def process_loop_sections(content: str, context: Dict[str, Any]) -> str:
+    """
+    Process loop sections in template content
+    
+    Syntax:
+    {% for item in items %}
+    Content to repeat for each item. Use {{item.property}} to access item data.
+    {% endfor %}
+    """
+    result = content
+    
+    # Find all loop blocks
+    while True:
+        start_match = LOOP_START_PATTERN.search(result)
+        if not start_match:
+            break
+        
+        # Find corresponding endfor
+        start_pos = start_match.end()
+        endfor_match = LOOP_END_PATTERN.search(result, start_pos)
+        if not endfor_match:
+            logger.warning("Found {% for %} without matching {% endfor %}")
+            break
+        
+        loop_var = start_match.group(1).strip()
+        collection_expr = start_match.group(2).strip()
+        loop_content = result[start_pos:endfor_match.start()]
+        
+        # Get the collection from context
+        try:
+            collection = evaluate_expression(collection_expr, context)
+            if not isinstance(collection, (list, tuple)):
+                logger.warning(f"Loop collection '{collection_expr}' is not iterable")
+                collection = []
+        except Exception as e:
+            logger.error(f"Error evaluating loop collection '{collection_expr}': {e}")
+            collection = []
+        
+        # Generate content for each item
+        repeated_content = ""
+        for i, item in enumerate(collection):
+            # Create item context
+            item_context = context.copy()
+            item_context[loop_var] = item
+            item_context[f"{loop_var}_index"] = i
+            item_context[f"{loop_var}_index0"] = i  # 0-based index
+            item_context[f"{loop_var}_first"] = (i == 0)
+            item_context[f"{loop_var}_last"] = (i == len(collection) - 1)
+            item_context[f"{loop_var}_length"] = len(collection)
+            
+            # Process the loop content with item context
+            item_content = process_template_content(loop_content, item_context)
+            repeated_content += item_content
+        
+        # Replace the entire loop block with the repeated content
+        result = result[:start_match.start()] + repeated_content + result[endfor_match.end():]
+    
+    return result
+
+
+def process_formatted_variables(content: str, context: Dict[str, Any]) -> Tuple[str, List[str]]:
+    """
+    Process variables with formatting in template content
+    
+    Syntax: {{ variable_name | format_spec }}
+    """
+    result = content
+    unresolved = []
+    
+    # Find all formatted variables
+    for match in FORMATTED_TOKEN_PATTERN.finditer(content):
+        var_name = match.group(1).strip()
+        format_spec = match.group(2).strip()
+        full_token = match.group(0)
+        
+        # Get variable value
+        value = context.get(var_name)
+        if value is None:
+            value = context.get(var_name.upper())
+        
+        if value is not None:
+            # Apply formatting
+            formatted_value = apply_variable_formatting(value, format_spec)
+            result = result.replace(full_token, formatted_value)
+        else:
+            unresolved.append(var_name)
+    
+    return result, unresolved
+
+
+def process_template_functions(content: str, context: Dict[str, Any]) -> Tuple[str, List[str]]:
+    """
+    Process template function calls
+    
+    Syntax: {{ function_name(arg1, arg2, ...) }}
+    """
+    result = content
+    unresolved = []
+    
+    for match in FUNCTION_PATTERN.finditer(content):
+        func_name = match.group(1).strip()
+        args_str = match.group(2).strip()
+        full_token = match.group(0)
+        
+        # Get function from context
+        func = context.get(func_name)
+        if func and callable(func):
+            try:
+                # Parse arguments
+                args = []
+                if args_str:
+                    # Simple argument parsing (supports strings, numbers, variables)
+                    arg_parts = [arg.strip() for arg in args_str.split(',')]
+                    for arg in arg_parts:
+                        if arg.startswith('"') and arg.endswith('"'):
+                            # String literal
+                            args.append(arg[1:-1])
+                        elif arg.startswith("'") and arg.endswith("'"):
+                            # String literal
+                            args.append(arg[1:-1])
+                        elif arg.replace('.', '').replace('-', '').isdigit():
+                            # Number literal
+                            args.append(float(arg) if '.' in arg else int(arg))
+                        else:
+                            # Variable reference
+                            var_value = context.get(arg, context.get(arg.upper(), arg))
+                            args.append(var_value)
+                
+                # Call function
+                func_result = func(*args)
+                result = result.replace(full_token, str(func_result))
+                
+            except Exception as e:
+                logger.error(f"Error calling function '{func_name}': {e}")
+                unresolved.append(f"{func_name}()")
+        else:
+            unresolved.append(f"{func_name}()")
+    
+    return result, unresolved
+
+
+def evaluate_condition(condition: str, context: Dict[str, Any]) -> bool:
+    """
+    Evaluate a conditional expression safely
+    """
+    try:
+        # Replace variables in condition
+        for var_name, value in context.items():
+            if var_name.startswith('_'):  # Skip internal variables
+                continue
+            condition = condition.replace(var_name, repr(value))
+        
+        # Safe evaluation with limited builtins
+        safe_context = {
+            '__builtins__': {},
+            'True': True,
+            'False': False,
+            'None': None,
+        }
+        
+        return bool(eval(condition, safe_context))
+    except Exception as e:
+        logger.error(f"Error evaluating condition '{condition}': {e}")
+        return False
+
+
+def evaluate_expression(expression: str, context: Dict[str, Any]) -> Any:
+    """
+    Evaluate an expression safely
+    """
+    try:
+        # Check if it's a simple variable reference
+        if expression in context:
+            return context[expression]
+        if expression.upper() in context:
+            return context[expression.upper()]
+        
+        # Try as a more complex expression
+        safe_context = {
+            '__builtins__': {},
+            **context
+        }
+        
+        return eval(expression, safe_context)
+    except Exception as e:
+        logger.error(f"Error evaluating expression '{expression}': {e}")
+        return None
+
+
+def process_template_content(content: str, context: Dict[str, Any]) -> str:
+    """
+    Process template content with all advanced features
+    """
+    # 1. Process conditional sections
+    content = process_conditional_sections(content, context)
+    
+    # 2. Process loop sections  
+    content = process_loop_sections(content, context)
+    
+    # 3. Process formatted variables
+    content, _ = process_formatted_variables(content, context)
+    
+    # 4. Process template functions
+    content, _ = process_template_functions(content, context)
+    
+    return content
+
+
+def convert_docx_to_pdf(docx_bytes: bytes) -> Optional[bytes]:
+    """
+    Convert DOCX to PDF using LibreOffice headless mode
+    """
+    try:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Save DOCX to temp file
+            docx_path = os.path.join(temp_dir, "document.docx")
+            with open(docx_path, "wb") as f:
+                f.write(docx_bytes)
+            
+            # Convert to PDF using LibreOffice
+            cmd = [
+                "libreoffice",
+                "--headless",
+                "--convert-to", "pdf",
+                "--outdir", temp_dir,
+                docx_path
+            ]
+            
+            result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
+            
+            if result.returncode == 0:
+                pdf_path = os.path.join(temp_dir, "document.pdf")
+                if os.path.exists(pdf_path):
+                    with open(pdf_path, "rb") as f:
+                        return f.read()
+            else:
+                logger.error(f"LibreOffice conversion failed: {result.stderr}")
+                
+    except subprocess.TimeoutExpired:
+        logger.error("LibreOffice conversion timed out")
+    except FileNotFoundError:
+        logger.warning("LibreOffice not found. PDF conversion not available.")
+    except Exception as e:
+        logger.error(f"Error converting DOCX to PDF: {e}")
+    
+    return None
+
+
 def render_docx(docx_bytes: bytes, context: Dict[str, Any]) -> bytes:
     if not DOCXTPL_AVAILABLE:
         # Return original bytes if docxtpl is not installed
         return docx_bytes
-    # Write to BytesIO for docxtpl
-    in_buffer = io.BytesIO(docx_bytes)
-    tpl = DocxTemplate(in_buffer)
-    tpl.render(context)
-    out_buffer = io.BytesIO()
-    tpl.save(out_buffer)
-    return out_buffer.getvalue()
+    
+    try:
+        # Write to BytesIO for docxtpl
+        in_buffer = io.BytesIO(docx_bytes)
+        tpl = DocxTemplate(in_buffer)
+        
+        # Enhanced context with template functions
+        enhanced_context = context.copy()
+        
+        # Render the template
+        tpl.render(enhanced_context)
+        
+        # Save to output buffer
+        out_buffer = io.BytesIO()
+        tpl.save(out_buffer)
+        return out_buffer.getvalue()
+        
+    except Exception as e:
+        logger.error(f"Error rendering DOCX template: {e}")
+        return docx_bytes
 
 

app/services/template_search.py

@@ -0,0 +1,308 @@
+"""
+TemplateSearchService centralizes query construction for templates search and
+keyword management, keeping API endpoints thin and consistent.
+
+Adds best-effort caching using Redis when available with an in-memory fallback.
+Cache keys are built from normalized query params.
+"""
+from __future__ import annotations
+
+from typing import List, Optional, Tuple, Dict, Any
+
+import json
+import time
+import threading
+
+from sqlalchemy import func, or_, exists
+from sqlalchemy.orm import Session
+
+from app.models.templates import DocumentTemplate, DocumentTemplateVersion, TemplateKeyword
+from app.services.cache import cache_get_json, cache_set_json, invalidate_prefix
+
+
+class TemplateSearchService:
+    _mem_cache: Dict[str, Tuple[float, Any]] = {}
+    _mem_lock = threading.RLock()
+    _SEARCH_TTL_SECONDS = 60  # Fallback TTL
+    _CATEGORIES_TTL_SECONDS = 120  # Fallback TTL
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+
+    async def search_templates(
+        self,
+        *,
+        q: Optional[str],
+        categories: Optional[List[str]],
+        keywords: Optional[List[str]],
+        keywords_mode: str,
+        has_keywords: Optional[bool],
+        skip: int,
+        limit: int,
+        sort_by: str,
+        sort_dir: str,
+        active_only: bool,
+        include_total: bool,
+    ) -> Tuple[List[Dict[str, Any]], Optional[int]]:
+        # Build normalized cache key parts
+        norm_categories = sorted({c for c in (categories or []) if c}) or None
+        norm_keywords = sorted({(kw or "").strip().lower() for kw in (keywords or []) if kw and kw.strip()}) or None
+        norm_mode = (keywords_mode or "any").lower()
+        if norm_mode not in ("any", "all"):
+            norm_mode = "any"
+        norm_sort_by = (sort_by or "name").lower()
+        if norm_sort_by not in ("name", "category", "updated"):
+            norm_sort_by = "name"
+        norm_sort_dir = (sort_dir or "asc").lower()
+        if norm_sort_dir not in ("asc", "desc"):
+            norm_sort_dir = "asc"
+
+        parts = {
+            "q": q or "",
+            "categories": norm_categories,
+            "keywords": norm_keywords,
+            "keywords_mode": norm_mode,
+            "has_keywords": has_keywords,
+            "skip": int(skip),
+            "limit": int(limit),
+            "sort_by": norm_sort_by,
+            "sort_dir": norm_sort_dir,
+            "active_only": bool(active_only),
+            "include_total": bool(include_total),
+        }
+
+        # Try cache first (local then adaptive)
+        cached = self._cache_get_local("templates", parts)
+        if cached is None:
+            try:
+                from app.services.adaptive_cache import adaptive_cache_get
+                cached = await adaptive_cache_get(
+                    cache_type="templates",
+                    cache_key="template_search",
+                    parts=parts
+                )
+            except Exception:
+                cached = await self._cache_get_redis("templates", parts)
+        if cached is not None:
+            return cached["items"], cached.get("total")
+
+        query = self.db.query(DocumentTemplate)
+        if active_only:
+            query = query.filter(DocumentTemplate.active == True)  # noqa: E712
+
+        if q:
+            like = f"%{q}%"
+            query = query.filter(
+                or_(
+                    DocumentTemplate.name.ilike(like),
+                    DocumentTemplate.description.ilike(like),
+                )
+            )
+
+        if norm_categories:
+            query = query.filter(DocumentTemplate.category.in_(norm_categories))
+
+        if norm_keywords:
+            query = query.join(TemplateKeyword, TemplateKeyword.template_id == DocumentTemplate.id)
+            if norm_mode == "any":
+                query = query.filter(TemplateKeyword.keyword.in_(norm_keywords)).distinct()
+            else:
+                query = query.filter(TemplateKeyword.keyword.in_(norm_keywords))
+                query = query.group_by(DocumentTemplate.id)
+                query = query.having(func.count(func.distinct(TemplateKeyword.keyword)) == len(norm_keywords))
+
+        if has_keywords is not None:
+            kw_exists = exists().where(TemplateKeyword.template_id == DocumentTemplate.id)
+            if has_keywords:
+                query = query.filter(kw_exists)
+            else:
+                query = query.filter(~kw_exists)
+
+        if norm_sort_by == "name":
+            order_col = DocumentTemplate.name
+        elif norm_sort_by == "category":
+            order_col = DocumentTemplate.category
+        else:
+            order_col = func.coalesce(DocumentTemplate.updated_at, DocumentTemplate.created_at)
+
+        if norm_sort_dir == "asc":
+            query = query.order_by(order_col.asc())
+        else:
+            query = query.order_by(order_col.desc())
+
+        total = query.count() if include_total else None
+        templates: List[DocumentTemplate] = query.offset(skip).limit(limit).all()
+
+        # Resolve latest version semver for current_version_id in bulk
+        current_ids = [t.current_version_id for t in templates if t.current_version_id]
+        latest_by_version_id: dict[int, str] = {}
+        if current_ids:
+            rows = (
+                self.db.query(DocumentTemplateVersion.id, DocumentTemplateVersion.semantic_version)
+                .filter(DocumentTemplateVersion.id.in_(current_ids))
+                .all()
+            )
+            latest_by_version_id = {row[0]: row[1] for row in rows}
+
+        items: List[Dict[str, Any]] = []
+        for tpl in templates:
+            latest_version = latest_by_version_id.get(int(tpl.current_version_id)) if tpl.current_version_id else None
+            items.append({
+                "id": tpl.id,
+                "name": tpl.name,
+                "category": tpl.category,
+                "active": tpl.active,
+                "latest_version": latest_version,
+            })
+
+        payload = {"items": items, "total": total}
+        # Store in caches (best-effort)
+        self._cache_set_local("templates", parts, payload, self._SEARCH_TTL_SECONDS)
+        
+        try:
+            from app.services.adaptive_cache import adaptive_cache_set
+            await adaptive_cache_set(
+                cache_type="templates",
+                cache_key="template_search",
+                value=payload,
+                parts=parts
+            )
+        except Exception:
+            await self._cache_set_redis("templates", parts, payload, self._SEARCH_TTL_SECONDS)
+        return items, total
+
+    async def list_categories(self, *, active_only: bool) -> List[tuple[Optional[str], int]]:
+        parts = {"active_only": bool(active_only)}
+        cached = self._cache_get_local("templates_categories", parts)
+        if cached is None:
+            cached = await self._cache_get_redis("templates_categories", parts)
+        if cached is not None:
+            items = cached.get("items") or []
+            return [(row[0], row[1]) for row in items]
+
+        query = self.db.query(DocumentTemplate.category, func.count(DocumentTemplate.id).label("count"))
+        if active_only:
+            query = query.filter(DocumentTemplate.active == True)  # noqa: E712
+        rows = query.group_by(DocumentTemplate.category).order_by(DocumentTemplate.category.asc()).all()
+        items = [(row[0], row[1]) for row in rows]
+        payload = {"items": items}
+        self._cache_set_local("templates_categories", parts, payload, self._CATEGORIES_TTL_SECONDS)
+        await self._cache_set_redis("templates_categories", parts, payload, self._CATEGORIES_TTL_SECONDS)
+        return items
+
+    def list_keywords(self, template_id: int) -> List[str]:
+        _ = self._get_template_or_404(template_id)
+        rows = (
+            self.db.query(TemplateKeyword)
+            .filter(TemplateKeyword.template_id == template_id)
+            .order_by(TemplateKeyword.keyword.asc())
+            .all()
+        )
+        return [r.keyword for r in rows]
+
+    async def add_keywords(self, template_id: int, keywords: List[str]) -> List[str]:
+        _ = self._get_template_or_404(template_id)
+        to_add = []
+        for kw in (keywords or []):
+            normalized = (kw or "").strip().lower()
+            if not normalized:
+                continue
+            exists_row = (
+                self.db.query(TemplateKeyword)
+                .filter(TemplateKeyword.template_id == template_id, TemplateKeyword.keyword == normalized)
+                .first()
+            )
+            if not exists_row:
+                to_add.append(TemplateKeyword(template_id=template_id, keyword=normalized))
+        if to_add:
+            self.db.add_all(to_add)
+            self.db.commit()
+            # Invalidate caches affected by keyword changes
+            await self.invalidate_all()
+        return self.list_keywords(template_id)
+
+    async def remove_keyword(self, template_id: int, keyword: str) -> List[str]:
+        _ = self._get_template_or_404(template_id)
+        normalized = (keyword or "").strip().lower()
+        if normalized:
+            self.db.query(TemplateKeyword).filter(
+                TemplateKeyword.template_id == template_id,
+                TemplateKeyword.keyword == normalized,
+            ).delete(synchronize_session=False)
+            self.db.commit()
+            await self.invalidate_all()
+        return self.list_keywords(template_id)
+
+    def _get_template_or_404(self, template_id: int) -> DocumentTemplate:
+        # Local import to avoid circular
+        from app.services.template_service import get_template_or_404 as _get
+
+        return _get(self.db, template_id)
+
+    # ---- Cache helpers ----
+    @classmethod
+    def _build_mem_key(cls, kind: str, parts: dict) -> str:
+        # Deterministic key
+        return f"search:{kind}:v1:{json.dumps(parts, sort_keys=True, separators=(",", ":"))}"
+
+    @classmethod
+    def _cache_get_local(cls, kind: str, parts: dict) -> Optional[dict]:
+        key = cls._build_mem_key(kind, parts)
+        now = time.time()
+        with cls._mem_lock:
+            entry = cls._mem_cache.get(key)
+            if not entry:
+                return None
+            expires_at, value = entry
+            if expires_at <= now:
+                try:
+                    del cls._mem_cache[key]
+                except Exception:
+                    pass
+                return None
+            return value
+
+    @classmethod
+    def _cache_set_local(cls, kind: str, parts: dict, value: dict, ttl_seconds: int) -> None:
+        key = cls._build_mem_key(kind, parts)
+        expires_at = time.time() + max(1, int(ttl_seconds))
+        with cls._mem_lock:
+            cls._mem_cache[key] = (expires_at, value)
+
+    @staticmethod
+    async def _cache_get_redis(kind: str, parts: dict) -> Optional[dict]:
+        try:
+            return await cache_get_json(kind, None, parts)
+        except Exception:
+            return None
+
+    @staticmethod
+    async def _cache_set_redis(kind: str, parts: dict, value: dict, ttl_seconds: int) -> None:
+        try:
+            await cache_set_json(kind, None, parts, value, ttl_seconds)
+        except Exception:
+            return
+
+    @classmethod
+    async def invalidate_all(cls) -> None:
+        # Clear in-memory
+        with cls._mem_lock:
+            cls._mem_cache.clear()
+        # Best-effort Redis invalidation
+        try:
+            await invalidate_prefix("search:templates:")
+            await invalidate_prefix("search:templates_categories:")
+        except Exception:
+            pass
+
+
+# Helper to run async cache calls from sync context
+def asyncio_run(aw):  # type: ignore
+    # Not used anymore; kept for backward compatibility if imported elsewhere
+    try:
+        import asyncio
+        return asyncio.run(aw)
+    except Exception:
+        return None
+
+

app/services/template_service.py

@@ -0,0 +1,147 @@
+"""
+Template service helpers extracted from API layer for document template and version operations.
+
+These functions centralize database lookups, validation, storage interactions, and
+preview/download resolution so that API endpoints remain thin.
+"""
+from __future__ import annotations
+
+from typing import Optional, List, Tuple, Dict, Any
+import os
+import hashlib
+
+from fastapi import HTTPException, status
+from sqlalchemy.orm import Session
+
+from app.models.templates import DocumentTemplate, DocumentTemplateVersion, TemplateKeyword
+from app.services.storage import get_default_storage
+from app.services.template_merge import extract_tokens_from_bytes, build_context, resolve_tokens, render_docx
+
+
+def get_template_or_404(db: Session, template_id: int) -> DocumentTemplate:
+    tpl = db.query(DocumentTemplate).filter(DocumentTemplate.id == template_id).first()
+    if not tpl:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Template not found")
+    return tpl
+
+
+def list_template_versions(db: Session, template_id: int) -> List[DocumentTemplateVersion]:
+    _ = get_template_or_404(db, template_id)
+    return (
+        db.query(DocumentTemplateVersion)
+        .filter(DocumentTemplateVersion.template_id == template_id)
+        .order_by(DocumentTemplateVersion.created_at.desc())
+        .all()
+    )
+
+
+def add_template_version(
+    db: Session,
+    *,
+    template_id: int,
+    semantic_version: str,
+    changelog: Optional[str],
+    approve: bool,
+    content: bytes,
+    filename_hint: str,
+    content_type: Optional[str],
+    created_by: Optional[str],
+) -> DocumentTemplateVersion:
+    tpl = get_template_or_404(db, template_id)
+    if not content:
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="No file uploaded")
+
+    sha256 = hashlib.sha256(content).hexdigest()
+    storage = get_default_storage()
+    storage_path = storage.save_bytes(content=content, filename_hint=filename_hint or "template.bin", subdir="templates")
+
+    version = DocumentTemplateVersion(
+        template_id=template_id,
+        semantic_version=semantic_version,
+        storage_path=storage_path,
+        mime_type=content_type,
+        size=len(content),
+        checksum=sha256,
+        changelog=changelog,
+        created_by=created_by,
+        is_approved=bool(approve),
+    )
+    db.add(version)
+    db.flush()
+    if approve:
+        tpl.current_version_id = version.id
+    db.commit()
+    return version
+
+
+def resolve_template_preview(
+    db: Session,
+    *,
+    template_id: int,
+    version_id: Optional[int],
+    context: Dict[str, Any],
+) -> Tuple[Dict[str, Any], List[str], bytes, str]:
+    tpl = get_template_or_404(db, template_id)
+    resolved_version_id = version_id or tpl.current_version_id
+    if not resolved_version_id:
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Template has no versions")
+
+    ver = (
+        db.query(DocumentTemplateVersion)
+        .filter(DocumentTemplateVersion.id == resolved_version_id)
+        .first()
+    )
+    if not ver:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Version not found")
+
+    storage = get_default_storage()
+    content = storage.open_bytes(ver.storage_path)
+    tokens = extract_tokens_from_bytes(content)
+    built_context = build_context(context or {}, "template", str(template_id))
+    resolved, unresolved = resolve_tokens(db, tokens, built_context)
+
+    output_bytes = content
+    output_mime = ver.mime_type
+    if ver.mime_type == "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+        output_bytes = render_docx(content, resolved)
+        output_mime = ver.mime_type
+
+    return resolved, unresolved, output_bytes, output_mime
+
+
+def get_download_payload(
+    db: Session,
+    *,
+    template_id: int,
+    version_id: Optional[int],
+) -> Tuple[bytes, str, str]:
+    tpl = get_template_or_404(db, template_id)
+    resolved_version_id = version_id or tpl.current_version_id
+    if not resolved_version_id:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Template has no approved version")
+
+    ver = (
+        db.query(DocumentTemplateVersion)
+        .filter(
+            DocumentTemplateVersion.id == resolved_version_id,
+            DocumentTemplateVersion.template_id == tpl.id,
+        )
+        .first()
+    )
+    if not ver:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Version not found")
+
+    storage = get_default_storage()
+    try:
+        content = storage.open_bytes(ver.storage_path)
+    except Exception:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Stored file not found")
+
+    base = os.path.basename(ver.storage_path)
+    if "_" in base:
+        original_name = base.split("_", 1)[1]
+    else:
+        original_name = base
+    return content, ver.mime_type, original_name
+
+

app/services/template_upload.py

@@ -0,0 +1,110 @@
+"""
+TemplateUploadService encapsulates validation, storage, and DB writes for
+template uploads to keep API endpoints thin and testable.
+"""
+from __future__ import annotations
+
+from typing import Optional
+
+import hashlib
+
+from fastapi import UploadFile
+from sqlalchemy.orm import Session
+
+from app.models.templates import DocumentTemplate, DocumentTemplateVersion
+from app.services.storage import get_default_storage
+from app.services.template_service import get_template_or_404
+from app.services.template_search import TemplateSearchService
+
+
+class TemplateUploadService:
+    """Service class for handling template uploads and initial version creation."""
+
+    def __init__(self, db: Session) -> None:
+        self.db = db
+
+    async def upload_template(
+        self,
+        *,
+        name: str,
+        category: Optional[str],
+        description: Optional[str],
+        semantic_version: str,
+        file: UploadFile,
+        created_by: Optional[str],
+    ) -> DocumentTemplate:
+        """Validate, store, and create a template with its first version."""
+        from app.utils.file_security import file_validator
+
+        # Validate upload and sanitize metadata
+        content, safe_filename, _file_ext, mime_type = await file_validator.validate_upload_file(
+            file, category="template"
+        )
+
+        checksum_sha256 = hashlib.sha256(content).hexdigest()
+        storage = get_default_storage()
+        storage_path = storage.save_bytes(
+            content=content,
+            filename_hint=safe_filename,
+            subdir="templates",
+        )
+
+        # Ensure unique template name by appending numeric suffix when duplicated
+        base_name = name
+        unique_name = base_name
+        suffix = 2
+        while (
+            self.db.query(DocumentTemplate).filter(DocumentTemplate.name == unique_name).first()
+            is not None
+        ):
+            unique_name = f"{base_name} ({suffix})"
+            suffix += 1
+
+        # Create template row
+        template = DocumentTemplate(
+            name=unique_name,
+            description=description,
+            category=category,
+            active=True,
+            created_by=created_by,
+        )
+        self.db.add(template)
+        self.db.flush()  # obtain template.id
+
+        # Create initial version row
+        version = DocumentTemplateVersion(
+            template_id=template.id,
+            semantic_version=semantic_version,
+            storage_path=storage_path,
+            mime_type=mime_type,
+            size=len(content),
+            checksum=checksum_sha256,
+            changelog=None,
+            created_by=created_by,
+            is_approved=True,
+        )
+        self.db.add(version)
+        self.db.flush()
+
+        # Point template to current approved version
+        template.current_version_id = version.id
+
+        # Persist and refresh
+        self.db.commit()
+        self.db.refresh(template)
+
+        # Invalidate search caches after upload
+        try:
+            # Best-effort: this is async API; call via service helper
+            import asyncio
+            service = TemplateSearchService(self.db)
+            if asyncio.get_event_loop().is_running():
+                asyncio.create_task(service.invalidate_all())  # type: ignore
+            else:
+                asyncio.run(service.invalidate_all())  # type: ignore
+        except Exception:
+            pass
+
+        return template
+
+

app/services/websocket_pool.py

@@ -0,0 +1,667 @@
+"""
+WebSocket Connection Pool and Management Service
+
+This module provides a centralized WebSocket connection pooling system for the Delphi Database
+application. It manages connections efficiently, handles cleanup of stale connections, 
+monitors connection health, and provides resource management to prevent memory leaks.
+
+Features:
+- Connection pooling by topic/channel
+- Automatic cleanup of inactive connections
+- Health monitoring and heartbeat management
+- Resource management and memory leak prevention
+- Integration with existing authentication
+- Structured logging for debugging
+"""
+
+import asyncio
+import time
+import uuid
+from typing import Dict, Set, Optional, Any, Callable, List, Union
+from datetime import datetime, timezone, timedelta
+from dataclasses import dataclass
+from enum import Enum
+from contextlib import asynccontextmanager
+
+from fastapi import WebSocket, WebSocketDisconnect
+from pydantic import BaseModel
+
+from app.utils.logging import StructuredLogger
+
+
+class ConnectionState(Enum):
+    """WebSocket connection states"""
+    CONNECTING = "connecting"
+    CONNECTED = "connected"
+    DISCONNECTING = "disconnecting"
+    DISCONNECTED = "disconnected"
+    ERROR = "error"
+
+
+class MessageType(Enum):
+    """WebSocket message types"""
+    PING = "ping"
+    PONG = "pong"
+    DATA = "data"
+    ERROR = "error"
+    HEARTBEAT = "heartbeat"
+    SUBSCRIBE = "subscribe"
+    UNSUBSCRIBE = "unsubscribe"
+
+
+@dataclass
+class ConnectionInfo:
+    """Information about a WebSocket connection"""
+    id: str
+    websocket: WebSocket
+    user_id: Optional[int]
+    topics: Set[str]
+    state: ConnectionState
+    created_at: datetime
+    last_activity: datetime
+    last_ping: Optional[datetime]
+    last_pong: Optional[datetime]
+    error_count: int
+    metadata: Dict[str, Any]
+
+    def is_alive(self) -> bool:
+        """Check if connection is alive based on state"""
+        return self.state in [ConnectionState.CONNECTED, ConnectionState.CONNECTING]
+
+    def is_stale(self, timeout_seconds: int = 300) -> bool:
+        """Check if connection is stale (no activity for timeout_seconds)"""
+        if not self.is_alive():
+            return True
+        return (datetime.now(timezone.utc) - self.last_activity).total_seconds() > timeout_seconds
+
+    def update_activity(self):
+        """Update last activity timestamp"""
+        self.last_activity = datetime.now(timezone.utc)
+
+
+class WebSocketMessage(BaseModel):
+    """Standard WebSocket message format"""
+    type: str
+    topic: Optional[str] = None
+    data: Optional[Dict[str, Any]] = None
+    timestamp: Optional[str] = None
+    error: Optional[str] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for JSON serialization"""
+        return self.model_dump(exclude_none=True)
+
+
+class WebSocketPool:
+    """
+    Centralized WebSocket connection pool manager
+    
+    Manages WebSocket connections by topics/channels, provides automatic cleanup,
+    health monitoring, and resource management.
+    """
+
+    def __init__(
+        self,
+        cleanup_interval: int = 60,  # seconds
+        connection_timeout: int = 300,  # seconds
+        heartbeat_interval: int = 30,  # seconds
+        max_connections_per_topic: int = 1000,
+        max_total_connections: int = 10000,
+    ):
+        self.cleanup_interval = cleanup_interval
+        self.connection_timeout = connection_timeout
+        self.heartbeat_interval = heartbeat_interval
+        self.max_connections_per_topic = max_connections_per_topic
+        self.max_total_connections = max_total_connections
+
+        # Connection storage
+        self._connections: Dict[str, ConnectionInfo] = {}
+        self._topics: Dict[str, Set[str]] = {}  # topic -> connection_ids
+        self._user_connections: Dict[int, Set[str]] = {}  # user_id -> connection_ids
+        
+        # Locks for thread safety
+        self._connections_lock = asyncio.Lock()
+        self._cleanup_task: Optional[asyncio.Task] = None
+        self._heartbeat_task: Optional[asyncio.Task] = None
+        
+        # Statistics
+        self._stats = {
+            "total_connections": 0,
+            "active_connections": 0,
+            "messages_sent": 0,
+            "messages_failed": 0,
+            "connections_cleaned": 0,
+            "last_cleanup": None,
+            "last_heartbeat": None,
+        }
+        
+        self.logger = StructuredLogger("websocket_pool", "INFO")
+        self.logger.info("WebSocket pool initialized", 
+                        cleanup_interval=cleanup_interval,
+                        connection_timeout=connection_timeout,
+                        heartbeat_interval=heartbeat_interval)
+
+    async def start(self):
+        """Start the WebSocket pool background tasks"""
+        # If no global pool exists, register this instance to satisfy contexts that
+        # rely on the module-level getter during tests and simple scripts
+        global _websocket_pool
+        if _websocket_pool is None:
+            _websocket_pool = self
+        if self._cleanup_task is None:
+            self._cleanup_task = asyncio.create_task(self._cleanup_worker())
+            self.logger.info("Started cleanup worker task")
+            
+        if self._heartbeat_task is None:
+            self._heartbeat_task = asyncio.create_task(self._heartbeat_worker())
+            self.logger.info("Started heartbeat worker task")
+
+    async def stop(self):
+        """Stop the WebSocket pool and cleanup all connections"""
+        self.logger.info("Stopping WebSocket pool")
+        
+        # Cancel background tasks
+        if self._cleanup_task:
+            self._cleanup_task.cancel()
+            try:
+                await self._cleanup_task
+            except asyncio.CancelledError:
+                pass
+            self._cleanup_task = None
+            
+        if self._heartbeat_task:
+            self._heartbeat_task.cancel()
+            try:
+                await self._heartbeat_task
+            except asyncio.CancelledError:
+                pass
+            self._heartbeat_task = None
+
+        # Close all connections
+        await self._close_all_connections()
+        
+        self.logger.info("WebSocket pool stopped")
+        # If this instance is the registered global, clear it
+        global _websocket_pool
+        if _websocket_pool is self:
+            _websocket_pool = None
+
+    async def add_connection(
+        self,
+        websocket: WebSocket,
+        user_id: Optional[int] = None,
+        topics: Optional[Set[str]] = None,
+        metadata: Optional[Dict[str, Any]] = None
+    ) -> str:
+        """
+        Add a new WebSocket connection to the pool
+        
+        Args:
+            websocket: WebSocket instance
+            user_id: Optional user ID for the connection
+            topics: Initial topics to subscribe to
+            metadata: Additional metadata for the connection
+            
+        Returns:
+            connection_id: Unique identifier for the connection
+            
+        Raises:
+            ValueError: If maximum connections exceeded
+        """
+        async with self._connections_lock:
+            # Check connection limits
+            if len(self._connections) >= self.max_total_connections:
+                raise ValueError(f"Maximum total connections ({self.max_total_connections}) exceeded")
+
+            # Generate unique connection ID
+            connection_id = f"ws_{uuid.uuid4().hex[:12]}"
+            
+            # Create connection info
+            connection_info = ConnectionInfo(
+                id=connection_id,
+                websocket=websocket,
+                user_id=user_id,
+                topics=topics or set(),
+                state=ConnectionState.CONNECTING,
+                created_at=datetime.now(timezone.utc),
+                last_activity=datetime.now(timezone.utc),
+                last_ping=None,
+                last_pong=None,
+                error_count=0,
+                metadata=metadata or {}
+            )
+            
+            # Store connection
+            self._connections[connection_id] = connection_info
+            
+            # Update topic subscriptions
+            for topic in connection_info.topics:
+                if topic not in self._topics:
+                    self._topics[topic] = set()
+                if len(self._topics[topic]) >= self.max_connections_per_topic:
+                    # Remove this connection and raise error
+                    del self._connections[connection_id]
+                    raise ValueError(f"Maximum connections per topic ({self.max_connections_per_topic}) exceeded for topic: {topic}")
+                self._topics[topic].add(connection_id)
+            
+            # Update user connections mapping
+            if user_id:
+                if user_id not in self._user_connections:
+                    self._user_connections[user_id] = set()
+                self._user_connections[user_id].add(connection_id)
+            
+            # Update statistics
+            self._stats["total_connections"] += 1
+            self._stats["active_connections"] = len(self._connections)
+            
+            self.logger.info("Added WebSocket connection",
+                           connection_id=connection_id,
+                           user_id=user_id,
+                           topics=list(connection_info.topics),
+                           total_connections=self._stats["active_connections"])
+            
+            return connection_id
+
+    async def remove_connection(self, connection_id: str, reason: str = "unknown"):
+        """Remove a WebSocket connection from the pool"""
+        async with self._connections_lock:
+            connection_info = self._connections.get(connection_id)
+            if not connection_info:
+                return
+            
+            # Update state
+            connection_info.state = ConnectionState.DISCONNECTING
+            
+            # Remove from topics
+            for topic in connection_info.topics:
+                if topic in self._topics:
+                    self._topics[topic].discard(connection_id)
+                    if not self._topics[topic]:
+                        del self._topics[topic]
+            
+            # Remove from user connections
+            if connection_info.user_id and connection_info.user_id in self._user_connections:
+                self._user_connections[connection_info.user_id].discard(connection_id)
+                if not self._user_connections[connection_info.user_id]:
+                    del self._user_connections[connection_info.user_id]
+            
+            # Remove from connections
+            del self._connections[connection_id]
+            
+            # Update statistics
+            self._stats["active_connections"] = len(self._connections)
+            
+            self.logger.info("Removed WebSocket connection",
+                           connection_id=connection_id,
+                           reason=reason,
+                           user_id=connection_info.user_id,
+                           total_connections=self._stats["active_connections"])
+
+    async def subscribe_to_topic(self, connection_id: str, topic: str) -> bool:
+        """Subscribe a connection to a topic"""
+        async with self._connections_lock:
+            connection_info = self._connections.get(connection_id)
+            if not connection_info or not connection_info.is_alive():
+                return False
+            
+            # Check topic connection limit
+            if topic not in self._topics:
+                self._topics[topic] = set()
+            if len(self._topics[topic]) >= self.max_connections_per_topic:
+                self.logger.warning("Topic connection limit exceeded",
+                                  topic=topic,
+                                  connection_id=connection_id,
+                                  current_count=len(self._topics[topic]))
+                return False
+            
+            # Add to topic and connection
+            self._topics[topic].add(connection_id)
+            connection_info.topics.add(topic)
+            connection_info.update_activity()
+            
+            self.logger.debug("Connection subscribed to topic",
+                            connection_id=connection_id,
+                            topic=topic,
+                            topic_subscribers=len(self._topics[topic]))
+            
+            return True
+
+    async def unsubscribe_from_topic(self, connection_id: str, topic: str) -> bool:
+        """Unsubscribe a connection from a topic"""
+        async with self._connections_lock:
+            connection_info = self._connections.get(connection_id)
+            if not connection_info:
+                return False
+            
+            # Remove from topic and connection
+            if topic in self._topics:
+                self._topics[topic].discard(connection_id)
+                if not self._topics[topic]:
+                    del self._topics[topic]
+            
+            connection_info.topics.discard(topic)
+            connection_info.update_activity()
+            
+            self.logger.debug("Connection unsubscribed from topic",
+                            connection_id=connection_id,
+                            topic=topic)
+            
+            return True
+
+    async def broadcast_to_topic(
+        self,
+        topic: str,
+        message: Union[WebSocketMessage, Dict[str, Any]],
+        exclude_connection_id: Optional[str] = None
+    ) -> int:
+        """
+        Broadcast a message to all connections subscribed to a topic
+        
+        Returns:
+            Number of successful sends
+        """
+        if isinstance(message, dict):
+            message = WebSocketMessage(**message)
+        
+        # Ensure timestamp is set
+        if not message.timestamp:
+            message.timestamp = datetime.now(timezone.utc).isoformat()
+        
+        # Get connection IDs for the topic
+        async with self._connections_lock:
+            connection_ids = list(self._topics.get(topic, set()))
+            if exclude_connection_id:
+                connection_ids = [cid for cid in connection_ids if cid != exclude_connection_id]
+        
+        if not connection_ids:
+            return 0
+        
+        # Send to all connections (outside the lock to avoid blocking)
+        success_count = 0
+        failed_connections = []
+        
+        for connection_id in connection_ids:
+            try:
+                success = await self._send_to_connection(connection_id, message)
+                if success:
+                    success_count += 1
+                else:
+                    failed_connections.append(connection_id)
+            except Exception as e:
+                self.logger.error("Error broadcasting to connection",
+                                connection_id=connection_id,
+                                topic=topic,
+                                error=str(e))
+                failed_connections.append(connection_id)
+        
+        # Update statistics
+        self._stats["messages_sent"] += success_count
+        self._stats["messages_failed"] += len(failed_connections)
+        
+        # Clean up failed connections
+        if failed_connections:
+            for connection_id in failed_connections:
+                await self.remove_connection(connection_id, "broadcast_failed")
+        
+        self.logger.debug("Broadcast completed",
+                         topic=topic,
+                         total_targets=len(connection_ids),
+                         successful=success_count,
+                         failed=len(failed_connections))
+        
+        return success_count
+
+    async def send_to_user(
+        self,
+        user_id: int,
+        message: Union[WebSocketMessage, Dict[str, Any]]
+    ) -> int:
+        """
+        Send a message to all connections for a specific user
+        
+        Returns:
+            Number of successful sends
+        """
+        if isinstance(message, dict):
+            message = WebSocketMessage(**message)
+        
+        # Get connection IDs for the user
+        async with self._connections_lock:
+            connection_ids = list(self._user_connections.get(user_id, set()))
+        
+        if not connection_ids:
+            return 0
+        
+        # Send to all user connections
+        success_count = 0
+        for connection_id in connection_ids:
+            try:
+                success = await self._send_to_connection(connection_id, message)
+                if success:
+                    success_count += 1
+            except Exception as e:
+                self.logger.error("Error sending to user connection",
+                                connection_id=connection_id,
+                                user_id=user_id,
+                                error=str(e))
+        
+        return success_count
+
+    async def _send_to_connection(self, connection_id: str, message: WebSocketMessage) -> bool:
+        """Send a message to a specific connection"""
+        async with self._connections_lock:
+            connection_info = self._connections.get(connection_id)
+            if not connection_info or not connection_info.is_alive():
+                return False
+            
+            websocket = connection_info.websocket
+        
+        try:
+            await websocket.send_json(message.to_dict())
+            connection_info.update_activity()
+            return True
+        except Exception as e:
+            connection_info.error_count += 1
+            connection_info.state = ConnectionState.ERROR
+            self.logger.warning("Failed to send message to connection",
+                              connection_id=connection_id,
+                              error=str(e),
+                              error_count=connection_info.error_count)
+            return False
+
+    async def ping_connection(self, connection_id: str) -> bool:
+        """Send a ping to a specific connection"""
+        ping_message = WebSocketMessage(
+            type=MessageType.PING.value,
+            timestamp=datetime.now(timezone.utc).isoformat()
+        )
+        
+        success = await self._send_to_connection(connection_id, ping_message)
+        if success:
+            async with self._connections_lock:
+                connection_info = self._connections.get(connection_id)
+                if connection_info:
+                    connection_info.last_ping = datetime.now(timezone.utc)
+        
+        return success
+
+    async def handle_pong(self, connection_id: str):
+        """Handle a pong response from a connection"""
+        async with self._connections_lock:
+            connection_info = self._connections.get(connection_id)
+            if connection_info:
+                connection_info.last_pong = datetime.now(timezone.utc)
+                connection_info.update_activity()
+                connection_info.state = ConnectionState.CONNECTED
+
+    async def get_connection_info(self, connection_id: str) -> Optional[ConnectionInfo]:
+        """Get information about a specific connection"""
+        async with self._connections_lock:
+            info = self._connections.get(connection_id)
+        # Fallback to global pool if this instance is not the registered one
+        # This supports tests that instantiate a local pool while the context
+        # manager uses the global pool created by app startup.
+        if info is None:
+            global _websocket_pool
+            if _websocket_pool is not None and _websocket_pool is not self:
+                return await _websocket_pool.get_connection_info(connection_id)
+        return info
+
+    async def get_topic_connections(self, topic: str) -> List[str]:
+        """Get all connection IDs subscribed to a topic"""
+        async with self._connections_lock:
+            return list(self._topics.get(topic, set()))
+
+    async def get_user_connections(self, user_id: int) -> List[str]:
+        """Get all connection IDs for a user"""
+        async with self._connections_lock:
+            return list(self._user_connections.get(user_id, set()))
+
+    async def get_stats(self) -> Dict[str, Any]:
+        """Get pool statistics"""
+        async with self._connections_lock:
+            active_by_state = {}
+            for conn in self._connections.values():
+                state = conn.state.value
+                active_by_state[state] = active_by_state.get(state, 0) + 1
+            
+            # Compute total unique users robustly (avoid falsey user_id like 0)
+            try:
+                unique_user_ids = {conn.user_id for conn in self._connections.values() if conn.user_id is not None}
+            except Exception:
+                unique_user_ids = set(self._user_connections.keys())
+
+            return {
+                **self._stats,
+                "active_connections": len(self._connections),
+                "total_topics": len(self._topics),
+                "total_users": len(unique_user_ids),
+                "connections_by_state": active_by_state,
+                "topic_distribution": {topic: len(conn_ids) for topic, conn_ids in self._topics.items()},
+            }
+
+    async def _cleanup_worker(self):
+        """Background task to clean up stale connections"""
+        while True:
+            try:
+                await asyncio.sleep(self.cleanup_interval)
+                await self._cleanup_stale_connections()
+                self._stats["last_cleanup"] = datetime.now(timezone.utc).isoformat()
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                self.logger.error("Error in cleanup worker", error=str(e))
+
+    async def _heartbeat_worker(self):
+        """Background task to send heartbeats to connections"""
+        while True:
+            try:
+                await asyncio.sleep(self.heartbeat_interval)
+                await self._send_heartbeats()
+                self._stats["last_heartbeat"] = datetime.now(timezone.utc).isoformat()
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                self.logger.error("Error in heartbeat worker", error=str(e))
+
+    async def _cleanup_stale_connections(self):
+        """Clean up stale and disconnected connections"""
+        stale_connections = []
+        
+        async with self._connections_lock:
+            for connection_id, connection_info in self._connections.items():
+                if connection_info.is_stale(self.connection_timeout):
+                    stale_connections.append(connection_id)
+        
+        # Remove stale connections
+        for connection_id in stale_connections:
+            await self.remove_connection(connection_id, "stale_connection")
+        
+        if stale_connections:
+            self._stats["connections_cleaned"] += len(stale_connections)
+            self.logger.info("Cleaned up stale connections",
+                           count=len(stale_connections),
+                           total_cleaned=self._stats["connections_cleaned"])
+
+    async def _send_heartbeats(self):
+        """Send heartbeats to all active connections"""
+        async with self._connections_lock:
+            connection_ids = list(self._connections.keys())
+        
+        heartbeat_message = WebSocketMessage(
+            type=MessageType.HEARTBEAT.value,
+            timestamp=datetime.now(timezone.utc).isoformat()
+        )
+        
+        failed_connections = []
+        for connection_id in connection_ids:
+            try:
+                success = await self._send_to_connection(connection_id, heartbeat_message)
+                if not success:
+                    failed_connections.append(connection_id)
+            except Exception:
+                failed_connections.append(connection_id)
+        
+        # Clean up failed connections
+        for connection_id in failed_connections:
+            await self.remove_connection(connection_id, "heartbeat_failed")
+
+    async def _close_all_connections(self):
+        """Close all active connections"""
+        async with self._connections_lock:
+            connection_ids = list(self._connections.keys())
+        
+        for connection_id in connection_ids:
+            await self.remove_connection(connection_id, "pool_shutdown")
+
+
+# Global WebSocket pool instance
+_websocket_pool: Optional[WebSocketPool] = None
+
+
+def get_websocket_pool() -> WebSocketPool:
+    """Get the global WebSocket pool instance"""
+    global _websocket_pool
+    if _websocket_pool is None:
+        _websocket_pool = WebSocketPool()
+    return _websocket_pool
+
+
+async def initialize_websocket_pool(**kwargs) -> WebSocketPool:
+    """Initialize and start the global WebSocket pool"""
+    global _websocket_pool
+    if _websocket_pool is None:
+        _websocket_pool = WebSocketPool(**kwargs)
+    await _websocket_pool.start()
+    return _websocket_pool
+
+
+async def shutdown_websocket_pool():
+    """Shutdown the global WebSocket pool"""
+    global _websocket_pool
+    if _websocket_pool is not None:
+        await _websocket_pool.stop()
+        _websocket_pool = None
+
+
+@asynccontextmanager
+async def websocket_connection(
+    websocket: WebSocket,
+    user_id: Optional[int] = None,
+    topics: Optional[Set[str]] = None,
+    metadata: Optional[Dict[str, Any]] = None
+):
+    """
+    Context manager for WebSocket connections
+    
+    Automatically handles connection registration and cleanup
+    """
+    pool = get_websocket_pool()
+    connection_id = None
+    
+    try:
+        connection_id = await pool.add_connection(websocket, user_id, topics, metadata)
+        yield connection_id, pool
+    finally:
+        if connection_id:
+            await pool.remove_connection(connection_id, "context_exit")

app/services/workflow_engine.py

@@ -0,0 +1,792 @@
+"""
+Document Workflow Execution Engine
+
+This service handles:
+- Event detection and processing
+- Workflow matching and triggering
+- Automated document generation
+- Action execution and error handling
+- Schedule management for time-based workflows
+"""
+from __future__ import annotations
+
+import json
+import uuid
+import asyncio
+from datetime import datetime, timedelta, timezone
+from typing import Dict, Any, List, Optional, Tuple
+import logging
+from croniter import croniter
+
+from sqlalchemy.orm import Session
+from sqlalchemy import and_, or_, func
+
+from app.models.document_workflows import (
+    DocumentWorkflow, WorkflowAction, WorkflowExecution, EventLog,
+    WorkflowTriggerType, WorkflowActionType, ExecutionStatus, WorkflowStatus
+)
+from app.models.files import File
+from app.models.deadlines import Deadline
+from app.models.templates import DocumentTemplate
+from app.models.user import User
+from app.services.advanced_variables import VariableProcessor
+from app.services.template_merge import build_context, resolve_tokens, render_docx
+from app.services.storage import get_default_storage
+from app.core.logging import get_logger
+from app.services.document_notifications import notify_processing, notify_completed, notify_failed
+
+logger = get_logger("workflow_engine")
+
+
+class WorkflowEngineError(Exception):
+    """Base exception for workflow engine errors"""
+    pass
+
+
+class WorkflowExecutionError(Exception):
+    """Exception for workflow execution failures"""
+    pass
+
+
+class EventProcessor:
+    """
+    Processes system events and triggers appropriate workflows
+    """
+    
+    def __init__(self, db: Session):
+        self.db = db
+    
+    async def log_event(
+        self,
+        event_type: str,
+        event_source: str,
+        file_no: Optional[str] = None,
+        client_id: Optional[str] = None,
+        user_id: Optional[int] = None,
+        resource_type: Optional[str] = None,
+        resource_id: Optional[str] = None,
+        event_data: Optional[Dict[str, Any]] = None,
+        previous_state: Optional[Dict[str, Any]] = None,
+        new_state: Optional[Dict[str, Any]] = None
+    ) -> str:
+        """
+        Log a system event that may trigger workflows
+        
+        Returns:
+            Event ID for tracking
+        """
+        event_id = str(uuid.uuid4())
+        
+        event_log = EventLog(
+            event_id=event_id,
+            event_type=event_type,
+            event_source=event_source,
+            file_no=file_no,
+            client_id=client_id,
+            user_id=user_id,
+            resource_type=resource_type,
+            resource_id=resource_id,
+            event_data=event_data or {},
+            previous_state=previous_state,
+            new_state=new_state,
+            occurred_at=datetime.now(timezone.utc)
+        )
+        
+        self.db.add(event_log)
+        self.db.commit()
+        
+        # Process the event asynchronously to find matching workflows
+        await self._process_event(event_log)
+        
+        return event_id
+    
+    async def _process_event(self, event: EventLog):
+        """
+        Process an event to find and trigger matching workflows
+        """
+        try:
+            triggered_workflows = []
+            
+            # Find workflows that match this event type
+            matching_workflows = self.db.query(DocumentWorkflow).filter(
+                DocumentWorkflow.status == WorkflowStatus.ACTIVE
+            ).all()
+            
+            # Filter workflows by trigger type (enum value comparison)
+            filtered_workflows = []
+            for workflow in matching_workflows:
+                if workflow.trigger_type.value == event.event_type:
+                    filtered_workflows.append(workflow)
+            
+            matching_workflows = filtered_workflows
+            
+            for workflow in matching_workflows:
+                if await self._should_trigger_workflow(workflow, event):
+                    execution_id = await self._trigger_workflow(workflow, event)
+                    if execution_id:
+                        triggered_workflows.append(workflow.id)
+            
+            # Update event log with triggered workflows
+            event.triggered_workflows = triggered_workflows
+            event.processed = True
+            event.processed_at = datetime.now(timezone.utc)
+            self.db.commit()
+            
+            logger.info(f"Event {event.event_id} processed, triggered {len(triggered_workflows)} workflows")
+            
+        except Exception as e:
+            logger.error(f"Error processing event {event.event_id}: {str(e)}")
+            event.processing_errors = [str(e)]
+            event.processed = True
+            event.processed_at = datetime.now(timezone.utc)
+            self.db.commit()
+    
+    async def _should_trigger_workflow(self, workflow: DocumentWorkflow, event: EventLog) -> bool:
+        """
+        Check if a workflow should be triggered for the given event
+        """
+        try:
+            # Check basic filters
+            if workflow.file_type_filter and event.file_no:
+                file_obj = self.db.query(File).filter(File.file_no == event.file_no).first()
+                if file_obj and file_obj.file_type not in workflow.file_type_filter:
+                    return False
+            
+            if workflow.status_filter and event.file_no:
+                file_obj = self.db.query(File).filter(File.file_no == event.file_no).first()
+                if file_obj and file_obj.status not in workflow.status_filter:
+                    return False
+            
+            if workflow.attorney_filter and event.file_no:
+                file_obj = self.db.query(File).filter(File.file_no == event.file_no).first()
+                if file_obj and file_obj.empl_num not in workflow.attorney_filter:
+                    return False
+            
+            if workflow.client_filter and event.client_id:
+                if event.client_id not in workflow.client_filter:
+                    return False
+            
+            # Check trigger conditions
+            if workflow.trigger_conditions:
+                return self._evaluate_trigger_conditions(workflow.trigger_conditions, event)
+            
+            return True
+            
+        except Exception as e:
+            logger.warning(f"Error evaluating workflow {workflow.id} for event {event.event_id}: {str(e)}")
+            return False
+    
+    def _evaluate_trigger_conditions(self, conditions: Dict[str, Any], event: EventLog) -> bool:
+        """
+        Evaluate complex trigger conditions against an event
+        """
+        try:
+            condition_type = conditions.get('type', 'simple')
+            
+            if condition_type == 'simple':
+                field = conditions.get('field')
+                operator = conditions.get('operator', 'equals')
+                expected_value = conditions.get('value')
+                
+                # Get actual value from event
+                actual_value = None
+                if field == 'event_type':
+                    actual_value = event.event_type
+                elif field == 'file_no':
+                    actual_value = event.file_no
+                elif field == 'client_id':
+                    actual_value = event.client_id
+                elif field.startswith('data.'):
+                    # Extract from event_data
+                    data_key = field[5:]  # Remove 'data.' prefix
+                    actual_value = event.event_data.get(data_key) if event.event_data else None
+                elif field.startswith('new_state.'):
+                    # Extract from new_state
+                    state_key = field[10:]  # Remove 'new_state.' prefix
+                    actual_value = event.new_state.get(state_key) if event.new_state else None
+                elif field.startswith('previous_state.'):
+                    # Extract from previous_state
+                    state_key = field[15:]  # Remove 'previous_state.' prefix
+                    actual_value = event.previous_state.get(state_key) if event.previous_state else None
+                
+                # Evaluate condition
+                return self._evaluate_simple_condition(actual_value, operator, expected_value)
+                
+            elif condition_type == 'compound':
+                operator = conditions.get('operator', 'and')
+                sub_conditions = conditions.get('conditions', [])
+                
+                if operator == 'and':
+                    return all(self._evaluate_trigger_conditions(cond, event) for cond in sub_conditions)
+                elif operator == 'or':
+                    return any(self._evaluate_trigger_conditions(cond, event) for cond in sub_conditions)
+                elif operator == 'not':
+                    return not self._evaluate_trigger_conditions(sub_conditions[0], event) if sub_conditions else False
+            
+            return False
+            
+        except Exception:
+            return False
+    
+    def _evaluate_simple_condition(self, actual_value: Any, operator: str, expected_value: Any) -> bool:
+        """
+        Evaluate a simple condition
+        """
+        try:
+            if operator == 'equals':
+                return actual_value == expected_value
+            elif operator == 'not_equals':
+                return actual_value != expected_value
+            elif operator == 'contains':
+                return str(expected_value) in str(actual_value) if actual_value else False
+            elif operator == 'starts_with':
+                return str(actual_value).startswith(str(expected_value)) if actual_value else False
+            elif operator == 'ends_with':
+                return str(actual_value).endswith(str(expected_value)) if actual_value else False
+            elif operator == 'is_empty':
+                return actual_value is None or str(actual_value).strip() == ''
+            elif operator == 'is_not_empty':
+                return actual_value is not None and str(actual_value).strip() != ''
+            elif operator == 'in':
+                return actual_value in expected_value if isinstance(expected_value, list) else False
+            elif operator == 'not_in':
+                return actual_value not in expected_value if isinstance(expected_value, list) else True
+            
+            # Numeric comparisons
+            elif operator in ['greater_than', 'less_than', 'greater_equal', 'less_equal']:
+                try:
+                    actual_num = float(actual_value) if actual_value is not None else 0
+                    expected_num = float(expected_value) if expected_value is not None else 0
+                    
+                    if operator == 'greater_than':
+                        return actual_num > expected_num
+                    elif operator == 'less_than':
+                        return actual_num < expected_num
+                    elif operator == 'greater_equal':
+                        return actual_num >= expected_num
+                    elif operator == 'less_equal':
+                        return actual_num <= expected_num
+                except (ValueError, TypeError):
+                    return False
+            
+            return False
+            
+        except Exception:
+            return False
+    
+    async def _trigger_workflow(self, workflow: DocumentWorkflow, event: EventLog) -> Optional[int]:
+        """
+        Trigger a workflow execution
+        
+        Returns:
+            Workflow execution ID if successful, None if failed
+        """
+        try:
+            execution = WorkflowExecution(
+                workflow_id=workflow.id,
+                triggered_by_event_id=event.event_id,
+                triggered_by_event_type=event.event_type,
+                context_file_no=event.file_no,
+                context_client_id=event.client_id,
+                context_user_id=event.user_id,
+                trigger_data=event.event_data,
+                status=ExecutionStatus.PENDING
+            )
+            
+            self.db.add(execution)
+            self.db.flush()  # Get the ID
+            
+            # Update workflow statistics
+            workflow.execution_count += 1
+            workflow.last_triggered_at = datetime.now(timezone.utc)
+            
+            self.db.commit()
+            
+            # Execute the workflow (possibly with delay)
+            if workflow.delay_minutes > 0:
+                # Schedule delayed execution
+                await _schedule_delayed_execution(execution.id, workflow.delay_minutes)
+            else:
+                # Execute immediately
+                await _execute_workflow(execution.id, self.db)
+            
+            return execution.id
+            
+        except Exception as e:
+            logger.error(f"Error triggering workflow {workflow.id}: {str(e)}")
+            self.db.rollback()
+            return None
+
+
+class WorkflowExecutor:
+    """
+    Executes individual workflow instances
+    """
+    
+    def __init__(self, db: Session):
+        self.db = db
+        self.variable_processor = VariableProcessor(db)
+    
+    async def execute_workflow(self, execution_id: int) -> bool:
+        """
+        Execute a workflow execution
+        
+        Returns:
+            True if successful, False if failed
+        """
+        execution = self.db.query(WorkflowExecution).filter(
+            WorkflowExecution.id == execution_id
+        ).first()
+        
+        if not execution:
+            logger.error(f"Workflow execution {execution_id} not found")
+            return False
+        
+        workflow = execution.workflow
+        if not workflow:
+            logger.error(f"Workflow for execution {execution_id} not found")
+            return False
+        
+        try:
+            # Update execution status
+            execution.status = ExecutionStatus.RUNNING
+            execution.started_at = datetime.now(timezone.utc)
+            self.db.commit()
+            
+            logger.info(f"Starting workflow execution {execution_id} for workflow '{workflow.name}'")
+            
+            # Build execution context
+            context = await self._build_execution_context(execution)
+            execution.execution_context = context
+            
+            # Execute actions in order
+            action_results = []
+            actions = sorted(workflow.actions, key=lambda a: a.action_order)
+            
+            for action in actions:
+                if await self._should_execute_action(action, context):
+                    result = await self._execute_action(action, context, execution)
+                    action_results.append(result)
+                    
+                    if not result.get('success', False) and not action.continue_on_failure:
+                        raise WorkflowExecutionError(f"Action {action.id} failed: {result.get('error', 'Unknown error')}")
+                else:
+                    action_results.append({
+                        'action_id': action.id,
+                        'skipped': True,
+                        'reason': 'Condition not met'
+                    })
+            
+            # Update execution with results
+            execution.action_results = action_results
+            execution.status = ExecutionStatus.COMPLETED
+            execution.completed_at = datetime.now(timezone.utc)
+            execution.execution_duration_seconds = int(
+                (execution.completed_at - execution.started_at).total_seconds()
+            )
+            
+            # Update workflow statistics
+            workflow.success_count += 1
+            
+            self.db.commit()
+            
+            logger.info(f"Workflow execution {execution_id} completed successfully")
+            return True
+            
+        except Exception as e:
+            # Handle execution failure
+            error_message = str(e)
+            logger.error(f"Workflow execution {execution_id} failed: {error_message}")
+            
+            execution.status = ExecutionStatus.FAILED
+            execution.error_message = error_message
+            execution.completed_at = datetime.now(timezone.utc)
+            
+            if execution.started_at:
+                execution.execution_duration_seconds = int(
+                    (execution.completed_at - execution.started_at).total_seconds()
+                )
+            
+            # Update workflow statistics
+            workflow.failure_count += 1
+            
+            # Check if we should retry
+            if execution.retry_count < workflow.max_retries:
+                execution.retry_count += 1
+                execution.next_retry_at = datetime.now(timezone.utc) + timedelta(
+                    minutes=workflow.retry_delay_minutes
+                )
+                execution.status = ExecutionStatus.RETRYING
+                logger.info(f"Scheduling retry {execution.retry_count} for execution {execution_id}")
+            
+            self.db.commit()
+            return False
+    
+    async def _build_execution_context(self, execution: WorkflowExecution) -> Dict[str, Any]:
+        """
+        Build context for workflow execution
+        """
+        context = {
+            'execution_id': execution.id,
+            'workflow_id': execution.workflow_id,
+            'event_id': execution.triggered_by_event_id,
+            'event_type': execution.triggered_by_event_type,
+            'trigger_data': execution.trigger_data or {},
+        }
+        
+        # Add file context if available
+        if execution.context_file_no:
+            file_obj = self.db.query(File).filter(
+                File.file_no == execution.context_file_no
+            ).first()
+            if file_obj:
+                context.update({
+                    'FILE_NO': file_obj.file_no,
+                    'CLIENT_ID': file_obj.id,
+                    'FILE_TYPE': file_obj.file_type,
+                    'FILE_STATUS': file_obj.status,
+                    'ATTORNEY': file_obj.empl_num,
+                    'MATTER': file_obj.regarding or '',
+                    'OPENED_DATE': file_obj.opened.isoformat() if file_obj.opened else '',
+                    'CLOSED_DATE': file_obj.closed.isoformat() if file_obj.closed else '',
+                    'HOURLY_RATE': str(file_obj.rate_per_hour),
+                })
+                
+                # Add client information
+                if file_obj.owner:
+                    context.update({
+                        'CLIENT_FIRST': file_obj.owner.first or '',
+                        'CLIENT_LAST': file_obj.owner.last or '',
+                        'CLIENT_FULL': f"{file_obj.owner.first or ''} {file_obj.owner.last or ''}".strip(),
+                        'CLIENT_COMPANY': file_obj.owner.company or '',
+                        'CLIENT_EMAIL': file_obj.owner.email or '',
+                        'CLIENT_PHONE': file_obj.owner.phone or '',
+                    })
+        
+        # Add user context if available
+        if execution.context_user_id:
+            user = self.db.query(User).filter(User.id == execution.context_user_id).first()
+            if user:
+                context.update({
+                    'USER_ID': str(user.id),
+                    'USERNAME': user.username,
+                    'USER_EMAIL': user.email or '',
+                })
+        
+        return context
+    
+    async def _should_execute_action(self, action: WorkflowAction, context: Dict[str, Any]) -> bool:
+        """
+        Check if an action should be executed based on its conditions
+        """
+        if not action.condition:
+            return True
+        
+        try:
+            # Use the same condition evaluation logic as trigger conditions
+            processor = EventProcessor(self.db)
+            # Create a mock event for condition evaluation
+            mock_event = type('MockEvent', (), {
+                'event_data': context.get('trigger_data', {}),
+                'new_state': context,
+                'previous_state': {},
+                'event_type': context.get('event_type'),
+                'file_no': context.get('FILE_NO'),
+                'client_id': context.get('CLIENT_ID'),
+            })()
+            
+            return processor._evaluate_trigger_conditions(action.condition, mock_event)
+            
+        except Exception as e:
+            logger.warning(f"Error evaluating action condition for action {action.id}: {str(e)}")
+            return True  # Default to executing the action
+    
+    async def _execute_action(
+        self,
+        action: WorkflowAction,
+        context: Dict[str, Any],
+        execution: WorkflowExecution
+    ) -> Dict[str, Any]:
+        """
+        Execute a specific workflow action
+        """
+        try:
+            if action.action_type == WorkflowActionType.GENERATE_DOCUMENT:
+                return await self._execute_document_generation(action, context, execution)
+            elif action.action_type == WorkflowActionType.SEND_EMAIL:
+                return await self._execute_send_email(action, context, execution)
+            elif action.action_type == WorkflowActionType.CREATE_DEADLINE:
+                return await self._execute_create_deadline(action, context, execution)
+            elif action.action_type == WorkflowActionType.UPDATE_FILE_STATUS:
+                return await self._execute_update_file_status(action, context, execution)
+            elif action.action_type == WorkflowActionType.CREATE_LEDGER_ENTRY:
+                return await self._execute_create_ledger_entry(action, context, execution)
+            elif action.action_type == WorkflowActionType.SEND_NOTIFICATION:
+                return await self._execute_send_notification(action, context, execution)
+            elif action.action_type == WorkflowActionType.EXECUTE_CUSTOM:
+                return await self._execute_custom_action(action, context, execution)
+            else:
+                return {
+                    'action_id': action.id,
+                    'success': False,
+                    'error': f'Unknown action type: {action.action_type.value}'
+                }
+                
+        except Exception as e:
+            logger.error(f"Error executing action {action.id}: {str(e)}")
+            return {
+                'action_id': action.id,
+                'success': False,
+                'error': str(e)
+            }
+    
+    async def _execute_document_generation(
+        self,
+        action: WorkflowAction,
+        context: Dict[str, Any],
+        execution: WorkflowExecution
+    ) -> Dict[str, Any]:
+        """
+        Execute document generation action
+        """
+        if not action.template_id:
+            return {
+                'action_id': action.id,
+                'success': False,
+                'error': 'No template specified'
+            }
+        
+        template = self.db.query(DocumentTemplate).filter(
+            DocumentTemplate.id == action.template_id
+        ).first()
+        
+        if not template or not template.current_version_id:
+            return {
+                'action_id': action.id,
+                'success': False,
+                'error': 'Template not found or has no current version'
+            }
+        
+        try:
+            # Get file number for notifications
+            file_no = context.get('FILE_NO')
+            if not file_no:
+                return {
+                    'action_id': action.id,
+                    'success': False,
+                    'error': 'No file number available for document generation'
+                }
+            
+            # Notify processing started
+            try:
+                await notify_processing(
+                    file_no=file_no,
+                    data={
+                        'action_id': action.id,
+                        'workflow_id': execution.workflow_id,
+                        'template_id': action.template_id,
+                        'template_name': template.name,
+                        'execution_id': execution.id
+                    }
+                )
+            except Exception:
+                # Don't fail workflow if notification fails
+                pass
+            
+            # Generate the document using the template system
+            from app.api.documents import generate_batch_documents
+            from app.models.documents import BatchGenerateRequest
+            
+            # Prepare the request
+            file_nos = [file_no]
+            
+            # Use the enhanced context for variable resolution
+            enhanced_context = build_context(
+                context,
+                context_type="file" if context.get('FILE_NO') else "global",
+                context_id=context.get('FILE_NO', 'default')
+            )
+            
+            # Here we would integrate with the document generation system
+            # For now, return a placeholder result
+            result = {
+                'action_id': action.id,
+                'success': True,
+                'template_id': action.template_id,
+                'template_name': template.name,
+                'generated_for_files': file_nos,
+                'output_format': action.output_format,
+                'generated_at': datetime.now(timezone.utc).isoformat()
+            }
+            
+            # Notify successful completion
+            try:
+                await notify_completed(
+                    file_no=file_no,
+                    data={
+                        'action_id': action.id,
+                        'workflow_id': execution.workflow_id,
+                        'template_id': action.template_id,
+                        'template_name': template.name,
+                        'execution_id': execution.id,
+                        'output_format': action.output_format,
+                        'generated_at': result['generated_at']
+                    }
+                )
+            except Exception:
+                # Don't fail workflow if notification fails
+                pass
+            
+            # Update execution with generated documents
+            if not execution.generated_documents:
+                execution.generated_documents = []
+            execution.generated_documents.append(result)
+            
+            return result
+            
+        except Exception as e:
+            # Notify failure
+            try:
+                await notify_failed(
+                    file_no=file_no,
+                    data={
+                        'action_id': action.id,
+                        'workflow_id': execution.workflow_id,
+                        'template_id': action.template_id,
+                        'template_name': template.name if 'template' in locals() else 'Unknown',
+                        'execution_id': execution.id,
+                        'error': str(e)
+                    }
+                )
+            except Exception:
+                # Don't fail workflow if notification fails
+                pass
+            
+            return {
+                'action_id': action.id,
+                'success': False,
+                'error': f'Document generation failed: {str(e)}'
+            }
+    
+    async def _execute_send_email(
+        self,
+        action: WorkflowAction,
+        context: Dict[str, Any],
+        execution: WorkflowExecution
+    ) -> Dict[str, Any]:
+        """
+        Execute send email action
+        """
+        # Placeholder for email sending functionality
+        return {
+            'action_id': action.id,
+            'success': True,
+            'email_sent': True,
+            'recipients': action.email_recipients or [],
+            'subject': action.email_subject_template or 'Automated notification'
+        }
+    
+    async def _execute_create_deadline(
+        self,
+        action: WorkflowAction,
+        context: Dict[str, Any],
+        execution: WorkflowExecution
+    ) -> Dict[str, Any]:
+        """
+        Execute create deadline action
+        """
+        # Placeholder for deadline creation functionality
+        return {
+            'action_id': action.id,
+            'success': True,
+            'deadline_created': True
+        }
+    
+    async def _execute_update_file_status(
+        self,
+        action: WorkflowAction,
+        context: Dict[str, Any],
+        execution: WorkflowExecution
+    ) -> Dict[str, Any]:
+        """
+        Execute update file status action
+        """
+        # Placeholder for file status update functionality
+        return {
+            'action_id': action.id,
+            'success': True,
+            'file_status_updated': True
+        }
+    
+    async def _execute_create_ledger_entry(
+        self,
+        action: WorkflowAction,
+        context: Dict[str, Any],
+        execution: WorkflowExecution
+    ) -> Dict[str, Any]:
+        """
+        Execute create ledger entry action
+        """
+        # Placeholder for ledger entry creation functionality
+        return {
+            'action_id': action.id,
+            'success': True,
+            'ledger_entry_created': True
+        }
+    
+    async def _execute_send_notification(
+        self,
+        action: WorkflowAction,
+        context: Dict[str, Any],
+        execution: WorkflowExecution
+    ) -> Dict[str, Any]:
+        """
+        Execute send notification action
+        """
+        # Placeholder for notification sending functionality
+        return {
+            'action_id': action.id,
+            'success': True,
+            'notification_sent': True
+        }
+    
+    async def _execute_custom_action(
+        self,
+        action: WorkflowAction,
+        context: Dict[str, Any],
+        execution: WorkflowExecution
+    ) -> Dict[str, Any]:
+        """
+        Execute custom action
+        """
+        # Placeholder for custom action execution
+        return {
+            'action_id': action.id,
+            'success': True,
+            'custom_action_executed': True
+        }
+
+
+# Helper functions for integration
+async def _execute_workflow(execution_id: int, db: Session = None):
+    """Execute a workflow (to be called asynchronously)"""
+    from app.database.base import get_db
+    
+    if db is None:
+        db = next(get_db())
+    
+    try:
+        executor = WorkflowExecutor(db)
+        success = await executor.execute_workflow(execution_id)
+        return success
+    except Exception as e:
+        logger.error(f"Error executing workflow {execution_id}: {str(e)}")
+        return False
+    finally:
+        if db:
+            db.close()
+
+async def _schedule_delayed_execution(execution_id: int, delay_minutes: int):
+    """Schedule delayed workflow execution"""
+    # This would be implemented with a proper scheduler in production
+    pass

app/services/workflow_integration.py

@@ -0,0 +1,519 @@
+"""
+Workflow Integration Service
+
+This service provides integration points for automatically logging events
+and triggering workflows from existing system operations.
+"""
+from __future__ import annotations
+
+import asyncio
+from datetime import datetime, timezone
+from typing import Dict, Any, Optional
+import logging
+
+from sqlalchemy.orm import Session
+
+from app.services.workflow_engine import EventProcessor
+from app.core.logging import get_logger
+
+logger = get_logger("workflow_integration")
+
+
+class WorkflowIntegration:
+    """
+    Helper service for integrating workflow automation with existing systems
+    """
+    
+    def __init__(self, db: Session):
+        self.db = db
+        self.event_processor = EventProcessor(db)
+    
+    async def log_file_status_change(
+        self,
+        file_no: str,
+        old_status: str,
+        new_status: str,
+        user_id: Optional[int] = None,
+        notes: Optional[str] = None
+    ):
+        """
+        Log a file status change event that may trigger workflows
+        """
+        try:
+            event_data = {
+                'old_status': old_status,
+                'new_status': new_status,
+                'notes': notes
+            }
+            
+            previous_state = {'status': old_status}
+            new_state = {'status': new_status}
+            
+            await self.event_processor.log_event(
+                event_type="file_status_change",
+                event_source="file_management",
+                file_no=file_no,
+                user_id=user_id,
+                resource_type="file",
+                resource_id=file_no,
+                event_data=event_data,
+                previous_state=previous_state,
+                new_state=new_state
+            )
+            
+            # Log specific status events
+            if new_status == "CLOSED":
+                await self.log_file_closed(file_no, user_id)
+            elif old_status in ["INACTIVE", "CLOSED"] and new_status == "ACTIVE":
+                await self.log_file_reopened(file_no, user_id)
+                
+        except Exception as e:
+            logger.error(f"Error logging file status change for {file_no}: {str(e)}")
+    
+    async def log_file_opened(
+        self,
+        file_no: str,
+        file_type: str,
+        client_id: str,
+        attorney: str,
+        user_id: Optional[int] = None
+    ):
+        """
+        Log a new file opening event
+        """
+        try:
+            event_data = {
+                'file_type': file_type,
+                'client_id': client_id,
+                'attorney': attorney
+            }
+            
+            await self.event_processor.log_event(
+                event_type="file_opened",
+                event_source="file_management",
+                file_no=file_no,
+                client_id=client_id,
+                user_id=user_id,
+                resource_type="file",
+                resource_id=file_no,
+                event_data=event_data
+            )
+            
+        except Exception as e:
+            logger.error(f"Error logging file opened for {file_no}: {str(e)}")
+    
+    async def log_file_closed(
+        self,
+        file_no: str,
+        user_id: Optional[int] = None,
+        final_balance: Optional[float] = None
+    ):
+        """
+        Log a file closure event
+        """
+        try:
+            event_data = {
+                'closed_by_user_id': user_id,
+                'final_balance': final_balance
+            }
+            
+            await self.event_processor.log_event(
+                event_type="file_closed",
+                event_source="file_management",
+                file_no=file_no,
+                user_id=user_id,
+                resource_type="file",
+                resource_id=file_no,
+                event_data=event_data
+            )
+            
+        except Exception as e:
+            logger.error(f"Error logging file closed for {file_no}: {str(e)}")
+    
+    async def log_file_reopened(
+        self,
+        file_no: str,
+        user_id: Optional[int] = None,
+        reason: Optional[str] = None
+    ):
+        """
+        Log a file reopening event
+        """
+        try:
+            event_data = {
+                'reopened_by_user_id': user_id,
+                'reason': reason
+            }
+            
+            await self.event_processor.log_event(
+                event_type="file_reopened",
+                event_source="file_management",
+                file_no=file_no,
+                user_id=user_id,
+                resource_type="file",
+                resource_id=file_no,
+                event_data=event_data
+            )
+            
+        except Exception as e:
+            logger.error(f"Error logging file reopened for {file_no}: {str(e)}")
+    
+    async def log_deadline_approaching(
+        self,
+        deadline_id: int,
+        file_no: Optional[str] = None,
+        client_id: Optional[str] = None,
+        days_until_deadline: int = 0,
+        deadline_type: Optional[str] = None
+    ):
+        """
+        Log a deadline approaching event
+        """
+        try:
+            event_data = {
+                'deadline_id': deadline_id,
+                'days_until_deadline': days_until_deadline,
+                'deadline_type': deadline_type
+            }
+            
+            await self.event_processor.log_event(
+                event_type="deadline_approaching",
+                event_source="deadline_management",
+                file_no=file_no,
+                client_id=client_id,
+                resource_type="deadline",
+                resource_id=str(deadline_id),
+                event_data=event_data
+            )
+            
+        except Exception as e:
+            logger.error(f"Error logging deadline approaching for {deadline_id}: {str(e)}")
+    
+    async def log_deadline_overdue(
+        self,
+        deadline_id: int,
+        file_no: Optional[str] = None,
+        client_id: Optional[str] = None,
+        days_overdue: int = 0,
+        deadline_type: Optional[str] = None
+    ):
+        """
+        Log a deadline overdue event
+        """
+        try:
+            event_data = {
+                'deadline_id': deadline_id,
+                'days_overdue': days_overdue,
+                'deadline_type': deadline_type
+            }
+            
+            await self.event_processor.log_event(
+                event_type="deadline_overdue",
+                event_source="deadline_management",
+                file_no=file_no,
+                client_id=client_id,
+                resource_type="deadline",
+                resource_id=str(deadline_id),
+                event_data=event_data
+            )
+            
+        except Exception as e:
+            logger.error(f"Error logging deadline overdue for {deadline_id}: {str(e)}")
+    
+    async def log_deadline_completed(
+        self,
+        deadline_id: int,
+        file_no: Optional[str] = None,
+        client_id: Optional[str] = None,
+        completed_by_user_id: Optional[int] = None,
+        completion_notes: Optional[str] = None
+    ):
+        """
+        Log a deadline completion event
+        """
+        try:
+            event_data = {
+                'deadline_id': deadline_id,
+                'completed_by_user_id': completed_by_user_id,
+                'completion_notes': completion_notes
+            }
+            
+            await self.event_processor.log_event(
+                event_type="deadline_completed",
+                event_source="deadline_management",
+                file_no=file_no,
+                client_id=client_id,
+                user_id=completed_by_user_id,
+                resource_type="deadline",
+                resource_id=str(deadline_id),
+                event_data=event_data
+            )
+            
+        except Exception as e:
+            logger.error(f"Error logging deadline completed for {deadline_id}: {str(e)}")
+    
+    async def log_payment_received(
+        self,
+        file_no: str,
+        amount: float,
+        payment_type: str,
+        payment_date: Optional[datetime] = None,
+        user_id: Optional[int] = None
+    ):
+        """
+        Log a payment received event
+        """
+        try:
+            event_data = {
+                'amount': amount,
+                'payment_type': payment_type,
+                'payment_date': payment_date.isoformat() if payment_date else None
+            }
+            
+            await self.event_processor.log_event(
+                event_type="payment_received",
+                event_source="billing",
+                file_no=file_no,
+                user_id=user_id,
+                resource_type="payment",
+                resource_id=f"{file_no}_{datetime.now().isoformat()}",
+                event_data=event_data
+            )
+            
+        except Exception as e:
+            logger.error(f"Error logging payment received for {file_no}: {str(e)}")
+    
+    async def log_payment_overdue(
+        self,
+        file_no: str,
+        amount_due: float,
+        days_overdue: int,
+        invoice_date: Optional[datetime] = None
+    ):
+        """
+        Log a payment overdue event
+        """
+        try:
+            event_data = {
+                'amount_due': amount_due,
+                'days_overdue': days_overdue,
+                'invoice_date': invoice_date.isoformat() if invoice_date else None
+            }
+            
+            await self.event_processor.log_event(
+                event_type="payment_overdue",
+                event_source="billing",
+                file_no=file_no,
+                resource_type="invoice",
+                resource_id=f"{file_no}_overdue",
+                event_data=event_data
+            )
+            
+        except Exception as e:
+            logger.error(f"Error logging payment overdue for {file_no}: {str(e)}")
+    
+    async def log_document_uploaded(
+        self,
+        file_no: str,
+        document_id: int,
+        filename: str,
+        document_type: Optional[str] = None,
+        user_id: Optional[int] = None
+    ):
+        """
+        Log a document upload event
+        """
+        try:
+            event_data = {
+                'document_id': document_id,
+                'filename': filename,
+                'document_type': document_type,
+                'uploaded_by_user_id': user_id
+            }
+            
+            await self.event_processor.log_event(
+                event_type="document_uploaded",
+                event_source="document_management",
+                file_no=file_no,
+                user_id=user_id,
+                resource_type="document",
+                resource_id=str(document_id),
+                event_data=event_data
+            )
+            
+        except Exception as e:
+            logger.error(f"Error logging document uploaded for {file_no}: {str(e)}")
+    
+    async def log_qdro_status_change(
+        self,
+        qdro_id: int,
+        file_no: str,
+        old_status: str,
+        new_status: str,
+        user_id: Optional[int] = None
+    ):
+        """
+        Log a QDRO status change event
+        """
+        try:
+            event_data = {
+                'qdro_id': qdro_id,
+                'old_status': old_status,
+                'new_status': new_status
+            }
+            
+            previous_state = {'status': old_status}
+            new_state = {'status': new_status}
+            
+            await self.event_processor.log_event(
+                event_type="qdro_status_change",
+                event_source="qdro_management",
+                file_no=file_no,
+                user_id=user_id,
+                resource_type="qdro",
+                resource_id=str(qdro_id),
+                event_data=event_data,
+                previous_state=previous_state,
+                new_state=new_state
+            )
+            
+        except Exception as e:
+            logger.error(f"Error logging QDRO status change for {qdro_id}: {str(e)}")
+    
+    async def log_custom_event(
+        self,
+        event_type: str,
+        event_source: str,
+        file_no: Optional[str] = None,
+        client_id: Optional[str] = None,
+        user_id: Optional[int] = None,
+        resource_type: Optional[str] = None,
+        resource_id: Optional[str] = None,
+        event_data: Optional[Dict[str, Any]] = None
+    ):
+        """
+        Log a custom event
+        """
+        try:
+            await self.event_processor.log_event(
+                event_type=event_type,
+                event_source=event_source,
+                file_no=file_no,
+                client_id=client_id,
+                user_id=user_id,
+                resource_type=resource_type,
+                resource_id=resource_id,
+                event_data=event_data or {}
+            )
+            
+        except Exception as e:
+            logger.error(f"Error logging custom event {event_type}: {str(e)}")
+
+
+# Helper functions for easy integration
+def create_workflow_integration(db: Session) -> WorkflowIntegration:
+    """
+    Create a workflow integration instance
+    """
+    return WorkflowIntegration(db)
+
+
+def run_async_event_logging(coro):
+    """
+    Helper to run async event logging in sync contexts
+    """
+    try:
+        # Try to get the current event loop
+        loop = asyncio.get_event_loop()
+        if loop.is_running():
+            # If loop is running, schedule the coroutine
+            asyncio.create_task(coro)
+        else:
+            # If no loop is running, run the coroutine
+            loop.run_until_complete(coro)
+    except RuntimeError:
+        # No event loop, create a new one
+        asyncio.run(coro)
+    except Exception as e:
+        logger.error(f"Error running async event logging: {str(e)}")
+
+
+# Sync wrapper functions for easy integration with existing code
+def log_file_status_change_sync(
+    db: Session,
+    file_no: str,
+    old_status: str,
+    new_status: str,
+    user_id: Optional[int] = None,
+    notes: Optional[str] = None
+):
+    """
+    Synchronous wrapper for file status change logging
+    """
+    integration = create_workflow_integration(db)
+    coro = integration.log_file_status_change(file_no, old_status, new_status, user_id, notes)
+    run_async_event_logging(coro)
+
+
+def log_file_opened_sync(
+    db: Session,
+    file_no: str,
+    file_type: str,
+    client_id: str,
+    attorney: str,
+    user_id: Optional[int] = None
+):
+    """
+    Synchronous wrapper for file opened logging
+    """
+    integration = create_workflow_integration(db)
+    coro = integration.log_file_opened(file_no, file_type, client_id, attorney, user_id)
+    run_async_event_logging(coro)
+
+
+def log_deadline_approaching_sync(
+    db: Session,
+    deadline_id: int,
+    file_no: Optional[str] = None,
+    client_id: Optional[str] = None,
+    days_until_deadline: int = 0,
+    deadline_type: Optional[str] = None
+):
+    """
+    Synchronous wrapper for deadline approaching logging
+    """
+    integration = create_workflow_integration(db)
+    coro = integration.log_deadline_approaching(deadline_id, file_no, client_id, days_until_deadline, deadline_type)
+    run_async_event_logging(coro)
+
+
+def log_payment_received_sync(
+    db: Session,
+    file_no: str,
+    amount: float,
+    payment_type: str,
+    payment_date: Optional[datetime] = None,
+    user_id: Optional[int] = None
+):
+    """
+    Synchronous wrapper for payment received logging
+    """
+    integration = create_workflow_integration(db)
+    coro = integration.log_payment_received(file_no, amount, payment_type, payment_date, user_id)
+    run_async_event_logging(coro)
+
+
+def log_document_uploaded_sync(
+    db: Session,
+    file_no: str,
+    document_id: int,
+    filename: str,
+    document_type: Optional[str] = None,
+    user_id: Optional[int] = None
+):
+    """
+    Synchronous wrapper for document uploaded logging
+    """
+    integration = create_workflow_integration(db)
+    coro = integration.log_document_uploaded(file_no, document_id, filename, document_type, user_id)
+    run_async_event_logging(coro)