572 lines
21 KiB
Python
572 lines
21 KiB
Python
"""
|
|
Advanced Variable Resolution Service
|
|
|
|
This service handles complex variable processing including:
|
|
- Conditional logic evaluation
|
|
- Mathematical calculations and formulas
|
|
- Dynamic data source queries
|
|
- Variable dependency resolution
|
|
- Caching and performance optimization
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import json
|
|
import math
|
|
import operator
|
|
from datetime import datetime, date, timedelta
|
|
from typing import Dict, Any, List, Optional, Tuple, Union
|
|
from decimal import Decimal, InvalidOperation
|
|
import logging
|
|
|
|
from sqlalchemy.orm import Session
|
|
from sqlalchemy import text
|
|
|
|
from app.models.template_variables import (
|
|
TemplateVariable, VariableContext, VariableAuditLog,
|
|
VariableType, VariableTemplate
|
|
)
|
|
from app.models.files import File
|
|
from app.models.rolodex import Rolodex
|
|
from app.core.logging import get_logger
|
|
|
|
logger = get_logger("advanced_variables")
|
|
|
|
|
|
class VariableProcessor:
|
|
"""
|
|
Handles advanced variable processing with conditional logic, calculations, and data sources
|
|
"""
|
|
|
|
def __init__(self, db: Session):
|
|
self.db = db
|
|
self._cache: Dict[str, Any] = {}
|
|
|
|
# Safe functions available in formula expressions
|
|
self.safe_functions = {
|
|
'abs': abs,
|
|
'round': round,
|
|
'min': min,
|
|
'max': max,
|
|
'sum': sum,
|
|
'len': len,
|
|
'str': str,
|
|
'int': int,
|
|
'float': float,
|
|
'math_ceil': math.ceil,
|
|
'math_floor': math.floor,
|
|
'math_sqrt': math.sqrt,
|
|
'today': lambda: date.today(),
|
|
'now': lambda: datetime.now(),
|
|
'days_between': lambda d1, d2: (d1 - d2).days if isinstance(d1, date) and isinstance(d2, date) else 0,
|
|
'format_currency': lambda x: f"${float(x):,.2f}" if x is not None else "$0.00",
|
|
'format_date': lambda d, fmt='%B %d, %Y': d.strftime(fmt) if isinstance(d, date) else str(d),
|
|
}
|
|
|
|
# Safe operators for formula evaluation
|
|
self.operators = {
|
|
'+': operator.add,
|
|
'-': operator.sub,
|
|
'*': operator.mul,
|
|
'/': operator.truediv,
|
|
'//': operator.floordiv,
|
|
'%': operator.mod,
|
|
'**': operator.pow,
|
|
'==': operator.eq,
|
|
'!=': operator.ne,
|
|
'<': operator.lt,
|
|
'<=': operator.le,
|
|
'>': operator.gt,
|
|
'>=': operator.ge,
|
|
'and': operator.and_,
|
|
'or': operator.or_,
|
|
'not': operator.not_,
|
|
}
|
|
|
|
def resolve_variables(
|
|
self,
|
|
variables: List[str],
|
|
context_type: str = "global",
|
|
context_id: str = "default",
|
|
base_context: Optional[Dict[str, Any]] = None
|
|
) -> Tuple[Dict[str, Any], List[str]]:
|
|
"""
|
|
Resolve a list of variables with their current values
|
|
|
|
Args:
|
|
variables: List of variable names to resolve
|
|
context_type: Context type (file, client, global, etc.)
|
|
context_id: Specific context identifier
|
|
base_context: Additional context values to use
|
|
|
|
Returns:
|
|
Tuple of (resolved_variables, unresolved_variables)
|
|
"""
|
|
resolved = {}
|
|
unresolved = []
|
|
processing_order = self._determine_processing_order(variables)
|
|
|
|
# Start with base context
|
|
if base_context:
|
|
resolved.update(base_context)
|
|
|
|
for var_name in processing_order:
|
|
try:
|
|
value = self._resolve_single_variable(
|
|
var_name, context_type, context_id, resolved
|
|
)
|
|
if value is not None:
|
|
resolved[var_name] = value
|
|
else:
|
|
unresolved.append(var_name)
|
|
except Exception as e:
|
|
logger.error(f"Error resolving variable {var_name}: {str(e)}")
|
|
unresolved.append(var_name)
|
|
|
|
return resolved, unresolved
|
|
|
|
def _resolve_single_variable(
|
|
self,
|
|
var_name: str,
|
|
context_type: str,
|
|
context_id: str,
|
|
current_context: Dict[str, Any]
|
|
) -> Any:
|
|
"""
|
|
Resolve a single variable based on its type and configuration
|
|
"""
|
|
# Get variable definition
|
|
var_def = self.db.query(TemplateVariable).filter(
|
|
TemplateVariable.name == var_name,
|
|
TemplateVariable.active == True
|
|
).first()
|
|
|
|
if not var_def:
|
|
return None
|
|
|
|
# Check for static value first
|
|
if var_def.static_value is not None:
|
|
return self._convert_value(var_def.static_value, var_def.variable_type)
|
|
|
|
# Check cache if enabled
|
|
cache_key = f"{var_name}:{context_type}:{context_id}"
|
|
if var_def.cache_duration_minutes > 0:
|
|
cached_value = self._get_cached_value(var_def, cache_key)
|
|
if cached_value is not None:
|
|
return cached_value
|
|
|
|
# Get context-specific value
|
|
context_value = self._get_context_value(var_def.id, context_type, context_id)
|
|
|
|
# Process based on variable type
|
|
if var_def.variable_type == VariableType.CALCULATED:
|
|
value = self._process_calculated_variable(var_def, current_context)
|
|
elif var_def.variable_type == VariableType.CONDITIONAL:
|
|
value = self._process_conditional_variable(var_def, current_context)
|
|
elif var_def.variable_type == VariableType.QUERY:
|
|
value = self._process_query_variable(var_def, current_context, context_type, context_id)
|
|
elif var_def.variable_type == VariableType.LOOKUP:
|
|
value = self._process_lookup_variable(var_def, current_context, context_type, context_id)
|
|
else:
|
|
# Simple variable types (string, number, date, boolean)
|
|
value = context_value if context_value is not None else var_def.default_value
|
|
value = self._convert_value(value, var_def.variable_type)
|
|
|
|
# Apply validation
|
|
if not self._validate_value(value, var_def):
|
|
logger.warning(f"Validation failed for variable {var_name}")
|
|
return var_def.default_value
|
|
|
|
# Cache the result
|
|
if var_def.cache_duration_minutes > 0:
|
|
self._cache_value(var_def, cache_key, value)
|
|
|
|
return value
|
|
|
|
def _process_calculated_variable(
|
|
self,
|
|
var_def: TemplateVariable,
|
|
context: Dict[str, Any]
|
|
) -> Any:
|
|
"""
|
|
Process a calculated variable using its formula
|
|
"""
|
|
if not var_def.formula:
|
|
return var_def.default_value
|
|
|
|
try:
|
|
# Create safe execution environment
|
|
safe_context = {
|
|
**self.safe_functions,
|
|
**context,
|
|
'__builtins__': {} # Disable built-ins for security
|
|
}
|
|
|
|
# Evaluate the formula
|
|
result = eval(var_def.formula, safe_context)
|
|
return result
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error evaluating formula for {var_def.name}: {str(e)}")
|
|
return var_def.default_value
|
|
|
|
def _process_conditional_variable(
|
|
self,
|
|
var_def: TemplateVariable,
|
|
context: Dict[str, Any]
|
|
) -> Any:
|
|
"""
|
|
Process a conditional variable using if/then/else logic
|
|
"""
|
|
if not var_def.conditional_logic:
|
|
return var_def.default_value
|
|
|
|
try:
|
|
logic = var_def.conditional_logic
|
|
if isinstance(logic, str):
|
|
logic = json.loads(logic)
|
|
|
|
# Process conditional rules
|
|
for rule in logic.get('rules', []):
|
|
condition = rule.get('condition')
|
|
if self._evaluate_condition(condition, context):
|
|
return self._convert_value(rule.get('value'), var_def.variable_type)
|
|
|
|
# No conditions matched, return default
|
|
return logic.get('default', var_def.default_value)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error processing conditional logic for {var_def.name}: {str(e)}")
|
|
return var_def.default_value
|
|
|
|
def _process_query_variable(
|
|
self,
|
|
var_def: TemplateVariable,
|
|
context: Dict[str, Any],
|
|
context_type: str,
|
|
context_id: str
|
|
) -> Any:
|
|
"""
|
|
Process a variable that gets its value from a database query
|
|
"""
|
|
if not var_def.data_source_query:
|
|
return var_def.default_value
|
|
|
|
try:
|
|
# Substitute context variables in the query
|
|
query = var_def.data_source_query
|
|
for key, value in context.items():
|
|
query = query.replace(f":{key}", str(value) if value is not None else "NULL")
|
|
|
|
# Add context parameters
|
|
query = query.replace(":context_id", context_id)
|
|
query = query.replace(":context_type", context_type)
|
|
|
|
# Execute query
|
|
result = self.db.execute(text(query)).first()
|
|
if result:
|
|
return result[0] if len(result) == 1 else dict(result)
|
|
return var_def.default_value
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error executing query for {var_def.name}: {str(e)}")
|
|
return var_def.default_value
|
|
|
|
def _process_lookup_variable(
|
|
self,
|
|
var_def: TemplateVariable,
|
|
context: Dict[str, Any],
|
|
context_type: str,
|
|
context_id: str
|
|
) -> Any:
|
|
"""
|
|
Process a variable that looks up values from a reference table
|
|
"""
|
|
if not all([var_def.lookup_table, var_def.lookup_key_field, var_def.lookup_value_field]):
|
|
return var_def.default_value
|
|
|
|
try:
|
|
# Get the lookup key from context
|
|
lookup_key = context.get(var_def.lookup_key_field)
|
|
if lookup_key is None and context_type == "file":
|
|
# Try to get from file context
|
|
file_obj = self.db.query(File).filter(File.file_no == context_id).first()
|
|
if file_obj:
|
|
lookup_key = getattr(file_obj, var_def.lookup_key_field, None)
|
|
|
|
if lookup_key is None:
|
|
return var_def.default_value
|
|
|
|
# Build and execute lookup query
|
|
query = text(f"""
|
|
SELECT {var_def.lookup_value_field}
|
|
FROM {var_def.lookup_table}
|
|
WHERE {var_def.lookup_key_field} = :lookup_key
|
|
LIMIT 1
|
|
""")
|
|
|
|
result = self.db.execute(query, {"lookup_key": lookup_key}).first()
|
|
return result[0] if result else var_def.default_value
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error processing lookup for {var_def.name}: {str(e)}")
|
|
return var_def.default_value
|
|
|
|
def _evaluate_condition(self, condition: Dict[str, Any], context: Dict[str, Any]) -> bool:
|
|
"""
|
|
Evaluate a conditional expression
|
|
"""
|
|
try:
|
|
field = condition.get('field')
|
|
operator_name = condition.get('operator', 'equals')
|
|
expected_value = condition.get('value')
|
|
|
|
actual_value = context.get(field)
|
|
|
|
# Convert values for comparison
|
|
if operator_name in ['equals', 'not_equals']:
|
|
return (actual_value == expected_value) if operator_name == 'equals' else (actual_value != expected_value)
|
|
elif operator_name in ['greater_than', 'less_than', 'greater_equal', 'less_equal']:
|
|
try:
|
|
actual_num = float(actual_value) if actual_value is not None else 0
|
|
expected_num = float(expected_value) if expected_value is not None else 0
|
|
|
|
if operator_name == 'greater_than':
|
|
return actual_num > expected_num
|
|
elif operator_name == 'less_than':
|
|
return actual_num < expected_num
|
|
elif operator_name == 'greater_equal':
|
|
return actual_num >= expected_num
|
|
elif operator_name == 'less_equal':
|
|
return actual_num <= expected_num
|
|
except (ValueError, TypeError):
|
|
return False
|
|
elif operator_name == 'contains':
|
|
return str(expected_value) in str(actual_value) if actual_value else False
|
|
elif operator_name == 'is_empty':
|
|
return actual_value is None or str(actual_value).strip() == ''
|
|
elif operator_name == 'is_not_empty':
|
|
return actual_value is not None and str(actual_value).strip() != ''
|
|
|
|
return False
|
|
|
|
except Exception:
|
|
return False
|
|
|
|
def _determine_processing_order(self, variables: List[str]) -> List[str]:
|
|
"""
|
|
Determine the order to process variables based on dependencies
|
|
"""
|
|
# Get all variable definitions
|
|
var_defs = self.db.query(TemplateVariable).filter(
|
|
TemplateVariable.name.in_(variables),
|
|
TemplateVariable.active == True
|
|
).all()
|
|
|
|
var_deps = {}
|
|
for var_def in var_defs:
|
|
deps = var_def.depends_on or []
|
|
if isinstance(deps, str):
|
|
deps = json.loads(deps)
|
|
var_deps[var_def.name] = [dep for dep in deps if dep in variables]
|
|
|
|
# Topological sort for dependency resolution
|
|
ordered = []
|
|
remaining = set(variables)
|
|
|
|
while remaining:
|
|
# Find variables with no unresolved dependencies
|
|
ready = [var for var in remaining if not any(dep in remaining for dep in var_deps.get(var, []))]
|
|
|
|
if not ready:
|
|
# Circular dependency or missing dependency, add remaining arbitrarily
|
|
ready = list(remaining)
|
|
|
|
ordered.extend(ready)
|
|
remaining -= set(ready)
|
|
|
|
return ordered
|
|
|
|
def _get_context_value(self, variable_id: int, context_type: str, context_id: str) -> Any:
|
|
"""
|
|
Get the context-specific value for a variable
|
|
"""
|
|
context = self.db.query(VariableContext).filter(
|
|
VariableContext.variable_id == variable_id,
|
|
VariableContext.context_type == context_type,
|
|
VariableContext.context_id == context_id
|
|
).first()
|
|
|
|
return context.computed_value if context and context.computed_value else (context.value if context else None)
|
|
|
|
def _convert_value(self, value: Any, var_type: VariableType) -> Any:
|
|
"""
|
|
Convert a value to the appropriate type
|
|
"""
|
|
if value is None:
|
|
return None
|
|
|
|
try:
|
|
if var_type == VariableType.NUMBER:
|
|
return float(value) if '.' in str(value) else int(value)
|
|
elif var_type == VariableType.BOOLEAN:
|
|
if isinstance(value, bool):
|
|
return value
|
|
return str(value).lower() in ('true', '1', 'yes', 'on')
|
|
elif var_type == VariableType.DATE:
|
|
if isinstance(value, date):
|
|
return value
|
|
# Try to parse date string
|
|
from dateutil.parser import parse
|
|
return parse(str(value)).date()
|
|
else:
|
|
return str(value)
|
|
except (ValueError, TypeError):
|
|
return value
|
|
|
|
def _validate_value(self, value: Any, var_def: TemplateVariable) -> bool:
|
|
"""
|
|
Validate a value against the variable's validation rules
|
|
"""
|
|
if var_def.required and (value is None or str(value).strip() == ''):
|
|
return False
|
|
|
|
if not var_def.validation_rules:
|
|
return True
|
|
|
|
try:
|
|
rules = var_def.validation_rules
|
|
if isinstance(rules, str):
|
|
rules = json.loads(rules)
|
|
|
|
# Apply validation rules
|
|
for rule_type, rule_value in rules.items():
|
|
if rule_type == 'min_length' and len(str(value)) < rule_value:
|
|
return False
|
|
elif rule_type == 'max_length' and len(str(value)) > rule_value:
|
|
return False
|
|
elif rule_type == 'pattern' and not re.match(rule_value, str(value)):
|
|
return False
|
|
elif rule_type == 'min_value' and float(value) < rule_value:
|
|
return False
|
|
elif rule_type == 'max_value' and float(value) > rule_value:
|
|
return False
|
|
|
|
return True
|
|
|
|
except Exception:
|
|
return True # Don't fail validation on rule processing errors
|
|
|
|
def _get_cached_value(self, var_def: TemplateVariable, cache_key: str) -> Any:
|
|
"""
|
|
Get cached value if still valid
|
|
"""
|
|
if not var_def.last_cached_at:
|
|
return None
|
|
|
|
cache_age = datetime.now() - var_def.last_cached_at
|
|
if cache_age.total_seconds() > (var_def.cache_duration_minutes * 60):
|
|
return None
|
|
|
|
return var_def.cached_value
|
|
|
|
def _cache_value(self, var_def: TemplateVariable, cache_key: str, value: Any):
|
|
"""
|
|
Cache a computed value
|
|
"""
|
|
var_def.cached_value = str(value) if value is not None else None
|
|
var_def.last_cached_at = datetime.now()
|
|
self.db.commit()
|
|
|
|
def set_variable_value(
|
|
self,
|
|
variable_name: str,
|
|
value: Any,
|
|
context_type: str = "global",
|
|
context_id: str = "default",
|
|
user_name: Optional[str] = None
|
|
) -> bool:
|
|
"""
|
|
Set a variable value in a specific context
|
|
"""
|
|
try:
|
|
var_def = self.db.query(TemplateVariable).filter(
|
|
TemplateVariable.name == variable_name,
|
|
TemplateVariable.active == True
|
|
).first()
|
|
|
|
if not var_def:
|
|
return False
|
|
|
|
# Get or create context
|
|
context = self.db.query(VariableContext).filter(
|
|
VariableContext.variable_id == var_def.id,
|
|
VariableContext.context_type == context_type,
|
|
VariableContext.context_id == context_id
|
|
).first()
|
|
|
|
old_value = context.value if context else None
|
|
|
|
if not context:
|
|
context = VariableContext(
|
|
variable_id=var_def.id,
|
|
context_type=context_type,
|
|
context_id=context_id,
|
|
value=str(value) if value is not None else None,
|
|
source="manual"
|
|
)
|
|
self.db.add(context)
|
|
else:
|
|
context.value = str(value) if value is not None else None
|
|
|
|
# Validate the value
|
|
converted_value = self._convert_value(value, var_def.variable_type)
|
|
context.is_valid = self._validate_value(converted_value, var_def)
|
|
|
|
# Log the change
|
|
audit_log = VariableAuditLog(
|
|
variable_id=var_def.id,
|
|
context_type=context_type,
|
|
context_id=context_id,
|
|
old_value=old_value,
|
|
new_value=context.value,
|
|
change_type="updated",
|
|
changed_by=user_name
|
|
)
|
|
self.db.add(audit_log)
|
|
|
|
self.db.commit()
|
|
return True
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error setting variable {variable_name}: {str(e)}")
|
|
self.db.rollback()
|
|
return False
|
|
|
|
def get_variables_for_template(self, template_id: int) -> List[Dict[str, Any]]:
|
|
"""
|
|
Get all variables associated with a template
|
|
"""
|
|
variables = self.db.query(TemplateVariable, VariableTemplate).join(
|
|
VariableTemplate, VariableTemplate.variable_id == TemplateVariable.id
|
|
).filter(
|
|
VariableTemplate.template_id == template_id,
|
|
TemplateVariable.active == True
|
|
).order_by(VariableTemplate.display_order, TemplateVariable.name).all()
|
|
|
|
result = []
|
|
for var_def, var_template in variables:
|
|
result.append({
|
|
'id': var_def.id,
|
|
'name': var_def.name,
|
|
'display_name': var_def.display_name or var_def.name,
|
|
'description': var_def.description,
|
|
'type': var_def.variable_type.value,
|
|
'required': var_template.override_required if var_template.override_required is not None else var_def.required,
|
|
'default_value': var_template.override_default or var_def.default_value,
|
|
'group_name': var_template.group_name,
|
|
'validation_rules': var_def.validation_rules
|
|
})
|
|
|
|
return result
|