changes
This commit is contained in:
HotSwapp
571
app/services/advanced_variables.py
Normal file
571
app/services/advanced_variables.py
Normal file
@@ -0,0 +1,571 @@
|
||||
"""
|
||||
Advanced Variable Resolution Service
|
||||
|
||||
This service handles complex variable processing including:
|
||||
- Conditional logic evaluation
|
||||
- Mathematical calculations and formulas
|
||||
- Dynamic data source queries
|
||||
- Variable dependency resolution
|
||||
- Caching and performance optimization
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import json
|
||||
import math
|
||||
import operator
|
||||
from datetime import datetime, date, timedelta
|
||||
from typing import Dict, Any, List, Optional, Tuple, Union
|
||||
from decimal import Decimal, InvalidOperation
|
||||
import logging
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.models.template_variables import (
|
||||
TemplateVariable, VariableContext, VariableAuditLog,
|
||||
VariableType, VariableTemplate
|
||||
)
|
||||
from app.models.files import File
|
||||
from app.models.rolodex import Rolodex
|
||||
from app.core.logging import get_logger
|
||||
|
||||
logger = get_logger("advanced_variables")
|
||||
|
||||
|
||||
class VariableProcessor:
|
||||
"""
|
||||
Handles advanced variable processing with conditional logic, calculations, and data sources
|
||||
"""
|
||||
|
||||
def __init__(self, db: Session):
|
||||
self.db = db
|
||||
self._cache: Dict[str, Any] = {}
|
||||
|
||||
# Safe functions available in formula expressions
|
||||
self.safe_functions = {
|
||||
'abs': abs,
|
||||
'round': round,
|
||||
'min': min,
|
||||
'max': max,
|
||||
'sum': sum,
|
||||
'len': len,
|
||||
'str': str,
|
||||
'int': int,
|
||||
'float': float,
|
||||
'math_ceil': math.ceil,
|
||||
'math_floor': math.floor,
|
||||
'math_sqrt': math.sqrt,
|
||||
'today': lambda: date.today(),
|
||||
'now': lambda: datetime.now(),
|
||||
'days_between': lambda d1, d2: (d1 - d2).days if isinstance(d1, date) and isinstance(d2, date) else 0,
|
||||
'format_currency': lambda x: f"${float(x):,.2f}" if x is not None else "$0.00",
|
||||
'format_date': lambda d, fmt='%B %d, %Y': d.strftime(fmt) if isinstance(d, date) else str(d),
|
||||
}
|
||||
|
||||
# Safe operators for formula evaluation
|
||||
self.operators = {
|
||||
'+': operator.add,
|
||||
'-': operator.sub,
|
||||
'*': operator.mul,
|
||||
'/': operator.truediv,
|
||||
'//': operator.floordiv,
|
||||
'%': operator.mod,
|
||||
'**': operator.pow,
|
||||
'==': operator.eq,
|
||||
'!=': operator.ne,
|
||||
'<': operator.lt,
|
||||
'<=': operator.le,
|
||||
'>': operator.gt,
|
||||
'>=': operator.ge,
|
||||
'and': operator.and_,
|
||||
'or': operator.or_,
|
||||
'not': operator.not_,
|
||||
}
|
||||
|
||||
def resolve_variables(
|
||||
self,
|
||||
variables: List[str],
|
||||
context_type: str = "global",
|
||||
context_id: str = "default",
|
||||
base_context: Optional[Dict[str, Any]] = None
|
||||
) -> Tuple[Dict[str, Any], List[str]]:
|
||||
"""
|
||||
Resolve a list of variables with their current values
|
||||
|
||||
Args:
|
||||
variables: List of variable names to resolve
|
||||
context_type: Context type (file, client, global, etc.)
|
||||
context_id: Specific context identifier
|
||||
base_context: Additional context values to use
|
||||
|
||||
Returns:
|
||||
Tuple of (resolved_variables, unresolved_variables)
|
||||
"""
|
||||
resolved = {}
|
||||
unresolved = []
|
||||
processing_order = self._determine_processing_order(variables)
|
||||
|
||||
# Start with base context
|
||||
if base_context:
|
||||
resolved.update(base_context)
|
||||
|
||||
for var_name in processing_order:
|
||||
try:
|
||||
value = self._resolve_single_variable(
|
||||
var_name, context_type, context_id, resolved
|
||||
)
|
||||
if value is not None:
|
||||
resolved[var_name] = value
|
||||
else:
|
||||
unresolved.append(var_name)
|
||||
except Exception as e:
|
||||
logger.error(f"Error resolving variable {var_name}: {str(e)}")
|
||||
unresolved.append(var_name)
|
||||
|
||||
return resolved, unresolved
|
||||
|
||||
def _resolve_single_variable(
|
||||
self,
|
||||
var_name: str,
|
||||
context_type: str,
|
||||
context_id: str,
|
||||
current_context: Dict[str, Any]
|
||||
) -> Any:
|
||||
"""
|
||||
Resolve a single variable based on its type and configuration
|
||||
"""
|
||||
# Get variable definition
|
||||
var_def = self.db.query(TemplateVariable).filter(
|
||||
TemplateVariable.name == var_name,
|
||||
TemplateVariable.active == True
|
||||
).first()
|
||||
|
||||
if not var_def:
|
||||
return None
|
||||
|
||||
# Check for static value first
|
||||
if var_def.static_value is not None:
|
||||
return self._convert_value(var_def.static_value, var_def.variable_type)
|
||||
|
||||
# Check cache if enabled
|
||||
cache_key = f"{var_name}:{context_type}:{context_id}"
|
||||
if var_def.cache_duration_minutes > 0:
|
||||
cached_value = self._get_cached_value(var_def, cache_key)
|
||||
if cached_value is not None:
|
||||
return cached_value
|
||||
|
||||
# Get context-specific value
|
||||
context_value = self._get_context_value(var_def.id, context_type, context_id)
|
||||
|
||||
# Process based on variable type
|
||||
if var_def.variable_type == VariableType.CALCULATED:
|
||||
value = self._process_calculated_variable(var_def, current_context)
|
||||
elif var_def.variable_type == VariableType.CONDITIONAL:
|
||||
value = self._process_conditional_variable(var_def, current_context)
|
||||
elif var_def.variable_type == VariableType.QUERY:
|
||||
value = self._process_query_variable(var_def, current_context, context_type, context_id)
|
||||
elif var_def.variable_type == VariableType.LOOKUP:
|
||||
value = self._process_lookup_variable(var_def, current_context, context_type, context_id)
|
||||
else:
|
||||
# Simple variable types (string, number, date, boolean)
|
||||
value = context_value if context_value is not None else var_def.default_value
|
||||
value = self._convert_value(value, var_def.variable_type)
|
||||
|
||||
# Apply validation
|
||||
if not self._validate_value(value, var_def):
|
||||
logger.warning(f"Validation failed for variable {var_name}")
|
||||
return var_def.default_value
|
||||
|
||||
# Cache the result
|
||||
if var_def.cache_duration_minutes > 0:
|
||||
self._cache_value(var_def, cache_key, value)
|
||||
|
||||
return value
|
||||
|
||||
def _process_calculated_variable(
|
||||
self,
|
||||
var_def: TemplateVariable,
|
||||
context: Dict[str, Any]
|
||||
) -> Any:
|
||||
"""
|
||||
Process a calculated variable using its formula
|
||||
"""
|
||||
if not var_def.formula:
|
||||
return var_def.default_value
|
||||
|
||||
try:
|
||||
# Create safe execution environment
|
||||
safe_context = {
|
||||
**self.safe_functions,
|
||||
**context,
|
||||
'__builtins__': {} # Disable built-ins for security
|
||||
}
|
||||
|
||||
# Evaluate the formula
|
||||
result = eval(var_def.formula, safe_context)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error evaluating formula for {var_def.name}: {str(e)}")
|
||||
return var_def.default_value
|
||||
|
||||
def _process_conditional_variable(
|
||||
self,
|
||||
var_def: TemplateVariable,
|
||||
context: Dict[str, Any]
|
||||
) -> Any:
|
||||
"""
|
||||
Process a conditional variable using if/then/else logic
|
||||
"""
|
||||
if not var_def.conditional_logic:
|
||||
return var_def.default_value
|
||||
|
||||
try:
|
||||
logic = var_def.conditional_logic
|
||||
if isinstance(logic, str):
|
||||
logic = json.loads(logic)
|
||||
|
||||
# Process conditional rules
|
||||
for rule in logic.get('rules', []):
|
||||
condition = rule.get('condition')
|
||||
if self._evaluate_condition(condition, context):
|
||||
return self._convert_value(rule.get('value'), var_def.variable_type)
|
||||
|
||||
# No conditions matched, return default
|
||||
return logic.get('default', var_def.default_value)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing conditional logic for {var_def.name}: {str(e)}")
|
||||
return var_def.default_value
|
||||
|
||||
def _process_query_variable(
|
||||
self,
|
||||
var_def: TemplateVariable,
|
||||
context: Dict[str, Any],
|
||||
context_type: str,
|
||||
context_id: str
|
||||
) -> Any:
|
||||
"""
|
||||
Process a variable that gets its value from a database query
|
||||
"""
|
||||
if not var_def.data_source_query:
|
||||
return var_def.default_value
|
||||
|
||||
try:
|
||||
# Substitute context variables in the query
|
||||
query = var_def.data_source_query
|
||||
for key, value in context.items():
|
||||
query = query.replace(f":{key}", str(value) if value is not None else "NULL")
|
||||
|
||||
# Add context parameters
|
||||
query = query.replace(":context_id", context_id)
|
||||
query = query.replace(":context_type", context_type)
|
||||
|
||||
# Execute query
|
||||
result = self.db.execute(text(query)).first()
|
||||
if result:
|
||||
return result[0] if len(result) == 1 else dict(result)
|
||||
return var_def.default_value
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error executing query for {var_def.name}: {str(e)}")
|
||||
return var_def.default_value
|
||||
|
||||
def _process_lookup_variable(
|
||||
self,
|
||||
var_def: TemplateVariable,
|
||||
context: Dict[str, Any],
|
||||
context_type: str,
|
||||
context_id: str
|
||||
) -> Any:
|
||||
"""
|
||||
Process a variable that looks up values from a reference table
|
||||
"""
|
||||
if not all([var_def.lookup_table, var_def.lookup_key_field, var_def.lookup_value_field]):
|
||||
return var_def.default_value
|
||||
|
||||
try:
|
||||
# Get the lookup key from context
|
||||
lookup_key = context.get(var_def.lookup_key_field)
|
||||
if lookup_key is None and context_type == "file":
|
||||
# Try to get from file context
|
||||
file_obj = self.db.query(File).filter(File.file_no == context_id).first()
|
||||
if file_obj:
|
||||
lookup_key = getattr(file_obj, var_def.lookup_key_field, None)
|
||||
|
||||
if lookup_key is None:
|
||||
return var_def.default_value
|
||||
|
||||
# Build and execute lookup query
|
||||
query = text(f"""
|
||||
SELECT {var_def.lookup_value_field}
|
||||
FROM {var_def.lookup_table}
|
||||
WHERE {var_def.lookup_key_field} = :lookup_key
|
||||
LIMIT 1
|
||||
""")
|
||||
|
||||
result = self.db.execute(query, {"lookup_key": lookup_key}).first()
|
||||
return result[0] if result else var_def.default_value
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing lookup for {var_def.name}: {str(e)}")
|
||||
return var_def.default_value
|
||||
|
||||
def _evaluate_condition(self, condition: Dict[str, Any], context: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
Evaluate a conditional expression
|
||||
"""
|
||||
try:
|
||||
field = condition.get('field')
|
||||
operator_name = condition.get('operator', 'equals')
|
||||
expected_value = condition.get('value')
|
||||
|
||||
actual_value = context.get(field)
|
||||
|
||||
# Convert values for comparison
|
||||
if operator_name in ['equals', 'not_equals']:
|
||||
return (actual_value == expected_value) if operator_name == 'equals' else (actual_value != expected_value)
|
||||
elif operator_name in ['greater_than', 'less_than', 'greater_equal', 'less_equal']:
|
||||
try:
|
||||
actual_num = float(actual_value) if actual_value is not None else 0
|
||||
expected_num = float(expected_value) if expected_value is not None else 0
|
||||
|
||||
if operator_name == 'greater_than':
|
||||
return actual_num > expected_num
|
||||
elif operator_name == 'less_than':
|
||||
return actual_num < expected_num
|
||||
elif operator_name == 'greater_equal':
|
||||
return actual_num >= expected_num
|
||||
elif operator_name == 'less_equal':
|
||||
return actual_num <= expected_num
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
elif operator_name == 'contains':
|
||||
return str(expected_value) in str(actual_value) if actual_value else False
|
||||
elif operator_name == 'is_empty':
|
||||
return actual_value is None or str(actual_value).strip() == ''
|
||||
elif operator_name == 'is_not_empty':
|
||||
return actual_value is not None and str(actual_value).strip() != ''
|
||||
|
||||
return False
|
||||
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _determine_processing_order(self, variables: List[str]) -> List[str]:
|
||||
"""
|
||||
Determine the order to process variables based on dependencies
|
||||
"""
|
||||
# Get all variable definitions
|
||||
var_defs = self.db.query(TemplateVariable).filter(
|
||||
TemplateVariable.name.in_(variables),
|
||||
TemplateVariable.active == True
|
||||
).all()
|
||||
|
||||
var_deps = {}
|
||||
for var_def in var_defs:
|
||||
deps = var_def.depends_on or []
|
||||
if isinstance(deps, str):
|
||||
deps = json.loads(deps)
|
||||
var_deps[var_def.name] = [dep for dep in deps if dep in variables]
|
||||
|
||||
# Topological sort for dependency resolution
|
||||
ordered = []
|
||||
remaining = set(variables)
|
||||
|
||||
while remaining:
|
||||
# Find variables with no unresolved dependencies
|
||||
ready = [var for var in remaining if not any(dep in remaining for dep in var_deps.get(var, []))]
|
||||
|
||||
if not ready:
|
||||
# Circular dependency or missing dependency, add remaining arbitrarily
|
||||
ready = list(remaining)
|
||||
|
||||
ordered.extend(ready)
|
||||
remaining -= set(ready)
|
||||
|
||||
return ordered
|
||||
|
||||
def _get_context_value(self, variable_id: int, context_type: str, context_id: str) -> Any:
|
||||
"""
|
||||
Get the context-specific value for a variable
|
||||
"""
|
||||
context = self.db.query(VariableContext).filter(
|
||||
VariableContext.variable_id == variable_id,
|
||||
VariableContext.context_type == context_type,
|
||||
VariableContext.context_id == context_id
|
||||
).first()
|
||||
|
||||
return context.computed_value if context and context.computed_value else (context.value if context else None)
|
||||
|
||||
def _convert_value(self, value: Any, var_type: VariableType) -> Any:
|
||||
"""
|
||||
Convert a value to the appropriate type
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
if var_type == VariableType.NUMBER:
|
||||
return float(value) if '.' in str(value) else int(value)
|
||||
elif var_type == VariableType.BOOLEAN:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
return str(value).lower() in ('true', '1', 'yes', 'on')
|
||||
elif var_type == VariableType.DATE:
|
||||
if isinstance(value, date):
|
||||
return value
|
||||
# Try to parse date string
|
||||
from dateutil.parser import parse
|
||||
return parse(str(value)).date()
|
||||
else:
|
||||
return str(value)
|
||||
except (ValueError, TypeError):
|
||||
return value
|
||||
|
||||
def _validate_value(self, value: Any, var_def: TemplateVariable) -> bool:
|
||||
"""
|
||||
Validate a value against the variable's validation rules
|
||||
"""
|
||||
if var_def.required and (value is None or str(value).strip() == ''):
|
||||
return False
|
||||
|
||||
if not var_def.validation_rules:
|
||||
return True
|
||||
|
||||
try:
|
||||
rules = var_def.validation_rules
|
||||
if isinstance(rules, str):
|
||||
rules = json.loads(rules)
|
||||
|
||||
# Apply validation rules
|
||||
for rule_type, rule_value in rules.items():
|
||||
if rule_type == 'min_length' and len(str(value)) < rule_value:
|
||||
return False
|
||||
elif rule_type == 'max_length' and len(str(value)) > rule_value:
|
||||
return False
|
||||
elif rule_type == 'pattern' and not re.match(rule_value, str(value)):
|
||||
return False
|
||||
elif rule_type == 'min_value' and float(value) < rule_value:
|
||||
return False
|
||||
elif rule_type == 'max_value' and float(value) > rule_value:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
except Exception:
|
||||
return True # Don't fail validation on rule processing errors
|
||||
|
||||
def _get_cached_value(self, var_def: TemplateVariable, cache_key: str) -> Any:
|
||||
"""
|
||||
Get cached value if still valid
|
||||
"""
|
||||
if not var_def.last_cached_at:
|
||||
return None
|
||||
|
||||
cache_age = datetime.now() - var_def.last_cached_at
|
||||
if cache_age.total_seconds() > (var_def.cache_duration_minutes * 60):
|
||||
return None
|
||||
|
||||
return var_def.cached_value
|
||||
|
||||
def _cache_value(self, var_def: TemplateVariable, cache_key: str, value: Any):
|
||||
"""
|
||||
Cache a computed value
|
||||
"""
|
||||
var_def.cached_value = str(value) if value is not None else None
|
||||
var_def.last_cached_at = datetime.now()
|
||||
self.db.commit()
|
||||
|
||||
def set_variable_value(
|
||||
self,
|
||||
variable_name: str,
|
||||
value: Any,
|
||||
context_type: str = "global",
|
||||
context_id: str = "default",
|
||||
user_name: Optional[str] = None
|
||||
) -> bool:
|
||||
"""
|
||||
Set a variable value in a specific context
|
||||
"""
|
||||
try:
|
||||
var_def = self.db.query(TemplateVariable).filter(
|
||||
TemplateVariable.name == variable_name,
|
||||
TemplateVariable.active == True
|
||||
).first()
|
||||
|
||||
if not var_def:
|
||||
return False
|
||||
|
||||
# Get or create context
|
||||
context = self.db.query(VariableContext).filter(
|
||||
VariableContext.variable_id == var_def.id,
|
||||
VariableContext.context_type == context_type,
|
||||
VariableContext.context_id == context_id
|
||||
).first()
|
||||
|
||||
old_value = context.value if context else None
|
||||
|
||||
if not context:
|
||||
context = VariableContext(
|
||||
variable_id=var_def.id,
|
||||
context_type=context_type,
|
||||
context_id=context_id,
|
||||
value=str(value) if value is not None else None,
|
||||
source="manual"
|
||||
)
|
||||
self.db.add(context)
|
||||
else:
|
||||
context.value = str(value) if value is not None else None
|
||||
|
||||
# Validate the value
|
||||
converted_value = self._convert_value(value, var_def.variable_type)
|
||||
context.is_valid = self._validate_value(converted_value, var_def)
|
||||
|
||||
# Log the change
|
||||
audit_log = VariableAuditLog(
|
||||
variable_id=var_def.id,
|
||||
context_type=context_type,
|
||||
context_id=context_id,
|
||||
old_value=old_value,
|
||||
new_value=context.value,
|
||||
change_type="updated",
|
||||
changed_by=user_name
|
||||
)
|
||||
self.db.add(audit_log)
|
||||
|
||||
self.db.commit()
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error setting variable {variable_name}: {str(e)}")
|
||||
self.db.rollback()
|
||||
return False
|
||||
|
||||
def get_variables_for_template(self, template_id: int) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get all variables associated with a template
|
||||
"""
|
||||
variables = self.db.query(TemplateVariable, VariableTemplate).join(
|
||||
VariableTemplate, VariableTemplate.variable_id == TemplateVariable.id
|
||||
).filter(
|
||||
VariableTemplate.template_id == template_id,
|
||||
TemplateVariable.active == True
|
||||
).order_by(VariableTemplate.display_order, TemplateVariable.name).all()
|
||||
|
||||
result = []
|
||||
for var_def, var_template in variables:
|
||||
result.append({
|
||||
'id': var_def.id,
|
||||
'name': var_def.name,
|
||||
'display_name': var_def.display_name or var_def.name,
|
||||
'description': var_def.description,
|
||||
'type': var_def.variable_type.value,
|
||||
'required': var_template.override_required if var_template.override_required is not None else var_def.required,
|
||||
'default_value': var_template.override_default or var_def.default_value,
|
||||
'group_name': var_template.group_name,
|
||||
'validation_rules': var_def.validation_rules
|
||||
})
|
||||
|
||||
return result
|
||||
Reference in New Issue
Block a user