changes

2025-08-18 20:20:04 -05:00
parent 89b2bc0aa2
commit bac8cc4bd5
114 changed files with 30258 additions and 1341 deletions
--- a/app/models/audit_enhanced.py
+++ b/app/models/audit_enhanced.py
@@ -0,0 +1,388 @@
+"""
+Enhanced audit logging models for P2 security features
+"""
+from datetime import datetime, timezone
+from typing import Optional, Dict, Any
+from enum import Enum
+import json
+
+from sqlalchemy import Column, Integer, String, DateTime, Boolean, ForeignKey, Text, Index
+from sqlalchemy.orm import relationship
+from sqlalchemy.dialects.postgresql import JSONB
+
+from app.models.base import BaseModel
+
+
+class SecurityEventType(str, Enum):
+    """Security event types for classification"""
+    # Authentication events
+    LOGIN_SUCCESS = "login_success"
+    LOGIN_FAILURE = "login_failure"
+    LOGOUT = "logout"
+    SESSION_EXPIRED = "session_expired"
+    PASSWORD_CHANGE = "password_change"
+    ACCOUNT_LOCKED = "account_locked"
+    
+    # Authorization events
+    ACCESS_DENIED = "access_denied"
+    PRIVILEGE_ESCALATION = "privilege_escalation"
+    UNAUTHORIZED_ACCESS = "unauthorized_access"
+    
+    # Data access events
+    DATA_READ = "data_read"
+    DATA_WRITE = "data_write"
+    DATA_DELETE = "data_delete"
+    DATA_EXPORT = "data_export"
+    BULK_OPERATION = "bulk_operation"
+    
+    # System events
+    CONFIGURATION_CHANGE = "configuration_change"
+    USER_CREATION = "user_creation"
+    USER_MODIFICATION = "user_modification"
+    USER_DELETION = "user_deletion"
+    
+    # Security events
+    SUSPICIOUS_ACTIVITY = "suspicious_activity"
+    ATTACK_DETECTED = "attack_detected"
+    SECURITY_VIOLATION = "security_violation"
+    IP_BLOCKED = "ip_blocked"
+    
+    # File events
+    FILE_UPLOAD = "file_upload"
+    FILE_DOWNLOAD = "file_download"
+    FILE_DELETION = "file_deletion"
+    FILE_MODIFICATION = "file_modification"
+    
+    # Integration events
+    API_ACCESS = "api_access"
+    EXTERNAL_SERVICE = "external_service"
+    IMPORT_OPERATION = "import_operation"
+
+
+class SecurityEventSeverity(str, Enum):
+    """Security event severity levels"""
+    CRITICAL = "critical"
+    HIGH = "high"
+    MEDIUM = "medium"
+    LOW = "low"
+    INFO = "info"
+
+
+class ComplianceStandard(str, Enum):
+    """Compliance standards for reporting"""
+    SOX = "sox"  # Sarbanes-Oxley
+    HIPAA = "hipaa"  # Health Insurance Portability and Accountability Act
+    GDPR = "gdpr"  # General Data Protection Regulation
+    SOC2 = "soc2"  # Service Organization Control 2
+    ISO27001 = "iso27001"  # Information Security Management
+    NIST = "nist"  # National Institute of Standards and Technology
+
+
+class EnhancedAuditLog(BaseModel):
+    """
+    Enhanced audit logging for comprehensive security monitoring
+    """
+    __tablename__ = "enhanced_audit_logs"
+
+    id = Column(Integer, primary_key=True, autoincrement=True, index=True)
+    
+    # Event identification
+    event_id = Column(String(64), nullable=False, unique=True, index=True)
+    event_type = Column(String(50), nullable=False, index=True)
+    event_category = Column(String(30), nullable=False, index=True)  # security, audit, compliance, system
+    severity = Column(String(20), nullable=False, index=True)
+    
+    # Event details
+    title = Column(String(255), nullable=False)
+    description = Column(Text, nullable=False)
+    outcome = Column(String(20), nullable=False, index=True)  # success, failure, error, blocked
+    
+    # User and session context
+    user_id = Column(Integer, ForeignKey("users.id"), nullable=True, index=True)
+    session_id = Column(String(128), nullable=True, index=True)
+    impersonated_user_id = Column(Integer, ForeignKey("users.id"), nullable=True)
+    
+    # Network context
+    source_ip = Column(String(45), nullable=True, index=True)
+    user_agent = Column(Text, nullable=True)
+    request_id = Column(String(64), nullable=True, index=True)
+    
+    # Geographic context
+    country = Column(String(5), nullable=True)
+    region = Column(String(100), nullable=True)
+    city = Column(String(100), nullable=True)
+    
+    # Technical context
+    endpoint = Column(String(255), nullable=True, index=True)
+    http_method = Column(String(10), nullable=True)
+    status_code = Column(Integer, nullable=True)
+    response_time_ms = Column(Integer, nullable=True)
+    
+    # Resource context
+    resource_type = Column(String(50), nullable=True, index=True)  # file, customer, document, etc.
+    resource_id = Column(String(100), nullable=True, index=True)
+    resource_name = Column(String(255), nullable=True)
+    
+    # Data context
+    data_before = Column(Text, nullable=True)  # JSON string of previous state
+    data_after = Column(Text, nullable=True)   # JSON string of new state
+    data_volume = Column(Integer, nullable=True)  # Bytes processed
+    record_count = Column(Integer, nullable=True)  # Number of records affected
+    
+    # Risk assessment
+    risk_score = Column(Integer, default=0, nullable=False, index=True)  # 0-100
+    risk_factors = Column(Text, nullable=True)  # JSON array of risk indicators
+    threat_indicators = Column(Text, nullable=True)  # JSON array of threat patterns
+    
+    # Compliance tracking
+    compliance_standards = Column(Text, nullable=True)  # JSON array of applicable standards
+    retention_period_days = Column(Integer, default=2555, nullable=False)  # 7 years default
+    
+    # Timestamp and tracking
+    timestamp = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False, index=True)
+    processed_at = Column(DateTime(timezone=True), nullable=True)
+    correlation_id = Column(String(64), nullable=True, index=True)  # For related events
+    
+    # Additional metadata
+    tags = Column(Text, nullable=True)  # JSON array of tags for categorization
+    custom_fields = Column(Text, nullable=True)  # JSON object for custom data
+    
+    # Relationships
+    user = relationship("User", foreign_keys=[user_id])
+    impersonated_user = relationship("User", foreign_keys=[impersonated_user_id])
+    
+    def set_data_before(self, data: Dict[str, Any]) -> None:
+        """Set data before change as JSON"""
+        self.data_before = json.dumps(data, default=str) if data else None
+    
+    def set_data_after(self, data: Dict[str, Any]) -> None:
+        """Set data after change as JSON"""
+        self.data_after = json.dumps(data, default=str) if data else None
+    
+    def get_data_before(self) -> Optional[Dict[str, Any]]:
+        """Get data before change from JSON"""
+        return json.loads(self.data_before) if self.data_before else None
+    
+    def get_data_after(self) -> Optional[Dict[str, Any]]:
+        """Get data after change from JSON"""
+        return json.loads(self.data_after) if self.data_after else None
+    
+    def set_risk_factors(self, factors: list) -> None:
+        """Set risk factors as JSON"""
+        self.risk_factors = json.dumps(factors) if factors else None
+    
+    def get_risk_factors(self) -> list:
+        """Get risk factors from JSON"""
+        return json.loads(self.risk_factors) if self.risk_factors else []
+    
+    def set_threat_indicators(self, indicators: list) -> None:
+        """Set threat indicators as JSON"""
+        self.threat_indicators = json.dumps(indicators) if indicators else None
+    
+    def get_threat_indicators(self) -> list:
+        """Get threat indicators from JSON"""
+        return json.loads(self.threat_indicators) if self.threat_indicators else []
+    
+    def set_compliance_standards(self, standards: list) -> None:
+        """Set compliance standards as JSON"""
+        self.compliance_standards = json.dumps(standards) if standards else None
+    
+    def get_compliance_standards(self) -> list:
+        """Get compliance standards from JSON"""
+        return json.loads(self.compliance_standards) if self.compliance_standards else []
+    
+    def set_tags(self, tags: list) -> None:
+        """Set tags as JSON"""
+        self.tags = json.dumps(tags) if tags else None
+    
+    def get_tags(self) -> list:
+        """Get tags from JSON"""
+        return json.loads(self.tags) if self.tags else []
+    
+    def set_custom_fields(self, fields: Dict[str, Any]) -> None:
+        """Set custom fields as JSON"""
+        self.custom_fields = json.dumps(fields, default=str) if fields else None
+    
+    def get_custom_fields(self) -> Optional[Dict[str, Any]]:
+        """Get custom fields from JSON"""
+        return json.loads(self.custom_fields) if self.custom_fields else None
+
+    # Add indexes for performance
+    __table_args__ = (
+        Index('idx_enhanced_audit_user_timestamp', 'user_id', 'timestamp'),
+        Index('idx_enhanced_audit_event_severity', 'event_type', 'severity'),
+        Index('idx_enhanced_audit_resource', 'resource_type', 'resource_id'),
+        Index('idx_enhanced_audit_ip_timestamp', 'source_ip', 'timestamp'),
+        Index('idx_enhanced_audit_correlation', 'correlation_id'),
+        Index('idx_enhanced_audit_risk_score', 'risk_score'),
+        Index('idx_enhanced_audit_compliance', 'compliance_standards'),
+    )
+
+
+class SecurityAlert(BaseModel):
+    """
+    Security alerts for real-time monitoring and incident response
+    """
+    __tablename__ = "security_alerts"
+
+    id = Column(Integer, primary_key=True, autoincrement=True, index=True)
+    
+    # Alert identification
+    alert_id = Column(String(64), nullable=False, unique=True, index=True)
+    rule_id = Column(String(64), nullable=False, index=True)
+    rule_name = Column(String(255), nullable=False)
+    
+    # Alert details
+    title = Column(String(255), nullable=False)
+    description = Column(Text, nullable=False)
+    severity = Column(String(20), nullable=False, index=True)
+    confidence = Column(Integer, default=100, nullable=False)  # 0-100 confidence score
+    
+    # Context
+    event_count = Column(Integer, default=1, nullable=False)  # Number of triggering events
+    time_window_minutes = Column(Integer, nullable=True)  # Time window for correlation
+    affected_users = Column(Text, nullable=True)  # JSON array of user IDs
+    affected_resources = Column(Text, nullable=True)  # JSON array of resource identifiers
+    
+    # Response tracking
+    status = Column(String(20), default="open", nullable=False, index=True)  # open, investigating, resolved, false_positive
+    assigned_to = Column(Integer, ForeignKey("users.id"), nullable=True)
+    resolved_by = Column(Integer, ForeignKey("users.id"), nullable=True)
+    resolution_notes = Column(Text, nullable=True)
+    
+    # Timestamps
+    first_seen = Column(DateTime(timezone=True), nullable=False, index=True)
+    last_seen = Column(DateTime(timezone=True), nullable=False, index=True)
+    acknowledged_at = Column(DateTime(timezone=True), nullable=True)
+    resolved_at = Column(DateTime(timezone=True), nullable=True)
+    
+    # Related audit logs
+    triggering_events = Column(Text, nullable=True)  # JSON array of audit log IDs
+    
+    # Additional metadata
+    tags = Column(Text, nullable=True)  # JSON array of tags
+    custom_fields = Column(Text, nullable=True)  # JSON object for custom data
+    
+    # Relationships
+    assignee = relationship("User", foreign_keys=[assigned_to])
+    resolver = relationship("User", foreign_keys=[resolved_by])
+
+
+class ComplianceReport(BaseModel):
+    """
+    Compliance reporting for various standards
+    """
+    __tablename__ = "compliance_reports"
+
+    id = Column(Integer, primary_key=True, autoincrement=True, index=True)
+    
+    # Report identification
+    report_id = Column(String(64), nullable=False, unique=True, index=True)
+    standard = Column(String(50), nullable=False, index=True)
+    report_type = Column(String(50), nullable=False, index=True)  # periodic, on_demand, incident
+    
+    # Report details
+    title = Column(String(255), nullable=False)
+    description = Column(Text, nullable=True)
+    
+    # Time range
+    start_date = Column(DateTime(timezone=True), nullable=False, index=True)
+    end_date = Column(DateTime(timezone=True), nullable=False, index=True)
+    
+    # Report content
+    summary = Column(Text, nullable=True)  # JSON summary of findings
+    details = Column(Text, nullable=True)  # JSON detailed findings
+    recommendations = Column(Text, nullable=True)  # JSON recommendations
+    
+    # Metrics
+    total_events = Column(Integer, default=0, nullable=False)
+    security_events = Column(Integer, default=0, nullable=False)
+    violations = Column(Integer, default=0, nullable=False)
+    high_risk_events = Column(Integer, default=0, nullable=False)
+    
+    # Status
+    status = Column(String(20), default="generating", nullable=False, index=True)  # generating, ready, delivered, archived
+    generated_by = Column(Integer, ForeignKey("users.id"), nullable=False)
+    generated_at = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False)
+    
+    # Delivery
+    recipients = Column(Text, nullable=True)  # JSON array of recipient emails
+    delivered_at = Column(DateTime(timezone=True), nullable=True)
+    
+    # File storage
+    file_path = Column(String(500), nullable=True)  # Path to generated report file
+    file_size = Column(Integer, nullable=True)  # File size in bytes
+    
+    # Relationships
+    generator = relationship("User", foreign_keys=[generated_by])
+
+
+class AuditRetentionPolicy(BaseModel):
+    """
+    Audit log retention policies for compliance
+    """
+    __tablename__ = "audit_retention_policies"
+
+    id = Column(Integer, primary_key=True, autoincrement=True, index=True)
+    
+    # Policy identification
+    policy_name = Column(String(255), nullable=False, unique=True, index=True)
+    event_types = Column(Text, nullable=True)  # JSON array of event types to apply to
+    compliance_standards = Column(Text, nullable=True)  # JSON array of applicable standards
+    
+    # Retention settings
+    retention_days = Column(Integer, nullable=False)  # Days to retain
+    archive_after_days = Column(Integer, nullable=True)  # Days before archiving
+    
+    # Policy details
+    description = Column(Text, nullable=True)
+    is_active = Column(Boolean, default=True, nullable=False)
+    priority = Column(Integer, default=100, nullable=False)  # Higher priority = more specific
+    
+    # Timestamps
+    created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False)
+    created_by = Column(Integer, ForeignKey("users.id"), nullable=False)
+    
+    # Relationships
+    creator = relationship("User")
+
+
+class SIEMIntegration(BaseModel):
+    """
+    SIEM integration configuration and status
+    """
+    __tablename__ = "siem_integrations"
+
+    id = Column(Integer, primary_key=True, autoincrement=True, index=True)
+    
+    # Integration identification
+    integration_name = Column(String(255), nullable=False, unique=True, index=True)
+    siem_type = Column(String(50), nullable=False, index=True)  # splunk, elk, qradar, etc.
+    
+    # Configuration
+    endpoint_url = Column(String(500), nullable=True)
+    api_key_hash = Column(String(255), nullable=True)  # Hashed API key
+    configuration = Column(Text, nullable=True)  # JSON configuration
+    
+    # Event filtering
+    event_types = Column(Text, nullable=True)  # JSON array of event types to send
+    severity_threshold = Column(String(20), default="medium", nullable=False)
+    
+    # Status
+    is_active = Column(Boolean, default=True, nullable=False)
+    is_healthy = Column(Boolean, default=True, nullable=False)
+    last_sync = Column(DateTime(timezone=True), nullable=True)
+    last_error = Column(Text, nullable=True)
+    
+    # Statistics
+    events_sent = Column(Integer, default=0, nullable=False)
+    errors_count = Column(Integer, default=0, nullable=False)
+    
+    # Timestamps
+    created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False)
+    updated_at = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False)
+    created_by = Column(Integer, ForeignKey("users.id"), nullable=False)
+    
+    # Relationships
+    creator = relationship("User")