Cloud Workload Protection: EDR for Cloud Environments

requirements.txt:

boto3>=1.34.0
botocore>=1.34.0
python-dateutil>=2.8.2

Complete Implementation:

#!/usr/bin/env python3
"""
Cloud Workload Protection - Monitoring Module
Production-ready cloud workload monitoring with comprehensive error handling
"""

import boto3
from botocore.exceptions import ClientError, BotoCoreError
from datetime import datetime, timedelta
from typing import List, Dict, Optional
import logging
import os
from dataclasses import dataclass
from enum import Enum
import json

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)


class MonitoringError(Exception):
    """Base exception for monitoring errors."""
    pass


class InvalidInstanceError(MonitoringError):
    """Raised when instance ID is invalid."""
    pass


class MetricsRetrievalError(MonitoringError):
    """Raised when metrics cannot be retrieved."""
    pass


@dataclass
class WorkloadMetrics:
    """Container for workload metrics data."""
    instance_id: str
    cpu_average: float
    cpu_maximum: float
    memory_utilization: Optional[float]
    network_in: Optional[float]
    network_out: Optional[float]
    timestamp: datetime
    region: str


class CloudWorkloadMonitor:
    """Monitors cloud workloads for security threats with comprehensive error handling."""
    
    def __init__(
        self, 
        region_name: str = 'us-east-1',
        aws_access_key_id: Optional[str] = None,
        aws_secret_access_key: Optional[str] = None
    ):
        """Initialize cloud workload monitor.
        
        Args:
            region_name: AWS region to monitor
            aws_access_key_id: AWS access key (defaults to env/credentials)
            aws_secret_access_key: AWS secret key (defaults to env/credentials)
            
        Raises:
            MonitoringError: If AWS clients cannot be initialized
        """
        self.region_name = region_name
        
        try:
            # Initialize AWS clients with credentials
            session = boto3.Session(
                aws_access_key_id=aws_access_key_id or os.getenv('AWS_ACCESS_KEY_ID'),
                aws_secret_access_key=aws_secret_access_key or os.getenv('AWS_SECRET_ACCESS_KEY'),
                region_name=region_name
            )
            
            self.cloudwatch = session.client('cloudwatch')
            self.ec2 = session.client('ec2')
            
            logger.info(f"Initialized CloudWorkloadMonitor for region: {region_name}")
            
        except (ClientError, BotoCoreError) as e:
            logger.error(f"Failed to initialize AWS clients: {e}")
            raise MonitoringError(f"Failed to initialize AWS clients: {e}") from e
    
    def _validate_instance_id(self, instance_id: str) -> None:
        """Validate EC2 instance ID format.
        
        Args:
            instance_id: Instance ID to validate
            
        Raises:
            InvalidInstanceError: If instance ID is invalid
        """
        if not instance_id:
            raise InvalidInstanceError("Instance ID cannot be empty")
        
        if not isinstance(instance_id, str):
            raise InvalidInstanceError(f"Instance ID must be string, got {type(instance_id)}")
        
        if not instance_id.startswith('i-') or len(instance_id) < 10:
            raise InvalidInstanceError(f"Invalid instance ID format: {instance_id}")
    
    def _get_metric_statistics_with_retry(
        self, 
        namespace: str,
        metric_name: str,
        dimensions: List[Dict[str, str]],
        start_time: datetime,
        end_time: datetime,
        period: int = 60,
        statistics: List[str] = None,
        max_retries: int = 3
    ) -> Dict:
        """Get metric statistics with retry logic.
        
        Args:
            namespace: CloudWatch namespace
            metric_name: Metric name to retrieve
            dimensions: Metric dimensions
            start_time: Start time for metrics
            end_time: End time for metrics
            period: Period in seconds
            statistics: List of statistics to retrieve
            max_retries: Maximum number of retry attempts
            
        Returns:
            CloudWatch metric statistics response
            
        Raises:
            MetricsRetrievalError: If metrics cannot be retrieved after retries
        """
        if statistics is None:
            statistics = ['Average', 'Maximum']
        
        last_exception = None
        
        for attempt in range(max_retries):
            try:
                response = self.cloudwatch.get_metric_statistics(
                    Namespace=namespace,
                    MetricName=metric_name,
                    Dimensions=dimensions,
                    StartTime=start_time,
                    EndTime=end_time,
                    Period=period,
                    Statistics=statistics
                )
                
                logger.debug(f"Successfully retrieved {metric_name} metrics (attempt {attempt + 1})")
                return response
                
            except ClientError as e:
                error_code = e.response.get('Error', {}).get('Code', '')
                last_exception = e
                
                # Don't retry on client errors (invalid parameters, etc.)
                if error_code in ['InvalidParameterValue', 'InvalidParameterCombination']:
                    logger.error(f"Invalid parameters for metric {metric_name}: {e}")
                    raise MetricsRetrievalError(f"Invalid parameters: {e}") from e
                
                # Retry on throttling or service errors
                if error_code in ['Throttling', 'ServiceUnavailable'] and attempt < max_retries - 1:
                    wait_time = 2 ** attempt  # Exponential backoff
                    logger.warning(f"Rate limited, retrying in {wait_time}s (attempt {attempt + 1}/{max_retries})")
                    import time
                    time.sleep(wait_time)
                    continue
                
                logger.error(f"Error retrieving metrics: {e}")
                raise MetricsRetrievalError(f"Failed to retrieve metrics: {e}") from e
            
            except BotoCoreError as e:
                last_exception = e
                if attempt < max_retries - 1:
                    wait_time = 2 ** attempt
                    logger.warning(f"Connection error, retrying in {wait_time}s (attempt {attempt + 1}/{max_retries})")
                    import time
                    time.sleep(wait_time)
                    continue
                raise MetricsRetrievalError(f"Connection error: {e}") from e
        
        # All retries exhausted
        raise MetricsRetrievalError(f"Failed to retrieve metrics after {max_retries} attempts") from last_exception
    
    def monitor_workload(
        self, 
        instance_id: str,
        time_window_minutes: int = 5
    ) -> Optional[WorkloadMetrics]:
        """Monitor workload metrics with comprehensive error handling.
        
        Args:
            instance_id: EC2 instance ID to monitor
            time_window_minutes: Time window for metrics in minutes
            
        Returns:
            WorkloadMetrics object with metrics, or None if no data available
            
        Raises:
            InvalidInstanceError: If instance_id is invalid
            MetricsRetrievalError: If metrics cannot be retrieved
            MonitoringError: For other monitoring errors
        """
        # Validate instance ID
        self._validate_instance_id(instance_id)
        
        try:
            # Calculate time range
            end_time = datetime.utcnow()
            start_time = end_time - timedelta(minutes=time_window_minutes)
            
            dimensions = [{'Name': 'InstanceId', 'Value': instance_id}]
            
            # Get CPU metrics
            logger.info(f"Retrieving CPU metrics for instance {instance_id}")
            cpu_metrics = self._get_metric_statistics_with_retry(
                namespace='AWS/EC2',
                metric_name='CPUUtilization',
                dimensions=dimensions,
                start_time=start_time,
                end_time=end_time,
                period=60,
                statistics=['Average', 'Maximum']
            )
            
            # Process CPU datapoints
            cpu_datapoints = cpu_metrics.get('Datapoints', [])
            if not cpu_datapoints:
                logger.warning(f"No CPU metrics found for instance {instance_id}")
                return None
            
            # Get most recent datapoint
            latest_cpu = max(cpu_datapoints, key=lambda x: x['Timestamp'])
            cpu_average = latest_cpu.get('Average', 0.0)
            cpu_maximum = latest_cpu.get('Maximum', 0.0)
            
            # Get memory metrics (if available)
            memory_utilization = None
            try:
                memory_metrics = self._get_metric_statistics_with_retry(
                    namespace='AWS/EC2',
                    metric_name='MemoryUtilization',
                    dimensions=dimensions,
                    start_time=start_time,
                    end_time=end_time,
                    period=60,
                    statistics=['Average']
                )
                memory_datapoints = memory_metrics.get('Datapoints', [])
                if memory_datapoints:
                    latest_memory = max(memory_datapoints, key=lambda x: x['Timestamp'])
                    memory_utilization = latest_memory.get('Average')
            except MetricsRetrievalError:
                logger.debug("Memory metrics not available (may require CloudWatch agent)")
            
            # Get network metrics
            network_in = None
            network_out = None
            try:
                network_in_metrics = self._get_metric_statistics_with_retry(
                    namespace='AWS/EC2',
                    metric_name='NetworkIn',
                    dimensions=dimensions,
                    start_time=start_time,
                    end_time=end_time,
                    period=60,
                    statistics=['Sum']
                )
                network_out_metrics = self._get_metric_statistics_with_retry(
                    namespace='AWS/EC2',
                    metric_name='NetworkOut',
                    dimensions=dimensions,
                    start_time=start_time,
                    end_time=end_time,
                    period=60,
                    statistics=['Sum']
                )
                
                network_in_datapoints = network_in_metrics.get('Datapoints', [])
                network_out_datapoints = network_out_metrics.get('Datapoints', [])
                
                if network_in_datapoints:
                    latest_net_in = max(network_in_datapoints, key=lambda x: x['Timestamp'])
                    network_in = latest_net_in.get('Sum', 0.0)
                
                if network_out_datapoints:
                    latest_net_out = max(network_out_datapoints, key=lambda x: x['Timestamp'])
                    network_out = latest_net_out.get('Sum', 0.0)
            except MetricsRetrievalError:
                logger.debug("Network metrics not available")
            
            # Create WorkloadMetrics object
            workload_metrics = WorkloadMetrics(
                instance_id=instance_id,
                cpu_average=float(cpu_average),
                cpu_maximum=float(cpu_maximum),
                memory_utilization=memory_utilization,
                network_in=network_in,
                network_out=network_out,
                timestamp=latest_cpu['Timestamp'],
                region=self.region_name
            )
            
            logger.info(f"Successfully monitored workload {instance_id}: CPU={cpu_average:.1f}%")
            return workload_metrics
            
        except (InvalidInstanceError, MetricsRetrievalError):
            raise
        except Exception as e:
            logger.error(f"Unexpected error monitoring workload {instance_id}: {e}", exc_info=True)
            raise MonitoringError(f"Unexpected error: {e}") from e
    
    def monitor_multiple_workloads(
        self, 
        instance_ids: List[str],
        time_window_minutes: int = 5
    ) -> Dict[str, Optional[WorkloadMetrics]]:
        """Monitor multiple workloads.
        
        Args:
            instance_ids: List of EC2 instance IDs to monitor
            time_window_minutes: Time window for metrics in minutes
            
        Returns:
            Dictionary mapping instance IDs to WorkloadMetrics objects
        """
        results = {}
        
        for instance_id in instance_ids:
            try:
                metrics = self.monitor_workload(instance_id, time_window_minutes)
                results[instance_id] = metrics
            except MonitoringError as e:
                logger.error(f"Failed to monitor {instance_id}: {e}")
                results[instance_id] = None
        
        return results


# Example usage
if __name__ == "__main__":
    # Initialize monitor
    monitor = CloudWorkloadMonitor(region_name='us-east-1')
    
    # Monitor single workload
    try:
        metrics = monitor.monitor_workload('i-1234567890abcdef0')
        if metrics:
            print(f"Instance: {metrics.instance_id}")
            print(f"CPU Average: {metrics.cpu_average:.2f}%")
            print(f"CPU Maximum: {metrics.cpu_maximum:.2f}%")
            print(f"Memory: {metrics.memory_utilization or 'N/A'}%")
            print(f"Timestamp: {metrics.timestamp}")
    except MonitoringError as e:
        print(f"Error: {e}")

Unit Tests:

# test_cloud_workload_monitor.py
import pytest
from unittest.mock import Mock, patch, MagicMock
from datetime import datetime, timedelta
from cloud_workload_monitor import (
    CloudWorkloadMonitor,
    WorkloadMetrics,
    InvalidInstanceError,
    MetricsRetrievalError,
    MonitoringError
)


class TestCloudWorkloadMonitor:
    """Unit tests for CloudWorkloadMonitor."""
    
    @pytest.fixture
    def mock_boto_session(self):
        """Create mock boto3 session."""
        with patch('cloud_workload_monitor.boto3.Session') as mock_session:
            mock_session_instance = Mock()
            mock_session.return_value = mock_session_instance
            yield mock_session_instance
    
    @pytest.fixture
    def monitor(self, mock_boto_session):
        """Create CloudWorkloadMonitor instance with mocked AWS clients."""
        mock_boto_session.client.return_value = Mock()
        monitor = CloudWorkloadMonitor(region_name='us-east-1')
        monitor.cloudwatch = Mock()
        monitor.ec2 = Mock()
        return monitor
    
    def test_validate_instance_id_valid(self, monitor):
        """Test validation with valid instance ID."""
        monitor._validate_instance_id('i-1234567890abcdef0')
        # Should not raise
    
    def test_validate_instance_id_invalid_empty(self, monitor):
        """Test validation with empty instance ID."""
        with pytest.raises(InvalidInstanceError, match="cannot be empty"):
            monitor._validate_instance_id('')
    
    def test_validate_instance_id_invalid_format(self, monitor):
        """Test validation with invalid format."""
        with pytest.raises(InvalidInstanceError, match="Invalid instance ID format"):
            monitor._validate_instance_id('invalid-id')
    
    def test_monitor_workload_success(self, monitor):
        """Test successful workload monitoring."""
        # Mock CloudWatch response
        mock_response = {
            'Datapoints': [
                {
                    'Timestamp': datetime.utcnow(),
                    'Average': 45.5,
                    'Maximum': 67.8,
                    'Unit': 'Percent'
                }
            ]
        }
        
        monitor.cloudwatch.get_metric_statistics.return_value = mock_response
        
        # Mock memory and network metrics (empty to test graceful handling)
        monitor._get_metric_statistics_with_retry = Mock(side_effect=[
            mock_response,  # CPU
            {'Datapoints': []},  # Memory
            {'Datapoints': []},  # Network In
            {'Datapoints': []},  # Network Out
        ])
        
        metrics = monitor.monitor_workload('i-1234567890abcdef0')
        
        assert metrics is not None
        assert metrics.instance_id == 'i-1234567890abcdef0'
        assert metrics.cpu_average == 45.5
        assert metrics.cpu_maximum == 67.8
    
    def test_monitor_workload_no_data(self, monitor):
        """Test monitoring when no metrics available."""
        monitor._get_metric_statistics_with_retry = Mock(return_value={'Datapoints': []})
        
        metrics = monitor.monitor_workload('i-1234567890abcdef0')
        
        assert metrics is None
    
    def test_monitor_workload_invalid_id(self, monitor):
        """Test monitoring with invalid instance ID."""
        with pytest.raises(InvalidInstanceError):
            monitor.monitor_workload('invalid')
    
    def test_get_metric_statistics_retry_on_throttling(self, monitor):
        """Test retry logic on throttling errors."""
        # First call raises throttling, second succeeds
        throttling_error = ClientError(
            {'Error': {'Code': 'Throttling', 'Message': 'Rate exceeded'}},
            'GetMetricStatistics'
        )
        success_response = {'Datapoints': []}
        
        monitor.cloudwatch.get_metric_statistics = Mock(side_effect=[
            throttling_error,
            success_response
        ])
        
        with patch('time.sleep'):  # Mock sleep to speed up test
            result = monitor._get_metric_statistics_with_retry(
                namespace='AWS/EC2',
                metric_name='CPUUtilization',
                dimensions=[{'Name': 'InstanceId', 'Value': 'i-123'}],
                start_time=datetime.utcnow(),
                end_time=datetime.utcnow(),
                max_retries=3
            )
        
        assert result == success_response
        assert monitor.cloudwatch.get_metric_statistics.call_count == 2


if __name__ == '__main__':
    pytest.main([__file__, '-v'])

Complete Threat Detection Implementation:

#!/usr/bin/env python3
"""
Cloud Workload Protection - Threat Detection Module
Production-ready threat detection with comprehensive analysis
"""

from enum import Enum
from dataclasses import dataclass, asdict
from typing import List, Optional, Dict
from datetime import datetime, timedelta
import logging
import json

logger = logging.getLogger(__name__)


class ThreatSeverity(Enum):
    """Threat severity levels."""
    LOW = "low"
    MEDIUM = "medium"
    HIGH = "high"
    CRITICAL = "critical"
    
    def __str__(self) -> str:
        return self.value


class ThreatType(Enum):
    """Types of threats detected."""
    HIGH_CPU = "high_cpu"
    CPU_SPIKE = "cpu_spike"
    HIGH_MEMORY = "high_memory"
    MEMORY_SPIKE = "memory_spike"
    NETWORK_ANOMALY = "network_anomaly"
    SUSPICIOUS_ACTIVITY = "suspicious_activity"
    RESOURCE_EXHAUSTION = "resource_exhaustion"
    
    def __str__(self) -> str:
        return self.value


@dataclass
class ThreatFinding:
    """Represents a security threat finding with complete metadata."""
    threat_type: str
    severity: ThreatSeverity
    value: float
    timestamp: datetime
    instance_id: str
    description: str
    region: str
    recommended_action: str
    confidence: float = 0.0  # Confidence score 0-1
    
    def to_dict(self) -> Dict:
        """Convert finding to dictionary for serialization."""
        result = asdict(self)
        result['severity'] = self.severity.value
        result['timestamp'] = self.timestamp.isoformat()
        return result
    
    def to_json(self) -> str:
        """Convert finding to JSON string."""
        return json.dumps(self.to_dict(), indent=2)


class ThreatDetector:
    """Detects security threats in cloud workloads with comprehensive analysis."""
    
    def __init__(
        self, 
        cpu_threshold: float = 90.0,
        cpu_critical_threshold: float = 98.0,
        memory_threshold: float = 85.0,
        memory_critical_threshold: float = 95.0,
        network_anomaly_multiplier: float = 3.0
    ):
        """Initialize threat detector with configurable thresholds.
        
        Args:
            cpu_threshold: CPU usage threshold for alerts (%)
            cpu_critical_threshold: CPU usage threshold for critical alerts (%)
            memory_threshold: Memory usage threshold for alerts (%)
            memory_critical_threshold: Memory usage threshold for critical alerts (%)
            network_anomaly_multiplier: Multiplier for network anomaly detection
        """
        if not 0 < cpu_threshold <= 100:
            raise ValueError(f"CPU threshold must be between 0 and 100, got {cpu_threshold}")
        if not 0 < memory_threshold <= 100:
            raise ValueError(f"Memory threshold must be between 0 and 100, got {memory_threshold}")
        
        self.cpu_threshold = cpu_threshold
        self.cpu_critical_threshold = cpu_critical_threshold
        self.memory_threshold = memory_threshold
        self.memory_critical_threshold = memory_critical_threshold
        self.network_anomaly_multiplier = network_anomaly_multiplier
        
        # Historical data for anomaly detection
        self.cpu_history: Dict[str, List[float]] = {}
        self.memory_history: Dict[str, List[float]] = {}
        self.network_history: Dict[str, List[Dict[str, float]]] = {}
        
        logger.info(
            f"Initialized ThreatDetector: CPU threshold={cpu_threshold}%, "
            f"Memory threshold={memory_threshold}%"
        )
    
    def _calculate_severity(self, value: float, threshold: float, critical_threshold: float) -> ThreatSeverity:
        """Calculate threat severity based on value and thresholds.
        
        Args:
            value: Current metric value
            threshold: Alert threshold
            critical_threshold: Critical threshold
            
        Returns:
            ThreatSeverity level
        """
        if value >= critical_threshold:
            return ThreatSeverity.CRITICAL
        elif value >= threshold * 1.5:  # 50% above threshold = HIGH
            return ThreatSeverity.HIGH
        elif value >= threshold:
            return ThreatSeverity.MEDIUM
        else:
            return ThreatSeverity.LOW
    
    def _calculate_confidence(self, value: float, threshold: float, historical_values: List[float]) -> float:
        """Calculate confidence score for a threat finding.
        
        Args:
            value: Current metric value
            threshold: Alert threshold
            historical_values: Historical values for comparison
            
        Returns:
            Confidence score between 0 and 1
        """
        if not historical_values:
            return 0.7  # Medium confidence without history
        
        # Calculate how many standard deviations above mean
        import statistics
        mean_val = statistics.mean(historical_values)
        if len(historical_values) > 1:
            stdev = statistics.stdev(historical_values)
        else:
            stdev = mean_val * 0.1  # Default 10% if only one value
        
        if stdev == 0:
            stdev = 1.0
        
        z_score = (value - mean_val) / stdev
        
        # Convert z-score to confidence (normalized to 0-1)
        # Higher z-score = higher confidence
        confidence = min(1.0, max(0.5, (z_score / 3.0) * 0.5 + 0.5))
        
        return confidence
    
    def _detect_cpu_threats(
        self, 
        workload_metrics, 
        instance_id: str
    ) -> List[ThreatFinding]:
        """Detect CPU-related threats.
        
        Args:
            workload_metrics: WorkloadMetrics object
            instance_id: Instance ID
            
        Returns:
            List of CPU-related threat findings
        """
        findings = []
        
        cpu_avg = workload_metrics.cpu_average
        cpu_max = workload_metrics.cpu_maximum
        
        # Update CPU history
        if instance_id not in self.cpu_history:
            self.cpu_history[instance_id] = []
        self.cpu_history[instance_id].append(cpu_avg)
        # Keep only last 100 values
        if len(self.cpu_history[instance_id]) > 100:
            self.cpu_history[instance_id] = self.cpu_history[instance_id][-100:]
        
        # High average CPU detection
        if cpu_avg > self.cpu_threshold:
            severity = self._calculate_severity(
                cpu_avg, 
                self.cpu_threshold, 
                self.cpu_critical_threshold
            )
            
            confidence = self._calculate_confidence(
                cpu_avg,
                self.cpu_threshold,
                self.cpu_history[instance_id][:-1]  # Exclude current value
            )
            
            findings.append(ThreatFinding(
                threat_type=ThreatType.HIGH_CPU.value,
                severity=severity,
                value=cpu_avg,
                timestamp=workload_metrics.timestamp,
                instance_id=instance_id,
                description=f"High CPU utilization: {cpu_avg:.1f}% (threshold: {self.cpu_threshold}%)",
                region=workload_metrics.region,
                recommended_action=self._get_cpu_recommendation(severity),
                confidence=confidence
            ))
        
        # CPU spike detection (sudden jump)
        if cpu_max > self.cpu_critical_threshold:
            # Check if this is a spike (sudden increase)
            is_spike = False
            if len(self.cpu_history[instance_id]) >= 2:
                previous_avg = self.cpu_history[instance_id][-2]
                if cpu_max > previous_avg * 1.5:  # 50% increase = spike
                    is_spike = True
            
            if is_spike or cpu_max >= 99.0:
                findings.append(ThreatFinding(
                    threat_type=ThreatType.CPU_SPIKE.value,
                    severity=ThreatSeverity.CRITICAL,
                    value=cpu_max,
                    timestamp=workload_metrics.timestamp,
                    instance_id=instance_id,
                    description=f"CPU spike detected: {cpu_max:.1f}% (critical threshold: {self.cpu_critical_threshold}%)",
                    region=workload_metrics.region,
                    recommended_action="Immediately investigate CPU usage. May indicate crypto-mining, DDoS, or resource exhaustion attack.",
                    confidence=0.95
                ))
        
        return findings
    
    def _detect_memory_threats(
        self, 
        workload_metrics, 
        instance_id: str
    ) -> List[ThreatFinding]:
        """Detect memory-related threats.
        
        Args:
            workload_metrics: WorkloadMetrics object
            instance_id: Instance ID
            
        Returns:
            List of memory-related threat findings
        """
        findings = []
        
        if workload_metrics.memory_utilization is None:
            return findings  # Memory metrics not available
        
        memory_util = workload_metrics.memory_utilization
        
        # Update memory history
        if instance_id not in self.memory_history:
            self.memory_history[instance_id] = []
        self.memory_history[instance_id].append(memory_util)
        if len(self.memory_history[instance_id]) > 100:
            self.memory_history[instance_id] = self.memory_history[instance_id][-100:]
        
        # High memory detection
        if memory_util > self.memory_threshold:
            severity = self._calculate_severity(
                memory_util,
                self.memory_threshold,
                self.memory_critical_threshold
            )
            
            confidence = self._calculate_confidence(
                memory_util,
                self.memory_threshold,
                self.memory_history[instance_id][:-1]
            )
            
            findings.append(ThreatFinding(
                threat_type=ThreatType.HIGH_MEMORY.value,
                severity=severity,
                value=memory_util,
                timestamp=workload_metrics.timestamp,
                instance_id=instance_id,
                description=f"High memory utilization: {memory_util:.1f}% (threshold: {self.memory_threshold}%)",
                region=workload_metrics.region,
                recommended_action=self._get_memory_recommendation(severity),
                confidence=confidence
            ))
        
        return findings
    
    def _detect_network_anomalies(
        self, 
        workload_metrics, 
        instance_id: str
    ) -> List[ThreatFinding]:
        """Detect network-related anomalies.
        
        Args:
            workload_metrics: WorkloadMetrics object
            instance_id: Instance ID
            
        Returns:
            List of network-related threat findings
        """
        findings = []
        
        if workload_metrics.network_in is None or workload_metrics.network_out is None:
            return findings
        
        # Update network history
        if instance_id not in self.network_history:
            self.network_history[instance_id] = []
        
        network_data = {
            'in': workload_metrics.network_in,
            'out': workload_metrics.network_out,
            'total': workload_metrics.network_in + workload_metrics.network_out
        }
        self.network_history[instance_id].append(network_data)
        if len(self.network_history[instance_id]) > 100:
            self.network_history[instance_id] = self.network_history[instance_id][-100:]
        
        # Detect network anomalies (sudden spikes)
        if len(self.network_history[instance_id]) >= 2:
            previous_data = self.network_history[instance_id][-2]
            current_total = network_data['total']
            previous_total = previous_data['total']
            
            # Check for significant increase
            if previous_total > 0 and current_total > previous_total * self.network_anomaly_multiplier:
                severity = ThreatSeverity.HIGH if current_total > previous_total * 5 else ThreatSeverity.MEDIUM
                
                findings.append(ThreatFinding(
                    threat_type=ThreatType.NETWORK_ANOMALY.value,
                    severity=severity,
                    value=current_total,
                    timestamp=workload_metrics.timestamp,
                    instance_id=instance_id,
                    description=f"Network traffic spike: {current_total:.0f} bytes ({self.network_anomaly_multiplier}x increase)",
                    region=workload_metrics.region,
                    recommended_action="Investigate network traffic. May indicate DDoS attack, data exfiltration, or botnet activity.",
                    confidence=0.85
                ))
        
        return findings
    
    def _get_cpu_recommendation(self, severity: ThreatSeverity) -> str:
        """Get recommendation for CPU threat based on severity."""
        recommendations = {
            ThreatSeverity.CRITICAL: "Immediate action required. Isolate workload, investigate process list, check for crypto-mining or DDoS.",
            ThreatSeverity.HIGH: "High priority. Review running processes, check for resource-intensive applications, consider scaling.",
            ThreatSeverity.MEDIUM: "Monitor closely. Review recent deployments or changes that may have increased CPU usage.",
            ThreatSeverity.LOW: "Review CPU usage patterns and consider optimization."
        }
        return recommendations.get(severity, "Monitor and investigate.")
    
    def _get_memory_recommendation(self, severity: ThreatSeverity) -> str:
        """Get recommendation for memory threat based on severity."""
        recommendations = {
            ThreatSeverity.CRITICAL: "Immediate action required. Check for memory leaks, review application logs, consider memory-intensive attacks.",
            ThreatSeverity.HIGH: "High priority. Review memory usage patterns, check for memory leaks, consider increasing memory allocation.",
            ThreatSeverity.MEDIUM: "Monitor closely. Review memory-intensive operations, optimize application memory usage.",
            ThreatSeverity.LOW: "Review memory usage patterns and optimize if needed."
        }
        return recommendations.get(severity, "Monitor and investigate.")
    
    def detect_anomalies(
        self, 
        workload_metrics
    ) -> List[ThreatFinding]:
        """Detect anomalous behavior in workloads with comprehensive analysis.
        
        Args:
            workload_metrics: WorkloadMetrics object from monitor
            
        Returns:
            List of threat findings
        """
        if workload_metrics is None:
            return []
        
        findings = []
        instance_id = workload_metrics.instance_id
        
        try:
            # Detect CPU threats
            cpu_findings = self._detect_cpu_threats(workload_metrics, instance_id)
            findings.extend(cpu_findings)
            
            # Detect memory threats
            memory_findings = self._detect_memory_threats(workload_metrics, instance_id)
            findings.extend(memory_findings)
            
            # Detect network anomalies
            network_findings = self._detect_network_anomalies(workload_metrics, instance_id)
            findings.extend(network_findings)
            
            # Log findings
            if findings:
                logger.warning(
                    f"Detected {len(findings)} threat(s) for instance {instance_id}: "
                    f"{[f.type for f in findings]}"
                )
            
        except Exception as e:
            logger.error(f"Error detecting anomalies for {instance_id}: {e}", exc_info=True)
            # Don't raise - return findings collected so far
        
        return findings
    
    def detect_anomalies_batch(
        self, 
        workload_metrics_list: List
    ) -> Dict[str, List[ThreatFinding]]:
        """Detect anomalies for multiple workloads.
        
        Args:
            workload_metrics_list: List of WorkloadMetrics objects
            
        Returns:
            Dictionary mapping instance IDs to lists of findings
        """
        results = {}
        
        for metrics in workload_metrics_list:
            if metrics is None:
                continue
            
            findings = self.detect_anomalies(metrics)
            results[metrics.instance_id] = findings
        
        return results


# Example usage
if __name__ == "__main__":
    # Initialize detector
    detector = ThreatDetector(
        cpu_threshold=90.0,
        memory_threshold=85.0
    )
    
    # Example: Create mock workload metrics
    from datetime import datetime
    from cloud_workload_monitor import WorkloadMetrics
    
    metrics = WorkloadMetrics(
        instance_id='i-1234567890abcdef0',
        cpu_average=95.5,
        cpu_maximum=98.2,
        memory_utilization=87.3,
        network_in=1000000.0,
        network_out=2000000.0,
        timestamp=datetime.utcnow(),
        region='us-east-1'
    )
    
    # Detect anomalies
    findings = detector.detect_anomalies(metrics)
    
    # Print findings
    for finding in findings:
        print(f"Threat: {finding.threat_type}")
        print(f"Severity: {finding.severity}")
        print(f"Description: {finding.description}")
        print(f"Recommendation: {finding.recommended_action}")
        print(f"Confidence: {finding.confidence:.2f}")
        print("---")

Unit Tests:

# test_threat_detector.py
import pytest
from datetime import datetime
from threat_detector import (
    ThreatDetector,
    ThreatFinding,
    ThreatSeverity,
    ThreatType
)
from cloud_workload_monitor import WorkloadMetrics


class TestThreatDetector:
    """Unit tests for ThreatDetector."""
    
    @pytest.fixture
    def detector(self):
        """Create ThreatDetector instance."""
        return ThreatDetector(
            cpu_threshold=90.0,
            memory_threshold=85.0
        )
    
    @pytest.fixture
    def sample_metrics(self):
        """Create sample WorkloadMetrics."""
        return WorkloadMetrics(
            instance_id='i-1234567890abcdef0',
            cpu_average=45.0,
            cpu_maximum=60.0,
            memory_utilization=50.0,
            network_in=100000.0,
            network_out=200000.0,
            timestamp=datetime.utcnow(),
            region='us-east-1'
        )
    
    def test_detect_high_cpu(self, detector, sample_metrics):
        """Test high CPU detection."""
        sample_metrics.cpu_average = 95.0
        sample_metrics.cpu_maximum = 96.0
        
        findings = detector.detect_anomalies(sample_metrics)
        
        assert len(findings) > 0
        cpu_findings = [f for f in findings if f.threat_type == ThreatType.HIGH_CPU.value]
        assert len(cpu_findings) == 1
        assert cpu_findings[0].severity in [ThreatSeverity.HIGH, ThreatSeverity.CRITICAL]
        assert cpu_findings[0].value == 95.0
    
    def test_detect_cpu_spike(self, detector, sample_metrics):
        """Test CPU spike detection."""
        # First call with normal CPU
        detector.detect_anomalies(sample_metrics)
        
        # Second call with spike
        sample_metrics.cpu_average = 50.0
        sample_metrics.cpu_maximum = 99.5
        
        findings = detector.detect_anomalies(sample_metrics)
        
        spike_findings = [f for f in findings if f.threat_type == ThreatType.CPU_SPIKE.value]
        assert len(spike_findings) > 0
        assert spike_findings[0].severity == ThreatSeverity.CRITICAL
    
    def test_detect_high_memory(self, detector, sample_metrics):
        """Test high memory detection."""
        sample_metrics.memory_utilization = 90.0
        
        findings = detector.detect_anomalies(sample_metrics)
        
        memory_findings = [f for f in findings if f.threat_type == ThreatType.HIGH_MEMORY.value]
        assert len(memory_findings) > 0
        assert memory_findings[0].value == 90.0
    
    def test_no_threats_normal_load(self, detector, sample_metrics):
        """Test that normal load doesn't trigger threats."""
        sample_metrics.cpu_average = 30.0
        sample_metrics.memory_utilization = 40.0
        
        findings = detector.detect_anomalies(sample_metrics)
        
        # Should have no critical or high severity findings
        critical_findings = [f for f in findings if f.severity in [ThreatSeverity.CRITICAL, ThreatSeverity.HIGH]]
        assert len(critical_findings) == 0
    
    def test_invalid_thresholds(self):
        """Test that invalid thresholds raise errors."""
        with pytest.raises(ValueError):
            ThreatDetector(cpu_threshold=-10.0)
        
        with pytest.raises(ValueError):
            ThreatDetector(cpu_threshold=150.0)
        
        with pytest.raises(ValueError):
            ThreatDetector(memory_threshold=-5.0)


if __name__ == '__main__':
    pytest.main([__file__, '-v'])

import pytest
from unittest.mock import Mock, patch
from datetime import datetime

class TestCloudWorkloadMonitor:
    """Unit tests for CloudWorkloadMonitor."""
    
    def test_monitor_workload_success(self):
        """Test successful workload monitoring."""
        monitor = CloudWorkloadMonitor()
        with patch.object(monitor.cloudwatch, 'get_metric_statistics') as mock_metrics:
            mock_metrics.return_value = {
                'Datapoints': [
                    {'Average': 50.0, 'Maximum': 60.0, 'Timestamp': datetime.utcnow()}
                ]
            }
            result = monitor.monitor_workload('i-1234567890abcdef0')
            assert result is not None
            assert result['instance_id'] == 'i-1234567890abcdef0'
    
    def test_monitor_workload_invalid_id(self):
        """Test monitoring with invalid instance ID."""
        monitor = CloudWorkloadMonitor()
        with pytest.raises(ValueError):
            monitor.monitor_workload('invalid')

class TestThreatDetector:
    """Unit tests for ThreatDetector."""
    
    def test_detect_high_cpu(self):
        """Test high CPU detection."""
        detector = ThreatDetector(cpu_threshold=90.0)
        metrics = {
            'instance_id': 'i-1234567890abcdef0',
            'metrics': [
                {'Average': 95.0, 'Maximum': 98.0, 'Timestamp': datetime.utcnow()}
            ]
        }
        findings = detector.detect_anomalies(metrics)
        assert len(findings) > 0
        assert any(f.threat_type == 'high_cpu' for f in findings)

# Remove CloudWatch alarms (if created)
aws cloudwatch delete-alarms --alarm-names workload-protection-*

# Remove IAM roles and policies (if created)
aws iam delete-role-policy --role-name WorkloadMonitorRole --policy-name WorkloadMonitorPolicy
aws iam delete-role --role-name WorkloadMonitorRole

# Remove test instances
aws ec2 terminate-instances --instance-ids i-test123

# Verify cleanup
aws cloudwatch describe-alarms --alarm-name-prefix workload-protection
# Should return empty list