Skip to content
HeliosDB HeliosDB Full

Self-Healing Database - API Usage Examples

Self-Healing Database - API Usage Examples

Feature ID: F5.2.1 Version: v5.2 Status: Production-Ready (190 tests passing) ARR Value: $15M Patent Status: 6 Invention Disclosures Filed

TABLE OF CONTENTS

QUICK START (5 MINUTES)

Minimal Setup

Get self-healing working in your database with just a few lines:

use heliosdb_self_healing::{SelfHealingEngine, SelfHealingConfig};


#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    // 1. Create engine with default configuration
    let config = SelfHealingConfig::default();
    let engine = SelfHealingEngine::new(config).await?;


    // 2. Start self-healing
    engine.start().await?;


    // 3. Self-healing is now active - monitor status
    println!("Self-healing status: {:?}", engine.get_health_status());


    // Keep running
    tokio::signal::ctrl_c().await?;


    // 4. Graceful shutdown
    engine.stop();


    Ok(())
}

That’s it! Your database now has:

  • Automatic health monitoring
  • Anomaly detection
  • Failure prediction
  • Autonomous recovery
  • 95%+ autonomous resolution rate

CORE COMPONENTS

ML Anomaly Detection

The ML-based anomaly detector identifies issues across 4 categories using statistical methods and isolation forests.

Basic Usage

use heliosdb_self_healing::anomaly_ml::{MlAnomalyDetector, DetectorConfig};
use std::collections::HashMap;


// Create detector with custom configuration
let config = DetectorConfig {
    window_size: 1000,          // Historical data points
    zscore_threshold: 3.0,      // Z-score sensitivity
    iqr_multiplier: 1.5,        // IQR outlier detection
    min_confidence: 0.7,        // Minimum confidence to report
    auto_baseline: true,        // Enable auto-learning
};


let detector = MlAnomalyDetector::new(config);

Performance Anomaly Detection

// Build baseline with normal metrics
for i in 0..100 {
    let mut metrics = HashMap::new();
    metrics.insert("query_latency_ms".to_string(), 50.0 + (i as f64 % 10.0));
    metrics.insert("cpu_usage".to_string(), 45.0 + (i as f64 % 5.0));


    detector.detect_performance(metrics).await;
}


// Detect anomaly in real-time
let mut current_metrics = HashMap::new();
current_metrics.insert("query_latency_ms".to_string(), 500.0); // High latency!
current_metrics.insert("cpu_usage".to_string(), 95.0);         // High CPU!


if let Some(anomaly) = detector.detect_performance(current_metrics).await {
    println!("Anomaly detected!");
    println!("  Category: {:?}", anomaly.category);
    println!("  Severity: {:?}", anomaly.severity);
    println!("  Score: {:.2}", anomaly.score);
    println!("  Description: {}", anomaly.description);
    println!("  Method: {:?}", anomaly.detection_method);
}

Output:

Anomaly detected!
  Category: Performance
  Severity: Critical
  Score: 0.92
  Description: performance anomaly detected: query_latency_ms = 500.00
               (z-score: 4.50, baseline: 55.00 ± 2.87)
  Method: ZScore

Security Anomaly Detection

// Monitor authentication patterns
let mut auth_metrics = HashMap::new();
auth_metrics.insert("failed_login_attempts".to_string(), 150.0);
auth_metrics.insert("unauthorized_access_count".to_string(), 25.0);
auth_metrics.insert("unusual_access_patterns".to_string(), 10.0);


if let Some(anomaly) = detector.detect_security(auth_metrics).await {
    // Alert security team
    alert_security_team(&anomaly);


    // Trigger automatic lockdown if critical
    if anomaly.severity == Severity::Critical {
        trigger_security_lockdown().await?;
    }
}

Resource Anomaly Detection

// Monitor system resources
let mut resource_metrics = HashMap::new();
resource_metrics.insert("cpu_percent".to_string(), 98.5);
resource_metrics.insert("memory_mb".to_string(), 15000.0);
resource_metrics.insert("disk_io_ops".to_string(), 10000.0);
resource_metrics.insert("network_mbps".to_string(), 950.0);


if let Some(anomaly) = detector.detect_resource(resource_metrics).await {
    match anomaly.severity {
        Severity::Critical => {
            // Immediate action required
            scale_resources_urgently().await?;
        }
        Severity::High => {
            // Schedule resource scaling
            schedule_resource_scaling().await?;
        }
        _ => {
            // Log for monitoring
            log_resource_anomaly(&anomaly);
        }
    }
}

Data Quality Anomaly Detection

// Monitor data integrity
let mut quality_metrics = HashMap::new();
quality_metrics.insert("null_value_percentage".to_string(), 45.0);
quality_metrics.insert("duplicate_row_count".to_string(), 5000.0);
quality_metrics.insert("schema_violations".to_string(), 250.0);
quality_metrics.insert("constraint_failures".to_string(), 100.0);


if let Some(anomaly) = detector.detect_data_quality(quality_metrics).await {
    // Data corruption detected
    log_data_quality_issue(&anomaly);


    // Trigger data validation pipeline
    trigger_data_validation().await?;


    // Consider restore from backup if severe
    if anomaly.score > 0.9 {
        consider_data_restore().await?;
    }
}

Statistics and Monitoring

// Get detection statistics
let stats = detector.get_stats().await;
println!("Total anomalies detected: {}", stats.total_anomalies);
println!("By category:");
for (category, count) in &stats.by_category {
    println!("  {:?}: {}", category, count);
}
println!("By severity:");
for (severity, count) in &stats.by_severity {
    println!("  {:?}: {}", severity, count);
}
println!("By method:");
for (method, count) in &stats.by_method {
    println!("  {:?}: {}", method, count);
}


// Reset baselines after major system changes
detector.reset_baselines().await;

Causal Inference

The causal inference engine builds a Bayesian network to identify root causes of failures through multi-hop reasoning.

Basic Setup

use heliosdb_self_healing::causal_inference::{
    CausalInferenceEngine, InferenceConfig, CausalNode, CausalEdge,
    NodeType, NodeState, EdgeType
};


let config = InferenceConfig {
    min_confidence: 0.7,           // Minimum confidence for root cause
    max_path_depth: 5,             // Maximum causal chain length
    min_edge_probability: 0.3,     // Minimum edge weight to consider
    multi_hop: true,               // Enable multi-hop reasoning
};


let engine = CausalInferenceEngine::new(config);

Building a Causal Network

// Add nodes representing system components and states
let disk_full = CausalNode {
    id: "disk_full".to_string(),
    name: "Disk Space Exhausted".to_string(),
    node_type: NodeType::Resource,
    state: NodeState::Failed,
};


let write_failure = CausalNode {
    id: "write_failure".to_string(),
    name: "Write Operations Failing".to_string(),
    node_type: NodeType::Component,
    state: NodeState::Failed,
};


let query_timeout = CausalNode {
    id: "query_timeout".to_string(),
    name: "Query Timeouts".to_string(),
    node_type: NodeType::Component,
    state: NodeState::Failed,
};


engine.add_node(disk_full).await;
engine.add_node(write_failure).await;
engine.add_node(query_timeout).await;


// Add causal edges with probabilities
engine.add_edge(CausalEdge {
    from: "disk_full".to_string(),
    to: "write_failure".to_string(),
    probability: 0.95,              // Very likely causal
    edge_type: EdgeType::DirectCause,
    observed_count: 150,            // Historical observations
}).await;


engine.add_edge(CausalEdge {
    from: "write_failure".to_string(),
    to: "query_timeout".to_string(),
    probability: 0.85,
    edge_type: EdgeType::Contributing,
    observed_count: 120,
}).await;

Root Cause Analysis

// Symptom detected: query timeouts
let symptom_id = "query_timeout";


// Infer root cause
if let Some(root_cause) = engine.infer_root_cause(symptom_id).await {
    println!("Root Cause Analysis");
    println!("==================");
    println!("Root cause: {}", root_cause.node.name);
    println!("Confidence: {:.1}%", root_cause.confidence * 100.0);
    println!("\nCausal path:");
    for (i, step) in root_cause.causal_path.iter().enumerate() {
        println!("  {}. {}", i + 1, step);
    }
    println!("\nSupporting evidence:");
    for evidence in &root_cause.evidence {
        println!("  • {}", evidence);
    }


    // Take action based on root cause
    match root_cause.node.node_type {
        NodeType::Resource => {
            scale_resources(&root_cause.node.id).await?;
        }
        NodeType::Component => {
            restart_component(&root_cause.node.id).await?;
        }
        NodeType::Dependency => {
            check_external_dependency(&root_cause.node.id).await?;
        }
        _ => {}
    }
}

Output:

Root Cause Analysis
==================
Root cause: Disk Space Exhausted
Confidence: 76.5%


Causal path:
  1. query_timeout
  2. write_failure
  3. disk_full


Supporting evidence:
  • disk_full → write_failure (95.0% confidence, 150 observations)
  • write_failure → query_timeout (85.0% confidence, 120 observations)

Complex Multi-Hop Scenario

// Build multi-level causal chain
// high_traffic -> high_cpu -> slow_queries -> connection_pool_exhaustion -> service_unavailable


let nodes = vec![
    ("high_traffic", "Unusual Traffic Spike", NodeType::UserAction),
    ("high_cpu", "CPU Saturation", NodeType::Resource),
    ("slow_queries", "Slow Query Execution", NodeType::Component),
    ("pool_exhausted", "Connection Pool Exhausted", NodeType::Configuration),
    ("service_down", "Service Unavailable", NodeType::Component),
];


for (id, name, node_type) in nodes {
    engine.add_node(CausalNode {
        id: id.to_string(),
        name: name.to_string(),
        node_type,
        state: NodeState::Failed,
    }).await;
}


// Add edges
let edges = vec![
    ("high_traffic", "high_cpu", 0.9, EdgeType::DirectCause, 200),
    ("high_cpu", "slow_queries", 0.85, EdgeType::Contributing, 180),
    ("slow_queries", "pool_exhausted", 0.8, EdgeType::DirectCause, 150),
    ("pool_exhausted", "service_down", 0.95, EdgeType::DirectCause, 160),
];


for (from, to, prob, edge_type, count) in edges {
    engine.add_edge(CausalEdge {
        from: from.to_string(),
        to: to.to_string(),
        probability: prob,
        edge_type,
        observed_count: count,
    }).await;
}


// Analyze from end symptom
if let Some(root) = engine.infer_root_cause("service_down").await {
    println!("Root cause: {} (confidence: {:.1}%)",
        root.node.name, root.confidence * 100.0);
}

Network Statistics

// Get network information
let (nodes, edges) = engine.get_network_info().await;
println!("Causal network: {} nodes, {} edges", nodes, edges);


// Get inference statistics
let stats = engine.get_stats().await;
println!("Total inferences: {}", stats.total_inferences);
println!("Root causes found: {}", stats.root_causes_found);
println!("Average confidence: {:.2}", stats.avg_confidence);
println!("Average path length: {:.2}", stats.avg_path_length);


// Reset network for new scenario
engine.reset().await;

RL Action Selection

The reinforcement learning module uses Proximal Policy Optimization (PPO) to learn optimal recovery strategies.

Basic Setup

use heliosdb_self_healing::rl_action_selector::{
    RLActionSelector, PPOConfig, RLState, RLAction, RewardCalculator
};
use heliosdb_self_healing::detector::FailureType;
use heliosdb_self_healing::health::ComponentType;


let config = PPOConfig {
    learning_rate: 0.001,          // Learning rate for policy updates
    gamma: 0.99,                   // Discount factor
    clip_epsilon: 0.2,             // PPO clipping parameter
    buffer_size: 10000,            // Experience replay buffer
    batch_size: 64,                // Training batch size
    epochs: 10,                    // Training epochs per update
};


let selector = RLActionSelector::new(config);

Creating RL State

// Capture current system state
let state = RLState {
    component: ComponentType::Database,
    failure_type: FailureType::PerformanceDegradation,
    cpu_usage: 0.85,               // 85% CPU
    memory_usage: 0.75,            // 75% memory
    disk_io: 0.90,                 // High I/O
    network_latency: 150.0,        // 150ms latency
    active_connections: 1500,      // Connection count
    error_rate: 10.0,              // 10 errors/sec
    time_since_failure: 120.0,     // 2 minutes
    recovery_attempts: 0,          // First attempt
};


// Convert to feature vector for neural network
let features = state.to_feature_vector();
println!("Feature vector: {:?}", features);

Action Selection

// Select action using learned policy
let action = selector.select_action(&state);
println!("Selected action: {:?}", action);


// Convert to recovery strategy
let strategy = action.to_recovery_strategy();
println!("Recovery strategy: {:?}", strategy);


// For evaluation, use greedy action
let greedy_action = selector.select_greedy_action(&state);
println!("Best action (greedy): {:?}", greedy_action);

Training the Policy

// Add experience after recovery attempt
let recovery_success = execute_recovery(&action).await?;
let recovery_time_ms = 1500;
let resource_cost = 0.2;


// Calculate reward
let calculator = RewardCalculator::default();
let reward = calculator.calculate_reward(
    recovery_success,
    recovery_time_ms,
    resource_cost,
);


println!("Reward: {:.2}", reward);


// Store experience
let next_state = capture_state_after_recovery().await;
selector.add_experience(
    state.clone(),
    action,
    reward,
    next_state,
    true,  // Episode done
);


// Train when enough samples collected
if selector.buffer_size() >= 64 {
    selector.train()?;
    println!("Policy updated!");
}

Complete Learning Loop

// Full reinforcement learning loop
async fn autonomous_learning_loop(
    selector: &RLActionSelector,
    iterations: usize,
) -> Result<(), Box<dyn std::error::Error>> {
    for i in 0..iterations {
        // 1. Detect failure
        let failure = detect_current_failure().await?;


        // 2. Create state representation
        let state = create_rl_state(&failure).await;


        // 3. Select action
        let action = selector.select_action(&state);


        // 4. Execute recovery
        let start_time = std::time::Instant::now();
        let success = match execute_recovery_action(&action, &failure).await {
            Ok(_) => true,
            Err(_) => false,
        };
        let duration = start_time.elapsed().as_millis() as u64;


        // 5. Calculate reward
        let calculator = RewardCalculator::default();
        let reward = calculator.calculate_reward(success, duration, 0.1);


        // 6. Get next state
        let next_state = create_rl_state(&failure).await;


        // 7. Store experience
        selector.add_experience(state, action, reward, next_state, true);


        // 8. Train periodically
        if i % 10 == 0 && selector.buffer_size() >= 64 {
            selector.train()?;
        }


        // 9. Monitor progress
        let stats = selector.get_stats();
        if i % 50 == 0 {
            println!("Iteration {}: Success rate: {:.1}%, Avg reward: {:.2}",
                i,
                (stats.successful_actions as f64 / stats.total_actions as f64) * 100.0,
                stats.avg_reward
            );
        }
    }


    Ok(())
}

Statistics and Monitoring

// Get learning statistics
let stats = selector.get_stats();
println!("RL Agent Statistics");
println!("==================");
println!("Total actions: {}", stats.total_actions);
println!("Successful: {} ({:.1}%)",
    stats.successful_actions,
    (stats.successful_actions as f64 / stats.total_actions as f64) * 100.0
);
println!("Failed: {}", stats.failed_actions);
println!("Total reward: {:.2}", stats.total_reward);
println!("Average reward: {:.2}", stats.avg_reward);
println!("Training episodes: {}", stats.training_episodes);


// Buffer management
println!("Buffer size: {}/{}",
    selector.buffer_size(),
    selector.config.buffer_size
);


// Clear buffer if needed
selector.clear_buffer();


// Reset statistics
selector.reset_stats();

Auto-Rollback

The auto-rollback manager creates state snapshots and automatically reverts failed recovery attempts.

Basic Setup

use heliosdb_self_healing::auto_rollback::{
    AutoRollbackManager, RollbackConfig, RollbackStatus
};
use heliosdb_self_healing::health::ComponentType;


let config = RollbackConfig {
    enabled: true,                      // Enable auto-rollback
    max_attempts: 3,                    // Max rollback retries
    timeout_secs: 30,                   // Rollback timeout
    verify_rollback: true,              // Verify after rollback
    max_depth: 3,                       // Max nested rollbacks
    snapshot_retention_secs: 3600,      // Keep snapshots for 1 hour
};


let manager = AutoRollbackManager::new(config);

Creating Snapshots

// Create snapshot before risky operation
let snapshot_id = manager.create_snapshot(
    ComponentType::Database,
    None,  // No parent snapshot
).await?;


println!("Created snapshot: {}", snapshot_id);


// Perform risky operation
match risky_operation().await {
    Ok(_) => {
        println!("Operation succeeded - cleaning up snapshot");
        manager.cleanup_snapshot(&snapshot_id).await?;
    }
    Err(e) => {
        println!("Operation failed - rolling back: {}", e);
        let result = manager.rollback(&snapshot_id).await?;
        println!("Rollback status: {:?}", result.status);
    }
}

Nested Rollbacks

// Create nested snapshots for multi-step operations
let snapshot1 = manager.create_snapshot(
    ComponentType::Database,
    None,
).await?;


// Perform step 1
if perform_step1().await.is_ok() {
    // Create nested snapshot for step 2
    let snapshot2 = manager.create_snapshot(
        ComponentType::Database,
        Some(snapshot1.clone()),
    ).await?;


    // Perform step 2
    if let Err(e) = perform_step2().await {
        // Rollback to snapshot2 (step 1 preserved)
        manager.rollback(&snapshot2).await?;
    } else {
        // Both steps succeeded
        manager.cleanup_snapshot(&snapshot2).await?;
        manager.cleanup_snapshot(&snapshot1).await?;
    }
}

Rollback with Verification

// Execute rollback with automatic verification
let snapshot_id = manager.create_snapshot(
    ComponentType::Storage,
    None,
).await?;


// ... operation fails ...


let result = manager.rollback(&snapshot_id).await?;


println!("Rollback completed:");
println!("  Status: {:?}", result.status);
println!("  Duration: {}ms", result.duration_ms);
println!("  Attempts: {}", result.attempts);
println!("  Verified: {}", result.verified);
println!("  Message: {}", result.message);


match result.status {
    RollbackStatus::Success => {
        println!("System restored successfully!");
    }
    RollbackStatus::Failed => {
        eprintln!("Rollback failed - manual intervention needed");
    }
    RollbackStatus::VerificationFailed => {
        eprintln!("Rollback completed but verification failed");
    }
    _ => {}
}

Automatic Cleanup

// Clean up old snapshots periodically
tokio::spawn(async move {
    let mut interval = tokio::time::interval(
        std::time::Duration::from_secs(3600)  // Every hour
    );


    loop {
        interval.tick().await;
        let cleaned = manager.cleanup_old_snapshots().await;
        println!("Cleaned up {} old snapshots", cleaned);
    }
});

Statistics

// Get rollback statistics
let stats = manager.get_stats().await;
println!("Rollback Statistics");
println!("==================");
println!("Total rollbacks: {}", stats.total_rollbacks);
println!("Successful: {} ({:.1}%)",
    stats.successful_rollbacks,
    stats.success_rate
);
println!("Failed: {}", stats.failed_rollbacks);
println!("Partial: {}", stats.partial_rollbacks);
println!("Average duration: {:.0}ms", stats.avg_duration_ms);
println!("Snapshots created: {}", stats.snapshots_created);
println!("Snapshots cleaned: {}", stats.snapshots_cleaned);


// Get rollback history
let history = manager.get_history().await;
for entry in history.iter().take(10) {
    println!("{}: {} - {:?} ({}ms)",
        entry.started_at.format("%Y-%m-%d %H:%M:%S"),
        entry.component,
        entry.status,
        entry.duration_ms
    );
}

Recovery History

The recovery history manager tracks all recovery attempts and identifies patterns for improved decision-making.

Basic Setup

use heliosdb_self_healing::recovery_history::{
    RecoveryHistoryManager, HistoryConfig
};
use heliosdb_self_healing::recovery::RecoveryResult;
use heliosdb_self_healing::detector::FailureType;
use heliosdb_self_healing::health::ComponentType;


let config = HistoryConfig {
    enabled: true,                      // Enable history tracking
    max_events: 10000,                  // Maximum events to store
    enable_pattern_analysis: true,      // Enable ML pattern detection
    pattern_threshold: 3,               // Min occurrences for pattern
    retention_secs: 86400 * 7,         // 7 days retention
};


let manager = RecoveryHistoryManager::new(config);

Logging Recovery Events

// Log recovery result
let result = RecoveryResult {
    failure_id: "fail-123".to_string(),
    strategy: RecoveryStrategy::Restart,
    status: RecoveryStatus::Success,
    duration_ms: 1250,
    started_at: chrono::Utc::now(),
    completed_at: Some(chrono::Utc::now()),
    message: "Successfully restarted database component".to_string(),
    attempts: 2,
};


let event_id = manager.log_event(
    &result,
    ComponentType::Database,
    FailureType::ComponentCrash,
).await?;


println!("Logged recovery event: {}", event_id);

Retrieving Historical Data

// Get all events
let all_events = manager.get_events().await;
println!("Total events: {}", all_events.len());


// Filter by component
let db_events = manager.get_events_for_component(
    ComponentType::Database
).await;
println!("Database events: {}", db_events.len());


// Filter by failure type
let crash_events = manager.get_events_for_failure(
    FailureType::ComponentCrash
).await;
println!("Crash events: {}", crash_events.len());


// Display recent events
for event in all_events.iter().take(5) {
    println!("{}: {:?} on {:?} using {:?} - {:?} ({}ms)",
        event.timestamp.format("%H:%M:%S"),
        event.failure_type,
        event.component,
        event.strategy,
        event.status,
        event.duration_ms
    );
}

Pattern Recognition

// Get detected patterns
let patterns = manager.get_patterns().await;
println!("Detected {} patterns", patterns.len());


for pattern in patterns {
    println!("\nPattern: {:?} on {:?}",
        pattern.failure_type,
        pattern.component
    );
    println!("  Best strategy: {:?}", pattern.best_strategy);
    println!("  Success rate: {:.1}%", pattern.success_rate);
    println!("  Occurrences: {}", pattern.occurrences);
    println!("  Avg duration: {:.0}ms", pattern.avg_duration_ms);
    println!("  Last seen: {}", pattern.last_seen);
}

Intelligent Strategy Selection

// Get best strategy based on historical data
let best_strategy = manager.get_best_strategy(
    ComponentType::Storage,
    FailureType::ResourceExhaustion,
).await;


if let Some(strategy) = best_strategy {
    println!("Recommended strategy for Storage/ResourceExhaustion: {:?}",
        strategy);


    // Use recommended strategy
    execute_recovery_with_strategy(strategy).await?;
} else {
    println!("No historical data - using default strategy");
    execute_recovery_with_strategy(RecoveryStrategy::Restart).await?;
}

Trend Analysis

// Analyze trends over time
let trends = manager.get_trends(7).await;  // 7 time periods


println!("Recovery Trends:");
println!("===============");
for trend in trends {
    println!("{}: {} recoveries, {:.1}% success, avg {}ms",
        trend.period,
        trend.recovery_count,
        trend.success_rate,
        trend.avg_duration_ms
    );
}


// Detect degrading performance
if trends.len() >= 2 {
    let recent = &trends[trends.len() - 1];
    let previous = &trends[trends.len() - 2];


    if recent.success_rate < previous.success_rate - 10.0 {
        eprintln!("WARNING: Recovery success rate declining!");
        alert_operations_team().await;
    }
}

Statistics

// Get comprehensive statistics
let stats = manager.get_stats().await;
println!("Recovery History Statistics");
println!("==========================");
println!("Total events: {}", stats.total_events);
println!("Successful recoveries: {} ({:.1}%)",
    stats.successful_recoveries,
    stats.overall_success_rate
);
println!("Failed recoveries: {}", stats.failed_recoveries);
println!("Average duration: {:.0}ms", stats.avg_recovery_duration_ms);
println!("Patterns detected: {}", stats.patterns_detected);


if let Some(failure_type) = stats.most_common_failure {
    println!("Most common failure: {:?}", failure_type);
}


if let Some(strategy) = stats.most_effective_strategy {
    println!("Most effective strategy: {:?}", strategy);
}

Cleanup

// Clean up old events periodically
let cleaned = manager.cleanup_old_events().await;
println!("Removed {} old events", cleaned);

Sandbox Testing

The sandbox manager provides isolated environments for safe recovery strategy testing.

Basic Setup

use heliosdb_self_healing::sandbox::{
    SandboxManager, SandboxConfig, IsolationLevel, TestOutcome
};
use heliosdb_self_healing::recovery::RecoveryStrategy;
use heliosdb_self_healing::detector::FailureType;
use heliosdb_self_healing::health::ComponentType;


let config = SandboxConfig {
    default_isolation: IsolationLevel::Complete,
    max_test_duration: std::time::Duration::from_secs(300),
    auto_rollback: true,
    max_concurrent_sandboxes: 5,
    performance_threshold: 10.0,       // Max 10% performance impact
};


let manager = SandboxManager::new(config);

Creating and Testing in Sandbox

// Create sandbox for testing
let sandbox_id = manager.create_sandbox(
    ComponentType::Database,
    FailureType::PerformanceDegradation,
    Some(IsolationLevel::ReadOnly),  // Read-only isolation
).await?;


println!("Created sandbox: {}", sandbox_id);


// Test recovery strategy
let result = manager.test_recovery(
    &sandbox_id,
    RecoveryStrategy::ClearCache,
).await?;


println!("Test Results:");
println!("  Outcome: {:?}", result.outcome);
println!("  Duration: {:?}", result.duration);
println!("  CPU overhead: {:.2}%", result.metrics.cpu_overhead);
println!("  Memory overhead: {:.2}MB", result.metrics.memory_overhead);
println!("  Latency impact: {:.2}ms", result.metrics.latency_impact);
println!("  Rolled back: {}", result.rolled_back);


// Clean up
manager.destroy_sandbox(&sandbox_id).await?;

Testing Multiple Strategies

// Compare different recovery strategies
let strategies = vec![
    RecoveryStrategy::Restart,
    RecoveryStrategy::ClearCache,
    RecoveryStrategy::ScaleResources,
    RecoveryStrategy::Failover,
];


let sandbox_id = manager.create_sandbox(
    ComponentType::Query,
    FailureType::SlowQuery,
    None,
).await?;


println!("Testing {} strategies...\n", strategies.len());


let mut best_strategy = None;
let mut best_score = 0.0;


for strategy in strategies {
    let result = manager.test_recovery(&sandbox_id, strategy).await?;


    // Calculate score (success + low overhead)
    let score = match result.outcome {
        TestOutcome::Success => {
            100.0 - result.metrics.cpu_overhead - result.metrics.latency_impact
        }
        _ => 0.0,
    };


    println!("{:?}: {:?} (score: {:.1})", strategy, result.outcome, score);


    if score > best_score {
        best_score = score;
        best_strategy = Some(strategy);
    }
}


if let Some(strategy) = best_strategy {
    println!("\nBest strategy: {:?} (score: {:.1})", strategy, best_score);
}


manager.destroy_sandbox(&sandbox_id).await?;

Isolation Levels

// Test with different isolation levels
let isolation_levels = vec![
    IsolationLevel::Complete,    // No production access
    IsolationLevel::ReadOnly,    // Read-only production data
    IsolationLevel::Shadow,      // Parallel execution
];


for level in isolation_levels {
    let sandbox_id = manager.create_sandbox(
        ComponentType::Storage,
        FailureType::ResourceExhaustion,
        Some(level),
    ).await?;


    println!("Testing with {:?} isolation", level);


    let result = manager.test_recovery(
        &sandbox_id,
        RecoveryStrategy::ScaleResources,
    ).await?;


    println!("  Result: {:?}", result.outcome);
    println!("  Performance impact: {:.2}%", result.metrics.throughput_impact);


    manager.destroy_sandbox(&sandbox_id).await?;
}

Performance Testing

// Test performance impact
let sandbox_id = manager.create_sandbox(
    ComponentType::Database,
    FailureType::ComponentCrash,
    None,
).await?;


let result = manager.test_recovery(
    &sandbox_id,
    RecoveryStrategy::Restart,
).await?;


println!("Performance Metrics:");
println!("  CPU overhead: {:.2}%", result.metrics.cpu_overhead);
println!("  Memory overhead: {:.2}MB", result.metrics.memory_overhead);
println!("  I/O overhead: {:.2} ops/s", result.metrics.io_overhead);
println!("  Latency impact: {:.2}ms", result.metrics.latency_impact);
println!("  Throughput impact: {:.2}%", result.metrics.throughput_impact);


// Check if within acceptable limits
if result.metrics.cpu_overhead > 10.0 {
    println!("WARNING: CPU overhead exceeds threshold!");
}


if result.metrics.latency_impact > 100.0 {
    println!("WARNING: Latency impact too high!");
}


manager.destroy_sandbox(&sandbox_id).await?;

Statistics

// Get sandbox statistics
let stats = manager.get_stats();
println!("Sandbox Statistics");
println!("=================");
println!("Total tests: {}", stats.total_tests);
println!("Successful: {} ({:.1}%)",
    stats.successful_tests,
    (stats.successful_tests as f64 / stats.total_tests as f64) * 100.0
);
println!("Failed: {}", stats.failed_tests);
println!("Timed out: {}", stats.timed_out_tests);
println!("Aborted: {}", stats.aborted_tests);
println!("Auto-rollbacks: {}", stats.total_rollbacks);
println!("Avg test duration: {:.0}ms", stats.avg_test_duration_ms);
println!("Avg performance overhead: {:.2}%", stats.avg_performance_overhead);


// Get test results
let results = manager.get_all_test_results();
println!("\nRecent test results:");
for result in results.iter().take(5) {
    println!("  {} - {:?}: {:?}",
        result.timestamp.format("%H:%M:%S"),
        result.strategy,
        result.outcome
    );
}

INTEGRATION EXAMPLES

Complete Self-Healing Pipeline

Integrate all 6 components for comprehensive autonomous healing:

use heliosdb_self_healing::*;
use std::sync::Arc;
use tokio::time::{interval, Duration};


struct ComprehensiveSelfHealing {
    anomaly_detector: Arc<MlAnomalyDetector>,
    causal_engine: Arc<CausalInferenceEngine>,
    rl_selector: Arc<RLActionSelector>,
    rollback_manager: Arc<AutoRollbackManager>,
    history_manager: Arc<RecoveryHistoryManager>,
    sandbox_manager: Arc<SandboxManager>,
}


impl ComprehensiveSelfHealing {
    async fn new() -> Result<Self, Box<dyn std::error::Error>> {
        Ok(Self {
            anomaly_detector: Arc::new(MlAnomalyDetector::new(
                DetectorConfig::default()
            )),
            causal_engine: Arc::new(CausalInferenceEngine::new(
                InferenceConfig::default()
            )),
            rl_selector: Arc::new(RLActionSelector::new(
                PPOConfig::default()
            )),
            rollback_manager: Arc::new(AutoRollbackManager::new(
                RollbackConfig::default()
            )),
            history_manager: Arc::new(RecoveryHistoryManager::new(
                HistoryConfig::default()
            )),
            sandbox_manager: Arc::new(SandboxManager::new(
                SandboxConfig::default()
            )),
        })
    }


    async fn monitor_and_heal(&self) -> Result<(), Box<dyn std::error::Error>> {
        let mut ticker = interval(Duration::from_secs(10));


        loop {
            ticker.tick().await;


            // 1. Collect metrics
            let metrics = collect_system_metrics().await?;


            // 2. Detect anomalies
            if let Some(anomaly) = self.anomaly_detector
                .detect_performance(metrics.clone())
                .await
            {
                println!("Anomaly detected: {:?}", anomaly);


                // 3. Identify root cause
                let symptom_id = format!("anomaly_{}", anomaly.id);
                if let Some(root_cause) = self.causal_engine
                    .infer_root_cause(&symptom_id)
                    .await
                {
                    println!("Root cause: {}", root_cause.node.name);


                    // 4. Create state snapshot
                    let snapshot_id = self.rollback_manager
                        .create_snapshot(ComponentType::Database, None)
                        .await?;


                    // 5. Check historical patterns
                    let best_strategy = self.history_manager
                        .get_best_strategy(
                            ComponentType::Database,
                            FailureType::PerformanceDegradation,
                        )
                        .await;


                    // 6. Select action (RL or historical)
                    let action = if let Some(strategy) = best_strategy {
                        // Use historical best strategy
                        RLAction::from_strategy(strategy)
                    } else {
                        // Use RL to select
                        let state = create_rl_state(&anomaly, &metrics);
                        self.rl_selector.select_action(&state)
                    };


                    // 7. Test in sandbox first
                    let sandbox_id = self.sandbox_manager
                        .create_sandbox(
                            ComponentType::Database,
                            FailureType::PerformanceDegradation,
                            None,
                        )
                        .await?;


                    let test_result = self.sandbox_manager
                        .test_recovery(&sandbox_id, action.to_recovery_strategy())
                        .await?;


                    self.sandbox_manager.destroy_sandbox(&sandbox_id).await?;


                    // 8. Execute if sandbox test passed
                    if test_result.outcome == TestOutcome::Success {
                        let start = std::time::Instant::now();


                        match execute_recovery_action(&action).await {
                            Ok(_) => {
                                let duration = start.elapsed().as_millis() as u64;


                                // Success - clean up snapshot
                                self.rollback_manager
                                    .cleanup_snapshot(&snapshot_id)
                                    .await?;


                                // Log success
                                let result = RecoveryResult {
                                    failure_id: anomaly.id.clone(),
                                    strategy: action.to_recovery_strategy(),
                                    status: RecoveryStatus::Success,
                                    duration_ms: duration,
                                    started_at: chrono::Utc::now(),
                                    completed_at: Some(chrono::Utc::now()),
                                    message: "Autonomous recovery successful".to_string(),
                                    attempts: 1,
                                };


                                self.history_manager.log_event(
                                    &result,
                                    ComponentType::Database,
                                    FailureType::PerformanceDegradation,
                                ).await?;


                                // Update RL with positive reward
                                let state = create_rl_state(&anomaly, &metrics);
                                let next_state = create_rl_state_after_recovery().await;
                                let calculator = RewardCalculator::default();
                                let reward = calculator.calculate_reward(true, duration, 0.1);


                                self.rl_selector.add_experience(
                                    state, action, reward, next_state, true
                                );


                                println!("Recovery successful!");
                            }
                            Err(e) => {
                                // Failure - rollback
                                println!("Recovery failed: {} - rolling back", e);


                                self.rollback_manager
                                    .rollback(&snapshot_id)
                                    .await?;


                                // Log failure
                                let result = RecoveryResult {
                                    failure_id: anomaly.id.clone(),
                                    strategy: action.to_recovery_strategy(),
                                    status: RecoveryStatus::Failed,
                                    duration_ms: 0,
                                    started_at: chrono::Utc::now(),
                                    completed_at: Some(chrono::Utc::now()),
                                    message: format!("Recovery failed: {}", e),
                                    attempts: 1,
                                };


                                self.history_manager.log_event(
                                    &result,
                                    ComponentType::Database,
                                    FailureType::PerformanceDegradation,
                                ).await?;


                                // Update RL with negative reward
                                let state = create_rl_state(&anomaly, &metrics);
                                self.rl_selector.add_experience(
                                    state,
                                    action,
                                    -5.0,
                                    state.clone(),
                                    true,
                                );
                            }
                        }


                        // Train RL periodically
                        if self.rl_selector.buffer_size() >= 64 {
                            self.rl_selector.train()?;
                        }
                    }
                }
            }
        }
    }
}


#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let healer = ComprehensiveSelfHealing::new().await?;
    healer.monitor_and_heal().await
}

COMMON USE CASES

Use Case 1: High CPU Detection and Resolution

async fn handle_high_cpu() -> Result<(), Box<dyn std::error::Error>> {
    let detector = MlAnomalyDetector::new(DetectorConfig::default());


    // Monitor CPU
    loop {
        let mut metrics = HashMap::new();
        metrics.insert("cpu_usage".to_string(), get_current_cpu_usage());


        if let Some(anomaly) = detector.detect_resource(metrics).await {
            if anomaly.severity >= Severity::High {
                println!("High CPU detected: {}", anomaly.description);


                // Immediate action
                scale_compute_resources().await?;


                // Analyze workload
                identify_heavy_queries().await?;


                break;
            }
        }


        tokio::time::sleep(Duration::from_secs(5)).await;
    }


    Ok(())
}

Use Case 2: Memory Leak Detection and Restart

async fn detect_and_fix_memory_leak() -> Result<(), Box<dyn std::error::Error>> {
    let detector = MlAnomalyDetector::new(DetectorConfig::default());
    let rollback_mgr = AutoRollbackManager::new(RollbackConfig::default());


    // Build memory usage baseline
    for _ in 0..100 {
        let mut metrics = HashMap::new();
        metrics.insert("memory_mb".to_string(), get_memory_usage());
        detector.detect_resource(metrics).await;
        tokio::time::sleep(Duration::from_secs(1)).await;
    }


    // Monitor for memory leak
    loop {
        let mut metrics = HashMap::new();
        let current_memory = get_memory_usage();
        metrics.insert("memory_mb".to_string(), current_memory);


        if let Some(anomaly) = detector.detect_resource(metrics).await {
            // Memory leak suspected
            println!("Possible memory leak: {}", anomaly.description);


            // Create snapshot
            let snapshot = rollback_mgr
                .create_snapshot(ComponentType::Database, None)
                .await?;


            // Attempt graceful restart
            match restart_component_gracefully().await {
                Ok(_) => {
                    println!("Component restarted successfully");
                    rollback_mgr.cleanup_snapshot(&snapshot).await?;
                }
                Err(e) => {
                    println!("Restart failed: {}", e);
                    rollback_mgr.rollback(&snapshot).await?;
                }
            }


            break;
        }


        tokio::time::sleep(Duration::from_secs(30)).await;
    }


    Ok(())
}

Use Case 3: Cascading Failure Prevention

async fn prevent_cascading_failure() -> Result<(), Box<dyn std::error::Error>> {
    let causal_engine = CausalInferenceEngine::new(InferenceConfig::default());


    // Build failure dependency graph
    build_system_dependency_graph(&causal_engine).await;


    // Detect primary failure
    let primary_failure = "database_connection_timeout";


    // Predict cascading effects
    if let Some(root_cause) = causal_engine
        .infer_root_cause(primary_failure)
        .await
    {
        println!("Primary failure will cascade from: {}", root_cause.node.name);


        // Take preemptive action
        for component_id in &root_cause.causal_path {
            preemptively_scale_component(component_id).await?;
        }


        println!("Preemptive scaling complete - cascade prevented");
    }


    Ok(())
}

Use Case 4: Autonomous Recovery Learning

async fn learn_recovery_strategies() -> Result<(), Box<dyn std::error::Error>> {
    let rl_selector = RLActionSelector::new(PPOConfig::default());
    let sandbox_mgr = SandboxManager::new(SandboxConfig::default());


    // Training loop
    for episode in 0..1000 {
        // Simulate failure
        let failure_type = random_failure_type();
        let component = random_component();


        // Create state
        let state = create_random_state(component, failure_type);


        // Select action
        let action = rl_selector.select_action(&state);


        // Test in sandbox
        let sandbox_id = sandbox_mgr
            .create_sandbox(component, failure_type, None)
            .await?;


        let result = sandbox_mgr
            .test_recovery(&sandbox_id, action.to_recovery_strategy())
            .await?;


        sandbox_mgr.destroy_sandbox(&sandbox_id).await?;


        // Calculate reward
        let reward = match result.outcome {
            TestOutcome::Success => {
                10.0 - (result.duration.as_secs_f64() * 0.01)
                     - result.metrics.cpu_overhead
            }
            TestOutcome::Failure => -5.0,
            TestOutcome::Timeout => -2.0,
            _ => 0.0,
        };


        // Store experience
        rl_selector.add_experience(
            state.clone(),
            action,
            reward,
            state,
            true,
        );


        // Train
        if episode % 10 == 0 && rl_selector.buffer_size() >= 64 {
            rl_selector.train()?;


            let stats = rl_selector.get_stats();
            println!("Episode {}: Avg reward: {:.2}, Success rate: {:.1}%",
                episode,
                stats.avg_reward,
                (stats.successful_actions as f64 / stats.total_actions as f64) * 100.0
            );
        }
    }


    println!("Training complete!");
    Ok(())
}

TROUBLESHOOTING

Issue: Anomaly Detector Not Detecting Issues

Symptoms: Known anomalies not being flagged

Solutions:

// 1. Check if baseline has been established
let baseline = detector.get_baseline(AnomalyCategory::Performance).await;
if baseline.is_none() {
    println!("Baseline not established - need more data points");
}


// 2. Lower thresholds temporarily
let config = DetectorConfig {
    zscore_threshold: 2.0,  // More sensitive
    min_confidence: 0.6,    // Lower bar
    ..Default::default()
};


// 3. Reset baseline if system changed significantly
detector.reset_baselines().await;

Issue: RL Agent Not Learning

Symptoms: No improvement in success rate over time

Solutions:

// 1. Check buffer size
if selector.buffer_size() < 64 {
    println!("Not enough training samples");
}


// 2. Adjust learning rate
let config = PPOConfig {
    learning_rate: 0.01,  // Increase if learning too slow
    ..Default::default()
};


// 3. Review reward function
let calculator = RewardCalculator {
    success_weight: 15.0,    // Increase reward for success
    failure_penalty: -10.0,  // Increase penalty for failure
    ..Default::default()
};


// 4. Check if training is being called
selector.train()?;
let stats = selector.get_stats();
println!("Training episodes: {}", stats.training_episodes);

Issue: Rollback Failing

Symptoms: Rollback operations not restoring state

Solutions:

// 1. Check if rollback is enabled
if !config.enabled {
    println!("Rollback is disabled!");
}


// 2. Verify snapshot exists
let snapshot = manager.create_snapshot(component, None).await?;
// ... later ...
match manager.rollback(&snapshot).await {
    Ok(result) => {
        if result.status != RollbackStatus::Success {
            eprintln!("Rollback failed: {}", result.message);
        }
    }
    Err(e) => eprintln!("Rollback error: {}", e),
}


// 3. Enable verification
let config = RollbackConfig {
    verify_rollback: true,
    ..Default::default()
};


// 4. Check max attempts
let config = RollbackConfig {
    max_attempts: 5,  // Increase retry attempts
    ..Default::default()
};

Issue: High Memory Usage

Symptoms: Self-healing system consuming too much memory

Solutions:

// 1. Limit history size
let history_config = HistoryConfig {
    max_events: 5000,  // Reduce from 10000
    ..Default::default()
};


// 2. Clean up old data
manager.cleanup_old_events().await;
rollback_mgr.cleanup_old_snapshots().await;


// 3. Limit RL buffer
let ppo_config = PPOConfig {
    buffer_size: 5000,  // Reduce from 10000
    ..Default::default()
};


// 4. Reduce detector window
let detector_config = DetectorConfig {
    window_size: 500,  // Reduce from 1000
    ..Default::default()
};

Issue: Sandbox Tests Taking Too Long

Symptoms: Sandbox operations timing out

Solutions:

// 1. Reduce test duration limit
let config = SandboxConfig {
    max_test_duration: Duration::from_secs(60),  // Reduce from 300
    ..Default::default()
};


// 2. Limit concurrent sandboxes
let config = SandboxConfig {
    max_concurrent_sandboxes: 2,  // Reduce from 5
    ..Default::default()
};


// 3. Check resource availability
let stats = manager.get_stats();
if stats.timed_out_tests > stats.successful_tests {
    println!("Too many timeouts - system may be overloaded");
}

Getting Help

If issues persist:

  1. Enable debug logging:
use tracing_subscriber;
tracing_subscriber::fmt::init();
  1. Check system report:
let report = engine.get_system_report();
println!("{:#?}", report);
  1. Review statistics:
println!("Recovery stats: {:#?}", engine.get_recovery_stats());
println!("Failure stats: {:#?}", engine.get_failure_stats());
println!("Prediction stats: {:#?}", engine.get_prediction_stats());
  1. Contact support with:
    • System report output
    • Logs from the last 24 hours
    • Configuration details
    • Example code that reproduces the issue

NEXT STEPS

Need help? Open an issue on GitHub or contact the HeliosDB team.