F5.3.3 Distributed Query Optimizer - Recommended Production Configuration

Quick Start Guide for Production Deployment

Standard Production Setup

use heliosdb_distributed_optimizer::*;
use std::sync::Arc;

// 1. Create cost model with default weights
let cost_model = Arc::new(DistributedCostModel::default());

// 2. Create optimizer with production configuration
let optimizer = QueryOptimizer::new(
    OptimizerConfig::production(),
    cost_model
);

// 3. Create statistics collector
let stats_collector = Arc::new(StatisticsCollector::new(
    StatisticsConfig::default()
));

// 4. Create adaptive planner with production settings
let adaptive_planner = AdaptivePlanner::new(
    stats_collector.clone(),
    AdaptiveConfig::production()
);

// 5. Use in query pipeline
fn optimize_and_execute(
    query: DistributedOperation,
    cluster: &ClusterStats,
    optimizer: &QueryOptimizer,
    planner: &AdaptivePlanner,
) -> Result<ExecutionResult> {
    // Optimize query
    let opt_result = optimizer.optimize(query, cluster)?;

    // Log optimization result
    log::info!(
        "Query optimized: {:.1}% improvement in {}μs",
        opt_result.improvement_ratio * 100.0,
        opt_result.optimization_time_us
    );

    // Execute optimized query
    let exec_result = execute_distributed_query(opt_result.plan.operation)?;

    // Learn from execution
    let execution = QueryExecution {
        query_id: QueryId::new(),
        operation: opt_result.plan.operation,
        execution_time_ms: exec_result.duration_ms,
        rows_processed: exec_result.rows_returned,
        bytes_transferred: exec_result.network_bytes,
        strategy: opt_result.strategy,
        timestamp: SystemTime::now(),
    };

    let actual_cost = DistributedCost {
        cpu_cost: exec_result.cpu_time_ms,
        network_cost: exec_result.network_time_ms,
        io_cost: exec_result.io_time_ms,
        memory_cost: exec_result.memory_gb,
        parallelism: exec_result.parallelism,
        estimated_time_ms: exec_result.duration_ms,
    };

    planner.learn_from_execution(&execution, &actual_cost);

    Ok(exec_result)
}

Production Configuration Values

OptimizerConfig::production()

OptimizerConfig {
    enable_join_reordering: true,
    enable_partition_pruning: true,
    enable_predicate_pushdown: true,
    enable_cost_based_optimization: true,
    max_join_reorder_size: 6,        // Reduced for faster optimization
    timeout_ms: 50,                   // Strict timeout
}

AdaptiveConfig::production()

AdaptiveConfig {
    learning_rate: 0.03,              // Conservative for stability
    min_samples_for_learning: 50,     // High confidence threshold
    enable_join_strategy_learning: true,
    enable_partition_strategy_learning: true,
    enable_cost_model_tuning: true,
}

Environment Variables

# Optional: Override default settings
export HELIOSDB_OPTIMIZER_MAX_JOIN_REORDER=6
export HELIOSDB_OPTIMIZER_TIMEOUT_MS=50
export HELIOSDB_ADAPTIVE_LEARNING_RATE=0.03
export HELIOSDB_ADAPTIVE_MIN_SAMPLES=50

# Monitoring
export HELIOSDB_ENABLE_METRICS=true
export HELIOSDB_METRICS_PORT=9090

# Logging
export RUST_LOG=heliosdb_distributed_optimizer=info

Monitoring Setup

use prometheus::{Registry, HistogramVec, GaugeVec};

// Create metrics
let registry = Registry::new();

let optimization_latency = HistogramVec::new(
    prometheus::HistogramOpts::new(
        "heliosdb_optimizer_latency_seconds",
        "Query optimization latency in seconds"
    ).buckets(vec![0.001, 0.005, 0.01, 0.05, 0.1, 0.5]),
    &["query_type"]
).unwrap();

let improvement_ratio = GaugeVec::new(
    prometheus::opts!(
        "heliosdb_optimizer_improvement_ratio",
        "Query cost improvement ratio"
    ),
    &["query_type"]
).unwrap();

registry.register(Box::new(optimization_latency.clone())).unwrap();
registry.register(Box::new(improvement_ratio.clone())).unwrap();

// Update metrics
let start = Instant::now();
let result = optimizer.optimize(query, cluster)?;
optimization_latency
    .with_label_values(&[&query_type])
    .observe(start.elapsed().as_secs_f64());
improvement_ratio
    .with_label_values(&[&query_type])
    .set(result.improvement_ratio);

Deployment Checklist

Quick Troubleshooting

High Optimization Latency (>50ms)

// Reduce max_join_reorder_size
let config = OptimizerConfig {
    max_join_reorder_size: 5,
    ..OptimizerConfig::production()
};

Unstable Cost Predictions

// Reduce learning rate
let config = AdaptiveConfig {
    learning_rate: 0.02,
    ..AdaptiveConfig::production()
};

Memory Growth

// Reset adaptive model periodically
if stats.total_samples > 10_000 {
    planner.reset();
}

Support

Tuning Guide: /docs/tuning/F5_3_3_QUERY_OPTIMIZER_TUNING.md
Performance Report: /docs/tuning/F5_3_3_PERFORMANCE_REPORT.md
Source Code: /heliosdb-distributed-optimizer/