Checkpoint Encryption - Quick Start Guide

Checkpoint Encryption - Quick Start Guide

5-Minute Setup

Step 1: Create Key Manager (30 seconds)

use heliosdb_streaming::{KmsConfig, KeyRotationPolicy, KeyManager};
use std::sync::Arc;

let key_manager = Arc::new(KeyManager::new(
    KmsConfig::Local {
        master_key: b"your-32-byte-production-key!!!!".to_vec(),
    },
    KeyRotationPolicy::default(), // 30-day rotation
).await?);

Step 2: Enable for Database Source (1 minute)

use heliosdb_streaming::connectors::database::DatabaseSource;

// Create source with encryption
let mut source = DatabaseSource::new_with_retry_and_key_manager(
    source_config,
    retry_policy,
    Some(key_manager.clone()), // ← encryption enabled!
).await?;

// Normal operations - encryption is automatic
let rows = source.read_batch().await?;

// Save encrypted checkpoint
let encrypted_checkpoint = source.serialize_checkpoint().await?;
std::fs::write("checkpoint.bin", &encrypted_checkpoint)?;

// Restore from encrypted checkpoint
let data = std::fs::read("checkpoint.bin")?;
source.deserialize_checkpoint(&data).await?;

Step 3: Enable for Database Sink (1 minute)

use heliosdb_streaming::connectors::database::DatabaseSinkConnector;

// Create sink with encryption
let mut sink = DatabaseSinkConnector::new_with_key_manager(
    sink_config,
    Some(key_manager.clone()), // ← encryption enabled!
).await?;

// Normal operations
sink.write(rows).await?;
sink.flush().await?;

// Save encrypted checkpoint
let encrypted_checkpoint = sink.serialize_checkpoint().await?;
std::fs::write("sink_checkpoint.bin", &encrypted_checkpoint)?;

// Restore from encrypted checkpoint
let data = std::fs::read("sink_checkpoint.bin")?;
sink.deserialize_checkpoint(&data).await?;

Step 4: Start Automatic Key Rotation (30 seconds)

// Start background rotation scheduler
let scheduler = key_manager.start_key_rotation_scheduler();

// That's it! Keys will rotate automatically every 30 days
// To stop: scheduler.stop().await;

---

Production Setup (AWS KMS)

use heliosdb_streaming::KmsConfig;

let key_manager = Arc::new(KeyManager::new(
    KmsConfig::AwsKms {
        key_id: "arn:aws:kms:us-east-1:123456789:key/your-key-id".to_string(),
        region: "us-east-1".to_string(),
    },
    KeyRotationPolicy::default(),
).await?);

// Use same as above - API is identical!

---

Production Setup (Azure Key Vault)

use heliosdb_streaming::KmsConfig;

let key_manager = Arc::new(KeyManager::new(
    KmsConfig::AzureKeyVault {
        vault_url: "https://your-vault.vault.azure.net".to_string(),
        key_name: "heliosdb-checkpoint-key".to_string(),
    },
    KeyRotationPolicy::default(),
).await?);

---

Production Setup (GCP Cloud KMS)

use heliosdb_streaming::KmsConfig;

let key_manager = Arc::new(KeyManager::new(
    KmsConfig::GcpKms {
        project_id: "your-project-id".to_string(),
        location: "us-central1".to_string(),
        key_ring: "heliosdb-keyring".to_string(),
        key_name: "checkpoint-key".to_string(),
        credentials_path: Some("/path/to/credentials.json".to_string()),
    },
    KeyRotationPolicy::default(),
).await?);

---

Common Operations

Manual Key Rotation

key_manager.rotate_key().await?;

Check Rotation Status

let status = key_manager.get_rotation_status();
println!("Current version: {}", status.current_version);
println!("Next rotation in: {}s", status.seconds_until_rotation);

Get Metrics

let stats = key_manager.get_metrics();
println!("Total operations: {}", stats.total_requests);
println!("Success rate: {}%",
    100.0 * stats.successful_requests as f64 / stats.total_requests as f64);

---

Security Checklist

• Production: Use AWS KMS, Azure Key Vault, or GCP Cloud KMS (NOT Local mode)
• Keys: Never hardcode production keys in code
• Rotation: Keep default 30-day rotation interval
• Access: Restrict KMS permissions to service accounts only
• Backups: Store encrypted checkpoints in durable storage (S3, GCS, etc.)
• Monitoring: Enable CloudTrail/Cloud Logging for KMS operations
• Testing: Test checkpoint restore before going live

---

Troubleshooting

”Decryption failed”

Cause: Tampered data or wrong key Fix: Verify checkpoint integrity; check key version matches

”Key version not found”

Cause: Key rotated beyond retention policy Fix: Increase max_previous_keys or re-encrypt old checkpoints

Checkpoint too large

Cause: Normal - encryption adds 32 bytes overhead Fix: No action needed (< 0.01% overhead for typical checkpoints)

---

Testing

# Run all encryption tests
cargo test --package heliosdb-streaming key_management

# Run example
cargo run --example checkpoint_encryption_example

---

Full Documentation

For complete documentation, see:

• CHECKPOINT_ENCRYPTION.md - Full documentation (400+ lines)
• IMPLEMENTATION_SUMMARY.md - Implementation details
• examples/checkpoint_encryption_example.rs - Working example

---

Support

Questions? Check:

1. Documentation: CHECKPOINT_ENCRYPTION.md
2. Example: Run cargo run --example checkpoint_encryption_example
3. Tests: Review test cases in key_management.rs

---

You’re now ready to use encrypted checkpoints in production! 🎉🔒