Hyperparameter Optimization (Kaizen) - Entrenar - Training & Optimization Library
Description: A comprehensive guide to building neural network training systems with autograd, optimizers, LoRA/QLoRA, and quantization using EXTREME TDD methodology

# Hyperparameter Optimization

Bayesian hyperparameter optimization with Tree-structured Parzen Estimator (TPE) for efficient search.

## Toyota Principle: Kaizen

Continuous improvement through systematic optimization. HPO automates the search for better hyperparameters.

## Quick Start

    #![allow(unused)]
    fn main() {
    use entrenar::optim::hpo::{
        BayesianOptimizer, SearchSpace, ParameterDef, Trial, TrialState
    };

    // Define search space
    let space = SearchSpace::new()
        .add("learning_rate", ParameterDef::LogUniform(1e-5, 1e-1))
        .add("batch_size", ParameterDef::Discrete(vec![16, 32, 64, 128]))
        .add("hidden_dim", ParameterDef::Uniform(64.0, 512.0))
        .add("dropout", ParameterDef::Uniform(0.0, 0.5));

    // Create optimizer
    let mut optimizer = BayesianOptimizer::new(space, 100); // 100 trials

    // Optimization loop
    while let Some(trial) = optimizer.ask()? {
        let params = &trial.params;

        // Train with these parameters
        let accuracy = train_model(
            params.get("learning_rate").unwrap().as_f64(),
            params.get("batch_size").unwrap().as_i64() as usize,
            params.get("hidden_dim").unwrap().as_f64() as usize,
            params.get("dropout").unwrap().as_f64(),
        )?;

        // Report result
        optimizer.tell(trial.id, accuracy, TrialState::Complete)?;
    }

    // Get best parameters
    let best = optimizer.best_trial()?;
    println!("Best accuracy: {}", best.value.unwrap());
    println!("Best params: {:?}", best.params);
    }

## Parameter Types

    #![allow(unused)]
    fn main() {
    use entrenar::optim::hpo::ParameterDef;

    // Continuous uniform [low, high]
    ParameterDef::Uniform(0.0, 1.0)

    // Log-uniform (good for learning rates)
    ParameterDef::LogUniform(1e-5, 1e-1)

    // Discrete choices
    ParameterDef::Discrete(vec![32, 64, 128, 256])

    // Categorical (strings)
    ParameterDef::Categorical(vec![
        "relu".to_string(),
        "gelu".to_string(),
        "swish".to_string(),
    ])

    // Integer range
    ParameterDef::IntUniform(1, 10)
    }

## Grid Search

For exhaustive search over discrete spaces:

    #![allow(unused)]
    fn main() {
    use entrenar::optim::hpo::GridSearch;

    let grid = GridSearch::new()
        .add("lr", vec![0.001, 0.01, 0.1])
        .add("batch_size", vec![32, 64])
        .add("optimizer", vec!["adam", "sgd"]);

    for config in grid.iter() {
        let result = train_with_config(&config)?;
        println!("{:?} -> {}", config, result);
    }
    }

## Random Search

For baseline comparison:

    #![allow(unused)]
    fn main() {
    use entrenar::optim::hpo::RandomSearch;

    let search = RandomSearch::new(space, 50); // 50 random trials

    for trial in search.iter() {
        let result = train_with_params(&trial.params)?;
        println!("Trial {}: {}", trial.id, result);
    }
    }

## Early Stopping

Prune unpromising trials:

    #![allow(unused)]
    fn main() {
    use entrenar::optim::hpo::{BayesianOptimizer, MedianPruner};

    let pruner = MedianPruner::new()
        .with_n_startup_trials(5)
        .with_n_warmup_steps(10);

    let mut optimizer = BayesianOptimizer::new(space, 100)
        .with_pruner(pruner);

    // During training
    for epoch in 0..100 {
        let loss = train_epoch(&model);

        // Check if should prune
        if optimizer.should_prune(trial_id, epoch, loss)? {
            optimizer.tell(trial_id, loss, TrialState::Pruned)?;
            break;
        }
    }
    }

## Multi-Objective Optimization

Optimize multiple objectives simultaneously:

    #![allow(unused)]
    fn main() {
    use entrenar::optim::hpo::MultiObjectiveOptimizer;

    let mut optimizer = MultiObjectiveOptimizer::new(space)
        .add_objective("accuracy", true)   // maximize
        .add_objective("latency", false);  // minimize

    while let Some(trial) = optimizer.ask()? {
        let (accuracy, latency) = evaluate(&trial.params)?;
        optimizer.tell(trial.id, vec![accuracy, latency])?;
    }

    // Get Pareto front
    let pareto_front = optimizer.pareto_front()?;
    }

## Parallel Trials

Run multiple trials concurrently:

    #![allow(unused)]
    fn main() {
    use entrenar::optim::hpo::BayesianOptimizer;
    use std::thread;

    let optimizer = Arc::new(Mutex::new(BayesianOptimizer::new(space, 100)));

    let handles: Vec<_> = (0..4).map(|_| {
        let opt = Arc::clone(&optimizer);
        thread::spawn(move || {
            loop {
                let trial = {
                    let mut opt = opt.lock().unwrap();
                    opt.ask()
                };

                if let Some(trial) = trial {
                    let result = train_with_params(&trial.params);
                    let mut opt = opt.lock().unwrap();
                    opt.tell(trial.id, result, TrialState::Complete);
                } else {
                    break;
                }
            }
        })
    }).collect();

    for handle in handles {
        handle.join().unwrap();
    }
    }

## Cargo Run Example

    # Run HPO sweep
    cargo run --example hpo_sweep

    # With specific search space
    cargo run --example hpo_sweep -- --trials 50 --space config/search_space.yaml

    # Resume from checkpoint
    cargo run --example hpo_sweep -- --resume hpo_checkpoint.json

## Visualization

    #![allow(unused)]
    fn main() {
    // Export trials for visualization
    let history = optimizer.history()?;

    // Save as JSON for plotting
    let json = serde_json::to_string_pretty(&history)?;
    std::fs::write("hpo_history.json", json)?;
    }

## Integration with Experiment Tracking

    #![allow(unused)]
    fn main() {
    use entrenar::storage::SqliteBackend;
    use entrenar::optim::hpo::BayesianOptimizer;

    let mut storage = SqliteBackend::open("experiments.db")?;
    let exp_id = storage.create_experiment("hpo-sweep", None)?;

    while let Some(trial) = optimizer.ask()? {
        let run_id = storage.create_run(&exp_id)?;
        storage.start_run(&run_id)?;

        // Log hyperparameters
        for (name, value) in &trial.params {
            storage.log_param(&run_id, name, value.clone().into())?;
        }

        // Train and evaluate
        let result = train_with_params(&trial.params)?;
        storage.log_metric(&run_id, "final_accuracy", 0, result)?;

        storage.complete_run(&run_id, RunStatus::Success)?;
        optimizer.tell(trial.id, result, TrialState::Complete)?;
    }
    }

## Best Practices

1. Start with random search - Establish baseline
2. Use log-uniform for learning rates - Spans orders of magnitude
3. Enable early stopping - Save compute on bad trials
4. Run enough trials - TPE needs ~20 trials to model well
5. Log everything - Integrate with experiment tracking

## See Also

- MLOps Overview
- Experiment Tracking
- Learning Rate Tuning