## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/bench_preflight.rs

    Line
    Count
    Source
    1
    //! Preflight Validation Protocol for Deterministic Benchmarking
    2
    //!
    3
    //! Implements the Toyota Way principles for benchmark quality:
    4
    //! - Jidoka: Fail-fast validation, stop on anomaly
    5
    //! - Poka-yoke: Error-proofing through type-safe configurations
    6
    //! - Genchi Genbutsu: Verify actual system state
    7
    //!
    8
    //! References:
    9
    //! - Hoefler & Belli SC'15 [1]: CV-based stopping
    10
    //! - Vitek & Kalibera EMSOFT'11 [7]: Reproducibility requirements
    11
    //! - Liker [9]: Toyota Way principles
    12
    13
    use std::fmt;
    14
    use std::time::Duration;
    15
    16
    use serde::{Deserialize, Serialize};
    17
    use thiserror::Error;
    18
    19
    // ============================================================================
    20
    // Specification Constants (per spec v1.0.1 Section 4.3)
    21
    // ============================================================================
    22
    23
    /// Canonical benchmark inputs - versioned for reproducibility
    24
    pub mod canonical_inputs {
    25
        /// Version: Changing inputs MUST increment this version
    26
        pub const VERSION: &str = "1.0.0";
    27
    28
        /// Fixed prompt for latency benchmarks
    29
        pub const LATENCY_PROMPT: &str = "Explain the concept of machine learning in one sentence.";
    30
    31
        /// Fixed token sequence for throughput benchmarks
    32
        pub const THROUGHPUT_TOKENS: &[u32] = &[1, 2, 3, 4, 5, 6, 7, 8];
    33
    34
        /// Fixed max tokens for generation benchmarks
    35
        pub const MAX_TOKENS: usize = 50;
    36
    37
        /// Specification version for metadata
    38
        pub const SPEC_VERSION: &str = "1.0.1";
    39
    }
    40
    41
    // ============================================================================
    42
    // Error Types (Poka-yoke: Make errors explicit and typed)
    43
    // ============================================================================
    44
    45
    /// Preflight validation errors with detailed context for diagnosis
    46
    #[derive(Debug, Error)]
    47
    pub enum PreflightError {
    48
        /// Server is not reachable at the specified URL
    49
        #[error("Server unreachable at {url}: {reason}")]
    50
        ServerUnreachable {
    51
            /// The URL that was unreachable
    52
            url: String,
    53
            /// The reason for the failure
    54
            reason: String,
    55
        },
    56
    57
        /// Health check endpoint failed
    58
        #[error("Health check failed at {url}: HTTP {status}")]
    59
        HealthCheckFailed {
    60
            /// The health check URL
    61
            url: String,
    62
            /// HTTP status code returned
    63
            status: u16,
    64
        },
    65
    66
        /// Required model not found in backend
    67
        #[error("Model not found: requested '{requested}', available: {available:?}")]
    68
        ModelNotFound {
    69
            /// Model name that was requested
    70
            requested: String,
    71
            /// List of available models
    72
            available: Vec<String>,
    73
        },
    74
    75
        /// Response schema does not match expected format
    76
        #[error("Schema mismatch: missing field '{missing_field}'")]
    77
        SchemaMismatch {
    78
            /// The field that was expected but missing
    79
            missing_field: String,
    80
        },
    81
    82
        /// Field type does not match expected type
    83
        #[error("Field type mismatch: '{field}' expected {expected}, got {actual}")]
    84
        FieldTypeMismatch {
    85
            /// Field name with type mismatch
    86
            field: String,
    87
            /// Expected type name
    88
            expected: String,
    89
            /// Actual type received
    90
            actual: String,
    91
        },
    92
    93
        /// Response parsing failed
    94
        #[error("Response parse error: {reason}")]
    95
        ResponseParseError {
    96
            /// Description of the parse failure
    97
            reason: String,
    98
        },
    99
    100
        /// Timeout during preflight check
    101
        #[error("Timeout after {duration:?} during {operation}")]
    102
        Timeout {
    103
            /// How long the operation waited before timing out
    104
            duration: Duration,
    105
            /// Name of the operation that timed out
    106
            operation: String,
    107
        },
    108
    109
        /// Configuration error
    110
        #[error("Configuration error: {reason}")]
    111
        ConfigError {
    112
            /// Description of the configuration error
    113
            reason: String,
    114
        },
    115
    }
    116
    117
    /// Result type for preflight operations
    118
    pub type PreflightResult<T> = Result<T, PreflightError>;
    119
    120
    // ============================================================================
    121
    // Preflight Check Trait (Jidoka: Quality built into every step)
    122
    // ============================================================================
    123
    124
    /// Trait for preflight validation checks
    125
    ///
    126
    /// Implements Jidoka principle: each check validates one aspect of system state
    127
    /// and fails fast if the condition is not met.
    128
    pub trait PreflightCheck: fmt::Debug + Send + Sync {
    129
        /// Unique identifier for this check (for logging and metrics)
    130
        fn name(&self) -> &'static str;
    131
    132
        /// Validate the condition, returning Ok(()) if passed
    133
        ///
    134
        /// # Errors
    135
        /// Returns `PreflightError` with detailed context if validation fails
    136
        fn validate(&self) -> PreflightResult<()>;
    137
    138
        /// Optional: Description of what this check validates
    139
    0
        fn description(&self) -> &'static str {
    140
    0
            "Preflight validation check"
    141
    0
        }
    142
    }
    143
    144
    // ============================================================================
    145
    // Deterministic Inference Configuration (per spec Section 4.1)
    146
    // ============================================================================
    147
    148
    /// Configuration for deterministic inference
    149
    ///
    150
    /// Per Fleming & Wallace [5], deterministic benchmarks require seed control
    151
    /// and elimination of randomness sources.
    152
    #[derive(Debug, Clone, Serialize, Deserialize)]
    153
    pub struct DeterministicInferenceConfig {
    154
        /// Temperature = 0.0 disables sampling randomness
    155
        pub temperature: f64,
    156
        /// Fixed seed for any remaining randomness
    157
        pub seed: u64,
    158
        /// Top-k = 1 forces greedy decoding
    159
        pub top_k: usize,
    160
        /// Top-p = 1.0 disables nucleus sampling
    161
        pub top_p: f64,
    162
    }
    163
    164
    impl Default for DeterministicInferenceConfig {
    165
    11
        fn default() -> Self {
    166
    11
            Self {
    167
    11
                temperature: 0.0, // Greedy decoding
    168
    11
                seed: 42,         // Fixed, reproducible seed
    169
    11
                top_k: 1,         // Deterministic token selection
    170
    11
                top_p: 1.0,       // Disable nucleus sampling
    171
    11
            }
    172
    11
        }
    173
    }
    174
    175
    impl DeterministicInferenceConfig {
    176
        /// Create a new deterministic config with custom seed
    177
        #[must_use]
    178
    1
        pub fn with_seed(seed: u64) -> Self {
    179
    1
            Self {
    180
    1
                seed,
    181
    1
                ..Default::default()
    182
    1
            }
    183
    1
        }
    184
    185
        /// Validate that this config is truly deterministic
    186
        ///
    187
        /// # Errors
    188
        /// Returns `PreflightError::ConfigError` if any parameter allows non-determinism:
    189
        /// - temperature > 0.0 (allows sampling randomness)
    190
        /// - top_k != 1 (allows multiple token choices)
    191
    10
        pub fn validate_determinism(&self) -> PreflightResult<()> {
    192
    10
            if self.temperature > 0.0 {
    193
    2
                return Err(PreflightError::ConfigError {
    194
    2
                    reason: format!(
    195
    2
                        "Temperature {} > 0.0 allows randomness; set to 0.0 for determinism",
    196
    2
                        self.temperature
    197
    2
                    ),
    198
    2
                });
    199
    8
            }
    200
    8
            if self.top_k != 1 {
    201
    1
                return Err(PreflightError::ConfigError {
    202
    1
                    reason: format!(
    203
    1
                        "top_k {} != 1 allows multiple token choices; set to 1 for determinism",
    204
    1
                        self.top_k
    205
    1
                    ),
    206
    1
                });
    207
    7
            }
    208
    7
            Ok(())
    209
    10
        }
    210
    }
    211
    212
    // ============================================================================
    213
    // CV-Based Stopping Criterion (per spec Section 3.1)
    214
    // ============================================================================
    215
    216
    /// Reason for stopping benchmark iteration
    217
    #[derive(Debug, Clone, PartialEq)]
    218
    pub enum StopReason {
    219
        /// CV threshold achieved (statistically sufficient)
    220
        CvConverged(f64),
    221
        /// Maximum samples reached (bounded resource usage)
    222
        MaxSamples,
    223
        /// Minimum samples not yet reached
    224
        Continue,
    225
    }
    226
    227
    /// Decision from stopping criterion
    228
    #[derive(Debug, Clone, PartialEq)]
    229
    pub enum StopDecision {
    230
        /// Continue collecting samples
    231
        Continue,
    232
        /// Stop with given reason
    233
        Stop(StopReason),
    234
    }
    235
    236
    /// CV-based stopping criterion per Hoefler & Belli SC'15 [1]
    237
    #[derive(Debug, Clone)]
    238
    pub struct CvStoppingCriterion {
    239
        /// Minimum samples before CV check (prevents premature stopping)
    240
        pub min_samples: usize,
    241
        /// Maximum samples (bounded resource usage)
    242
        pub max_samples: usize,
    243
        /// Target CV threshold (e.g., 0.05 = 5%)
    244
        pub cv_threshold: f64,
    245
    }
    246
    247
    impl Default for CvStoppingCriterion {
    248
    4
        fn default() -> Self {
    249
    4
            Self {
    250
    4
                min_samples: 5,
    251
    4
                max_samples: 30,
    252
    4
                cv_threshold: 0.05, // 5% CV target per SC'15
    253
    4
            }
    254
    4
        }
    255
    }
    256
    257
    impl CvStoppingCriterion {
    258
        /// Create new criterion with custom parameters
    259
        #[must_use]
    260
    4
        pub fn new(min_samples: usize, max_samples: usize, cv_threshold: f64) -> Self {
    261
    4
            Self {
    262
    4
                min_samples,
    263
    4
                max_samples,
    264
    4
                cv_threshold,
    265
    4
            }
    266
    4
        }
    267
    268
        /// Evaluate whether to stop based on current samples
    269
        #[must_use]
    270
    4
        pub fn should_stop(&self, samples: &[f64]) -> StopDecision {
    271
    4
            if samples.len() < self.min_samples {
    272
    1
                return StopDecision::Continue;
    273
    3
            }
    274
    3
            if samples.len() >= self.max_samples {
    275
    1
                return StopDecision::Stop(StopReason::MaxSamples);
    276
    2
            }
    277
    278
    2
            let cv = self.calculate_cv(samples);
    279
    2
            if cv < self.cv_threshold {
    280
    1
                StopDecision::Stop(StopReason::CvConverged(cv))
    281
            } else {
    282
    1
                StopDecision::Continue
    283
            }
    284
    4
        }
    285
    286
        /// Calculate coefficient of variation (std_dev / mean)
    287
        #[must_use]
    288
    6
        pub fn calculate_cv(&self, samples: &[f64]) -> f64 {
    289
    6
            if samples.len() < 2 {
    290
    2
                return f64::MAX;
    291
    4
            }
    292
    293
    4
            let n = samples.len() as f64;
    294
    4
            let mean = samples.iter().sum::<f64>() / n;
    295
    296
    4
            if mean.abs() < f64::EPSILON {
    297
    0
                return f64::MAX;
    298
    4
            }
    299
    300
    20
            let variance4 = samples4.iter4().map4(|x| (x - mean).powi(2)).sum4::<f64>() / (n - 1.0)4;
    301
    4
            let std_dev = variance.sqrt();
    302
    4
            std_dev / mean
    303
    6
        }
    304
    }
    305
    306
    // ============================================================================
    307
    // Outlier Detection (per spec Section 3.3)
    308
    // ============================================================================
    309
    310
    /// Outlier detection using Median Absolute Deviation (MAD)
    311
    ///
    312
    /// Per Chen et al. [4], MAD is more robust than standard deviation
    313
    /// for samples with potential outliers.
    314
    pub struct OutlierDetector {
    315
        /// k-factor for threshold (3.0 = ~99.7% for normal distribution)
    316
        pub k_factor: f64,
    317
    }
    318
    319
    impl Default for OutlierDetector {
    320
    5
        fn default() -> Self {
    321
    5
            Self { k_factor: 3.0 }
    322
    5
        }
    323
    }
    324
    325
    impl OutlierDetector {
    326
        /// Create detector with custom k-factor
    327
        #[must_use]
    328
    0
        pub fn new(k_factor: f64) -> Self {
    329
    0
            Self { k_factor }
    330
    0
        }
    331
    332
        /// Detect outliers in samples
    333
        ///
    334
        /// Returns a boolean vector where `true` indicates an outlier
    335
        #[must_use]
    336
    4
        pub fn detect(&self, samples: &[f64]) -> Vec<bool> {
    337
    4
            if samples.len() < 3 {
    338
    1
                return vec![false; samples.len()];
    339
    3
            }
    340
    341
    3
            let median = Self::percentile(samples, 50.0);
    342
    15
            let deviations3: Vec<f64>3 = samples3.iter3().map3(|x| (x - median).abs()).collect3();
    343
    3
            let mad = Self::percentile(&deviations, 50.0);
    344
    345
            // 1.4826 scales MAD to equivalent std dev for normal distribution
    346
    3
            let threshold = self.k_factor * mad * 1.4826;
    347
    348
    3
            samples
    349
    3
                .iter()
    350
    15
                .map3(|x| (x - median).abs() > threshold)
    351
    3
                .collect()
    352
    4
        }
    353
    354
        /// Filter outliers from samples, returning clean samples
    355
        #[must_use]
    356
    1
        pub fn filter(&self, samples: &[f64]) -> Vec<f64> {
    357
    1
            let outliers = self.detect(samples);
    358
    1
            samples
    359
    1
                .iter()
    360
    1
                .zip(outliers.iter())
    361
    5
                .filter1(|(_, is_outlier)| !**is_outlier)
    362
    1
                .map(|(sample, _)| *sample)
    363
    1
                .collect()
    364
    1
        }
    365
    366
        /// Calculate percentile of samples
    367
    8
        fn percentile(samples: &[f64], p: f64) -> f64 {
    368
    8
            if samples.is_empty() {
    369
    0
                return 0.0;
    370
    8
            }
    371
    372
    8
            let mut sorted = samples.to_vec();
    373
    59
            sorted8.sort_by8(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
    374
    375
    8
            let idx = ((p / 100.0) * (sorted.len() - 1) as f64).round() as usize;
    376
    8
            sorted[idx.min(sorted.len() - 1)]
    377
    8
        }
    378
    }
    379
    380
    // ============================================================================
    381
    // Quality Metrics (per spec Section 6.1)
    382
    // ============================================================================
    383
    384
    /// Quality metrics for benchmark results
    385
    #[derive(Debug, Clone, Serialize, Deserialize)]
    386
    pub struct QualityMetrics {
    387
        /// CV at the point where benchmark stopped
    388
        pub cv_at_stop: f64,
    389
        /// Whether CV threshold was achieved
    390
        pub cv_converged: bool,
    391
        /// Number of outliers detected
    392
        pub outliers_detected: usize,
    393
        /// Number of outliers excluded from statistics
    394
        pub outliers_excluded: usize,
    395
        /// List of preflight checks that passed
    396
        pub preflight_checks_passed: Vec<String>,
    397
    }
    398
    399
    impl Default for QualityMetrics {
    400
    1
        fn default() -> Self {
    401
    1
            Self {
    402
    1
                cv_at_stop: f64::MAX,
    403
    1
                cv_converged: false,
    404
    1
                outliers_detected: 0,
    405
    1
                outliers_excluded: 0,
    406
    1
                preflight_checks_passed: Vec::new(),
    407
    1
            }
    408
    1
        }
    409
    }
    410
    411
    // ============================================================================
    412
    // Preflight Check Implementations
    413
    // ============================================================================
    414
    415
    /// Check that a configuration is deterministic
    416
    #[derive(Debug)]
    417
    pub struct DeterminismCheck {
    418
        config: DeterministicInferenceConfig,
    419
    }
    420
    421
    impl DeterminismCheck {
    422
        /// Create a new determinism check for the given config
    423
        #[must_use]
    424
    6
        pub fn new(config: DeterministicInferenceConfig) -> Self {
    425
    6
            Self { config }
    426
    6
        }
    427
    }
    428
    429
    impl PreflightCheck for DeterminismCheck {
    430
    5
        fn name(&self) -> &'static str {
    431
    5
            "determinism_check"
    432
    5
        }
    433
    434
    0
        fn description(&self) -> &'static str {
    435
    0
            "Validates inference configuration ensures deterministic output"
    436
    0
        }
    437
    438
    6
        fn validate(&self) -> PreflightResult<()> {
    439
    6
            self.config.validate_determinism()
    440
    6
        }
    441
    }
    442
    443
    /// Server availability check - validates server URL and health endpoint response
    444
    ///
    445
    /// Per spec Section 4.2: Verify server is reachable before benchmark
    446
    #[derive(Debug)]
    447
    pub struct ServerAvailabilityCheck {
    448
        /// Server URL to check
    449
        url: String,
    450
        /// Health endpoint path (e.g., "/health")
    451
        health_path: String,
    452
        /// Cached health check result (status code, None if not yet checked)
    453
        health_status: Option<u16>,
    454
    }
    455
    456
    impl ServerAvailabilityCheck {
    457
        /// Create a new server availability check
    458
        #[must_use]
    459
    11
        pub fn new(url: String, health_path: String) -> Self {
    460
    11
            Self {
    461
    11
                url,
    462
    11
                health_path,
    463
    11
                health_status: None,
    464
    11
            }
    465
    11
        }
    466
    467
        /// Create with llama.cpp defaults (port 8082, /health)
    468
        #[must_use]
    469
    8
        pub fn llama_cpp(port: u16) -> Self {
    470
    8
            Self::new(format!("http://127.0.0.1:{port}"), "/health".to_string())
    471
    8
        }
    472
    473
        /// Create with Ollama defaults (port 11434, /api/tags)
    474
        #[must_use]
    475
    1
        pub fn ollama(port: u16) -> Self {
    476
    1
            Self::new(format!("http://127.0.0.1:{port}"), "/api/tags".to_string())
    477
    1
        }
    478
    479
        /// Set the health check result (called after HTTP request)
    480
    6
        pub fn set_health_status(&mut self, status: u16) {
    481
    6
            self.health_status = Some(status);
    482
    6
        }
    483
    484
        /// Get the full health URL
    485
        #[must_use]
    486
    6
        pub fn health_url(&self) -> String {
    487
    6
            format!("{}{}", self.url, self.health_path)
    488
    6
        }
    489
    490
        /// Check if URL is well-formed
    491
    9
        fn validate_url(&self) -> PreflightResult<()> {
    492
    9
            if self.url.is_empty() {
    493
    1
                return Err(PreflightError::ConfigError {
    494
    1
                    reason: "Server URL cannot be empty".to_string(),
    495
    1
                });
    496
    8
            }
    497
    8
            if !self.url.starts_with("http://") && !self.url.starts_with("https://")1 {
    498
    1
                return Err(PreflightError::ConfigError {
    499
    1
                    reason: format!(
    500
    1
                        "Server URL must start with http:// or https://, got: {}",
    501
    1
                        self.url
    502
    1
                    ),
    503
    1
                });
    504
    7
            }
    505
    7
            Ok(())
    506
    9
        }
    507
    }
    508
    509
    impl PreflightCheck for ServerAvailabilityCheck {
    510
    1
        fn name(&self) -> &'static str {
    511
    1
            "server_availability_check"
    512
    1
        }
    513
    514
    0
        fn description(&self) -> &'static str {
    515
    0
            "Validates server is reachable at the configured URL"
    516
    0
        }
    517
    518
    9
        fn validate(&self) -> PreflightResult<()> {
    519
            // First validate URL format
    520
    9
            self.validate_url()?2;
    521
    522
            // Check if health status has been set
    523
    6
            match self.health_status {
    524
    6
                Some(status2) if status >= 200 && status5 < 3002 => Ok(())2,
    525
    4
                Some(status) => Err(PreflightError::HealthCheckFailed {
    526
    4
                    url: self.health_url(),
    527
    4
                    status,
    528
    4
                }),
    529
    1
                None => Err(PreflightError::ConfigError {
    530
    1
                    reason: "Health check not performed - call set_health_status() first".to_string(),
    531
    1
                }),
    532
            }
    533
    9
        }
    534
    }
    535
    536
    /// Model availability check - validates requested model exists
    537
    ///
    538
    /// Per spec Section 4.2: Verify model is loaded before benchmark
    539
    #[derive(Debug)]
    540
    pub struct ModelAvailabilityCheck {
    541
        /// Model name that is requested
    542
        requested_model: String,
    543
        /// List of available models (populated after query)
    544
        available_models: Vec<String>,
    545
    }
    546
    547
    impl ModelAvailabilityCheck {
    548
        /// Create a new model availability check
    549
        #[must_use]
    550
    6
        pub fn new(requested_model: String) -> Self {
    551
    6
            Self {
    552
    6
                requested_model,
    553
    6
                available_models: Vec::new(),
    554
    6
            }
    555
    6
        }
    556
    557
        /// Set the list of available models (called after querying server)
    558
    3
        pub fn set_available_models(&mut self, models: Vec<String>) {
    559
    3
            self.available_models = models;
    560
    3
        }
    561
    562
        /// Get the requested model name
    563
        #[must_use]
    564
    1
        pub fn requested_model(&self) -> &str {
    565
    1
            &self.requested_model
    566
    1
        }
    567
    }
    568
    569
    impl PreflightCheck for ModelAvailabilityCheck {
    570
    1
        fn name(&self) -> &'static str {
    571
    1
            "model_availability_check"
    572
    1
        }
    573
    574
    0
        fn description(&self) -> &'static str {
    575
    0
            "Validates requested model is available on the server"
    576
    0
        }
    577
    578
    5
        fn validate(&self) -> PreflightResult<()> {
    579
    5
            if self.requested_model.is_empty() {
    580
    1
                return Err(PreflightError::ConfigError {
    581
    1
                    reason: "Model name cannot be empty".to_string(),
    582
    1
                });
    583
    4
            }
    584
    585
    4
            if self.available_models.is_empty() {
    586
    1
                return Err(PreflightError::ConfigError {
    587
    1
                    reason: "Available models list not set - call set_available_models() first"
    588
    1
                        .to_string(),
    589
    1
                });
    590
    3
            }
    591
    592
            // Check for exact match or partial match (model:tag format)
    593
    4
            let found3 = self.available_models.iter()3.any3(|m| {
    594
    4
                m == &self.requested_model
    595
    3
                    || m.starts_with(&format!("{}:", self.requested_model))
    596
    2
                    || self.requested_model.starts_with(&format!("{m}:"))
    597
    4
            });
    598
    599
    3
            if found {
    600
    2
                Ok(())
    601
            } else {
    602
    1
                Err(PreflightError::ModelNotFound {
    603
    1
                    requested: self.requested_model.clone(),
    604
    1
                    available: self.available_models.clone(),
    605
    1
                })
    606
            }
    607
    5
        }
    608
    }
    609
    610
    /// Response schema check - validates JSON response has required fields
    611
    ///
    612
    /// Per spec Section 4.3: Verify response format matches expected schema
    613
    #[derive(Debug)]
    614
    pub struct ResponseSchemaCheck {
    615
        /// Required field names that must be present
    616
        required_fields: Vec<String>,
    617
        /// Optional field type constraints (field_name -> expected_type)
    618
        field_types: std::collections::HashMap<String, String>,
    619
    }
    620
    621
    impl ResponseSchemaCheck {
    622
        /// Create a new response schema check with required fields
    623
        #[must_use]
    624
    9
        pub fn new(required_fields: Vec<String>) -> Self {
    625
    9
            Self {
    626
    9
                required_fields,
    627
    9
                field_types: std::collections::HashMap::new(),
    628
    9
            }
    629
    9
        }
    630
    631
        /// Create schema check for llama.cpp /completion response
    632
        #[must_use]
    633
    1
        pub fn llama_cpp_completion() -> Self {
    634
    1
            let mut check = Self::new(vec![
    635
    1
                "content".to_string(),
    636
    1
                "tokens_predicted".to_string(),
    637
    1
                "timings".to_string(),
    638
            ]);
    639
    1
            check
    640
    1
                .field_types
    641
    1
                .insert("tokens_predicted".to_string(), "number".to_string());
    642
    1
            check
    643
    1
                .field_types
    644
    1
                .insert("content".to_string(), "string".to_string());
    645
    1
            check
    646
    1
        }
    647
    648
        /// Create schema check for Ollama /api/generate response
    649
        #[must_use]
    650
    1
        pub fn ollama_generate() -> Self {
    651
    1
            let mut check = Self::new(vec!["response".to_string(), "done".to_string()]);
    652
    1
            check
    653
    1
                .field_types
    654
    1
                .insert("response".to_string(), "string".to_string());
    655
    1
            check
    656
    1
                .field_types
    657
    1
                .insert("done".to_string(), "boolean".to_string());
    658
    1
            check
    659
    1
        }
    660
    661
        /// Add a type constraint for a field
    662
        #[must_use]
    663
    1
        pub fn with_type_constraint(mut self, field: String, expected_type: String) -> Self {
    664
    1
            self.field_types.insert(field, expected_type);
    665
    1
            self
    666
    1
        }
    667
    668
        /// Validate a JSON value against this schema
    669
        ///
    670
        /// # Errors
    671
        /// Returns `PreflightError` if:
    672
        /// - The JSON is not an object
    673
        /// - A required field is missing
    674
        /// - A field has an unexpected type
    675
    5
        pub fn validate_json(&self, json: &serde_json::Value) -> PreflightResult<()> {
    676
    5
            let obj4 = json
    677
    5
                .as_object()
    678
    5
                .ok_or_else(|| PreflightError::ResponseParseError {
    679
    1
                    reason: "Expected JSON object at root".to_string(),
    680
    1
                })?;
    681
    682
            // Check required fields exist
    683
    9
            for field6 in &self.required_fields {
    684
    6
                if !obj.contains_key(field) {
    685
    1
                    return Err(PreflightError::SchemaMismatch {
    686
    1
                        missing_field: field.clone(),
    687
    1
                    });
    688
    5
                }
    689
            }
    690
    691
            // Check field types
    692
    4
            for (field2, expected_type2) in &self.field_types {
    693
    2
                if let Some(value) = obj.get(field) {
    694
    2
                    let actual_type = match value {
    695
    0
                        serde_json::Value::Null => "null",
    696
    0
                        serde_json::Value::Bool(_) => "boolean",
    697
    1
                        serde_json::Value::Number(_) => "number",
    698
    1
                        serde_json::Value::String(_) => "string",
    699
    0
                        serde_json::Value::Array(_) => "array",
    700
    0
                        serde_json::Value::Object(_) => "object",
    701
                    };
    702
    703
    2
                    if actual_type != expected_type {
    704
    1
                        return Err(PreflightError::FieldTypeMismatch {
    705
    1
                            field: field.clone(),
    706
    1
                            expected: expected_type.clone(),
    707
    1
                            actual: actual_type.to_string(),
    708
    1
                        });
    709
    1
                    }
    710
    0
                }
    711
            }
    712
    713
    2
            Ok(())
    714
    5
        }
    715
    }
    716
    717
    impl PreflightCheck for ResponseSchemaCheck {
    718
    2
        fn name(&self) -> &'static str {
    719
    2
            "response_schema_check"
    720
    2
        }
    721
    722
    0
        fn description(&self) -> &'static str {
    723
    0
            "Validates response JSON matches expected schema"
    724
    0
        }
    725
    726
    5
        fn validate(&self) -> PreflightResult<()> {
    727
            // Standalone validation - just checks configuration is valid
    728
    5
            if self.required_fields.is_empty() {
    729
    2
                return Err(PreflightError::ConfigError {
    730
    2
                    reason: "At least one required field must be specified".to_string(),
    731
    2
                });
    732
    3
            }
    733
    3
            Ok(())
    734
    5
        }
    735
    }
    736
    737
    /// Preflight validation runner - executes all checks in sequence
    738
    ///
    739
    /// Per Jidoka principle: Stop immediately on first failure
    740
    #[derive(Debug, Default)]
    741
    pub struct PreflightRunner {
    742
        /// Checks to execute in order
    743
        checks: Vec<Box<dyn PreflightCheck>>,
    744
        /// Names of passed checks (populated during run)
    745
        passed: Vec<String>,
    746
    }
    747
    748
    impl PreflightRunner {
    749
        /// Create a new preflight runner
    750
        #[must_use]
    751
    4
        pub fn new() -> Self {
    752
    4
            Self::default()
    753
    4
        }
    754
    755
        /// Add a check to the runner
    756
    6
        pub fn add_check(&mut self, check: Box<dyn PreflightCheck>) {
    757
    6
            self.checks.push(check);
    758
    6
        }
    759
    760
        /// Run all checks, stopping on first failure (Jidoka)
    761
        ///
    762
        /// Returns list of passed check names on success
    763
        ///
    764
        /// # Errors
    765
        /// Returns the `PreflightError` from the first check that fails.
    766
        /// Per Jidoka principle, execution stops immediately on first failure.
    767
    5
        pub fn run(&mut self) -> PreflightResult<Vec<String>> {
    768
    5
            self.passed.clear();
    769
    770
    10
            for check6 in &self.checks {
    771
    6
                check.validate()?1;
    772
    5
                self.passed.push(check.name().to_string());
    773
            }
    774
    775
    4
            Ok(self.passed.clone())
    776
    5
        }
    777
    778
        /// Get passed checks (after run)
    779
        #[must_use]
    780
    3
        pub fn passed_checks(&self) -> &[String] {
    781
    3
            &self.passed
    782
    3
        }
    783
    }
    784
    785
    // ============================================================================
    786
    // Tests (EXTREME TDD: Tests written FIRST)
    787
    // ============================================================================
    788
    789
    #[cfg(test)]
    790
    mod tests {
    791
        use super::*;
    792
    793
        // =========================================================================
    794
        // Canonical Inputs Tests
    795
        // =========================================================================
    796
    797
        #[test]
    798
    1
        fn test_canonical_inputs_version_is_semver() {
    799
    1
            let version = canonical_inputs::VERSION;
    800
    1
            let parts: Vec<&str> = version.split('.').collect();
    801
    1
            assert_eq!(
    802
    1
                parts.len(),
    803
                3,
    804
    0
                "Version should be semver (major.minor.patch)"
    805
            );
    806
    4
            for part3 in parts {
    807
    3
                assert!(
    808
    3
                    part.parse::<u32>().is_ok(),
    809
    0
                    "Version part '{}' should be numeric",
    810
                    part
    811
                );
    812
            }
    813
    1
        }
    814
    815
        #[test]
    816
    1
        fn test_canonical_inputs_prompt_not_empty() {
    817
            // Verify prompt has reasonable length for benchmarking
    818
    1
            let prompt_len = canonical_inputs::LATENCY_PROMPT.len();
    819
    1
            assert!(
    820
    1
                prompt_len >= 10,
    821
    0
                "Latency prompt should have at least 10 chars, got {}",
    822
                prompt_len
    823
            );
    824
    1
        }
    825
    826
        #[test]
    827
    1
        fn test_canonical_inputs_tokens_not_empty() {
    828
            // Verify we have enough tokens for throughput testing
    829
    1
            let token_count = canonical_inputs::THROUGHPUT_TOKENS.len();
    830
    1
            assert!(
    831
    1
                token_count >= 4,
    832
    0
                "Throughput tokens should have at least 4 tokens, got {}",
    833
                token_count
    834
            );
    835
    1
        }
    836
    837
        #[test]
    838
    1
        fn test_canonical_inputs_max_tokens_reasonable() {
    839
            // Verify max tokens is in a sensible range
    840
    1
            let max_tokens = canonical_inputs::MAX_TOKENS;
    841
    1
            assert!(
    842
    1
                max_tokens > 0,
    843
    0
                "Max tokens should be positive, got {}",
    844
                max_tokens
    845
            );
    846
    1
            assert!(
    847
    1
                max_tokens <= 1000,
    848
    0
                "Max tokens should be <= 1000, got {}",
    849
                max_tokens
    850
            );
    851
    1
        }
    852
    853
        // =========================================================================
    854
        // DeterministicInferenceConfig Tests
    855
        // =========================================================================
    856
    857
        #[test]
    858
    1
        fn test_deterministic_config_default_is_deterministic() {
    859
    1
            let config = DeterministicInferenceConfig::default();
    860
    1
            assert!(
    861
    1
                config.validate_determinism().is_ok(),
    862
    0
                "Default config should be deterministic"
    863
            );
    864
    1
        }
    865
    866
        #[test]
    867
    1
        fn test_deterministic_config_default_values() {
    868
    1
            let config = DeterministicInferenceConfig::default();
    869
    1
            assert_eq!(config.temperature, 0.0);
    870
    1
            assert_eq!(config.seed, 42);
    871
    1
            assert_eq!(config.top_k, 1);
    872
    1
            assert_eq!(config.top_p, 1.0);
    873
    1
        }
    874
    875
        #[test]
    876
    1
        fn test_deterministic_config_with_seed() {
    877
    1
            let config = DeterministicInferenceConfig::with_seed(12345);
    878
    1
            assert_eq!(config.seed, 12345);
    879
    1
            assert!(config.validate_determinism().is_ok());
    880
    1
        }
    881
    882
        #[test]
    883
    1
        fn test_deterministic_config_rejects_nonzero_temperature() {
    884
    1
            let config = DeterministicInferenceConfig {
    885
    1
                temperature: 0.7,
    886
    1
                ..Default::default()
    887
    1
            };
    888
    1
            let result = config.validate_determinism();
    889
    1
            assert!(result.is_err());
    890
    1
            let err = result.unwrap_err();
    891
    1
            assert!(matches!0(err, PreflightError::ConfigError { .. }));
    892
    1
        }
    893
    894
        #[test]
    895
    1
        fn test_deterministic_config_rejects_topk_not_one() {
    896
    1
            let config = DeterministicInferenceConfig {
    897
    1
                top_k: 50,
    898
    1
                ..Default::default()
    899
    1
            };
    900
    1
            let result = config.validate_determinism();
    901
    1
            assert!(result.is_err());
    902
    1
        }
    903
    904
        // =========================================================================
    905
        // CvStoppingCriterion Tests
    906
        // =========================================================================
    907
    908
        #[test]
    909
    1
        fn test_cv_criterion_default_values() {
    910
    1
            let criterion = CvStoppingCriterion::default();
    911
    1
            assert_eq!(criterion.min_samples, 5);
    912
    1
            assert_eq!(criterion.max_samples, 30);
    913
    1
            assert!((criterion.cv_threshold - 0.05).abs() < 0.001);
    914
    1
        }
    915
    916
        #[test]
    917
    1
        fn test_cv_criterion_continues_below_min_samples() {
    918
    1
            let criterion = CvStoppingCriterion::new(5, 30, 0.05);
    919
    1
            let samples = vec![100.0, 100.0, 100.0]; // Only 3 samples
    920
    1
            assert_eq!(criterion.should_stop(&samples), StopDecision::Continue);
    921
    1
        }
    922
    923
        #[test]
    924
    1
        fn test_cv_criterion_stops_at_max_samples() {
    925
    1
            let criterion = CvStoppingCriterion::new(5, 10, 0.01); // Very tight CV
    926
    10
            let samples1: Vec<f64>1 = (1..=101).map1(|x| x as f64 * 10.0).collect1();
    927
    1
            assert_eq!(
    928
    1
                criterion.should_stop(&samples),
    929
                StopDecision::Stop(StopReason::MaxSamples)
    930
            );
    931
    1
        }
    932
    933
        #[test]
    934
    1
        fn test_cv_criterion_converges_on_identical_values() {
    935
    1
            let criterion = CvStoppingCriterion::new(5, 30, 0.05);
    936
    1
            let samples = vec![100.0, 100.0, 100.0, 100.0, 100.0];
    937
    1
            let cv = criterion.calculate_cv(&samples);
    938
    1
            assert!(
    939
    1
                cv < 0.001,
    940
    0
                "CV of identical values should be ~0, got {}",
    941
                cv
    942
            );
    943
    944
    1
            match criterion.should_stop(&samples) {
    945
    1
                StopDecision::Stop(StopReason::CvConverged(cv)) => {
    946
    1
                    assert!(cv < 0.05);
    947
                },
    948
    0
                other => panic!("Expected CvConverged, got {:?}", other),
    949
            }
    950
    1
        }
    951
    952
        #[test]
    953
    1
        fn test_cv_criterion_continues_on_high_variance() {
    954
    1
            let criterion = CvStoppingCriterion::new(5, 30, 0.05);
    955
            // High variance: 10, 100, 10, 100, 10 - CV >> 0.05
    956
    1
            let samples = vec![10.0, 100.0, 10.0, 100.0, 10.0];
    957
    1
            assert_eq!(criterion.should_stop(&samples), StopDecision::Continue);
    958
    1
        }
    959
    960
        #[test]
    961
    1
        fn test_cv_calculation_single_value() {
    962
    1
            let criterion = CvStoppingCriterion::default();
    963
    1
            let samples = vec![100.0];
    964
    1
            let cv = criterion.calculate_cv(&samples);
    965
    1
            assert_eq!(cv, f64::MAX);
    966
    1
        }
    967
    968
        #[test]
    969
    1
        fn test_cv_calculation_empty() {
    970
    1
            let criterion = CvStoppingCriterion::default();
    971
    1
            let samples: Vec<f64> = vec![];
    972
    1
            let cv = criterion.calculate_cv(&samples);
    973
    1
            assert_eq!(cv, f64::MAX);
    974
    1
        }
    975
    976
        #[test]
    977
    1
        fn test_cv_calculation_known_values() {
    978
    1
            let criterion = CvStoppingCriterion::default();
    979
            // Mean = 100, values deviate by ~7.9, so CV ~0.079
    980
    1
            let samples = vec![90.0, 95.0, 100.0, 105.0, 110.0];
    981
    1
            let cv = criterion.calculate_cv(&samples);
    982
    1
            assert!(cv > 0.07 && cv < 0.09, "Expected CV ~0.079, got {}"0, cv);
    983
    1
        }
    984
    985
        // =========================================================================
    986
        // OutlierDetector Tests
    987
        // =========================================================================
    988
    989
        #[test]
    990
    1
        fn test_outlier_detector_default_k_factor() {
    991
    1
            let detector = OutlierDetector::default();
    992
    1
            assert!((detector.k_factor - 3.0).abs() < 0.001);
    993
    1
        }
    994
    995
        #[test]
    996
    1
        fn test_outlier_detector_no_outliers_uniform() {
    997
    1
            let detector = OutlierDetector::default();
    998
    1
            let samples = vec![100.0, 101.0, 99.0, 100.5, 99.5];
    999
    1
            let outliers = detector.detect(&samples);
    1000
    1
            assert!(
    1001
    1
                !outliers.iter().any(|&x| x),
    1002
    0
                "Uniform samples should have no outliers"
    1003
            );
    1004
    1
        }
    1005
    1006
        #[test]
    1007
    1
        fn test_outlier_detector_finds_extreme_outlier() {
    1008
    1
            let detector = OutlierDetector::default();
    1009
    1
            let samples = vec![100.0, 101.0, 99.0, 100.0, 1000.0]; // 1000 is extreme
    1010
    1
            let outliers = detector.detect(&samples);
    1011
    1
            assert!(outliers[4], "1000.0 should be detected as outlier"0);
    1012
    1
        }
    1013
    1014
        #[test]
    1015
    1
        fn test_outlier_detector_filter_removes_outliers() {
    1016
    1
            let detector = OutlierDetector::default();
    1017
    1
            let samples = vec![100.0, 101.0, 99.0, 100.0, 1000.0];
    1018
    1
            let filtered = detector.filter(&samples);
    1019
    1
            assert!(
    1020
    1
                !filtered.contains(&1000.0),
    1021
    0
                "Filtered should not contain outlier"
    1022
            );
    1023
    1
            assert_eq!(filtered.len(), 4);
    1024
    1
        }
    1025
    1026
        #[test]
    1027
    1
        fn test_outlier_detector_handles_small_samples() {
    1028
    1
            let detector = OutlierDetector::default();
    1029
    1
            let samples = vec![100.0, 200.0]; // Only 2 samples
    1030
    1
            let outliers = detector.detect(&samples);
    1031
    1
            assert_eq!(
    1032
                outliers,
    1033
    1
                vec![false, false],
    1034
    0
                "Should not detect outliers with < 3 samples"
    1035
            );
    1036
    1
        }
    1037
    1038
        #[test]
    1039
    1
        fn test_outlier_detector_percentile() {
    1040
            // Uses nearest-rank method: idx = round((p/100) * (n-1))
    1041
    1
            let samples = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
    1042
    1043
            // p50 of 10 elements: idx = round(0.5 * 9) = round(4.5) = 5 → value 6.0
    1044
    1
            let p50 = OutlierDetector::percentile(&samples, 50.0);
    1045
    1
            assert!(
    1046
    1
                (p50 - 5.5).abs() < 1.0,
    1047
    0
                "p50 should be ~5.5 (nearest rank gives 6), got {}",
    1048
                p50
    1049
            );
    1050
    1051
            // p99: idx = round(0.99 * 9) = round(8.91) = 9 → value 10.0
    1052
    1
            let p99 = OutlierDetector::percentile(&samples, 99.0);
    1053
    1
            assert!((p99 - 10.0).abs() < 0.5, "p99 should be ~10.0, got {}"0, p99);
    1054
    1
        }
    1055
    1056
        // =========================================================================
    1057
        // QualityMetrics Tests
    1058
        // =========================================================================
    1059
    1060
        #[test]
    1061
    1
        fn test_quality_metrics_default() {
    1062
    1
            let metrics = QualityMetrics::default();
    1063
    1
            assert_eq!(metrics.cv_at_stop, f64::MAX);
    1064
    1
            assert!(!metrics.cv_converged);
    1065
    1
            assert_eq!(metrics.outliers_detected, 0);
    1066
    1
            assert!(metrics.preflight_checks_passed.is_empty());
    1067
    1
        }
    1068
    1069
        #[test]
    1070
    1
        fn test_quality_metrics_serialization() {
    1071
    1
            let metrics = QualityMetrics {
    1072
    1
                cv_at_stop: 0.03,
    1073
    1
                cv_converged: true,
    1074
    1
                outliers_detected: 2,
    1075
    1
                outliers_excluded: 1,
    1076
    1
                preflight_checks_passed: vec!["server_check".to_string()],
    1077
    1
            };
    1078
    1
            let json = serde_json::to_string(&metrics).expect("serialization");
    1079
    1
            assert!(json.contains("0.03"));
    1080
    1
            assert!(json.contains("server_check"));
    1081
    1
        }
    1082
    1083
        // =========================================================================
    1084
        // DeterminismCheck Tests
    1085
        // =========================================================================
    1086
    1087
        #[test]
    1088
    1
        fn test_determinism_check_trait_impl() {
    1089
    1
            let config = DeterministicInferenceConfig::default();
    1090
    1
            let check = DeterminismCheck::new(config);
    1091
    1
            assert_eq!(check.name(), "determinism_check");
    1092
    1
            assert!(check.validate().is_ok());
    1093
    1
        }
    1094
    1095
        #[test]
    1096
    1
        fn test_determinism_check_fails_on_bad_config() {
    1097
    1
            let config = DeterministicInferenceConfig {
    1098
    1
                temperature: 0.5,
    1099
    1
                ..Default::default()
    1100
    1
            };
    1101
    1
            let check = DeterminismCheck::new(config);
    1102
    1
            assert!(check.validate().is_err());
    1103
    1
        }
    1104
    1105
        // =========================================================================
    1106
        // PreflightError Tests
    1107
        // =========================================================================
    1108
    1109
        #[test]
    1110
    1
        fn test_preflight_error_display() {
    1111
    1
            let err = PreflightError::ModelNotFound {
    1112
    1
                requested: "phi".to_string(),
    1113
    1
                available: vec!["phi2:2.7b".to_string(), "llama2".to_string()],
    1114
    1
            };
    1115
    1
            let msg = format!("{}", err);
    1116
    1
            assert!(msg.contains("phi"));
    1117
    1
            assert!(msg.contains("phi2:2.7b"));
    1118
    1
        }
    1119
    1120
        #[test]
    1121
    1
        fn test_preflight_error_schema_mismatch() {
    1122
    1
            let err = PreflightError::SchemaMismatch {
    1123
    1
                missing_field: "eval_count".to_string(),
    1124
    1
            };
    1125
    1
            let msg = format!("{}", err);
    1126
    1
            assert!(msg.contains("eval_count"));
    1127
    1
        }
    1128
    1129
        #[test]
    1130
    1
        fn test_preflight_error_type_mismatch() {
    1131
    1
            let err = PreflightError::FieldTypeMismatch {
    1132
    1
                field: "tokens".to_string(),
    1133
    1
                expected: "number".to_string(),
    1134
    1
                actual: "string".to_string(),
    1135
    1
            };
    1136
    1
            let msg = format!("{}", err);
    1137
    1
            assert!(msg.contains("tokens"));
    1138
    1
            assert!(msg.contains("number"));
    1139
    1
        }
    1140
    1141
        // =========================================================================
    1142
        // ServerAvailabilityCheck Tests
    1143
        // =========================================================================
    1144
    1145
        #[test]
    1146
    1
        fn test_server_check_llama_cpp_defaults() {
    1147
    1
            let check = ServerAvailabilityCheck::llama_cpp(8082);
    1148
    1
            assert_eq!(check.health_url(), "http://127.0.0.1:8082/health");
    1149
    1
            assert_eq!(check.name(), "server_availability_check");
    1150
    1
        }
    1151
    1152
        #[test]
    1153
    1
        fn test_server_check_ollama_defaults() {
    1154
    1
            let check = ServerAvailabilityCheck::ollama(11434);
    1155
    1
            assert_eq!(check.health_url(), "http://127.0.0.1:11434/api/tags");
    1156
    1
        }
    1157
    1158
        #[test]
    1159
    1
        fn test_server_check_validates_url_format() {
    1160
    1
            let check = ServerAvailabilityCheck::new("invalid-url".to_string(), "/health".to_string());
    1161
    1
            let result = check.validate();
    1162
    1
            assert!(result.is_err());
    1163
    1
        }
    1164
    1165
        #[test]
    1166
    1
        fn test_server_check_rejects_empty_url() {
    1167
    1
            let check = ServerAvailabilityCheck::new(String::new(), "/health".to_string());
    1168
    1
            let result = check.validate();
    1169
    1
            assert!(matches!0(result, Err(PreflightError::ConfigError { .. })));
    1170
    1
        }
    1171
    1172
        #[test]
    1173
    1
        fn test_server_check_requires_health_status() {
    1174
    1
            let check = ServerAvailabilityCheck::llama_cpp(8082);
    1175
            // No health status set yet
    1176
    1
            let result = check.validate();
    1177
    1
            assert!(result.is_err());
    1178
    1
        }
    1179
    1180
        #[test]
    1181
    1
        fn test_server_check_accepts_200_status() {
    1182
    1
            let mut check = ServerAvailabilityCheck::llama_cpp(8082);
    1183
    1
            check.set_health_status(200);
    1184
    1
            assert!(check.validate().is_ok());
    1185
    1
        }
    1186
    1187
        #[test]
    1188
    1
        fn test_server_check_accepts_204_status() {
    1189
    1
            let mut check = ServerAvailabilityCheck::llama_cpp(8082);
    1190
    1
            check.set_health_status(204); // No Content is valid
    1191
    1
            assert!(check.validate().is_ok());
    1192
    1
        }
    1193
    1194
        #[test]
    1195
    1
        fn test_server_check_rejects_500_status() {
    1196
    1
            let mut check = ServerAvailabilityCheck::llama_cpp(8082);
    1197
    1
            check.set_health_status(500);
    1198
    1
            let result = check.validate();
    1199
    1
            assert!(matches!0(
    1200
    1
                result,
    1201
                Err(PreflightError::HealthCheckFailed { status: 500, .. })
    1202
            ));
    1203
    1
        }
    1204
    1205
        #[test]
    1206
    1
        fn test_server_check_rejects_404_status() {
    1207
    1
            let mut check = ServerAvailabilityCheck::llama_cpp(8082);
    1208
    1
            check.set_health_status(404);
    1209
    1
            let result = check.validate();
    1210
    1
            assert!(result.is_err());
    1211
    1
        }
    1212
    1213
        // =========================================================================
    1214
        // ModelAvailabilityCheck Tests
    1215
        // =========================================================================
    1216
    1217
        #[test]
    1218
    1
        fn test_model_check_finds_exact_match() {
    1219
    1
            let mut check = ModelAvailabilityCheck::new("phi2:2.7b".to_string());
    1220
    1
            check.set_available_models(vec!["phi2:2.7b".to_string(), "llama2".to_string()]);
    1221
    1
            assert!(check.validate().is_ok());
    1222
    1
        }
    1223
    1224
        #[test]
    1225
    1
        fn test_model_check_finds_partial_match() {
    1226
    1
            let mut check = ModelAvailabilityCheck::new("phi2".to_string());
    1227
    1
            check.set_available_models(vec!["phi2:2.7b".to_string(), "llama2".to_string()]);
    1228
    1
            assert!(check.validate().is_ok());
    1229
    1
        }
    1230
    1231
        #[test]
    1232
    1
        fn test_model_check_fails_on_missing_model() {
    1233
    1
            let mut check = ModelAvailabilityCheck::new("gpt4".to_string());
    1234
    1
            check.set_available_models(vec!["phi2:2.7b".to_string(), "llama2".to_string()]);
    1235
    1
            let result = check.validate();
    1236
    1
            assert!(matches!0(result, Err(PreflightError::ModelNotFound { .. })));
    1237
    1
        }
    1238
    1239
        #[test]
    1240
    1
        fn test_model_check_rejects_empty_model_name() {
    1241
    1
            let check = ModelAvailabilityCheck::new(String::new());
    1242
    1
            let result = check.validate();
    1243
    1
            assert!(result.is_err());
    1244
    1
        }
    1245
    1246
        #[test]
    1247
    1
        fn test_model_check_requires_available_models() {
    1248
    1
            let check = ModelAvailabilityCheck::new("phi2".to_string());
    1249
            // No available models set
    1250
    1
            let result = check.validate();
    1251
    1
            assert!(result.is_err());
    1252
    1
        }
    1253
    1254
        #[test]
    1255
    1
        fn test_model_check_name() {
    1256
    1
            let check = ModelAvailabilityCheck::new("phi2".to_string());
    1257
    1
            assert_eq!(check.name(), "model_availability_check");
    1258
    1
            assert_eq!(check.requested_model(), "phi2");
    1259
    1
        }
    1260
    1261
        // =========================================================================
    1262
        // ResponseSchemaCheck Tests
    1263
        // =========================================================================
    1264
    1265
        #[test]
    1266
    1
        fn test_schema_check_llama_cpp_completion() {
    1267
    1
            let check = ResponseSchemaCheck::llama_cpp_completion();
    1268
    1
            assert_eq!(check.name(), "response_schema_check");
    1269
    1
            assert!(check.validate().is_ok()); // Has required fields
    1270
    1
        }
    1271
    1272
        #[test]
    1273
    1
        fn test_schema_check_ollama_generate() {
    1274
    1
            let check = ResponseSchemaCheck::ollama_generate();
    1275
    1
            assert!(check.validate().is_ok());
    1276
    1
        }
    1277
    1278
        #[test]
    1279
    1
        fn test_schema_check_validates_required_fields() {
    1280
    1
            let check = ResponseSchemaCheck::new(vec!["content".to_string(), "tokens".to_string()]);
    1281
    1
            let json: serde_json::Value = serde_json::json!({
    1282
    1
                "content": "Hello",
    1283
    1
                "tokens": 5
    1284
            });
    1285
    1
            assert!(check.validate_json(&json).is_ok());
    1286
    1
        }
    1287
    1288
        #[test]
    1289
    1
        fn test_schema_check_fails_on_missing_field() {
    1290
    1
            let check = ResponseSchemaCheck::new(vec!["content".to_string(), "tokens".to_string()]);
    1291
    1
            let json: serde_json::Value = serde_json::json!({
    1292
    1
                "content": "Hello"
    1293
                // missing "tokens"
    1294
            });
    1295
    1
            let result = check.validate_json(&json);
    1296
    1
            assert!(
    1297
    1
                matches!(result, Err(PreflightError::SchemaMismatch { missing_field }) if missing_field == "tokens")
    1298
            );
    1299
    1
        }
    1300
    1301
        #[test]
    1302
    1
        fn test_schema_check_validates_field_types() {
    1303
    1
            let check = ResponseSchemaCheck::new(vec!["count".to_string()])
    1304
    1
                .with_type_constraint("count".to_string(), "number".to_string());
    1305
    1306
            // Correct type
    1307
    1
            let json: serde_json::Value = serde_json::json!({ "count": 42 });
    1308
    1
            assert!(check.validate_json(&json).is_ok());
    1309
    1310
            // Wrong type
    1311
    1
            let json: serde_json::Value = serde_json::json!({ "count": "42" });
    1312
    1
            let result = check.validate_json(&json);
    1313
    1
            assert!(matches!0(
    1314
    1
                result,
    1315
                Err(PreflightError::FieldTypeMismatch { .. })
    1316
            ));
    1317
    1
        }
    1318
    1319
        #[test]
    1320
    1
        fn test_schema_check_rejects_non_object() {
    1321
    1
            let check = ResponseSchemaCheck::new(vec!["content".to_string()]);
    1322
    1
            let json: serde_json::Value = serde_json::json!("not an object");
    1323
    1
            let result = check.validate_json(&json);
    1324
    1
            assert!(matches!0(
    1325
    1
                result,
    1326
                Err(PreflightError::ResponseParseError { .. })
    1327
            ));
    1328
    1
        }
    1329
    1330
        #[test]
    1331
    1
        fn test_schema_check_rejects_empty_required_fields() {
    1332
    1
            let check = ResponseSchemaCheck::new(vec![]);
    1333
    1
            let result = check.validate();
    1334
    1
            assert!(result.is_err());
    1335
    1
        }
    1336
    1337
        // =========================================================================
    1338
        // PreflightRunner Tests
    1339
        // =========================================================================
    1340
    1341
        #[test]
    1342
    1
        fn test_runner_runs_all_checks() {
    1343
    1
            let mut runner = PreflightRunner::new();
    1344
    1345
            // Add a passing check
    1346
    1
            let config = DeterministicInferenceConfig::default();
    1347
    1
            runner.add_check(Box::new(DeterminismCheck::new(config)));
    1348
    1349
            // Add another passing check
    1350
    1
            let schema = ResponseSchemaCheck::new(vec!["foo".to_string()]);
    1351
    1
            runner.add_check(Box::new(schema));
    1352
    1353
    1
            let result = runner.run();
    1354
    1
            assert!(result.is_ok());
    1355
    1
            assert_eq!(result.expect("test").len(), 2);
    1356
    1
        }
    1357
    1358
        #[test]
    1359
    1
        fn test_runner_stops_on_first_failure_jidoka() {
    1360
    1
            let mut runner = PreflightRunner::new();
    1361
    1362
            // Add a passing check
    1363
    1
            let config = DeterministicInferenceConfig::default();
    1364
    1
            runner.add_check(Box::new(DeterminismCheck::new(config)));
    1365
    1366
            // Add a failing check (empty required fields)
    1367
    1
            let schema = ResponseSchemaCheck::new(vec![]);
    1368
    1
            runner.add_check(Box::new(schema));
    1369
    1370
            // Add another check that won't run
    1371
    1
            let config2 = DeterministicInferenceConfig::default();
    1372
    1
            runner.add_check(Box::new(DeterminismCheck::new(config2)));
    1373
    1374
    1
            let result = runner.run();
    1375
    1
            assert!(result.is_err());
    1376
            // Only first check passed before failure
    1377
    1
            assert_eq!(runner.passed_checks().len(), 1);
    1378
    1
        }
    1379
    1380
        #[test]
    1381
    1
        fn test_runner_empty_passes() {
    1382
    1
            let mut runner = PreflightRunner::new();
    1383
    1
            let result = runner.run();
    1384
    1
            assert!(result.is_ok());
    1385
    1
            assert!(result.expect("test").is_empty());
    1386
    1
        }
    1387
    1388
        #[test]
    1389
    1
        fn test_runner_clears_passed_on_rerun() {
    1390
    1
            let mut runner = PreflightRunner::new();
    1391
    1392
    1
            let config = DeterministicInferenceConfig::default();
    1393
    1
            runner.add_check(Box::new(DeterminismCheck::new(config)));
    1394
    1395
            // First run
    1396
    1
            let _ = runner.run();
    1397
    1
            assert_eq!(runner.passed_checks().len(), 1);
    1398
    1399
            // Second run should clear and repopulate
    1400
    1
            let _ = runner.run();
    1401
    1
            assert_eq!(runner.passed_checks().len(), 1);
    1402
    1
        }
    1403
    1404
        // =========================================================================
    1405
        // IMP-143: Real-World Server Verification Tests (EXTREME TDD)
    1406
        // =========================================================================
    1407
        // These tests verify actual connectivity to external servers.
    1408
        // Run with: cargo test test_imp_143 --lib -- --ignored
    1409
    1410
        /// IMP-143a: Verify llama.cpp server preflight check works with real server
    1411
        #[test]
    1412
        #[ignore = "Requires running llama.cpp server on port 8082"]
    1413
    0
        fn test_imp_143a_llamacpp_real_server_check() {
    1414
            // This test requires: llama-server -m model.gguf --host 127.0.0.1 --port 8082
    1415
    0
            let mut check = ServerAvailabilityCheck::llama_cpp(8082);
    1416
    1417
            // Attempt real connection
    1418
    0
            let client = reqwest::blocking::Client::builder()
    1419
    0
                .timeout(std::time::Duration::from_secs(5))
    1420
    0
                .build()
    1421
    0
                .expect("IMP-143a: Should create HTTP client");
    1422
    1423
    0
            let health_url = check.health_url();
    1424
    0
            match client.get(&health_url).send() {
    1425
    0
                Ok(response) => {
    1426
    0
                    let status = response.status().as_u16();
    1427
    0
                    check.set_health_status(status);
    1428
    1429
                    // IMP-143a: If server is running, check should pass
    1430
    0
                    let result = check.validate();
    1431
    0
                    assert!(
    1432
    0
                        result.is_ok(),
    1433
    0
                        "IMP-143a: llama.cpp server check should pass when server is running. Status: {}, Error: {:?}",
    1434
                        status,
    1435
    0
                        result.err()
    1436
                    );
    1437
                },
    1438
    0
                Err(e) => {
    1439
    0
                    panic!(
    1440
    0
                        "IMP-143a: Could not connect to llama.cpp server at {}. \
    1441
    0
                        Start with: llama-server -m model.gguf --host 127.0.0.1 --port 8082. \
    1442
    0
                        Error: {}",
    1443
                        health_url, e
    1444
                    );
    1445
                },
    1446
            }
    1447
    0
        }
    1448
    1449
        /// IMP-143b: Verify Ollama server preflight check works with real server
    1450
        #[test]
    1451
        #[ignore = "Requires running Ollama server on port 11434"]
    1452
    0
        fn test_imp_143b_ollama_real_server_check() {
    1453
            // This test requires: ollama serve
    1454
    0
            let mut check = ServerAvailabilityCheck::ollama(11434);
    1455
    1456
    0
            let client = reqwest::blocking::Client::builder()
    1457
    0
                .timeout(std::time::Duration::from_secs(5))
    1458
    0
                .build()
    1459
    0
                .expect("IMP-143b: Should create HTTP client");
    1460
    1461
    0
            let health_url = check.health_url();
    1462
    0
            match client.get(&health_url).send() {
    1463
    0
                Ok(response) => {
    1464
    0
                    let status = response.status().as_u16();
    1465
    0
                    check.set_health_status(status);
    1466
    1467
                    // IMP-143b: If server is running, check should pass
    1468
    0
                    let result = check.validate();
    1469
    0
                    assert!(
    1470
    0
                        result.is_ok(),
    1471
    0
                        "IMP-143b: Ollama server check should pass when server is running. Status: {}, Error: {:?}",
    1472
                        status,
    1473
    0
                        result.err()
    1474
                    );
    1475
                },
    1476
    0
                Err(e) => {
    1477
    0
                    panic!(
    1478
    0
                        "IMP-143b: Could not connect to Ollama server at {}. \
    1479
    0
                        Start with: ollama serve. \
    1480
    0
                        Error: {}",
    1481
                        health_url, e
    1482
                    );
    1483
                },
    1484
            }
    1485
    0
        }
    1486
    1487
        /// IMP-143c: Preflight runner should detect server availability
    1488
        #[test]
    1489
    1
        fn test_imp_143c_preflight_detects_unavailable_server() {
    1490
            // Use a port that's unlikely to have a server running
    1491
    1
            let mut check = ServerAvailabilityCheck::llama_cpp(59999);
    1492
    1
            check.set_health_status(0); // Connection refused test
    1493
    1494
            // IMP-143c: Check should fail for unavailable server
    1495
    1
            let result = check.validate();
    1496
    1
            assert!(
    1497
    1
                result.is_err(),
    1498
    0
                "IMP-143c: Preflight should detect unavailable server"
    1499
            );
    1500
    1
        }
    1501
    1502
        /// IMP-143d: Preflight reports correct error for connection failures
    1503
        #[test]
    1504
    1
        fn test_imp_143d_preflight_error_reporting() {
    1505
    1
            let mut check = ServerAvailabilityCheck::llama_cpp(59998);
    1506
    1
            check.set_health_status(503); // Service unavailable
    1507
    1508
    1
            let result = check.validate();
    1509
    1
            match result {
    1510
    1
                Err(PreflightError::HealthCheckFailed { status, url, .. }) => {
    1511
    1
                    assert_eq!(status, 503, "IMP-143d: Should report correct status code"0);
    1512
    1
                    assert!(url.contains("59998"), "IMP-143d: Should report correct URL"0);
    1513
                },
    1514
    0
                _ => panic!("IMP-143d: Should return HealthCheckFailed error"),
    1515
            }
    1516
    1
        }
    1517
    }