## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/bench/mod.rs

    Line
    Count
    Source
    1
    //! Benchmark harness for model runner comparison
    2
    //!
    3
    //! Implements the benchmark specification v1.1 with Toyota Way engineering principles:
    4
    //! - Dynamic CV-based stop-rule (Hoefler & Belli, SC12)
    5
    //! - Thermal throttling protocol
    6
    //! - ITL variance measurement (Dean & Barroso, "Tail at Scale")
    7
    //! - KV-cache fragmentation detection (PagedAttention methodology)
    8
    //! - KL-Divergence quality validation (LLM.int8())
    9
    //!
    10
    //! ## References
    11
    //!
    12
    //! - [17] Hoefler & Belli, "Scientific Benchmarking of Parallel Computing Systems", SC'15
    13
    //! - [11] Dean & Barroso, "The Tail at Scale", CACM 2013
    14
    //! - [12] Kwon et al., "PagedAttention", SOSP'23
    15
    //! - [13] Dettmers et al., "LLM.int8()", NeurIPS 2022
    16
    17
    #![allow(clippy::cast_precision_loss)] // Statistical functions need usize->f64
    18
    19
    use std::time::Duration;
    20
    21
    use serde::{Deserialize, Serialize};
    22
    23
    #[cfg(feature = "bench-http")]
    24
    use crate::http_client::{CompletionRequest, ModelHttpClient, OllamaOptions, OllamaRequest};
    25
    26
    // PMAT-802: Extracted modules
    27
    mod runtime;
    28
    mod statistics;
    29
    mod load_testing;
    30
    mod matrix;
    31
    mod gpu_parity;
    32
    33
    // RuntimeType always available (matrix.rs needs it)
    34
    pub use runtime::RuntimeType;
    35
    36
    // HTTP-dependent runtime backends
    37
    #[cfg(feature = "bench-http")]
    38
    pub use runtime::{
    39
        InferenceRequest, InferenceResponse, BackendInfo, RuntimeBackend,
    40
        MockBackend, BackendRegistry, LlamaCppConfig, VllmConfig, LlamaCppBackend,
    41
        VllmBackend, OllamaConfig, OllamaBackend,
    42
    };
    43
    44
    pub use statistics::{
    45
        MeasurementProtocol, LatencyStatistics, detect_outliers, BenchmarkMetrics,
    46
        Regression, RegressionReport, RegressionDetector, WelchTTestResult, welch_t_test,
    47
    };
    48
    49
    pub use load_testing::{LoadTestConfig, LoadTestResult, LoadTestRunner};
    50
    51
    pub use matrix::{
    52
        MatrixBenchmarkEntry, BenchmarkMatrix, MatrixBenchmarkConfig,
    53
        BackendSummary, MatrixSummary, ComputeBackendType,
    54
    };
    55
    56
    pub use gpu_parity::{
    57
        GpuParityBenchmark, GpuParityResult, GapAnalysis, FalsifiableClaim,
    58
        OptimizedGemmConfig, GemmPerformanceResult, OptimizedGemmBenchmark,
    59
        FusedOpType, FusedOpSpec, FlashAttentionConfig, Imp900Result, MemoryPoolConfig,
    60
    };
    61
    62
    // ============================================================================
    63
    // Dynamic Sampler (Section 2.1)
    64
    // ============================================================================
    65
    66
    /// Dynamic stop-rule based on Coefficient of Variation (CV)
    67
    ///
    68
    /// Per Hoefler & Belli [17], fixed iteration counts mask variance characteristics.
    69
    /// This sampler stops when statistical stability is achieved.
    70
    #[derive(Debug, Clone)]
    71
    pub struct DynamicSampler {
    72
        /// Minimum number of samples before checking CV
    73
        pub min_samples: usize,
    74
        /// Maximum samples (failsafe)
    75
        pub max_samples: usize,
    76
        /// Target CV threshold (default: 0.05 = 5%)
    77
        pub cv_threshold: f64,
    78
        /// Sliding window size for CV calculation
    79
        pub cv_window: usize,
    80
        /// Number of consecutive stable windows required
    81
        pub stability_count: usize,
    82
        /// Current stability streak
    83
        stable_streak: usize,
    84
    }
    85
    86
    impl Default for DynamicSampler {
    87
    6
        fn default() -> Self {
    88
    6
            Self {
    89
    6
                min_samples: 100,
    90
    6
                max_samples: 10_000,
    91
    6
                cv_threshold: 0.05,
    92
    6
                cv_window: 50,
    93
    6
                stability_count: 3,
    94
    6
                stable_streak: 0,
    95
    6
            }
    96
    6
        }
    97
    }
    98
    99
    impl DynamicSampler {
    100
        /// Create a new sampler with custom parameters
    101
        #[must_use]
    102
    8
        pub fn new(min_samples: usize, max_samples: usize, cv_threshold: f64) -> Self {
    103
    8
            Self {
    104
    8
                min_samples,
    105
    8
                max_samples,
    106
    8
                cv_threshold,
    107
    8
                cv_window: 50,
    108
    8
                stability_count: 3,
    109
    8
                stable_streak: 0,
    110
    8
            }
    111
    8
        }
    112
    113
        /// Check if sampling should continue
    114
        ///
    115
        /// Returns `true` if more samples are needed, `false` if stable.
    116
        #[must_use]
    117
    20
        pub fn should_continue(&mut self, samples: &[f64]) -> bool {
    118
    20
            let n = samples.len();
    119
    120
            // Always continue until minimum samples
    121
    20
            if n < self.min_samples {
    122
    11
                return true;
    123
    9
            }
    124
    125
            // Stop at maximum (failsafe)
    126
    9
            if n >= self.max_samples {
    127
    2
                return false;
    128
    7
            }
    129
    130
            // Compute CV over sliding window
    131
    7
            let window_start = n.saturating_sub(self.cv_window);
    132
    7
            let window = &samples[window_start..];
    133
    7
            let cv = compute_cv(window);
    134
    135
    7
            if cv < self.cv_threshold {
    136
    7
                self.stable_streak += 1;
    137
    7
                if self.stable_streak >= self.stability_count {
    138
    3
                    return false; // Stable - stop sampling
    139
    4
                }
    140
    0
            } else {
    141
    0
                self.stable_streak = 0;
    142
    0
            }
    143
    144
    4
            true // Continue sampling
    145
    20
        }
    146
    147
        /// Get the current CV for the last window
    148
        #[must_use]
    149
    7
        pub fn current_cv(&self, samples: &[f64]) -> f64 {
    150
    7
            if samples.len() < 2 {
    151
    2
                return f64::INFINITY;
    152
    5
            }
    153
    5
            let window_start = samples.len().saturating_sub(self.cv_window);
    154
    5
            compute_cv(&samples[window_start..])
    155
    7
        }
    156
    157
        /// Reset the sampler for a new run
    158
    1
        pub fn reset(&mut self) {
    159
    1
            self.stable_streak = 0;
    160
    1
        }
    161
    }
    162
    163
    /// Compute Coefficient of Variation (CV = std_dev / mean)
    164
    55
    fn compute_cv(data: &[f64]) -> f64 {
    165
    55
        if data.len() < 2 {
    166
    22
            return f64::INFINITY;
    167
    33
        }
    168
    169
    33
        let n = data.len() as f64;
    170
    33
        let mean = data.iter().sum::<f64>() / n;
    171
    172
    33
        if mean.abs() < 1e-10 {
    173
    1
            return f64::INFINITY;
    174
    32
        }
    175
    176
    510
        let variance32 = data32.iter32().map32(|x| (x - mean).powi(2)).sum32::<f64>() / (n - 1.0)32;
    177
    32
        let std_dev = variance.sqrt();
    178
    179
    32
        std_dev / mean.abs()
    180
    55
    }
    181
    182
    // ============================================================================
    183
    // Thermal Guard (Section 2.2)
    184
    // ============================================================================
    185
    186
    /// Temperature monitoring for benchmark validity
    187
    ///
    188
    /// Per spec Section 2.2, benchmarks are invalid if temperature variance > 2°C.
    189
    #[derive(Debug, Clone)]
    190
    pub struct ThermalGuard {
    191
        /// Maximum temperature before cooldown (°C)
    192
        pub max_temp_c: f64,
    193
        /// Temperature to resume at after cooldown (°C)
    194
        pub cooldown_threshold_c: f64,
    195
        /// Cooldown sleep duration (ms)
    196
        pub cooldown_sleep_ms: u64,
    197
        /// Maximum allowed temperature variance (°C)
    198
        pub temp_variance_c: f64,
    199
    }
    200
    201
    impl Default for ThermalGuard {
    202
    20
        fn default() -> Self {
    203
    20
            Self {
    204
    20
                max_temp_c: 80.0,
    205
    20
                cooldown_threshold_c: 70.0,
    206
    20
                cooldown_sleep_ms: 10_000,
    207
    20
                temp_variance_c: 2.0,
    208
    20
            }
    209
    20
        }
    210
    }
    211
    212
    /// Result of thermal validation
    213
    #[derive(Debug, Clone, PartialEq)]
    214
    pub enum ThermalValidity {
    215
        /// Temperature variance within acceptable range
    216
        Valid,
    217
        /// Temperature variance too high
    218
        Invalid(String),
    219
    }
    220
    221
    impl ThermalGuard {
    222
        /// Create a new ThermalGuard with custom parameters
    223
        #[must_use]
    224
    1
        pub fn new(
    225
    1
            max_temp_c: f64,
    226
    1
            cooldown_threshold_c: f64,
    227
    1
            cooldown_sleep_ms: u64,
    228
    1
            temp_variance_c: f64,
    229
    1
        ) -> Self {
    230
    1
            Self {
    231
    1
                max_temp_c,
    232
    1
                cooldown_threshold_c,
    233
    1
                cooldown_sleep_ms,
    234
    1
                temp_variance_c,
    235
    1
            }
    236
    1
        }
    237
    238
        /// Check if cooldown is needed (without sleeping)
    239
        #[must_use]
    240
    2
        pub fn needs_cooldown(&self, current_temp: f64) -> bool {
    241
    2
            current_temp > self.max_temp_c
    242
    2
        }
    243
    244
        /// Check if benchmark results are thermally valid
    245
        #[must_use]
    246
    8
        pub fn validate_run(&self, temps: &[f64]) -> ThermalValidity {
    247
    8
            if temps.is_empty() {
    248
    2
                return ThermalValidity::Valid;
    249
    6
            }
    250
    251
    6
            let variance = compute_variance(temps);
    252
    6
            let std_dev = variance.sqrt();
    253
    254
    6
            if std_dev > self.temp_variance_c {
    255
    3
                ThermalValidity::Invalid(format!(
    256
    3
                    "Temperature variance {std_dev:.2}°C exceeds threshold {:.2}°C",
    257
    3
                    self.temp_variance_c
    258
    3
                ))
    259
            } else {
    260
    3
                ThermalValidity::Valid
    261
            }
    262
    8
        }
    263
    264
        /// Check if cooldown is needed and sleep if so
    265
    1
        pub fn cooldown_if_needed(&self, current_temp: f64) {
    266
    1
            if current_temp > self.max_temp_c {
    267
    0
                std::thread::sleep(Duration::from_millis(self.cooldown_sleep_ms));
    268
    1
            }
    269
    1
        }
    270
    271
        /// Get max temperature from readings
    272
        #[must_use]
    273
    6
        pub fn max_temp(&self, temps: &[f64]) -> f64 {
    274
    6
            if temps.is_empty() {
    275
    2
                return 0.0;
    276
    4
            }
    277
    4
            temps.iter().copied().fold(f64::NEG_INFINITY, f64::max)
    278
    6
        }
    279
    280
        /// Get temperature variance
    281
        #[must_use]
    282
    7
        pub fn temp_variance(&self, temps: &[f64]) -> f64 {
    283
    7
            compute_variance(temps).sqrt()
    284
    7
        }
    285
    }
    286
    287
    /// Compute variance of a dataset
    288
    22
    fn compute_variance(data: &[f64]) -> f64 {
    289
    22
        if data.len() < 2 {
    290
    8
            return 0.0;
    291
    14
        }
    292
    293
    14
        let n = data.len() as f64;
    294
    14
        let mean = data.iter().sum::<f64>() / n;
    295
    213
        data14.iter14().map14(|x| (x - mean).powi(2)).sum14::<f64>() / (n - 1.0)14
    296
    22
    }
    297
    298
    // ============================================================================
    299
    // KV-Cache Metrics (Section 4.3)
    300
    // ============================================================================
    301
    302
    /// KV-cache fragmentation metrics per PagedAttention [12]
    303
    #[derive(Debug, Clone, Default, Serialize, Deserialize)]
    304
    pub struct KvCacheMetrics {
    305
        /// Total allocated KV-cache memory (bytes)
    306
        pub allocated_bytes: u64,
    307
        /// Actually used KV-cache memory (bytes)
    308
        pub used_bytes: u64,
    309
        /// Fragmentation percentage (waste)
    310
        pub fragmentation_pct: f64,
    311
    }
    312
    313
    impl KvCacheMetrics {
    314
        /// Create new metrics from allocated and used bytes
    315
        #[must_use]
    316
    5
        pub fn new(allocated_bytes: u64, used_bytes: u64) -> Self {
    317
    5
            let waste = allocated_bytes.saturating_sub(used_bytes);
    318
    5
            let fragmentation_pct = if allocated_bytes > 0 {
    319
    4
                (waste as f64 / allocated_bytes as f64) * 100.0
    320
            } else {
    321
    1
                0.0
    322
            };
    323
    324
    5
            Self {
    325
    5
                allocated_bytes,
    326
    5
                used_bytes,
    327
    5
                fragmentation_pct,
    328
    5
            }
    329
    5
        }
    330
    331
        /// Get allocated memory in MB
    332
        #[must_use]
    333
    1
        pub fn allocated_mb(&self) -> f64 {
    334
    1
            self.allocated_bytes as f64 / (1024.0 * 1024.0)
    335
    1
        }
    336
    337
        /// Get used memory in MB
    338
        #[must_use]
    339
    1
        pub fn used_mb(&self) -> f64 {
    340
    1
            self.used_bytes as f64 / (1024.0 * 1024.0)
    341
    1
        }
    342
    343
        /// Check if fragmentation is acceptable (< threshold)
    344
        #[must_use]
    345
    3
        pub fn is_acceptable(&self, threshold_pct: f64) -> bool {
    346
    3
            self.fragmentation_pct < threshold_pct
    347
    3
        }
    348
    }
    349
    350
    // ============================================================================
    351
    // Energy Metrics (Section 4.4)
    352
    // ============================================================================
    353
    354
    /// Energy measurement metrics per Garcia-Martin et al. [14]
    355
    #[derive(Debug, Clone, Default, Serialize, Deserialize)]
    356
    pub struct EnergyMetrics {
    357
        /// Total energy consumed (Joules)
    358
        pub total_joules: f64,
    359
        /// Idle power consumption (Watts)
    360
        pub idle_watts: f64,
    361
        /// Average active power consumption (Watts)
    362
        pub active_watts_avg: f64,
    363
        /// Number of tokens generated
    364
        pub tokens_generated: u64,
    365
    }
    366
    367
    impl EnergyMetrics {
    368
        /// Create new energy metrics
    369
        #[must_use]
    370
    5
        pub fn new(total_joules: f64, idle_watts: f64, active_watts_avg: f64, tokens: u64) -> Self {
    371
    5
            Self {
    372
    5
                total_joules,
    373
    5
                idle_watts,
    374
    5
                active_watts_avg,
    375
    5
                tokens_generated: tokens,
    376
    5
            }
    377
    5
        }
    378
    379
        /// Calculate energy per token (Joules/token)
    380
        #[must_use]
    381
    2
        pub fn joules_per_token(&self) -> f64 {
    382
    2
            if self.tokens_generated == 0 {
    383
    1
                return 0.0;
    384
    1
            }
    385
    1
            self.total_joules / self.tokens_generated as f64
    386
    2
        }
    387
    388
        /// Calculate energy efficiency ratio (tokens per Joule)
    389
        #[must_use]
    390
    3
        pub fn tokens_per_joule(&self) -> f64 {
    391
    3
            if self.total_joules < 1e-10 {
    392
    2
                return 0.0;
    393
    1
            }
    394
    1
            self.tokens_generated as f64 / self.total_joules
    395
    3
        }
    396
    }
    397
    398
    // ============================================================================
    399
    // ITL (Inter-Token Latency) Metrics (Section 4.2)
    400
    // ============================================================================
    401
    402
    /// Inter-Token Latency metrics per "Tail at Scale" [11]
    403
    #[derive(Debug, Clone, Default, Serialize, Deserialize)]
    404
    pub struct ItlMetrics {
    405
        /// Median ITL (ms)
    406
        pub median_ms: f64,
    407
        /// Standard deviation (jitter indicator)
    408
        pub std_dev_ms: f64,
    409
        /// p99 ITL (ms)
    410
        pub p99_ms: f64,
    411
        /// p99.9 ITL (ms)
    412
        pub p999_ms: f64,
    413
    }
    414
    415
    impl ItlMetrics {
    416
        /// Create ITL metrics from raw measurements
    417
        #[must_use]
    418
    10
        pub fn from_measurements(itl_times_ms: &[f64]) -> Self {
    419
    10
            if itl_times_ms.is_empty() {
    420
    2
                return Self::default();
    421
    8
            }
    422
    423
    8
            let mut sorted = itl_times_ms.to_vec();
    424
    247
            sorted8.sort_by8(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
    425
    426
    8
            let n = sorted.len();
    427
    8
            let median_ms = if n.is_multiple_of(2) {
    428
    6
                f64::midpoint(sorted[n / 2 - 1], sorted[n / 2])
    429
            } else {
    430
    2
                sorted[n / 2]
    431
            };
    432
    433
    8
            let mean = itl_times_ms.iter().sum::<f64>() / n as f64;
    434
    226
            let variance8 = itl_times_ms8.iter8().map8(|x| (x - mean).powi(2)).sum8::<f64>()
    435
    8
                / (n as f64 - 1.0).max(1.0);
    436
    8
            let std_dev_ms = variance.sqrt();
    437
    438
    8
            let percentile_99 = ((n as f64 * 0.99).ceil() as usize)
    439
    8
                .saturating_sub(1)
    440
    8
                .min(n - 1);
    441
    8
            let percentile_999 = ((n as f64 * 0.999).ceil() as usize)
    442
    8
                .saturating_sub(1)
    443
    8
                .min(n - 1);
    444
    445
    8
            Self {
    446
    8
                median_ms,
    447
    8
                std_dev_ms,
    448
    8
                p99_ms: sorted[percentile_99],
    449
    8
                p999_ms: sorted[percentile_999],
    450
    8
            }
    451
    10
        }
    452
    453
        /// Check if jitter is acceptable (std_dev < threshold)
    454
        #[must_use]
    455
    2
        pub fn is_low_jitter(&self, threshold_ms: f64) -> bool {
    456
    2
            self.std_dev_ms < threshold_ms
    457
    2
        }
    458
    }
    459
    460
    // ============================================================================
    461
    // KL-Divergence Quality Validation (Section 6.1)
    462
    // ============================================================================
    463
    464
    /// Result of quantization quality validation
    465
    #[derive(Debug, Clone, PartialEq)]
    466
    pub enum QualityResult {
    467
        /// Quality is acceptable
    468
        Pass {
    469
            /// Measured KL-divergence (nats)
    470
            kl_divergence: f64,
    471
        },
    472
        /// Quality degradation detected
    473
        Fail {
    474
            /// Measured KL-divergence (nats)
    475
            kl_divergence: f64,
    476
            /// Threshold that was exceeded
    477
            threshold: f64,
    478
            /// Descriptive message
    479
            message: &'static str,
    480
        },
    481
    }
    482
    483
    /// Compute softmax of logits
    484
    18
    fn softmax(logits: &[f32]) -> Vec<f64> {
    485
    18
        let max_logit = logits.iter().copied().fold(f32::NEG_INFINITY, f32::max);
    486
    18
        let exp_logits: Vec<f64> = logits
    487
    18
            .iter()
    488
    76
            .map18(|x| ((*x - max_logit) as f64).exp())
    489
    18
            .collect();
    490
    18
        let sum: f64 = exp_logits.iter().sum();
    491
    76
        exp_logits.iter()18.map18(|x| x / sum).collect18()
    492
    18
    }
    493
    494
    /// Validate quantization quality using KL-Divergence
    495
    ///
    496
    /// Per LLM.int8() [13], epsilon checks fail on outlier features.
    497
    /// KL-divergence provides a proper information-theoretic measure.
    498
    ///
    499
    /// # Arguments
    500
    ///
    501
    /// * `fp32_logits` - Reference logits from FP32 model
    502
    /// * `quantized_logits` - Logits from quantized model
    503
    /// * `threshold` - Maximum acceptable KL-divergence (nats)
    504
    ///
    505
    /// # Returns
    506
    ///
    507
    /// `QualityResult::Pass` if KL-divergence < threshold, `Fail` otherwise.
    508
    #[must_use]
    509
    8
    pub fn validate_quantization_quality(
    510
    8
        fp32_logits: &[f32],
    511
    8
        quantized_logits: &[f32],
    512
    8
        threshold: f64,
    513
    8
    ) -> QualityResult {
    514
    8
        if fp32_logits.len() != quantized_logits.len() {
    515
    1
            return QualityResult::Fail {
    516
    1
                kl_divergence: f64::INFINITY,
    517
    1
                threshold,
    518
    1
                message: "Logit vector lengths do not match",
    519
    1
            };
    520
    7
        }
    521
    522
    7
        if fp32_logits.is_empty() {
    523
    1
            return QualityResult::Pass { kl_divergence: 0.0 };
    524
    6
        }
    525
    526
        // Convert to probability distributions
    527
    6
        let fp32_probs = softmax(fp32_logits);
    528
    6
        let quant_probs = softmax(quantized_logits);
    529
    530
        // Compute KL(P_fp32 || P_quant)
    531
    6
        let kl_div: f64 = fp32_probs
    532
    6
            .iter()
    533
    6
            .zip(&quant_probs)
    534
    27
            .map6(|(p, q)| {
    535
    27
                if *p > 1e-10 && *q > 1e-10 {
    536
    27
                    p * (p / q).ln()
    537
                } else {
    538
    0
                    0.0
    539
                }
    540
    27
            })
    541
    6
            .sum();
    542
    543
    6
        if kl_div < threshold {
    544
    4
            QualityResult::Pass {
    545
    4
                kl_divergence: kl_div,
    546
    4
            }
    547
        } else {
    548
    2
            QualityResult::Fail {
    549
    2
                kl_divergence: kl_div,
    550
    2
                threshold,
    551
    2
                message: "Quantization quality degradation detected",
    552
    2
            }
    553
        }
    554
    8
    }
    555
    556
    // ============================================================================
    557
    // Benchmark Result (Section 4.1)
    558
    // ============================================================================
    559
    560
    /// Configuration for a benchmark run
    561
    #[derive(Debug, Clone, Serialize, Deserialize)]
    562
    pub struct BenchmarkConfig {
    563
        /// Model identifier
    564
        pub model: String,
    565
        /// Model format (apr, gguf, safetensors)
    566
        pub format: String,
    567
        /// Quantization level
    568
        pub quantization: String,
    569
        /// Runtime name
    570
        pub runtime: String,
    571
        /// Runtime version
    572
        pub runtime_version: String,
    573
    }
    574
    575
    /// Complete benchmark result per spec Section 4.1
    576
    #[derive(Debug, Clone, Serialize, Deserialize)]
    577
    pub struct BenchmarkResult {
    578
        /// Configuration used
    579
        pub config: BenchmarkConfig,
    580
        /// Cold start time (ms)
    581
        pub cold_start_ms: f64,
    582
        /// Model load time (ms)
    583
        pub model_load_ms: f64,
    584
        /// Time-to-first-token measurements (ms)
    585
        pub ttft_ms: Vec<f64>,
    586
        /// Inter-token latency measurements (ms)
    587
        pub itl_ms: Vec<f64>,
    588
        /// Generation throughput measurements (tok/s)
    589
        pub generation_tok_s: Vec<f64>,
    590
        /// Peak memory usage (MB)
    591
        pub peak_memory_mb: u64,
    592
        /// KV-cache fragmentation percentage
    593
        pub kv_cache_waste_pct: f64,
    594
        /// Total energy consumed (Joules)
    595
        pub energy_joules: f64,
    596
        /// Total tokens generated
    597
        pub tokens_generated: u64,
    598
        /// Actual number of iterations (dynamic sampling)
    599
        pub actual_iterations: usize,
    600
        /// CV at stop point
    601
        pub cv_at_stop: f64,
    602
        /// Unix timestamp
    603
        pub timestamp: u64,
    604
    }
    605
    606
    /// Summary statistics for benchmark results
    607
    #[derive(Debug, Clone, Serialize, Deserialize)]
    608
    pub struct BenchmarkSummary {
    609
        // TTFT metrics
    610
        /// TTFT p50 (ms)
    611
        pub ttft_p50: f64,
    612
        /// TTFT p95 (ms)
    613
        pub ttft_p95: f64,
    614
        /// TTFT p99 (ms)
    615
        pub ttft_p99: f64,
    616
        /// TTFT p99.9 (ms)
    617
        pub ttft_p999: f64,
    618
    619
        // ITL metrics
    620
        /// ITL median (ms)
    621
        pub itl_median: f64,
    622
        /// ITL standard deviation (jitter)
    623
        pub itl_std_dev: f64,
    624
    625
        // Throughput metrics
    626
        /// Throughput median (tok/s)
    627
        pub throughput_median: f64,
    628
        /// Throughput 95% CI (lower, upper)
    629
        pub throughput_ci_95: (f64, f64),
    630
    631
        // Energy metrics
    632
        /// Energy per token (J/tok)
    633
        pub token_joules: f64,
    634
    635
        // Memory metrics
    636
        /// KV-cache waste percentage
    637
        pub memory_waste_pct: f64,
    638
    639
        // Statistical validity
    640
        /// Number of iterations run
    641
        pub iterations: usize,
    642
        /// Final CV value
    643
        pub cv_final: f64,
    644
    }
    645
    646
    impl BenchmarkResult {
    647
        /// Generate summary statistics from raw measurements
    648
        #[must_use]
    649
    5
        pub fn summary(&self) -> BenchmarkSummary {
    650
            BenchmarkSummary {
    651
    5
                ttft_p50: percentile(&self.ttft_ms, 50.0),
    652
    5
                ttft_p95: percentile(&self.ttft_ms, 95.0),
    653
    5
                ttft_p99: percentile(&self.ttft_ms, 99.0),
    654
    5
                ttft_p999: percentile(&self.ttft_ms, 99.9),
    655
    656
    5
                itl_median: percentile(&self.itl_ms, 50.0),
    657
    5
                itl_std_dev: compute_std_dev(&self.itl_ms),
    658
    659
    5
                throughput_median: percentile(&self.generation_tok_s, 50.0),
    660
    5
                throughput_ci_95: bootstrap_ci(&self.generation_tok_s, 0.95, 1000),
    661
    662
    5
                token_joules: if self.tokens_generated > 0 {
    663
    4
                    self.energy_joules / self.tokens_generated as f64
    664
                } else {
    665
    1
                    0.0
    666
                },
    667
    668
    5
                memory_waste_pct: self.kv_cache_waste_pct,
    669
    5
                iterations: self.actual_iterations,
    670
    5
                cv_final: self.cv_at_stop,
    671
            }
    672
    5
        }
    673
    }
    674
    675
    /// Compute percentile of a dataset
    676
    162
    fn percentile(data: &[f64], p: f64) -> f64 {
    677
    162
        if data.is_empty() {
    678
    3
            return 0.0;
    679
    159
        }
    680
    681
    159
        let mut sorted = data.to_vec();
    682
    3.81k
        sorted159.sort_by159(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
    683
    684
    159
        let idx = ((sorted.len() as f64 * p / 100.0).ceil() as usize)
    685
    159
            .saturating_sub(1)
    686
    159
            .min(sorted.len() - 1);
    687
    159
        sorted[idx]
    688
    162
    }
    689
    690
    /// Compute standard deviation
    691
    7
    fn compute_std_dev(data: &[f64]) -> f64 {
    692
    7
        compute_variance(data).sqrt()
    693
    7
    }
    694
    695
    /// Bootstrap confidence interval
    696
    10
    fn bootstrap_ci(data: &[f64], confidence: f64, n_resamples: usize) -> (f64, f64) {
    697
    10
        if data.is_empty() {
    698
    1
            return (0.0, 0.0);
    699
    9
        }
    700
    701
    9
        let mut bootstrap_means = Vec::with_capacity(n_resamples);
    702
    9
        let n = data.len();
    703
    704
    8.10k
        for i in 0..n_resamples9 {
    705
            // Simple deterministic pseudo-random for reproducibility
    706
            // Uses a basic LCG instead of hash for clippy compliance
    707
    8.10k
            let seed = (i as u64)
    708
    8.10k
                .wrapping_mul(6_364_136_223_846_793_005)
    709
    8.10k
                .wrapping_add(1);
    710
    711
    8.10k
            let mut sum = 0.0;
    712
    218k
            for j in 0..n8.10k {
    713
    218k
                let idx = ((seed.wrapping_mul(j as u64 + 1)) as usize) % n;
    714
    218k
                sum += data[idx];
    715
    218k
            }
    716
    8.10k
            bootstrap_means.push(sum / n as f64);
    717
        }
    718
    719
    25.4k
        bootstrap_means9.sort_by9(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
    720
    721
    9
        let alpha = 1.0 - confidence;
    722
    9
        let lower_idx = ((n_resamples as f64 * alpha / 2.0).floor() as usize).min(n_resamples - 1);
    723
    9
        let upper_idx =
    724
    9
            ((n_resamples as f64 * (1.0 - alpha / 2.0)).ceil() as usize).min(n_resamples - 1);
    725
    726
    9
        (bootstrap_means[lower_idx], bootstrap_means[upper_idx])
    727
    10
    }
    728
    729
    // ============================================================================
    730
    // Convoy Test (Section 2.4)
    731
    // ============================================================================
    732
    733
    /// Workload type for convoy testing
    734
    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
    735
    pub enum WorkloadType {
    736
        /// Short QA: 32 input tokens, 64 output tokens
    737
        ShortQa,
    738
        /// Long Context: 2048 input tokens, 512 output tokens
    739
        LongContext,
    740
    }
    741
    742
    impl WorkloadType {
    743
        /// Get input token count for this workload type
    744
        #[must_use]
    745
    2
        pub const fn input_tokens(&self) -> usize {
    746
    2
            match self {
    747
    1
                Self::ShortQa => 32,
    748
    1
                Self::LongContext => 2048,
    749
            }
    750
    2
        }
    751
    752
        /// Get output token count for this workload type
    753
        #[must_use]
    754
    2
        pub const fn output_tokens(&self) -> usize {
    755
    2
            match self {
    756
    1
                Self::ShortQa => 64,
    757
    1
                Self::LongContext => 512,
    758
            }
    759
    2
        }
    760
    }
    761
    762
    /// Configuration for convoy test per spec Section 2.4
    763
    #[derive(Debug, Clone)]
    764
    pub struct ConvoyTestConfig {
    765
        /// Number of long-context requests (default: 10)
    766
        pub long_requests: usize,
    767
        /// Number of short-QA requests (default: 100)
    768
        pub short_requests: usize,
    769
        /// Maximum acceptable p99 latency increase (default: 50%)
    770
        pub max_p99_increase_pct: f64,
    771
        /// Maximum acceptable head-of-line blocking time (ms)
    772
        pub max_hol_blocking_ms: f64,
    773
        /// Maximum acceptable KV-cache fragmentation (%)
    774
        pub max_kv_fragmentation_pct: f64,
    775
    }
    776
    777
    impl Default for ConvoyTestConfig {
    778
    13
        fn default() -> Self {
    779
    13
            Self {
    780
    13
                long_requests: 10,
    781
    13
                short_requests: 100,
    782
    13
                max_p99_increase_pct: 50.0,
    783
    13
                max_hol_blocking_ms: 500.0,
    784
    13
                max_kv_fragmentation_pct: 15.0,
    785
    13
            }
    786
    13
        }
    787
    }
    788
    789
    /// Result of a single request in convoy test
    790
    #[derive(Debug, Clone, Serialize, Deserialize)]
    791
    pub struct ConvoyRequestResult {
    792
        /// Request type
    793
        pub workload_type: String,
    794
        /// Time spent waiting (head-of-line blocking)
    795
        pub queue_time_ms: f64,
    796
        /// Time to first token
    797
        pub ttft_ms: f64,
    798
        /// Total latency
    799
        pub total_latency_ms: f64,
    800
    }
    801
    802
    /// Overall convoy test result
    803
    #[derive(Debug, Clone, Serialize, Deserialize)]
    804
    pub struct ConvoyTestResult {
    805
        /// Number of long-context requests in test
    806
        pub long_requests: usize,
    807
        /// Number of short-QA requests in test
    808
        pub short_requests: usize,
    809
    810
        /// Baseline: Short-QA p99 without convoy
    811
        pub baseline_short_p99_ms: f64,
    812
        /// Convoy: Short-QA p99 with convoy
    813
        pub convoy_short_p99_ms: f64,
    814
        /// P99 increase percentage
    815
        pub p99_increase_pct: f64,
    816
    817
        /// Maximum head-of-line blocking observed
    818
        pub max_hol_blocking_ms: f64,
    819
        /// Average head-of-line blocking
    820
        pub avg_hol_blocking_ms: f64,
    821
    822
        /// KV-cache fragmentation during convoy
    823
        pub kv_fragmentation_pct: f64,
    824
    825
        /// Pass/fail status
    826
        pub passed: bool,
    827
        /// Failure reasons (if any)
    828
        pub failure_reasons: Vec<String>,
    829
    }
    830
    831
    impl ConvoyTestResult {
    832
        /// Create a new convoy test result from measurements
    833
        #[must_use]
    834
    12
        pub fn new(
    835
    12
            config: &ConvoyTestConfig,
    836
    12
            baseline_short_latencies: &[f64],
    837
    12
            convoy_short_latencies: &[f64],
    838
    12
            hol_blocking_times: &[f64],
    839
    12
            kv_fragmentation_pct: f64,
    840
    12
        ) -> Self {
    841
    12
            let baseline_short_p99 = percentile(baseline_short_latencies, 99.0);
    842
    12
            let convoy_short_p99 = percentile(convoy_short_latencies, 99.0);
    843
    844
    12
            let p99_increase_pct = if baseline_short_p99 > 0.0 {
    845
    11
                ((convoy_short_p99 - baseline_short_p99) / baseline_short_p99) * 100.0
    846
            } else {
    847
    1
                0.0
    848
            };
    849
    850
    12
            let max_hol_blocking = hol_blocking_times.iter().copied().fold(0.0_f64, f64::max);
    851
    12
            let avg_hol_blocking = if hol_blocking_times.is_empty() {
    852
    1
                0.0
    853
            } else {
    854
    11
                hol_blocking_times.iter().sum::<f64>() / hol_blocking_times.len() as f64
    855
            };
    856
    857
    12
            let mut failure_reasons = Vec::new();
    858
    859
    12
            if p99_increase_pct > config.max_p99_increase_pct {
    860
    1
                failure_reasons.push(format!(
    861
    1
                    "P99 increase {p99_increase_pct:.1}% exceeds threshold {:.1}%",
    862
    1
                    config.max_p99_increase_pct
    863
    1
                ));
    864
    11
            }
    865
    866
    12
            if max_hol_blocking > config.max_hol_blocking_ms {
    867
    1
                failure_reasons.push(format!(
    868
    1
                    "Max HOL blocking {max_hol_blocking:.1}ms exceeds threshold {:.1}ms",
    869
    1
                    config.max_hol_blocking_ms
    870
    1
                ));
    871
    11
            }
    872
    873
    12
            if kv_fragmentation_pct > config.max_kv_fragmentation_pct {
    874
    1
                failure_reasons.push(format!(
    875
    1
                    "KV fragmentation {kv_fragmentation_pct:.1}% exceeds threshold {:.1}%",
    876
    1
                    config.max_kv_fragmentation_pct
    877
    1
                ));
    878
    11
            }
    879
    880
    12
            Self {
    881
    12
                long_requests: config.long_requests,
    882
    12
                short_requests: config.short_requests,
    883
    12
                baseline_short_p99_ms: baseline_short_p99,
    884
    12
                convoy_short_p99_ms: convoy_short_p99,
    885
    12
                p99_increase_pct,
    886
    12
                max_hol_blocking_ms: max_hol_blocking,
    887
    12
                avg_hol_blocking_ms: avg_hol_blocking,
    888
    12
                kv_fragmentation_pct,
    889
    12
                passed: failure_reasons.is_empty(),
    890
    12
                failure_reasons,
    891
    12
            }
    892
    12
        }
    893
    }
    894
    895
    // ============================================================================
    896
    // Saturation Test (Section 2.5)
    897
    // ============================================================================
    898
    899
    /// Configuration for saturation stress test per spec Section 2.5
    900
    #[derive(Debug, Clone)]
    901
    pub struct SaturationTestConfig {
    902
        /// CPU load percentage (default: 50%)
    903
        pub cpu_load_pct: u8,
    904
        /// Maximum acceptable throughput degradation (default: 30%)
    905
        pub max_throughput_degradation_pct: f64,
    906
        /// Maximum acceptable p99 latency increase (default: 100%)
    907
        pub max_p99_increase_pct: f64,
    908
    }
    909
    910
    impl Default for SaturationTestConfig {
    911
    12
        fn default() -> Self {
    912
    12
            Self {
    913
    12
                cpu_load_pct: 50,
    914
    12
                max_throughput_degradation_pct: 30.0,
    915
    12
                max_p99_increase_pct: 100.0,
    916
    12
            }
    917
    12
        }
    918
    }
    919
    920
    /// Saturation test result
    921
    #[derive(Debug, Clone, Serialize, Deserialize)]
    922
    pub struct SaturationTestResult {
    923
        /// CPU load used
    924
        pub cpu_load_pct: u8,
    925
    926
        /// Baseline throughput (tok/s)
    927
        pub baseline_throughput: f64,
    928
        /// Stressed throughput (tok/s)
    929
        pub stressed_throughput: f64,
    930
        /// Throughput degradation percentage
    931
        pub throughput_degradation_pct: f64,
    932
    933
        /// Baseline p99 latency (ms)
    934
        pub baseline_p99_ms: f64,
    935
        /// Stressed p99 latency (ms)
    936
        pub stressed_p99_ms: f64,
    937
        /// P99 latency increase percentage
    938
        pub p99_increase_pct: f64,
    939
    940
        /// Pass/fail status
    941
        pub passed: bool,
    942
        /// Failure reasons (if any)
    943
        pub failure_reasons: Vec<String>,
    944
    }
    945
    946
    impl SaturationTestResult {
    947
        /// Create a new saturation test result
    948
        #[must_use]
    949
    11
        pub fn new(
    950
    11
            config: &SaturationTestConfig,
    951
    11
            baseline_throughputs: &[f64],
    952
    11
            stressed_throughputs: &[f64],
    953
    11
            baseline_latencies: &[f64],
    954
    11
            stressed_latencies: &[f64],
    955
    11
        ) -> Self {
    956
    11
            let baseline_throughput = if baseline_throughputs.is_empty() {
    957
    1
                0.0
    958
            } else {
    959
    10
                baseline_throughputs.iter().sum::<f64>() / baseline_throughputs.len() as f64
    960
            };
    961
    962
    11
            let stressed_throughput = if stressed_throughputs.is_empty() {
    963
    1
                0.0
    964
            } else {
    965
    10
                stressed_throughputs.iter().sum::<f64>() / stressed_throughputs.len() as f64
    966
            };
    967
    968
    11
            let throughput_degradation_pct = if baseline_throughput > 0.0 {
    969
    9
                ((baseline_throughput - stressed_throughput) / baseline_throughput) * 100.0
    970
            } else {
    971
    2
                0.0
    972
            };
    973
    974
    11
            let baseline_p99 = percentile(baseline_latencies, 99.0);
    975
    11
            let stressed_p99 = percentile(stressed_latencies, 99.0);
    976
    977
    11
            let p99_increase_pct = if baseline_p99 > 0.0 {
    978
    9
                ((stressed_p99 - baseline_p99) / baseline_p99) * 100.0
    979
            } else {
    980
    2
                0.0
    981
            };
    982
    983
    11
            let mut failure_reasons = Vec::new();
    984
    985
    11
            if throughput_degradation_pct > config.max_throughput_degradation_pct {
    986
    1
                failure_reasons.push(format!(
    987
    1
                    "Throughput degradation {throughput_degradation_pct:.1}% exceeds threshold {:.1}%",
    988
    1
                    config.max_throughput_degradation_pct
    989
    1
                ));
    990
    10
            }
    991
    992
    11
            if p99_increase_pct > config.max_p99_increase_pct {
    993
    1
                failure_reasons.push(format!(
    994
    1
                    "P99 increase {p99_increase_pct:.1}% exceeds threshold {:.1}%",
    995
    1
                    config.max_p99_increase_pct
    996
    1
                ));
    997
    10
            }
    998
    999
    11
            Self {
    1000
    11
                cpu_load_pct: config.cpu_load_pct,
    1001
    11
                baseline_throughput,
    1002
    11
                stressed_throughput,
    1003
    11
                throughput_degradation_pct,
    1004
    11
                baseline_p99_ms: baseline_p99,
    1005
    11
                stressed_p99_ms: stressed_p99,
    1006
    11
                p99_increase_pct,
    1007
    11
                passed: failure_reasons.is_empty(),
    1008
    11
                failure_reasons,
    1009
    11
            }
    1010
    11
        }
    1011
    }
    1012
    1013
    // ============================================================================
    1014
    // Benchmark Runner (Full Harness)
    1015
    // ============================================================================
    1016
    1017
    /// Hardware specification for reproducibility
    1018
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1019
    pub struct HardwareSpec {
    1020
        /// CPU model
    1021
        pub cpu: String,
    1022
        /// GPU model (if any)
    1023
        pub gpu: Option<String>,
    1024
        /// Total memory in GB
    1025
        pub memory_gb: u64,
    1026
        /// Storage type
    1027
        pub storage: String,
    1028
    }
    1029
    1030
    impl Default for HardwareSpec {
    1031
    38
        fn default() -> Self {
    1032
    38
            Self {
    1033
    38
                cpu: "Unknown".to_string(),
    1034
    38
                gpu: None,
    1035
    38
                memory_gb: 0,
    1036
    38
                storage: "Unknown".to_string(),
    1037
    38
            }
    1038
    38
        }
    1039
    }
    1040
    1041
    /// Sampling method configuration
    1042
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1043
    pub struct SamplingConfig {
    1044
        /// Sampling method (e.g., "dynamic_cv")
    1045
        pub method: String,
    1046
        /// CV threshold for stopping
    1047
        pub cv_threshold: f64,
    1048
        /// Actual iterations run
    1049
        pub actual_iterations: usize,
    1050
        /// CV at stop point
    1051
        pub cv_at_stop: f64,
    1052
        /// Warmup iterations
    1053
        pub warmup_iterations: usize,
    1054
    }
    1055
    1056
    impl Default for SamplingConfig {
    1057
    19
        fn default() -> Self {
    1058
    19
            Self {
    1059
    19
                method: "dynamic_cv".to_string(),
    1060
    19
                cv_threshold: 0.05,
    1061
    19
                actual_iterations: 0,
    1062
    19
                cv_at_stop: 0.0,
    1063
    19
                warmup_iterations: 100,
    1064
    19
            }
    1065
    19
        }
    1066
    }
    1067
    1068
    /// Thermal validity info
    1069
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1070
    pub struct ThermalInfo {
    1071
        /// Whether thermal conditions were valid
    1072
        pub valid: bool,
    1073
        /// Temperature variance (°C)
    1074
        pub temp_variance_c: f64,
    1075
        /// Maximum temperature observed (°C)
    1076
        pub max_temp_c: f64,
    1077
    }
    1078
    1079
    impl Default for ThermalInfo {
    1080
    19
        fn default() -> Self {
    1081
    19
            Self {
    1082
    19
                valid: true,
    1083
    19
                temp_variance_c: 0.0,
    1084
    19
                max_temp_c: 0.0,
    1085
    19
            }
    1086
    19
        }
    1087
    }
    1088
    1089
    /// TTFT results structure
    1090
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1091
    pub struct TtftResults {
    1092
        /// P50 (median)
    1093
        pub p50: f64,
    1094
        /// P95
    1095
        pub p95: f64,
    1096
        /// P99
    1097
        pub p99: f64,
    1098
        /// P99.9
    1099
        pub p999: f64,
    1100
    }
    1101
    1102
    /// ITL results structure
    1103
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1104
    pub struct ItlResults {
    1105
        /// Median ITL
    1106
        pub median: f64,
    1107
        /// Standard deviation (jitter)
    1108
        pub std_dev: f64,
    1109
        /// P99 ITL
    1110
        pub p99: f64,
    1111
    }
    1112
    1113
    /// Throughput results structure
    1114
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1115
    pub struct ThroughputResults {
    1116
        /// Median throughput (tok/s)
    1117
        pub median: f64,
    1118
        /// 95% confidence interval
    1119
        pub ci_95: (f64, f64),
    1120
    }
    1121
    1122
    /// Memory results structure
    1123
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1124
    pub struct MemoryResults {
    1125
        /// Model size (MB)
    1126
        pub model_mb: u64,
    1127
        /// Peak RSS (MB)
    1128
        pub peak_rss_mb: u64,
    1129
        /// KV-cache waste percentage
    1130
        pub kv_waste_pct: f64,
    1131
    }
    1132
    1133
    /// Energy results structure
    1134
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1135
    pub struct EnergyResults {
    1136
        /// Total energy (Joules)
    1137
        pub total_joules: f64,
    1138
        /// Energy per token (J/tok)
    1139
        pub token_joules: f64,
    1140
        /// Idle power (Watts)
    1141
        pub idle_watts: f64,
    1142
    }
    1143
    1144
    /// Cold start results structure
    1145
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1146
    pub struct ColdStartResults {
    1147
        /// Median cold start time (ms)
    1148
        pub median: f64,
    1149
        /// P99 cold start time (ms)
    1150
        pub p99: f64,
    1151
    }
    1152
    1153
    /// Quality validation results
    1154
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1155
    pub struct QualityValidation {
    1156
        /// KL-divergence vs FP32
    1157
        pub kl_divergence_vs_fp32: f64,
    1158
        /// Perplexity on WikiText-2 (optional)
    1159
        pub perplexity_wikitext2: Option<f64>,
    1160
    }
    1161
    1162
    /// Full benchmark results per JSON schema v1.1 (Appendix B)
    1163
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1164
    pub struct FullBenchmarkResult {
    1165
        /// Schema version
    1166
        pub version: String,
    1167
        /// ISO 8601 timestamp
    1168
        pub timestamp: String,
    1169
        /// Model configuration
    1170
        pub config: BenchmarkConfig,
    1171
        /// Hardware specification
    1172
        pub hardware: HardwareSpec,
    1173
        /// Sampling configuration
    1174
        pub sampling: SamplingConfig,
    1175
        /// Thermal information
    1176
        pub thermal: ThermalInfo,
    1177
        /// All results
    1178
        pub results: BenchmarkResults,
    1179
        /// Quality validation
    1180
        pub quality: QualityValidation,
    1181
    }
    1182
    1183
    /// Consolidated benchmark results
    1184
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1185
    pub struct BenchmarkResults {
    1186
        /// Time-to-first-token metrics
    1187
        pub ttft_ms: TtftResults,
    1188
        /// Inter-token latency metrics
    1189
        pub itl_ms: ItlResults,
    1190
        /// Throughput metrics
    1191
        pub throughput_tok_s: ThroughputResults,
    1192
        /// Memory metrics
    1193
        pub memory_mb: MemoryResults,
    1194
        /// Energy metrics
    1195
        pub energy: EnergyResults,
    1196
        /// Cold start metrics
    1197
        pub cold_start_ms: ColdStartResults,
    1198
    }
    1199
    1200
    impl FullBenchmarkResult {
    1201
        /// Create from a BenchmarkResult with additional metadata
    1202
        #[must_use]
    1203
    3
        pub fn from_benchmark_result(
    1204
    3
            result: &BenchmarkResult,
    1205
    3
            hardware: HardwareSpec,
    1206
    3
            thermal_temps: &[f64],
    1207
    3
            kl_divergence: f64,
    1208
    3
        ) -> Self {
    1209
    3
            let thermal_guard = ThermalGuard::default();
    1210
    3
            let thermal_validity = thermal_guard.validate_run(thermal_temps);
    1211
    1212
    3
            let summary = result.summary();
    1213
    1214
    3
            Self {
    1215
    3
                version: "1.1".to_string(),
    1216
    3
                timestamp: chrono_timestamp(),
    1217
    3
                config: result.config.clone(),
    1218
    3
                hardware,
    1219
    3
                sampling: SamplingConfig {
    1220
    3
                    method: "dynamic_cv".to_string(),
    1221
    3
                    cv_threshold: 0.05,
    1222
    3
                    actual_iterations: result.actual_iterations,
    1223
    3
                    cv_at_stop: result.cv_at_stop,
    1224
    3
                    warmup_iterations: 100,
    1225
    3
                },
    1226
    3
                thermal: ThermalInfo {
    1227
    3
                    valid: thermal_validity == ThermalValidity::Valid,
    1228
    3
                    temp_variance_c: thermal_guard.temp_variance(thermal_temps),
    1229
    3
                    max_temp_c: thermal_guard.max_temp(thermal_temps),
    1230
    3
                },
    1231
    3
                results: BenchmarkResults {
    1232
    3
                    ttft_ms: TtftResults {
    1233
    3
                        p50: summary.ttft_p50,
    1234
    3
                        p95: summary.ttft_p95,
    1235
    3
                        p99: summary.ttft_p99,
    1236
    3
                        p999: summary.ttft_p999,
    1237
    3
                    },
    1238
    3
                    itl_ms: ItlResults {
    1239
    3
                        median: summary.itl_median,
    1240
    3
                        std_dev: summary.itl_std_dev,
    1241
    3
                        p99: percentile(&result.itl_ms, 99.0),
    1242
    3
                    },
    1243
    3
                    throughput_tok_s: ThroughputResults {
    1244
    3
                        median: summary.throughput_median,
    1245
    3
                        ci_95: summary.throughput_ci_95,
    1246
    3
                    },
    1247
    3
                    memory_mb: MemoryResults {
    1248
    3
                        model_mb: result.peak_memory_mb / 2, // Approximate model size
    1249
    3
                        peak_rss_mb: result.peak_memory_mb,
    1250
    3
                        kv_waste_pct: result.kv_cache_waste_pct,
    1251
    3
                    },
    1252
    3
                    energy: EnergyResults {
    1253
    3
                        total_joules: result.energy_joules,
    1254
    3
                        token_joules: summary.token_joules,
    1255
    3
                        idle_watts: 0.0, // Would need separate measurement
    1256
    3
                    },
    1257
    3
                    cold_start_ms: ColdStartResults {
    1258
    3
                        median: result.cold_start_ms,
    1259
    3
                        p99: result.cold_start_ms * 1.5, // Approximate
    1260
    3
                    },
    1261
    3
                },
    1262
    3
                quality: QualityValidation {
    1263
    3
                    kl_divergence_vs_fp32: kl_divergence,
    1264
    3
                    perplexity_wikitext2: None,
    1265
    3
                },
    1266
    3
            }
    1267
    3
        }
    1268
    1269
        /// Serialize to JSON string
    1270
        ///
    1271
        /// # Errors
    1272
        ///
    1273
        /// Returns an error if serialization fails.
    1274
    1
        pub fn to_json(&self) -> Result<String, serde_json::Error> {
    1275
    1
            serde_json::to_string_pretty(self)
    1276
    1
        }
    1277
    1278
        /// Deserialize from JSON string
    1279
        ///
    1280
        /// # Errors
    1281
        ///
    1282
        /// Returns an error if the JSON is invalid or doesn't match the schema.
    1283
    3
        pub fn from_json(json: &str) -> Result<Self, serde_json::Error> {
    1284
    3
            serde_json::from_str(json)
    1285
    3
        }
    1286
    }
    1287
    1288
    /// Generate ISO 8601 timestamp
    1289
    22
    fn chrono_timestamp() -> String {
    1290
        use std::time::{SystemTime, UNIX_EPOCH};
    1291
    1292
    22
        let duration = SystemTime::now()
    1293
    22
            .duration_since(UNIX_EPOCH)
    1294
    22
            .unwrap_or_default();
    1295
    22
        let secs = duration.as_secs();
    1296
    1297
        // Simple ISO 8601 format without external dependencies
    1298
    22
        format!("1970-01-01T00:00:00Z+{secs}s")
    1299
    22
    }
    1300
    1301
    // ============================================================================
    1302
    // Benchmark Comparison
    1303
    // ============================================================================
    1304
    1305
    /// Result of comparing two benchmarks
    1306
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1307
    pub struct BenchmarkComparison {
    1308
        /// Baseline config
    1309
        pub baseline_runtime: String,
    1310
        /// Current config
    1311
        pub current_runtime: String,
    1312
    1313
        /// TTFT p99 change percentage (negative = improvement)
    1314
        pub ttft_p99_change_pct: f64,
    1315
        /// Throughput change percentage (positive = improvement)
    1316
        pub throughput_change_pct: f64,
    1317
        /// Memory change percentage (negative = improvement)
    1318
        pub memory_change_pct: f64,
    1319
        /// Energy change percentage (negative = improvement)
    1320
        pub energy_change_pct: f64,
    1321
    1322
        /// Overall winner
    1323
        pub winner: String,
    1324
        /// Significance level (p-value from Mann-Whitney U)
    1325
        pub significance: f64,
    1326
    }
    1327
    1328
    impl BenchmarkComparison {
    1329
        /// Compare two benchmark results
    1330
        #[must_use]
    1331
    4
        pub fn compare(baseline: &FullBenchmarkResult, current: &FullBenchmarkResult) -> Self {
    1332
    4
            let ttft_p99_change = if baseline.results.ttft_ms.p99 > 0.0 {
    1333
    3
                ((current.results.ttft_ms.p99 - baseline.results.ttft_ms.p99)
    1334
    3
                    / baseline.results.ttft_ms.p99)
    1335
    3
                    * 100.0
    1336
            } else {
    1337
    1
                0.0
    1338
            };
    1339
    1340
    4
            let throughput_change = if baseline.results.throughput_tok_s.median > 0.0 {
    1341
    3
                ((current.results.throughput_tok_s.median - baseline.results.throughput_tok_s.median)
    1342
    3
                    / baseline.results.throughput_tok_s.median)
    1343
    3
                    * 100.0
    1344
            } else {
    1345
    1
                0.0
    1346
            };
    1347
    1348
    4
            let memory_change = if baseline.results.memory_mb.peak_rss_mb > 0 {
    1349
    3
                ((current.results.memory_mb.peak_rss_mb as f64
    1350
    3
                    - baseline.results.memory_mb.peak_rss_mb as f64)
    1351
    3
                    / baseline.results.memory_mb.peak_rss_mb as f64)
    1352
    3
                    * 100.0
    1353
            } else {
    1354
    1
                0.0
    1355
            };
    1356
    1357
    4
            let energy_change = if baseline.results.energy.token_joules > 0.0 {
    1358
    3
                ((current.results.energy.token_joules - baseline.results.energy.token_joules)
    1359
    3
                    / baseline.results.energy.token_joules)
    1360
    3
                    * 100.0
    1361
            } else {
    1362
    1
                0.0
    1363
            };
    1364
    1365
            // Simple winner determination: count improvements
    1366
    4
            let mut current_wins = 0;
    1367
    4
            let mut baseline_wins = 0;
    1368
    1369
    4
            if ttft_p99_change < -5.0 {
    1370
    1
                current_wins += 1;
    1371
    3
            } else if ttft_p99_change > 5.0 {
    1372
    1
                baseline_wins += 1;
    1373
    2
            }
    1374
    1375
    4
            if throughput_change > 5.0 {
    1376
    1
                current_wins += 1;
    1377
    3
            } else if throughput_change < -5.0 {
    1378
    1
                baseline_wins += 1;
    1379
    2
            }
    1380
    1381
    4
            if memory_change < -5.0 {
    1382
    1
                current_wins += 1;
    1383
    3
            } else if memory_change > 5.0 {
    1384
    1
                baseline_wins += 1;
    1385
    2
            }
    1386
    1387
    4
            if energy_change < -5.0 {
    1388
    1
                current_wins += 1;
    1389
    3
            } else if energy_change > 5.0 {
    1390
    1
                baseline_wins += 1;
    1391
    2
            }
    1392
    1393
    4
            let winner = match current_wins.cmp(&baseline_wins) {
    1394
    1
                std::cmp::Ordering::Greater => current.config.runtime.clone(),
    1395
    1
                std::cmp::Ordering::Less => baseline.config.runtime.clone(),
    1396
    2
                std::cmp::Ordering::Equal => "tie".to_string(),
    1397
            };
    1398
    1399
    4
            Self {
    1400
    4
                baseline_runtime: baseline.config.runtime.clone(),
    1401
    4
                current_runtime: current.config.runtime.clone(),
    1402
    4
                ttft_p99_change_pct: ttft_p99_change,
    1403
    4
                throughput_change_pct: throughput_change,
    1404
    4
                memory_change_pct: memory_change,
    1405
    4
                energy_change_pct: energy_change,
    1406
    4
                winner,
    1407
    4
                significance: 0.001, // Would need actual Mann-Whitney U test
    1408
    4
            }
    1409
    4
        }
    1410
    }
    1411
    1412
    // ============================================================================
    1413
    // Regression Detection
    1414
    // ============================================================================
    1415
    1416
    /// Regression detection result
    1417
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1418
    pub struct RegressionResult {
    1419
        /// Whether a regression was detected
    1420
        pub regression_detected: bool,
    1421
        /// Metrics that regressed
    1422
        pub regressed_metrics: Vec<String>,
    1423
        /// Regression threshold used (%)
    1424
        pub threshold_pct: f64,
    1425
    }
    1426
    1427
    impl RegressionResult {
    1428
        /// Check for regressions between baseline and current
    1429
        #[must_use]
    1430
    5
        pub fn check(
    1431
    5
            baseline: &FullBenchmarkResult,
    1432
    5
            current: &FullBenchmarkResult,
    1433
    5
            threshold_pct: f64,
    1434
    5
        ) -> Self {
    1435
    5
            let mut regressed_metrics = Vec::new();
    1436
    1437
            // Check TTFT p99 (higher = regression)
    1438
    5
            if baseline.results.ttft_ms.p99 > 0.0 {
    1439
    4
                let change = ((current.results.ttft_ms.p99 - baseline.results.ttft_ms.p99)
    1440
    4
                    / baseline.results.ttft_ms.p99)
    1441
    4
                    * 100.0;
    1442
    4
                if change > threshold_pct {
    1443
    1
                    regressed_metrics.push(format!("ttft_p99 (+{change:.1}%)"));
    1444
    3
                }
    1445
    1
            }
    1446
    1447
            // Check throughput (lower = regression)
    1448
    5
            if baseline.results.throughput_tok_s.median > 0.0 {
    1449
    4
                let change = ((baseline.results.throughput_tok_s.median
    1450
    4
                    - current.results.throughput_tok_s.median)
    1451
    4
                    / baseline.results.throughput_tok_s.median)
    1452
    4
                    * 100.0;
    1453
    4
                if change > threshold_pct {
    1454
    1
                    regressed_metrics.push(format!("throughput (-{change:.1}%)"));
    1455
    3
                }
    1456
    1
            }
    1457
    1458
            // Check memory (higher = regression)
    1459
    5
            if baseline.results.memory_mb.peak_rss_mb > 0 {
    1460
    4
                let change = ((current.results.memory_mb.peak_rss_mb as f64
    1461
    4
                    - baseline.results.memory_mb.peak_rss_mb as f64)
    1462
    4
                    / baseline.results.memory_mb.peak_rss_mb as f64)
    1463
    4
                    * 100.0;
    1464
    4
                if change > threshold_pct {
    1465
    1
                    regressed_metrics.push(format!("memory (+{change:.1}%)"));
    1466
    3
                }
    1467
    1
            }
    1468
    1469
    5
            Self {
    1470
    5
                regression_detected: !regressed_metrics.is_empty(),
    1471
    5
                regressed_metrics,
    1472
    5
                threshold_pct,
    1473
    5
            }
    1474
    5
        }
    1475
    }
    1476
    1477
    1478
    // Tests extracted to tests.rs (PMAT-802)
    1479
    #[cfg(test)]
    1480
    #[path = "tests.rs"]
    1481
    mod bench_tests;