## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/apr_transformer/benchmark.rs

    Line
    Count
    Source
    1
    //! APR Benchmark Infrastructure (Y6: Format Parity Validation)
    2
    //!
    3
    //! Provides standardized benchmarking for APR transformers following the benchmark spec:
    4
    //! - Dynamic CV-based sampling
    5
    //! - Statistical metrics (p50, p99, std_dev)
    6
    //! - Throughput and memory measurement
    7
    //!
    8
    //! Extracted from apr_transformer.rs (PMAT-802)
    9
    10
    use crate::error::Result;
    11
    12
    use super::{AprTransformer, GenerateConfig};
    13
    14
    // ============================================================================
    15
    // Y6: APR Benchmark Infrastructure (Format Parity Validation)
    16
    // ============================================================================
    17
    18
    /// CPU decode threshold: 50 tok/s per spec Y6
    19
    pub const APR_CPU_DECODE_THRESHOLD_TOK_S: f64 = 50.0;
    20
    21
    /// Prefill threshold: 100 tok/s per spec Y8
    22
    pub const APR_PREFILL_THRESHOLD_TOK_S: f64 = 100.0;
    23
    24
    /// Parity threshold: 95% of baseline per spec Y6
    25
    pub const APR_PARITY_THRESHOLD_PCT: f64 = 95.0;
    26
    27
    /// Result of an APR benchmark run
    28
    #[derive(Debug, Clone, Default)]
    29
    pub struct AprBenchmarkResult {
    30
        /// Number of tokens generated
    31
        pub tokens_generated: usize,
    32
        /// Total time in milliseconds
    33
        pub total_time_ms: f64,
    34
        /// Throughput in tokens per second
    35
        pub tokens_per_second: f64,
    36
        /// Median throughput (p50)
    37
        pub throughput_p50: f64,
    38
        /// 99th percentile throughput (worst case)
    39
        pub throughput_p99: f64,
    40
        /// Standard deviation of throughput
    41
        pub throughput_std_dev: f64,
    42
        /// Peak memory usage in MB
    43
        pub peak_memory_mb: f64,
    44
        /// Model memory in MB
    45
        pub model_memory_mb: f64,
    46
    }
    47
    48
    impl AprBenchmarkResult {
    49
        /// Check if benchmark meets the given throughput threshold
    50
        #[must_use]
    51
    0
        pub fn meets_threshold(&self, threshold_tok_s: f64) -> bool {
    52
    0
            self.tokens_per_second >= threshold_tok_s
    53
    0
        }
    54
    55
        /// Compare this result to a baseline
    56
        #[must_use]
    57
    0
        pub fn compare_to_baseline(&self, baseline: &AprBenchmarkResult) -> AprParityComparison {
    58
    0
            let throughput_ratio = if baseline.tokens_per_second > 0.0 {
    59
    0
                self.tokens_per_second / baseline.tokens_per_second
    60
            } else {
    61
    0
                1.0
    62
            };
    63
    64
    0
            let memory_ratio = if baseline.peak_memory_mb > 0.0 {
    65
    0
                self.peak_memory_mb / baseline.peak_memory_mb
    66
            } else {
    67
    0
                1.0
    68
            };
    69
    70
    0
            AprParityComparison {
    71
    0
                throughput_ratio,
    72
    0
                memory_ratio,
    73
    0
                parity_threshold_pct: APR_PARITY_THRESHOLD_PCT,
    74
    0
            }
    75
    0
        }
    76
    }
    77
    78
    /// Result of prefill benchmark
    79
    #[derive(Debug, Clone, Default)]
    80
    pub struct AprPrefillResult {
    81
        /// Number of prompt tokens processed
    82
        pub prompt_tokens: usize,
    83
        /// Prefill time in milliseconds
    84
        pub prefill_time_ms: f64,
    85
        /// Prefill throughput in tokens per second
    86
        pub prefill_tok_s: f64,
    87
    }
    88
    89
    /// Result of load time benchmark
    90
    #[derive(Debug, Clone, Default)]
    91
    pub struct AprLoadResult {
    92
        /// Load time in milliseconds
    93
        pub load_time_ms: f64,
    94
    }
    95
    96
    /// Comparison of APR benchmark to baseline (for parity validation)
    97
    #[derive(Debug, Clone)]
    98
    pub struct AprParityComparison {
    99
        /// Ratio of APR throughput to baseline
    100
        pub throughput_ratio: f64,
    101
        /// Ratio of APR memory to baseline
    102
        pub memory_ratio: f64,
    103
        /// Parity threshold percentage
    104
        pub parity_threshold_pct: f64,
    105
    }
    106
    107
    impl AprParityComparison {
    108
        /// Check if APR achieves parity with baseline
    109
        #[must_use]
    110
    0
        pub fn is_parity(&self) -> bool {
    111
    0
            self.throughput_ratio >= (self.parity_threshold_pct / 100.0)
    112
    0
        }
    113
    }
    114
    115
    /// Benchmark runner for APR transformers (Y6)
    116
    ///
    117
    /// Provides standardized benchmarking following the benchmark spec:
    118
    /// - Dynamic CV-based sampling
    119
    /// - Statistical metrics (p50, p99, std_dev)
    120
    /// - Throughput and memory measurement
    121
    #[derive(Debug)]
    122
    pub struct AprBenchmarkRunner {
    123
        /// The transformer to benchmark
    124
        transformer: AprTransformer,
    125
        /// Number of warmup iterations
    126
        warmup_iterations: usize,
    127
        /// Number of measurement iterations
    128
        measure_iterations: usize,
    129
    }
    130
    131
    impl AprBenchmarkRunner {
    132
        /// Create a new benchmark runner for the given transformer
    133
        #[must_use]
    134
    0
        pub fn new(transformer: AprTransformer) -> Self {
    135
    0
            Self {
    136
    0
                transformer,
    137
    0
                warmup_iterations: 3,
    138
    0
                measure_iterations: 10,
    139
    0
            }
    140
    0
        }
    141
    142
        /// Get warmup iterations
    143
        #[must_use]
    144
    0
        pub fn warmup_iterations(&self) -> usize {
    145
    0
            self.warmup_iterations
    146
    0
        }
    147
    148
        /// Get measure iterations
    149
        #[must_use]
    150
    0
        pub fn measure_iterations(&self) -> usize {
    151
    0
            self.measure_iterations
    152
    0
        }
    153
    154
        /// Set warmup iterations
    155
    0
        pub fn set_warmup_iterations(&mut self, n: usize) {
    156
    0
            self.warmup_iterations = n;
    157
    0
        }
    158
    159
        /// Set measure iterations
    160
    0
        pub fn set_measure_iterations(&mut self, n: usize) {
    161
    0
            self.measure_iterations = n.max(1);
    162
    0
        }
    163
    164
        /// Benchmark decode throughput
    165
        ///
    166
        /// # Arguments
    167
        ///
    168
        /// * `prompt` - Initial token IDs
    169
        /// * `num_tokens` - Number of tokens to generate
    170
        ///
    171
        /// # Returns
    172
        ///
    173
        /// Benchmark result with throughput metrics
    174
    0
        pub fn benchmark_decode(
    175
    0
            &mut self,
    176
    0
            prompt: &[u32],
    177
    0
            num_tokens: usize,
    178
    0
        ) -> Result<AprBenchmarkResult> {
    179
            use std::time::Instant;
    180
    181
            // Warmup
    182
    0
            for _ in 0..self.warmup_iterations {
    183
    0
                let gen_config = GenerateConfig {
    184
    0
                    max_tokens: num_tokens.min(5),
    185
    0
                    temperature: 0.0,
    186
    0
                    ..Default::default()
    187
    0
                };
    188
    0
                let _ = self.transformer.generate_with_cache(prompt, &gen_config)?;
    189
            }
    190
    191
            // Measurement runs
    192
    0
            let mut throughputs = Vec::with_capacity(self.measure_iterations);
    193
    0
            let mut total_tokens = 0usize;
    194
    0
            let mut total_time_ms = 0.0f64;
    195
    196
    0
            for _ in 0..self.measure_iterations {
    197
    0
                let gen_config = GenerateConfig {
    198
    0
                    max_tokens: num_tokens,
    199
    0
                    temperature: 0.0,
    200
    0
                    ..Default::default()
    201
    0
                };
    202
    203
    0
                let start = Instant::now();
    204
    0
                let output = self.transformer.generate_with_cache(prompt, &gen_config)?;
    205
    0
                let elapsed = start.elapsed();
    206
    207
    0
                let generated = output.len().saturating_sub(prompt.len());
    208
    0
                let time_ms = elapsed.as_secs_f64() * 1000.0;
    209
    0
                let throughput = if time_ms > 0.0 {
    210
    0
                    (generated as f64) / (time_ms / 1000.0)
    211
                } else {
    212
    0
                    0.0
    213
                };
    214
    215
    0
                throughputs.push(throughput);
    216
    0
                total_tokens += generated;
    217
    0
                total_time_ms += time_ms;
    218
            }
    219
    220
            // Calculate statistics
    221
    0
            let mean_throughput = if !throughputs.is_empty() {
    222
    0
                throughputs.iter().sum::<f64>() / throughputs.len() as f64
    223
            } else {
    224
    0
                0.0
    225
            };
    226
    227
    0
            let mut sorted = throughputs.clone();
    228
    0
            sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
    229
    230
    0
            let p50 = if !sorted.is_empty() {
    231
    0
                sorted[sorted.len() / 2]
    232
            } else {
    233
    0
                0.0
    234
            };
    235
    236
    0
            let p99_idx =
    237
    0
                ((sorted.len() as f64 * 0.01).floor() as usize).min(sorted.len().saturating_sub(1));
    238
    0
            let p99 = if !sorted.is_empty() {
    239
    0
                sorted[p99_idx]
    240
            } else {
    241
    0
                0.0
    242
            };
    243
    244
    0
            let std_dev = if throughputs.len() > 1 {
    245
    0
                let variance = throughputs
    246
    0
                    .iter()
    247
    0
                    .map(|t| (t - mean_throughput).powi(2))
    248
    0
                    .sum::<f64>()
    249
    0
                    / (throughputs.len() - 1) as f64;
    250
    0
                variance.sqrt()
    251
            } else {
    252
    0
                0.0
    253
            };
    254
    255
            // Memory estimation
    256
    0
            let model_memory_mb = (self.transformer.memory_size() as f64) / (1024.0 * 1024.0);
    257
    258
    0
            Ok(AprBenchmarkResult {
    259
    0
                tokens_generated: total_tokens / self.measure_iterations.max(1),
    260
    0
                total_time_ms: total_time_ms / self.measure_iterations.max(1) as f64,
    261
    0
                tokens_per_second: mean_throughput,
    262
    0
                throughput_p50: p50,
    263
    0
                throughput_p99: p99,
    264
    0
                throughput_std_dev: std_dev,
    265
    0
                peak_memory_mb: model_memory_mb * 1.5, // Estimate: model + KV cache
    266
    0
                model_memory_mb,
    267
    0
            })
    268
    0
        }
    269
    270
        /// Benchmark prefill throughput
    271
        ///
    272
        /// # Arguments
    273
        ///
    274
        /// * `prompt` - Tokens to prefill
    275
        ///
    276
        /// # Returns
    277
        ///
    278
        /// Prefill benchmark result
    279
    0
        pub fn benchmark_prefill(&mut self, prompt: &[u32]) -> Result<AprPrefillResult> {
    280
            use std::time::Instant;
    281
    282
            // Warmup
    283
    0
            for _ in 0..self.warmup_iterations {
    284
    0
                let _ = self.transformer.forward(prompt)?;
    285
            }
    286
    287
            // Measurement runs
    288
    0
            let mut prefill_times_ms = Vec::with_capacity(self.measure_iterations);
    289
    290
    0
            for _ in 0..self.measure_iterations {
    291
    0
                let start = Instant::now();
    292
    0
                let _ = self.transformer.forward(prompt)?;
    293
    0
                let elapsed = start.elapsed();
    294
    0
                prefill_times_ms.push(elapsed.as_secs_f64() * 1000.0);
    295
            }
    296
    297
    0
            let mean_time_ms = if !prefill_times_ms.is_empty() {
    298
    0
                prefill_times_ms.iter().sum::<f64>() / prefill_times_ms.len() as f64
    299
            } else {
    300
    0
                0.0
    301
            };
    302
    303
    0
            let prefill_tok_s = if mean_time_ms > 0.0 {
    304
    0
                (prompt.len() as f64) / (mean_time_ms / 1000.0)
    305
            } else {
    306
    0
                0.0
    307
            };
    308
    309
    0
            Ok(AprPrefillResult {
    310
    0
                prompt_tokens: prompt.len(),
    311
    0
                prefill_time_ms: mean_time_ms,
    312
    0
                prefill_tok_s,
    313
    0
            })
    314
    0
        }
    315
    316
        /// Benchmark model load time
    317
        ///
    318
        /// # Arguments
    319
        ///
    320
        /// * `loader` - Closure that creates the transformer
    321
        ///
    322
        /// # Returns
    323
        ///
    324
        /// Load time result
    325
    0
        pub fn benchmark_load<F>(loader: F) -> Result<AprLoadResult>
    326
    0
        where
    327
    0
            F: Fn() -> AprTransformer,
    328
        {
    329
            use std::time::Instant;
    330
    331
            // Single measurement (load is typically done once)
    332
    0
            let start = Instant::now();
    333
    0
            let _transformer = loader();
    334
    0
            let elapsed = start.elapsed();
    335
    336
    0
            Ok(AprLoadResult {
    337
    0
                load_time_ms: elapsed.as_secs_f64() * 1000.0,
    338
    0
            })
    339
    0
        }
    340
    }