## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/bench/matrix.rs

    Line
    Count
    Source
    1
    //! Backend benchmark matrix for comparing compute implementations
    2
    //!
    3
    //! Extracted from bench/mod.rs (PMAT-802) to reduce module size.
    4
    //! Contains:
    5
    //! - ComputeBackendType enum (CPU, wgpu, CUDA)
    6
    //! - BenchmarkMatrix for cross-backend comparison
    7
    //! - MatrixBenchmarkConfig and related types
    8
    9
    #![allow(clippy::cast_precision_loss)]
    10
    11
    use std::fmt::Write;
    12
    13
    use serde::{Deserialize, Serialize};
    14
    15
    use super::{chrono_timestamp, compute_cv, percentile, HardwareSpec, RuntimeType};
    16
    17
    // ============================================================================
    18
    // Backend Benchmark Matrix (per Hoefler & Belli SC'15)
    19
    // ============================================================================
    20
    21
    /// Compute backend type for benchmark matrix
    22
    ///
    23
    /// Represents the different compute backends that can be benchmarked:
    24
    /// - CPU: Scalar/SIMD operations via trueno CPU backend
    25
    /// - Wgpu: Cross-platform GPU via trueno wgpu backend
    26
    /// - Cuda: NVIDIA GPU via trueno-gpu PTX execution
    27
    #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
    28
    pub enum ComputeBackendType {
    29
        /// CPU backend (scalar/SIMD via trueno)
    30
        Cpu,
    31
        /// wgpu GPU backend (cross-platform via trueno)
    32
        Wgpu,
    33
        /// CUDA GPU backend (NVIDIA via trueno-gpu)
    34
        Cuda,
    35
    }
    36
    37
    impl std::fmt::Display for ComputeBackendType {
    38
    24
        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
    39
    24
            match self {
    40
    15
                Self::Cpu => write!(f, "cpu"),
    41
    5
                Self::Wgpu => write!(f, "wgpu"),
    42
    4
                Self::Cuda => write!(f, "cuda"),
    43
            }
    44
    24
        }
    45
    }
    46
    47
    impl ComputeBackendType {
    48
        /// Parse from string
    49
        #[must_use]
    50
    7
        pub fn parse(s: &str) -> Option<Self> {
    51
    7
            match s.to_lowercase().as_str() {
    52
    7
                "cpu" => Some(Self::Cpu)2,
    53
    5
                "wgpu" | "gpu"4 => Some(Self::Wgpu)2,
    54
    3
                "cuda" | "nvidia"2 => Some(Self::Cuda)2,
    55
    1
                _ => None,
    56
            }
    57
    7
        }
    58
    59
        /// All available backend types
    60
        #[must_use]
    61
    4
        pub fn all() -> Vec<Self> {
    62
    4
            vec![Self::Cpu, Self::Wgpu, Self::Cuda]
    63
    4
        }
    64
    }
    65
    66
    /// Single entry in the benchmark matrix
    67
    ///
    68
    /// Represents results for one (runtime, backend) combination.
    69
    #[derive(Debug, Clone, Serialize, Deserialize)]
    70
    pub struct MatrixBenchmarkEntry {
    71
        /// Runtime type (realizar, llama-cpp, ollama, vllm)
    72
        pub runtime: RuntimeType,
    73
        /// Compute backend (cpu, wgpu, cuda)
    74
        pub backend: ComputeBackendType,
    75
        /// Model name/identifier
    76
        pub model: String,
    77
        /// Whether this configuration is available
    78
        pub available: bool,
    79
        /// p50 latency in milliseconds
    80
        pub p50_latency_ms: f64,
    81
        /// p99 latency in milliseconds
    82
        pub p99_latency_ms: f64,
    83
        /// Throughput in tokens per second
    84
        pub throughput_tps: f64,
    85
        /// Cold start time in milliseconds
    86
        pub cold_start_ms: f64,
    87
        /// Number of samples collected
    88
        pub samples: usize,
    89
        /// Final CV at stop
    90
        pub cv_at_stop: f64,
    91
        /// Additional notes (e.g., "GPU layers: 99")
    92
        pub notes: String,
    93
    }
    94
    95
    impl Default for MatrixBenchmarkEntry {
    96
    5
        fn default() -> Self {
    97
    5
            Self {
    98
    5
                runtime: RuntimeType::Realizar,
    99
    5
                backend: ComputeBackendType::Cpu,
    100
    5
                model: String::new(),
    101
    5
                available: false,
    102
    5
                p50_latency_ms: 0.0,
    103
    5
                p99_latency_ms: 0.0,
    104
    5
                throughput_tps: 0.0,
    105
    5
                cold_start_ms: 0.0,
    106
    5
                samples: 0,
    107
    5
                cv_at_stop: 0.0,
    108
    5
                notes: String::new(),
    109
    5
            }
    110
    5
        }
    111
    }
    112
    113
    impl MatrixBenchmarkEntry {
    114
        /// Create a new unavailable entry (placeholder)
    115
        #[must_use]
    116
    5
        pub fn unavailable(runtime: RuntimeType, backend: ComputeBackendType) -> Self {
    117
    5
            Self {
    118
    5
                runtime,
    119
    5
                backend,
    120
    5
                available: false,
    121
    5
                notes: "Backend not available".to_string(),
    122
    5
                ..Default::default()
    123
    5
            }
    124
    5
        }
    125
    126
        /// Create entry from raw latency samples
    127
        #[must_use]
    128
    39
        pub fn from_samples(
    129
    39
            runtime: RuntimeType,
    130
    39
            backend: ComputeBackendType,
    131
    39
            model: &str,
    132
    39
            latencies_ms: &[f64],
    133
    39
            throughputs_tps: &[f64],
    134
    39
            cold_start_ms: f64,
    135
    39
        ) -> Self {
    136
    39
            let samples = latencies_ms.len();
    137
    39
            if samples == 0 {
    138
    1
                return Self::unavailable(runtime, backend);
    139
    38
            }
    140
    141
    38
            let p50_latency = percentile(latencies_ms, 50.0);
    142
    38
            let p99_latency = percentile(latencies_ms, 99.0);
    143
    38
            let throughput = if throughputs_tps.is_empty() {
    144
    0
                0.0
    145
            } else {
    146
    38
                throughputs_tps.iter().sum::<f64>() / throughputs_tps.len() as f64
    147
            };
    148
    38
            let cv = compute_cv(latencies_ms);
    149
    150
    38
            Self {
    151
    38
                runtime,
    152
    38
                backend,
    153
    38
                model: model.to_string(),
    154
    38
                available: true,
    155
    38
                p50_latency_ms: p50_latency,
    156
    38
                p99_latency_ms: p99_latency,
    157
    38
                throughput_tps: throughput,
    158
    38
                cold_start_ms,
    159
    38
                samples,
    160
    38
                cv_at_stop: cv,
    161
    38
                notes: String::new(),
    162
    38
            }
    163
    39
        }
    164
    165
        /// Add notes to the entry
    166
        #[must_use]
    167
    1
        pub fn with_notes(mut self, notes: &str) -> Self {
    168
    1
            self.notes = notes.to_string();
    169
    1
            self
    170
    1
        }
    171
    }
    172
    173
    /// Complete benchmark matrix comparing runtimes across backends
    174
    ///
    175
    /// Per Hoefler & Belli SC'15, this matrix enables:
    176
    /// - Reproducible comparisons across configurations
    177
    /// - Statistical validity via CV-based stopping
    178
    /// - Clear identification of performance characteristics
    179
    #[derive(Debug, Clone, Serialize, Deserialize)]
    180
    pub struct BenchmarkMatrix {
    181
        /// Schema version
    182
        pub version: String,
    183
        /// ISO 8601 timestamp
    184
        pub timestamp: String,
    185
        /// Model used for benchmarking
    186
        pub model: String,
    187
        /// Hardware specification
    188
        pub hardware: HardwareSpec,
    189
        /// Benchmark methodology
    190
        pub methodology: String,
    191
        /// CV threshold used
    192
        pub cv_threshold: f64,
    193
        /// Matrix entries indexed by (runtime, backend)
    194
        pub entries: Vec<MatrixBenchmarkEntry>,
    195
    }
    196
    197
    impl BenchmarkMatrix {
    198
        /// Create a new empty matrix
    199
        #[must_use]
    200
    18
        pub fn new(model: &str, hardware: HardwareSpec) -> Self {
    201
    18
            Self {
    202
    18
                version: "1.1".to_string(),
    203
    18
                timestamp: chrono_timestamp(),
    204
    18
                model: model.to_string(),
    205
    18
                hardware,
    206
    18
                methodology: "CV-based stopping (Hoefler & Belli SC'15)".to_string(),
    207
    18
                cv_threshold: 0.05,
    208
    18
                entries: Vec::new(),
    209
    18
            }
    210
    18
        }
    211
    212
        /// Add an entry to the matrix
    213
    32
        pub fn add_entry(&mut self, entry: MatrixBenchmarkEntry) {
    214
            // Remove existing entry for same (runtime, backend) if present
    215
    32
            self.entries
    216
    32
                .retain(|e| e.runtime != entry.runtime23 || e.backend != entry.backend6);
    217
    32
            self.entries.push(entry);
    218
    32
        }
    219
    220
        /// Get entry for specific (runtime, backend) combination
    221
        #[must_use]
    222
    2
        pub fn get_entry(
    223
    2
            &self,
    224
    2
            runtime: RuntimeType,
    225
    2
            backend: ComputeBackendType,
    226
    2
        ) -> Option<&MatrixBenchmarkEntry> {
    227
    2
            self.entries
    228
    2
                .iter()
    229
    2
                .find(|e| e.runtime == runtime && e.backend == backend1)
    230
    2
        }
    231
    232
        /// Get all entries for a specific runtime
    233
        #[must_use]
    234
    2
        pub fn entries_for_runtime(&self, runtime: RuntimeType) -> Vec<&MatrixBenchmarkEntry> {
    235
    2
            self.entries
    236
    2
                .iter()
    237
    6
                .filter2(|e| e.runtime == runtime)
    238
    2
                .collect()
    239
    2
        }
    240
    241
        /// Get all entries for a specific backend
    242
        #[must_use]
    243
    14
        pub fn entries_for_backend(&self, backend: ComputeBackendType) -> Vec<&MatrixBenchmarkEntry> {
    244
    14
            self.entries
    245
    14
                .iter()
    246
    40
                .filter14(|e| e.backend == backend)
    247
    14
                .collect()
    248
    14
        }
    249
    250
        /// Find the fastest runtime for a given backend (by p50 latency)
    251
        #[must_use]
    252
    1
        pub fn fastest_for_backend(
    253
    1
            &self,
    254
    1
            backend: ComputeBackendType,
    255
    1
        ) -> Option<&MatrixBenchmarkEntry> {
    256
    1
            self.entries_for_backend(backend)
    257
    1
                .into_iter()
    258
    1
                .filter(|e| e.available)
    259
    1
                .min_by(|a, b| {
    260
    1
                    a.p50_latency_ms
    261
    1
                        .partial_cmp(&b.p50_latency_ms)
    262
    1
                        .expect("test")
    263
    1
                })
    264
    1
        }
    265
    266
        /// Find the highest throughput runtime for a given backend
    267
        #[must_use]
    268
    1
        pub fn highest_throughput_for_backend(
    269
    1
            &self,
    270
    1
            backend: ComputeBackendType,
    271
    1
        ) -> Option<&MatrixBenchmarkEntry> {
    272
    1
            self.entries_for_backend(backend)
    273
    1
                .into_iter()
    274
    1
                .filter(|e| e.available)
    275
    1
                .max_by(|a, b| {
    276
    1
                    a.throughput_tps
    277
    1
                        .partial_cmp(&b.throughput_tps)
    278
    1
                        .expect("test")
    279
    1
                })
    280
    1
        }
    281
    282
        /// Generate markdown table for README
    283
        #[must_use]
    284
    6
        pub fn to_markdown_table(&self) -> String {
    285
    6
            let mut table = String::new();
    286
    287
            // Header
    288
    6
            table.push_str("| Runtime | Backend | p50 Latency | p99 Latency | Throughput | Cold Start | Samples | CV |\n");
    289
    6
            table.push_str("|---------|---------|-------------|-------------|------------|------------|---------|----|\n");
    290
    291
            // Sort entries by runtime, then backend
    292
    6
            let mut sorted_entries = self.entries.clone();
    293
    6
            sorted_entries.sort_by(|a, b| {2
    294
    2
                let runtime_cmp = format!("{:?}", a.runtime).cmp(&format!("{:?}", b.runtime));
    295
    2
                if runtime_cmp == std::cmp::Ordering::Equal {
    296
    1
                    format!("{}", a.backend).cmp(&format!("{}", b.backend))
    297
                } else {
    298
    1
                    runtime_cmp
    299
                }
    300
    2
            });
    301
    302
    14
            for entry8 in &sorted_entries {
    303
    8
                if entry.available {
    304
    7
                    let _ = writeln!(
    305
    7
                        table,
    306
    7
                        "| **{}** | {} | {:.1}ms | {:.1}ms | {:.1} tok/s | {:.0}ms | {} | {:.3} |",
    307
    7
                        format!("{:?}", entry.runtime).to_lowercase(),
    308
    7
                        entry.backend,
    309
    7
                        entry.p50_latency_ms,
    310
    7
                        entry.p99_latency_ms,
    311
    7
                        entry.throughput_tps,
    312
    7
                        entry.cold_start_ms,
    313
    7
                        entry.samples,
    314
    7
                        entry.cv_at_stop,
    315
    7
                    );
    316
    7
                } else {
    317
    1
                    let _ = writeln!(
    318
    1
                        table,
    319
    1
                        "| {} | {} | - | - | - | - | - | - |",
    320
    1
                        format!("{:?}", entry.runtime).to_lowercase(),
    321
    1
                        entry.backend,
    322
    1
                    );
    323
    1
                }
    324
            }
    325
    326
    6
            table
    327
    6
        }
    328
    329
        /// Serialize to JSON
    330
        ///
    331
        /// # Errors
    332
        ///
    333
        /// Returns error if serialization fails.
    334
    1
        pub fn to_json(&self) -> Result<String, serde_json::Error> {
    335
    1
            serde_json::to_string_pretty(self)
    336
    1
        }
    337
    338
        /// Deserialize from JSON
    339
        ///
    340
        /// # Errors
    341
        ///
    342
        /// Returns error if JSON is invalid.
    343
    1
        pub fn from_json(json: &str) -> Result<Self, serde_json::Error> {
    344
    1
            serde_json::from_str(json)
    345
    1
        }
    346
    }
    347
    348
    /// Matrix benchmark runner configuration
    349
    #[derive(Debug, Clone)]
    350
    pub struct MatrixBenchmarkConfig {
    351
        /// Runtimes to benchmark
    352
        pub runtimes: Vec<RuntimeType>,
    353
        /// Backends to benchmark
    354
        pub backends: Vec<ComputeBackendType>,
    355
        /// Model path
    356
        pub model_path: String,
    357
        /// Prompt for benchmarking
    358
        pub prompt: String,
    359
        /// Max tokens to generate
    360
        pub max_tokens: usize,
    361
        /// CV threshold for stopping
    362
        pub cv_threshold: f64,
    363
        /// Minimum samples
    364
        pub min_samples: usize,
    365
        /// Maximum samples (failsafe)
    366
        pub max_samples: usize,
    367
        /// Warmup iterations
    368
        pub warmup_iterations: usize,
    369
    }
    370
    371
    impl Default for MatrixBenchmarkConfig {
    372
    1
        fn default() -> Self {
    373
    1
            Self {
    374
    1
                runtimes: vec![
    375
    1
                    RuntimeType::Realizar,
    376
    1
                    RuntimeType::LlamaCpp,
    377
    1
                    RuntimeType::Ollama,
    378
    1
                ],
    379
    1
                backends: vec![ComputeBackendType::Cpu, ComputeBackendType::Wgpu],
    380
    1
                model_path: String::new(),
    381
    1
                prompt: "Explain machine learning in one sentence.".to_string(),
    382
    1
                max_tokens: 50,
    383
    1
                cv_threshold: 0.05,
    384
    1
                min_samples: 30,
    385
    1
                max_samples: 200,
    386
    1
                warmup_iterations: 5,
    387
    1
            }
    388
    1
        }
    389
    }
    390
    391
    /// Summary statistics for a single matrix column (backend)
    392
    #[derive(Debug, Clone, Serialize, Deserialize)]
    393
    pub struct BackendSummary {
    394
        /// Backend type
    395
        pub backend: ComputeBackendType,
    396
        /// Number of available runtimes
    397
        pub available_runtimes: usize,
    398
        /// Fastest runtime (by p50 latency)
    399
        pub fastest_runtime: Option<String>,
    400
        /// Fastest p50 latency
    401
        pub fastest_p50_ms: f64,
    402
        /// Highest throughput runtime
    403
        pub highest_throughput_runtime: Option<String>,
    404
        /// Highest throughput (tok/s)
    405
        pub highest_throughput_tps: f64,
    406
    }
    407
    408
    /// Summary of the entire benchmark matrix
    409
    #[derive(Debug, Clone, Serialize, Deserialize)]
    410
    pub struct MatrixSummary {
    411
        /// Total entries in matrix
    412
        pub total_entries: usize,
    413
        /// Number of available entries
    414
        pub available_entries: usize,
    415
        /// Per-backend summaries
    416
        pub backend_summaries: Vec<BackendSummary>,
    417
        /// Overall fastest (runtime, backend) combination
    418
        pub overall_fastest: Option<(String, String)>,
    419
        /// Overall highest throughput (runtime, backend)
    420
        pub overall_highest_throughput: Option<(String, String)>,
    421
    }
    422
    423
    impl BenchmarkMatrix {
    424
        /// Generate summary statistics
    425
        #[must_use]
    426
    3
        pub fn summary(&self) -> MatrixSummary {
    427
    3
            let total_entries = self.entries.len();
    428
    3
            let available_entries = self.entries.iter().filter(|e| e.available).count();
    429
    430
    3
            let mut backend_summaries = Vec::new();
    431
    9
            for backend in ComputeBackendType::all3() {
    432
    9
                let entries: Vec<_> = self.entries_for_backend(backend);
    433
    9
                let available: Vec<_> = entries.iter().filter(|e| e.available).collect();
    434
    435
    9
                let fastest = available.iter().min_by(|a, b| {4
    436
    4
                    a.p50_latency_ms
    437
    4
                        .partial_cmp(&b.p50_latency_ms)
    438
    4
                        .expect("test")
    439
    4
                });
    440
    9
                let highest_tp = available.iter().max_by(|a, b| {4
    441
    4
                    a.throughput_tps
    442
    4
                        .partial_cmp(&b.throughput_tps)
    443
    4
                        .expect("test")
    444
    4
                });
    445
    446
    9
                backend_summaries.push(BackendSummary {
    447
    9
                    backend,
    448
    9
                    available_runtimes: available.len(),
    449
    9
                    fastest_runtime: fastest.map(|e| format!4("{:?}"4, e.runtime).to_lowercase4()),
    450
    9
                    fastest_p50_ms: fastest.map_or(0.0, |e| e.p50_latency_ms),
    451
    9
                    highest_throughput_runtime: highest_tp
    452
    9
                        .map(|e| format!4("{:?}"4, e.runtime).to_lowercase4()),
    453
    9
                    highest_throughput_tps: highest_tp.map_or(0.0, |e| e.throughput_tps),
    454
                });
    455
            }
    456
    457
    3
            let available = self.entries.iter().filter(|e| e.available);
    458
    3
            let overall_fastest = available
    459
    3
                .clone()
    460
    5
                .min_by3(|a, b| {
    461
    5
                    a.p50_latency_ms
    462
    5
                        .partial_cmp(&b.p50_latency_ms)
    463
    5
                        .expect("test")
    464
    5
                })
    465
    3
                .map(|e| {
    466
    3
                    (
    467
    3
                        format!("{:?}", e.runtime).to_lowercase(),
    468
    3
                        e.backend.to_string(),
    469
    3
                    )
    470
    3
                });
    471
    3
            let overall_highest_throughput = available
    472
    5
                .max_by3(|a, b| {
    473
    5
                    a.throughput_tps
    474
    5
                        .partial_cmp(&b.throughput_tps)
    475
    5
                        .expect("test")
    476
    5
                })
    477
    3
                .map(|e| {
    478
    3
                    (
    479
    3
                        format!("{:?}", e.runtime).to_lowercase(),
    480
    3
                        e.backend.to_string(),
    481
    3
                    )
    482
    3
                });
    483
    484
    3
            MatrixSummary {
    485
    3
                total_entries,
    486
    3
                available_entries,
    487
    3
                backend_summaries,
    488
    3
                overall_fastest,
    489
    3
                overall_highest_throughput,
    490
    3
            }
    491
    3
        }
    492
    }