/home/noah/src/realizar/src/bench/matrix.rs

Source
//! Backend benchmark matrix for comparing compute implementations
//!
//! Extracted from bench/mod.rs (PMAT-802) to reduce module size.
//! Contains:
//! - ComputeBackendType enum (CPU, wgpu, CUDA)
//! - BenchmarkMatrix for cross-backend comparison
//! - MatrixBenchmarkConfig and related types

#![allow(clippy::cast_precision_loss)]

use std::fmt::Write;

use serde::{Deserialize, Serialize};

use super::{chrono_timestamp, compute_cv, percentile, HardwareSpec, RuntimeType};

// ============================================================================
// Backend Benchmark Matrix (per Hoefler & Belli SC'15)
// ============================================================================

/// Compute backend type for benchmark matrix
///
/// Represents the different compute backends that can be benchmarked:
/// - CPU: Scalar/SIMD operations via trueno CPU backend
/// - Wgpu: Cross-platform GPU via trueno wgpu backend
/// - Cuda: NVIDIA GPU via trueno-gpu PTX execution
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum ComputeBackendType {
    /// CPU backend (scalar/SIMD via trueno)
    Cpu,
    /// wgpu GPU backend (cross-platform via trueno)
    Wgpu,
    /// CUDA GPU backend (NVIDIA via trueno-gpu)
    Cuda,
}

impl std::fmt::Display for ComputeBackendType {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            Self::Cpu => write!(f, "cpu"),
            Self::Wgpu => write!(f, "wgpu"),
            Self::Cuda => write!(f, "cuda"),
        }
    }
}

impl ComputeBackendType {
    /// Parse from string
    #[must_use]
    pub fn parse(s: &str) -> Option<Self> {
        match s.to_lowercase().as_str() {
            "cpu" => Some(Self::Cpu)2,
            "wgpu" | "gpu"4 => Some(Self::Wgpu)2,
            "cuda" | "nvidia"2 => Some(Self::Cuda)2,
            _ => None,
        }
    }

    /// All available backend types
    #[must_use]
    pub fn all() -> Vec<Self> {
        vec![Self::Cpu, Self::Wgpu, Self::Cuda]
    }
}

/// Single entry in the benchmark matrix
///
/// Represents results for one (runtime, backend) combination.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MatrixBenchmarkEntry {
    /// Runtime type (realizar, llama-cpp, ollama, vllm)
    pub runtime: RuntimeType,
    /// Compute backend (cpu, wgpu, cuda)
    pub backend: ComputeBackendType,
    /// Model name/identifier
    pub model: String,
    /// Whether this configuration is available
    pub available: bool,
    /// p50 latency in milliseconds
    pub p50_latency_ms: f64,
    /// p99 latency in milliseconds
    pub p99_latency_ms: f64,
    /// Throughput in tokens per second
    pub throughput_tps: f64,
    /// Cold start time in milliseconds
    pub cold_start_ms: f64,
    /// Number of samples collected
    pub samples: usize,
    /// Final CV at stop
    pub cv_at_stop: f64,
    /// Additional notes (e.g., "GPU layers: 99")
    pub notes: String,
}

impl Default for MatrixBenchmarkEntry {
    fn default() -> Self {
        Self {
            runtime: RuntimeType::Realizar,
            backend: ComputeBackendType::Cpu,
            model: String::new(),
            available: false,
            p50_latency_ms: 0.0,
            p99_latency_ms: 0.0,
            throughput_tps: 0.0,
            cold_start_ms: 0.0,
            samples: 0,
            cv_at_stop: 0.0,
            notes: String::new(),
        }
    }
}

impl MatrixBenchmarkEntry {
    /// Create a new unavailable entry (placeholder)
    #[must_use]
    pub fn unavailable(runtime: RuntimeType, backend: ComputeBackendType) -> Self {
        Self {
            runtime,
            backend,
            available: false,
            notes: "Backend not available".to_string(),
            ..Default::default()
        }
    }

    /// Create entry from raw latency samples
    #[must_use]
    pub fn from_samples(
        runtime: RuntimeType,
        backend: ComputeBackendType,
        model: &str,
        latencies_ms: &[f64],
        throughputs_tps: &[f64],
        cold_start_ms: f64,
    ) -> Self {
        let samples = latencies_ms.len();
        if samples == 0 {
            return Self::unavailable(runtime, backend);
        }

        let p50_latency = percentile(latencies_ms, 50.0);
        let p99_latency = percentile(latencies_ms, 99.0);
        let throughput = if throughputs_tps.is_empty() {
            0.0
        } else {
            throughputs_tps.iter().sum::<f64>() / throughputs_tps.len() as f64
        };
        let cv = compute_cv(latencies_ms);

        Self {
            runtime,
            backend,
            model: model.to_string(),
            available: true,
            p50_latency_ms: p50_latency,
            p99_latency_ms: p99_latency,
            throughput_tps: throughput,
            cold_start_ms,
            samples,
            cv_at_stop: cv,
            notes: String::new(),
        }
    }

    /// Add notes to the entry
    #[must_use]
    pub fn with_notes(mut self, notes: &str) -> Self {
        self.notes = notes.to_string();
        self
    }
}

/// Complete benchmark matrix comparing runtimes across backends
///
/// Per Hoefler & Belli SC'15, this matrix enables:
/// - Reproducible comparisons across configurations
/// - Statistical validity via CV-based stopping
/// - Clear identification of performance characteristics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BenchmarkMatrix {
    /// Schema version
    pub version: String,
    /// ISO 8601 timestamp
    pub timestamp: String,
    /// Model used for benchmarking
    pub model: String,
    /// Hardware specification
    pub hardware: HardwareSpec,
    /// Benchmark methodology
    pub methodology: String,
    /// CV threshold used
    pub cv_threshold: f64,
    /// Matrix entries indexed by (runtime, backend)
    pub entries: Vec<MatrixBenchmarkEntry>,
}

impl BenchmarkMatrix {
    /// Create a new empty matrix
    #[must_use]
    pub fn new(model: &str, hardware: HardwareSpec) -> Self {
        Self {
            version: "1.1".to_string(),
            timestamp: chrono_timestamp(),
            model: model.to_string(),
            hardware,
            methodology: "CV-based stopping (Hoefler & Belli SC'15)".to_string(),
            cv_threshold: 0.05,
            entries: Vec::new(),
        }
    }

    /// Add an entry to the matrix
    pub fn add_entry(&mut self, entry: MatrixBenchmarkEntry) {
        // Remove existing entry for same (runtime, backend) if present
        self.entries
            .retain(|e| e.runtime != entry.runtime23 || e.backend != entry.backend6);
        self.entries.push(entry);
    }

    /// Get entry for specific (runtime, backend) combination
    #[must_use]
    pub fn get_entry(
        &self,
        runtime: RuntimeType,
        backend: ComputeBackendType,
    ) -> Option<&MatrixBenchmarkEntry> {
        self.entries
            .iter()
            .find(|e| e.runtime == runtime && e.backend == backend1)
    }

    /// Get all entries for a specific runtime
    #[must_use]
    pub fn entries_for_runtime(&self, runtime: RuntimeType) -> Vec<&MatrixBenchmarkEntry> {
        self.entries
            .iter()
            .filter2(|e| e.runtime == runtime)
            .collect()
    }

    /// Get all entries for a specific backend
    #[must_use]
    pub fn entries_for_backend(&self, backend: ComputeBackendType) -> Vec<&MatrixBenchmarkEntry> {
        self.entries
            .iter()
            .filter14(|e| e.backend == backend)
            .collect()
    }

    /// Find the fastest runtime for a given backend (by p50 latency)
    #[must_use]
    pub fn fastest_for_backend(
        &self,
        backend: ComputeBackendType,
    ) -> Option<&MatrixBenchmarkEntry> {
        self.entries_for_backend(backend)
            .into_iter()
            .filter(|e| e.available)
            .min_by(|a, b| {
                a.p50_latency_ms
                    .partial_cmp(&b.p50_latency_ms)
                    .expect("test")
            })
    }

    /// Find the highest throughput runtime for a given backend
    #[must_use]
    pub fn highest_throughput_for_backend(
        &self,
        backend: ComputeBackendType,
    ) -> Option<&MatrixBenchmarkEntry> {
        self.entries_for_backend(backend)
            .into_iter()
            .filter(|e| e.available)
            .max_by(|a, b| {
                a.throughput_tps
                    .partial_cmp(&b.throughput_tps)
                    .expect("test")
            })
    }

    /// Generate markdown table for README
    #[must_use]
    pub fn to_markdown_table(&self) -> String {
        let mut table = String::new();

        // Header
        table.push_str("| Runtime | Backend | p50 Latency | p99 Latency | Throughput | Cold Start | Samples | CV |\n");
        table.push_str("|---------|---------|-------------|-------------|------------|------------|---------|----|\n");

        // Sort entries by runtime, then backend
        let mut sorted_entries = self.entries.clone();
        sorted_entries.sort_by(|a, b| {2
            let runtime_cmp = format!("{:?}", a.runtime).cmp(&format!("{:?}", b.runtime));
            if runtime_cmp == std::cmp::Ordering::Equal {
                format!("{}", a.backend).cmp(&format!("{}", b.backend))
            } else {
                runtime_cmp
            }
        });

        for entry8 in &sorted_entries {
            if entry.available {
                let _ = writeln!(
                    table,
                    "| **{}** | {} | {:.1}ms | {:.1}ms | {:.1} tok/s | {:.0}ms | {} | {:.3} |",
                    format!("{:?}", entry.runtime).to_lowercase(),
                    entry.backend,
                    entry.p50_latency_ms,
                    entry.p99_latency_ms,
                    entry.throughput_tps,
                    entry.cold_start_ms,
                    entry.samples,
                    entry.cv_at_stop,
                );
            } else {
                let _ = writeln!(
                    table,
                    "| {} | {} | - | - | - | - | - | - |",
                    format!("{:?}", entry.runtime).to_lowercase(),
                    entry.backend,
                );
            }
        }

        table
    }

    /// Serialize to JSON
    ///
    /// # Errors
    ///
    /// Returns error if serialization fails.
    pub fn to_json(&self) -> Result<String, serde_json::Error> {
        serde_json::to_string_pretty(self)
    }

    /// Deserialize from JSON
    ///
    /// # Errors
    ///
    /// Returns error if JSON is invalid.
    pub fn from_json(json: &str) -> Result<Self, serde_json::Error> {
        serde_json::from_str(json)
    }
}

/// Matrix benchmark runner configuration
#[derive(Debug, Clone)]
pub struct MatrixBenchmarkConfig {
    /// Runtimes to benchmark
    pub runtimes: Vec<RuntimeType>,
    /// Backends to benchmark
    pub backends: Vec<ComputeBackendType>,
    /// Model path
    pub model_path: String,
    /// Prompt for benchmarking
    pub prompt: String,
    /// Max tokens to generate
    pub max_tokens: usize,
    /// CV threshold for stopping
    pub cv_threshold: f64,
    /// Minimum samples
    pub min_samples: usize,
    /// Maximum samples (failsafe)
    pub max_samples: usize,
    /// Warmup iterations
    pub warmup_iterations: usize,
}

impl Default for MatrixBenchmarkConfig {
    fn default() -> Self {
        Self {
            runtimes: vec![
                RuntimeType::Realizar,
                RuntimeType::LlamaCpp,
                RuntimeType::Ollama,
            ],
            backends: vec![ComputeBackendType::Cpu, ComputeBackendType::Wgpu],
            model_path: String::new(),
            prompt: "Explain machine learning in one sentence.".to_string(),
            max_tokens: 50,
            cv_threshold: 0.05,
            min_samples: 30,
            max_samples: 200,
            warmup_iterations: 5,
        }
    }
}

/// Summary statistics for a single matrix column (backend)
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BackendSummary {
    /// Backend type
    pub backend: ComputeBackendType,
    /// Number of available runtimes
    pub available_runtimes: usize,
    /// Fastest runtime (by p50 latency)
    pub fastest_runtime: Option<String>,
    /// Fastest p50 latency
    pub fastest_p50_ms: f64,
    /// Highest throughput runtime
    pub highest_throughput_runtime: Option<String>,
    /// Highest throughput (tok/s)
    pub highest_throughput_tps: f64,
}

/// Summary of the entire benchmark matrix
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MatrixSummary {
    /// Total entries in matrix
    pub total_entries: usize,
    /// Number of available entries
    pub available_entries: usize,
    /// Per-backend summaries
    pub backend_summaries: Vec<BackendSummary>,
    /// Overall fastest (runtime, backend) combination
    pub overall_fastest: Option<(String, String)>,
    /// Overall highest throughput (runtime, backend)
    pub overall_highest_throughput: Option<(String, String)>,
}

impl BenchmarkMatrix {
    /// Generate summary statistics
    #[must_use]
    pub fn summary(&self) -> MatrixSummary {
        let total_entries = self.entries.len();
        let available_entries = self.entries.iter().filter(|e| e.available).count();

        let mut backend_summaries = Vec::new();
        for backend in ComputeBackendType::all3() {
            let entries: Vec<_> = self.entries_for_backend(backend);
            let available: Vec<_> = entries.iter().filter(|e| e.available).collect();

            let fastest = available.iter().min_by(|a, b| {4
                a.p50_latency_ms
                    .partial_cmp(&b.p50_latency_ms)
                    .expect("test")
            });
            let highest_tp = available.iter().max_by(|a, b| {4
                a.throughput_tps
                    .partial_cmp(&b.throughput_tps)
                    .expect("test")
            });

            backend_summaries.push(BackendSummary {
                backend,
                available_runtimes: available.len(),
                fastest_runtime: fastest.map(|e| format!4("{:?}"4, e.runtime).to_lowercase4()),
                fastest_p50_ms: fastest.map_or(0.0, |e| e.p50_latency_ms),
                highest_throughput_runtime: highest_tp
                    .map(|e| format!4("{:?}"4, e.runtime).to_lowercase4()),
                highest_throughput_tps: highest_tp.map_or(0.0, |e| e.throughput_tps),
            });
        }

        let available = self.entries.iter().filter(|e| e.available);
        let overall_fastest = available
            .clone()
            .min_by3(|a, b| {
                a.p50_latency_ms
                    .partial_cmp(&b.p50_latency_ms)
                    .expect("test")
            })
            .map(|e| {
                (
                    format!("{:?}", e.runtime).to_lowercase(),
                    e.backend.to_string(),
                )
            });
        let overall_highest_throughput = available
            .max_by3(|a, b| {
                a.throughput_tps
                    .partial_cmp(&b.throughput_tps)
                    .expect("test")
            })
            .map(|e| {
                (
                    format!("{:?}", e.runtime).to_lowercase(),
                    e.backend.to_string(),
                )
            });

        MatrixSummary {
            total_entries,
            available_entries,
            backend_summaries,
            overall_fastest,
            overall_highest_throughput,
        }
    }
}

Coverage Report

Created: 2026-01-25 15:05

Line	Count	Source
1		//! Backend benchmark matrix for comparing compute implementations
2		//!
3		//! Extracted from bench/mod.rs (PMAT-802) to reduce module size.
4		//! Contains:
5		//! - ComputeBackendType enum (CPU, wgpu, CUDA)
6		//! - BenchmarkMatrix for cross-backend comparison
7		//! - MatrixBenchmarkConfig and related types
8
9		#![allow(clippy::cast_precision_loss)]
10
11		use std::fmt::Write;
12
13		use serde::{Deserialize, Serialize};
14
15		use super::{chrono_timestamp, compute_cv, percentile, HardwareSpec, RuntimeType};
16
17		// ============================================================================
18		// Backend Benchmark Matrix (per Hoefler & Belli SC'15)
19		// ============================================================================
20
21		/// Compute backend type for benchmark matrix
22		///
23		/// Represents the different compute backends that can be benchmarked:
24		/// - CPU: Scalar/SIMD operations via trueno CPU backend
25		/// - Wgpu: Cross-platform GPU via trueno wgpu backend
26		/// - Cuda: NVIDIA GPU via trueno-gpu PTX execution
27		#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
28		pub enum ComputeBackendType {
29		/// CPU backend (scalar/SIMD via trueno)
30		Cpu,
31		/// wgpu GPU backend (cross-platform via trueno)
32		Wgpu,
33		/// CUDA GPU backend (NVIDIA via trueno-gpu)
34		Cuda,
35		}
36
37		impl std::fmt::Display for ComputeBackendType {
38	24	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
39	24	match self {
40	15	Self::Cpu => write!(f, "cpu"),
41	5	Self::Wgpu => write!(f, "wgpu"),
42	4	Self::Cuda => write!(f, "cuda"),
43		}
44	24	}
45		}
46
47		impl ComputeBackendType {
48		/// Parse from string
49		#[must_use]
50	7	pub fn parse(s: &str) -> Option<Self> {
51	7	match s.to_lowercase().as_str() {
52	7	"cpu" => Some(Self::Cpu)2 ,
53	5	"wgpu" \| "gpu"4 => Some(Self::Wgpu)2 ,
54	3	"cuda" \| "nvidia"2 => Some(Self::Cuda)2 ,
55	1	_ => None,
56		}
57	7	}
58
59		/// All available backend types
60		#[must_use]
61	4	pub fn all() -> Vec<Self> {
62	4	vec![Self::Cpu, Self::Wgpu, Self::Cuda]
63	4	}
64		}
65
66		/// Single entry in the benchmark matrix
67		///
68		/// Represents results for one (runtime, backend) combination.
69		#[derive(Debug, Clone, Serialize, Deserialize)]
70		pub struct MatrixBenchmarkEntry {
71		/// Runtime type (realizar, llama-cpp, ollama, vllm)
72		pub runtime: RuntimeType,
73		/// Compute backend (cpu, wgpu, cuda)
74		pub backend: ComputeBackendType,
75		/// Model name/identifier
76		pub model: String,
77		/// Whether this configuration is available
78		pub available: bool,
79		/// p50 latency in milliseconds
80		pub p50_latency_ms: f64,
81		/// p99 latency in milliseconds
82		pub p99_latency_ms: f64,
83		/// Throughput in tokens per second
84		pub throughput_tps: f64,
85		/// Cold start time in milliseconds
86		pub cold_start_ms: f64,
87		/// Number of samples collected
88		pub samples: usize,
89		/// Final CV at stop
90		pub cv_at_stop: f64,
91		/// Additional notes (e.g., "GPU layers: 99")
92		pub notes: String,
93		}
94
95		impl Default for MatrixBenchmarkEntry {
96	5	fn default() -> Self {
97	5	Self {
98	5	runtime: RuntimeType::Realizar,
99	5	backend: ComputeBackendType::Cpu,
100	5	model: String::new(),
101	5	available: false,
102	5	p50_latency_ms: 0.0,
103	5	p99_latency_ms: 0.0,
104	5	throughput_tps: 0.0,
105	5	cold_start_ms: 0.0,
106	5	samples: 0,
107	5	cv_at_stop: 0.0,
108	5	notes: String::new(),
109	5	}
110	5	}
111		}
112
113		impl MatrixBenchmarkEntry {
114		/// Create a new unavailable entry (placeholder)
115		#[must_use]
116	5	pub fn unavailable(runtime: RuntimeType, backend: ComputeBackendType) -> Self {
117	5	Self {
118	5	runtime,
119	5	backend,
120	5	available: false,
121	5	notes: "Backend not available".to_string(),
122	5	..Default::default()
123	5	}
124	5	}
125
126		/// Create entry from raw latency samples
127		#[must_use]
128	39	pub fn from_samples(
129	39	runtime: RuntimeType,
130	39	backend: ComputeBackendType,
131	39	model: &str,
132	39	latencies_ms: &[f64],
133	39	throughputs_tps: &[f64],
134	39	cold_start_ms: f64,
135	39	) -> Self {
136	39	let samples = latencies_ms.len();
137	39	if samples == 0 {
138	1	return Self::unavailable(runtime, backend);
139	38	}
140
141	38	let p50_latency = percentile(latencies_ms, 50.0);
142	38	let p99_latency = percentile(latencies_ms, 99.0);
143	38	let throughput = if throughputs_tps.is_empty() {
144	0	0.0
145		} else {
146	38	throughputs_tps.iter().sum::<f64>() / throughputs_tps.len() as f64
147		};
148	38	let cv = compute_cv(latencies_ms);
149
150	38	Self {
151	38	runtime,
152	38	backend,
153	38	model: model.to_string(),
154	38	available: true,
155	38	p50_latency_ms: p50_latency,
156	38	p99_latency_ms: p99_latency,
157	38	throughput_tps: throughput,
158	38	cold_start_ms,
159	38	samples,
160	38	cv_at_stop: cv,
161	38	notes: String::new(),
162	38	}
163	39	}
164
165		/// Add notes to the entry
166		#[must_use]
167	1	pub fn with_notes(mut self, notes: &str) -> Self {
168	1	self.notes = notes.to_string();
169	1	self
170	1	}
171		}
172
173		/// Complete benchmark matrix comparing runtimes across backends
174		///
175		/// Per Hoefler & Belli SC'15, this matrix enables:
176		/// - Reproducible comparisons across configurations
177		/// - Statistical validity via CV-based stopping
178		/// - Clear identification of performance characteristics
179		#[derive(Debug, Clone, Serialize, Deserialize)]
180		pub struct BenchmarkMatrix {
181		/// Schema version
182		pub version: String,
183		/// ISO 8601 timestamp
184		pub timestamp: String,
185		/// Model used for benchmarking
186		pub model: String,
187		/// Hardware specification
188		pub hardware: HardwareSpec,
189		/// Benchmark methodology
190		pub methodology: String,
191		/// CV threshold used
192		pub cv_threshold: f64,
193		/// Matrix entries indexed by (runtime, backend)
194		pub entries: Vec<MatrixBenchmarkEntry>,
195		}
196
197		impl BenchmarkMatrix {
198		/// Create a new empty matrix
199		#[must_use]
200	18	pub fn new(model: &str, hardware: HardwareSpec) -> Self {
201	18	Self {
202	18	version: "1.1".to_string(),
203	18	timestamp: chrono_timestamp(),
204	18	model: model.to_string(),
205	18	hardware,
206	18	methodology: "CV-based stopping (Hoefler & Belli SC'15)".to_string(),
207	18	cv_threshold: 0.05,
208	18	entries: Vec::new(),
209	18	}
210	18	}
211
212		/// Add an entry to the matrix
213	32	pub fn add_entry(&mut self, entry: MatrixBenchmarkEntry) {
214		// Remove existing entry for same (runtime, backend) if present
215	32	self.entries
216	32	.retain(\|e\| e.runtime != entry.runtime23 \|\| e.backend != entry.backend6 );
217	32	self.entries.push(entry);
218	32	}
219
220		/// Get entry for specific (runtime, backend) combination
221		#[must_use]
222	2	pub fn get_entry(
223	2	&self,
224	2	runtime: RuntimeType,
225	2	backend: ComputeBackendType,
226	2	) -> Option<&MatrixBenchmarkEntry> {
227	2	self.entries
228	2	.iter()
229	2	.find(\|e\| e.runtime == runtime && e.backend == backend1 )
230	2	}
231
232		/// Get all entries for a specific runtime
233		#[must_use]
234	2	pub fn entries_for_runtime(&self, runtime: RuntimeType) -> Vec<&MatrixBenchmarkEntry> {
235	2	self.entries
236	2	.iter()
237	6	. filter2 (\|e\| e.runtime == runtime)
238	2	.collect()
239	2	}
240
241		/// Get all entries for a specific backend
242		#[must_use]
243	14	pub fn entries_for_backend(&self, backend: ComputeBackendType) -> Vec<&MatrixBenchmarkEntry> {
244	14	self.entries
245	14	.iter()
246	40	. filter14 (\|e\| e.backend == backend)
247	14	.collect()
248	14	}
249
250		/// Find the fastest runtime for a given backend (by p50 latency)
251		#[must_use]
252	1	pub fn fastest_for_backend(
253	1	&self,
254	1	backend: ComputeBackendType,
255	1	) -> Option<&MatrixBenchmarkEntry> {
256	1	self.entries_for_backend(backend)
257	1	.into_iter()
258	1	.filter(\|e\| e.available)
259	1	.min_by(\|a, b\| {
260	1	a.p50_latency_ms
261	1	.partial_cmp(&b.p50_latency_ms)
262	1	.expect("test")
263	1	})
264	1	}
265
266		/// Find the highest throughput runtime for a given backend
267		#[must_use]
268	1	pub fn highest_throughput_for_backend(
269	1	&self,
270	1	backend: ComputeBackendType,
271	1	) -> Option<&MatrixBenchmarkEntry> {
272	1	self.entries_for_backend(backend)
273	1	.into_iter()
274	1	.filter(\|e\| e.available)
275	1	.max_by(\|a, b\| {
276	1	a.throughput_tps
277	1	.partial_cmp(&b.throughput_tps)
278	1	.expect("test")
279	1	})
280	1	}
281
282		/// Generate markdown table for README
283		#[must_use]
284	6	pub fn to_markdown_table(&self) -> String {
285	6	let mut table = String::new();
286
287		// Header
288	6	table.push_str("\| Runtime \| Backend \| p50 Latency \| p99 Latency \| Throughput \| Cold Start \| Samples \| CV \|\n");
289	6	table.push_str("\|---------\|---------\|-------------\|-------------\|------------\|------------\|---------\|----\|\n");
290
291		// Sort entries by runtime, then backend
292	6	let mut sorted_entries = self.entries.clone();
293	6	sorted_entries.sort_by(\|a, b\| {2
294	2	let runtime_cmp = format!("{:?}", a.runtime).cmp(&format!("{:?}", b.runtime));
295	2	if runtime_cmp == std::cmp::Ordering::Equal {
296	1	format!("{}", a.backend).cmp(&format!("{}", b.backend))
297		} else {
298	1	runtime_cmp
299		}
300	2	});
301
302	14	for entry8 in &sorted_entries {
303	8	if entry.available {
304	7	let _ = writeln!(
305	7	table,
306	7	"\| {} \| {} \| {:.1}ms \| {:.1}ms \| {:.1} tok/s \| {:.0}ms \| {} \| {:.3} \|",
307	7	format!("{:?}", entry.runtime).to_lowercase(),
308	7	entry.backend,
309	7	entry.p50_latency_ms,
310	7	entry.p99_latency_ms,
311	7	entry.throughput_tps,
312	7	entry.cold_start_ms,
313	7	entry.samples,
314	7	entry.cv_at_stop,
315	7	);
316	7	} else {
317	1	let _ = writeln!(
318	1	table,
319	1	"\| {} \| {} \| - \| - \| - \| - \| - \| - \|",
320	1	format!("{:?}", entry.runtime).to_lowercase(),
321	1	entry.backend,
322	1	);
323	1	}
324		}
325
326	6	table
327	6	}
328
329		/// Serialize to JSON
330		///
331		/// # Errors
332		///
333		/// Returns error if serialization fails.
334	1	pub fn to_json(&self) -> Result<String, serde_json::Error> {
335	1	serde_json::to_string_pretty(self)
336	1	}
337
338		/// Deserialize from JSON
339		///
340		/// # Errors
341		///
342		/// Returns error if JSON is invalid.
343	1	pub fn from_json(json: &str) -> Result<Self, serde_json::Error> {
344	1	serde_json::from_str(json)
345	1	}
346		}
347
348		/// Matrix benchmark runner configuration
349		#[derive(Debug, Clone)]
350		pub struct MatrixBenchmarkConfig {
351		/// Runtimes to benchmark
352		pub runtimes: Vec<RuntimeType>,
353		/// Backends to benchmark
354		pub backends: Vec<ComputeBackendType>,
355		/// Model path
356		pub model_path: String,
357		/// Prompt for benchmarking
358		pub prompt: String,
359		/// Max tokens to generate
360		pub max_tokens: usize,
361		/// CV threshold for stopping
362		pub cv_threshold: f64,
363		/// Minimum samples
364		pub min_samples: usize,
365		/// Maximum samples (failsafe)
366		pub max_samples: usize,
367		/// Warmup iterations
368		pub warmup_iterations: usize,
369		}
370
371		impl Default for MatrixBenchmarkConfig {
372	1	fn default() -> Self {
373	1	Self {
374	1	runtimes: vec![
375	1	RuntimeType::Realizar,
376	1	RuntimeType::LlamaCpp,
377	1	RuntimeType::Ollama,
378	1	],
379	1	backends: vec![ComputeBackendType::Cpu, ComputeBackendType::Wgpu],
380	1	model_path: String::new(),
381	1	prompt: "Explain machine learning in one sentence.".to_string(),
382	1	max_tokens: 50,
383	1	cv_threshold: 0.05,
384	1	min_samples: 30,
385	1	max_samples: 200,
386	1	warmup_iterations: 5,
387	1	}
388	1	}
389		}
390
391		/// Summary statistics for a single matrix column (backend)
392		#[derive(Debug, Clone, Serialize, Deserialize)]
393		pub struct BackendSummary {
394		/// Backend type
395		pub backend: ComputeBackendType,
396		/// Number of available runtimes
397		pub available_runtimes: usize,
398		/// Fastest runtime (by p50 latency)
399		pub fastest_runtime: Option<String>,
400		/// Fastest p50 latency
401		pub fastest_p50_ms: f64,
402		/// Highest throughput runtime
403		pub highest_throughput_runtime: Option<String>,
404		/// Highest throughput (tok/s)
405		pub highest_throughput_tps: f64,
406		}
407
408		/// Summary of the entire benchmark matrix
409		#[derive(Debug, Clone, Serialize, Deserialize)]
410		pub struct MatrixSummary {
411		/// Total entries in matrix
412		pub total_entries: usize,
413		/// Number of available entries
414		pub available_entries: usize,
415		/// Per-backend summaries
416		pub backend_summaries: Vec<BackendSummary>,
417		/// Overall fastest (runtime, backend) combination
418		pub overall_fastest: Option<(String, String)>,
419		/// Overall highest throughput (runtime, backend)
420		pub overall_highest_throughput: Option<(String, String)>,
421		}
422
423		impl BenchmarkMatrix {
424		/// Generate summary statistics
425		#[must_use]
426	3	pub fn summary(&self) -> MatrixSummary {
427	3	let total_entries = self.entries.len();
428	3	let available_entries = self.entries.iter().filter(\|e\| e.available).count();
429
430	3	let mut backend_summaries = Vec::new();
431	9	for backend in ComputeBackendType::all3 () {
432	9	let entries: Vec<_> = self.entries_for_backend(backend);
433	9	let available: Vec<_> = entries.iter().filter(\|e\| e.available).collect();
434
435	9	let fastest = available.iter().min_by(\|a, b\| {4
436	4	a.p50_latency_ms
437	4	.partial_cmp(&b.p50_latency_ms)
438	4	.expect("test")
439	4	});
440	9	let highest_tp = available.iter().max_by(\|a, b\| {4
441	4	a.throughput_tps
442	4	.partial_cmp(&b.throughput_tps)
443	4	.expect("test")
444	4	});
445
446	9	backend_summaries.push(BackendSummary {
447	9	backend,
448	9	available_runtimes: available.len(),
449	9	fastest_runtime: fastest.map(\|e\| format!4 ( "{:?}"4 , e.runtime). to_lowercase4 ()),
450	9	fastest_p50_ms: fastest.map_or(0.0, \|e\| e.p50_latency_ms),
451	9	highest_throughput_runtime: highest_tp
452	9	.map(\|e\| format!4 ( "{:?}"4 , e.runtime). to_lowercase4 ()),
453	9	highest_throughput_tps: highest_tp.map_or(0.0, \|e\| e.throughput_tps),
454		});
455		}
456
457	3	let available = self.entries.iter().filter(\|e\| e.available);
458	3	let overall_fastest = available
459	3	.clone()
460	5	. min_by3 (\|a, b\| {
461	5	a.p50_latency_ms
462	5	.partial_cmp(&b.p50_latency_ms)
463	5	.expect("test")
464	5	})
465	3	.map(\|e\| {
466	3	(
467	3	format!("{:?}", e.runtime).to_lowercase(),
468	3	e.backend.to_string(),
469	3	)
470	3	});
471	3	let overall_highest_throughput = available
472	5	. max_by3 (\|a, b\| {
473	5	a.throughput_tps
474	5	.partial_cmp(&b.throughput_tps)
475	5	.expect("test")
476	5	})
477	3	.map(\|e\| {
478	3	(
479	3	format!("{:?}", e.runtime).to_lowercase(),
480	3	e.backend.to_string(),
481	3	)
482	3	});
483
484	3	MatrixSummary {
485	3	total_entries,
486	3	available_entries,
487	3	backend_summaries,
488	3	overall_fastest,
489	3	overall_highest_throughput,
490	3	}
491	3	}
492		}