/home/noah/src/realizar/src/apr_transformer/benchmark.rs

Source
//! APR Benchmark Infrastructure (Y6: Format Parity Validation)
//!
//! Provides standardized benchmarking for APR transformers following the benchmark spec:
//! - Dynamic CV-based sampling
//! - Statistical metrics (p50, p99, std_dev)
//! - Throughput and memory measurement
//!
//! Extracted from apr_transformer.rs (PMAT-802)

use crate::error::Result;

use super::{AprTransformer, GenerateConfig};

// ============================================================================
// Y6: APR Benchmark Infrastructure (Format Parity Validation)
// ============================================================================

/// CPU decode threshold: 50 tok/s per spec Y6
pub const APR_CPU_DECODE_THRESHOLD_TOK_S: f64 = 50.0;

/// Prefill threshold: 100 tok/s per spec Y8
pub const APR_PREFILL_THRESHOLD_TOK_S: f64 = 100.0;

/// Parity threshold: 95% of baseline per spec Y6
pub const APR_PARITY_THRESHOLD_PCT: f64 = 95.0;

/// Result of an APR benchmark run
#[derive(Debug, Clone, Default)]
pub struct AprBenchmarkResult {
    /// Number of tokens generated
    pub tokens_generated: usize,
    /// Total time in milliseconds
    pub total_time_ms: f64,
    /// Throughput in tokens per second
    pub tokens_per_second: f64,
    /// Median throughput (p50)
    pub throughput_p50: f64,
    /// 99th percentile throughput (worst case)
    pub throughput_p99: f64,
    /// Standard deviation of throughput
    pub throughput_std_dev: f64,
    /// Peak memory usage in MB
    pub peak_memory_mb: f64,
    /// Model memory in MB
    pub model_memory_mb: f64,
}

impl AprBenchmarkResult {
    /// Check if benchmark meets the given throughput threshold
    #[must_use]
    pub fn meets_threshold(&self, threshold_tok_s: f64) -> bool {
        self.tokens_per_second >= threshold_tok_s
    }

    /// Compare this result to a baseline
    #[must_use]
    pub fn compare_to_baseline(&self, baseline: &AprBenchmarkResult) -> AprParityComparison {
        let throughput_ratio = if baseline.tokens_per_second > 0.0 {
            self.tokens_per_second / baseline.tokens_per_second
        } else {
            1.0
        };

        let memory_ratio = if baseline.peak_memory_mb > 0.0 {
            self.peak_memory_mb / baseline.peak_memory_mb
        } else {
            1.0
        };

        AprParityComparison {
            throughput_ratio,
            memory_ratio,
            parity_threshold_pct: APR_PARITY_THRESHOLD_PCT,
        }
    }
}

/// Result of prefill benchmark
#[derive(Debug, Clone, Default)]
pub struct AprPrefillResult {
    /// Number of prompt tokens processed
    pub prompt_tokens: usize,
    /// Prefill time in milliseconds
    pub prefill_time_ms: f64,
    /// Prefill throughput in tokens per second
    pub prefill_tok_s: f64,
}

/// Result of load time benchmark
#[derive(Debug, Clone, Default)]
pub struct AprLoadResult {
    /// Load time in milliseconds
    pub load_time_ms: f64,
}

/// Comparison of APR benchmark to baseline (for parity validation)
#[derive(Debug, Clone)]
pub struct AprParityComparison {
    /// Ratio of APR throughput to baseline
    pub throughput_ratio: f64,
    /// Ratio of APR memory to baseline
    pub memory_ratio: f64,
    /// Parity threshold percentage
    pub parity_threshold_pct: f64,
}

impl AprParityComparison {
    /// Check if APR achieves parity with baseline
    #[must_use]
    pub fn is_parity(&self) -> bool {
        self.throughput_ratio >= (self.parity_threshold_pct / 100.0)
    }
}

/// Benchmark runner for APR transformers (Y6)
///
/// Provides standardized benchmarking following the benchmark spec:
/// - Dynamic CV-based sampling
/// - Statistical metrics (p50, p99, std_dev)
/// - Throughput and memory measurement
#[derive(Debug)]
pub struct AprBenchmarkRunner {
    /// The transformer to benchmark
    transformer: AprTransformer,
    /// Number of warmup iterations
    warmup_iterations: usize,
    /// Number of measurement iterations
    measure_iterations: usize,
}

impl AprBenchmarkRunner {
    /// Create a new benchmark runner for the given transformer
    #[must_use]
    pub fn new(transformer: AprTransformer) -> Self {
        Self {
            transformer,
            warmup_iterations: 3,
            measure_iterations: 10,
        }
    }

    /// Get warmup iterations
    #[must_use]
    pub fn warmup_iterations(&self) -> usize {
        self.warmup_iterations
    }

    /// Get measure iterations
    #[must_use]
    pub fn measure_iterations(&self) -> usize {
        self.measure_iterations
    }

    /// Set warmup iterations
    pub fn set_warmup_iterations(&mut self, n: usize) {
        self.warmup_iterations = n;
    }

    /// Set measure iterations
    pub fn set_measure_iterations(&mut self, n: usize) {
        self.measure_iterations = n.max(1);
    }

    /// Benchmark decode throughput
    ///
    /// # Arguments
    ///
    /// * `prompt` - Initial token IDs
    /// * `num_tokens` - Number of tokens to generate
    ///
    /// # Returns
    ///
    /// Benchmark result with throughput metrics
    pub fn benchmark_decode(
        &mut self,
        prompt: &[u32],
        num_tokens: usize,
    ) -> Result<AprBenchmarkResult> {
        use std::time::Instant;

        // Warmup
        for _ in 0..self.warmup_iterations {
            let gen_config = GenerateConfig {
                max_tokens: num_tokens.min(5),
                temperature: 0.0,
                ..Default::default()
            };
            let _ = self.transformer.generate_with_cache(prompt, &gen_config)?;
        }

        // Measurement runs
        let mut throughputs = Vec::with_capacity(self.measure_iterations);
        let mut total_tokens = 0usize;
        let mut total_time_ms = 0.0f64;

        for _ in 0..self.measure_iterations {
            let gen_config = GenerateConfig {
                max_tokens: num_tokens,
                temperature: 0.0,
                ..Default::default()
            };

            let start = Instant::now();
            let output = self.transformer.generate_with_cache(prompt, &gen_config)?;
            let elapsed = start.elapsed();

            let generated = output.len().saturating_sub(prompt.len());
            let time_ms = elapsed.as_secs_f64() * 1000.0;
            let throughput = if time_ms > 0.0 {
                (generated as f64) / (time_ms / 1000.0)
            } else {
                0.0
            };

            throughputs.push(throughput);
            total_tokens += generated;
            total_time_ms += time_ms;
        }

        // Calculate statistics
        let mean_throughput = if !throughputs.is_empty() {
            throughputs.iter().sum::<f64>() / throughputs.len() as f64
        } else {
            0.0
        };

        let mut sorted = throughputs.clone();
        sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));

        let p50 = if !sorted.is_empty() {
            sorted[sorted.len() / 2]
        } else {
            0.0
        };

        let p99_idx =
            ((sorted.len() as f64 * 0.01).floor() as usize).min(sorted.len().saturating_sub(1));
        let p99 = if !sorted.is_empty() {
            sorted[p99_idx]
        } else {
            0.0
        };

        let std_dev = if throughputs.len() > 1 {
            let variance = throughputs
                .iter()
                .map(|t| (t - mean_throughput).powi(2))
                .sum::<f64>()
                / (throughputs.len() - 1) as f64;
            variance.sqrt()
        } else {
            0.0
        };

        // Memory estimation
        let model_memory_mb = (self.transformer.memory_size() as f64) / (1024.0 * 1024.0);

        Ok(AprBenchmarkResult {
            tokens_generated: total_tokens / self.measure_iterations.max(1),
            total_time_ms: total_time_ms / self.measure_iterations.max(1) as f64,
            tokens_per_second: mean_throughput,
            throughput_p50: p50,
            throughput_p99: p99,
            throughput_std_dev: std_dev,
            peak_memory_mb: model_memory_mb * 1.5, // Estimate: model + KV cache
            model_memory_mb,
        })
    }

    /// Benchmark prefill throughput
    ///
    /// # Arguments
    ///
    /// * `prompt` - Tokens to prefill
    ///
    /// # Returns
    ///
    /// Prefill benchmark result
    pub fn benchmark_prefill(&mut self, prompt: &[u32]) -> Result<AprPrefillResult> {
        use std::time::Instant;

        // Warmup
        for _ in 0..self.warmup_iterations {
            let _ = self.transformer.forward(prompt)?;
        }

        // Measurement runs
        let mut prefill_times_ms = Vec::with_capacity(self.measure_iterations);

        for _ in 0..self.measure_iterations {
            let start = Instant::now();
            let _ = self.transformer.forward(prompt)?;
            let elapsed = start.elapsed();
            prefill_times_ms.push(elapsed.as_secs_f64() * 1000.0);
        }

        let mean_time_ms = if !prefill_times_ms.is_empty() {
            prefill_times_ms.iter().sum::<f64>() / prefill_times_ms.len() as f64
        } else {
            0.0
        };

        let prefill_tok_s = if mean_time_ms > 0.0 {
            (prompt.len() as f64) / (mean_time_ms / 1000.0)
        } else {
            0.0
        };

        Ok(AprPrefillResult {
            prompt_tokens: prompt.len(),
            prefill_time_ms: mean_time_ms,
            prefill_tok_s,
        })
    }

    /// Benchmark model load time
    ///
    /// # Arguments
    ///
    /// * `loader` - Closure that creates the transformer
    ///
    /// # Returns
    ///
    /// Load time result
    pub fn benchmark_load<F>(loader: F) -> Result<AprLoadResult>
    where
        F: Fn() -> AprTransformer,
    {
        use std::time::Instant;

        // Single measurement (load is typically done once)
        let start = Instant::now();
        let _transformer = loader();
        let elapsed = start.elapsed();

        Ok(AprLoadResult {
            load_time_ms: elapsed.as_secs_f64() * 1000.0,
        })
    }
}

Coverage Report

Created: 2026-01-25 15:05

Line	Count	Source
1		//! APR Benchmark Infrastructure (Y6: Format Parity Validation)
2		//!
3		//! Provides standardized benchmarking for APR transformers following the benchmark spec:
4		//! - Dynamic CV-based sampling
5		//! - Statistical metrics (p50, p99, std_dev)
6		//! - Throughput and memory measurement
7		//!
8		//! Extracted from apr_transformer.rs (PMAT-802)
9
10		use crate::error::Result;
11
12		use super::{AprTransformer, GenerateConfig};
13
14		// ============================================================================
15		// Y6: APR Benchmark Infrastructure (Format Parity Validation)
16		// ============================================================================
17
18		/// CPU decode threshold: 50 tok/s per spec Y6
19		pub const APR_CPU_DECODE_THRESHOLD_TOK_S: f64 = 50.0;
20
21		/// Prefill threshold: 100 tok/s per spec Y8
22		pub const APR_PREFILL_THRESHOLD_TOK_S: f64 = 100.0;
23
24		/// Parity threshold: 95% of baseline per spec Y6
25		pub const APR_PARITY_THRESHOLD_PCT: f64 = 95.0;
26
27		/// Result of an APR benchmark run
28		#[derive(Debug, Clone, Default)]
29		pub struct AprBenchmarkResult {
30		/// Number of tokens generated
31		pub tokens_generated: usize,
32		/// Total time in milliseconds
33		pub total_time_ms: f64,
34		/// Throughput in tokens per second
35		pub tokens_per_second: f64,
36		/// Median throughput (p50)
37		pub throughput_p50: f64,
38		/// 99th percentile throughput (worst case)
39		pub throughput_p99: f64,
40		/// Standard deviation of throughput
41		pub throughput_std_dev: f64,
42		/// Peak memory usage in MB
43		pub peak_memory_mb: f64,
44		/// Model memory in MB
45		pub model_memory_mb: f64,
46		}
47
48		impl AprBenchmarkResult {
49		/// Check if benchmark meets the given throughput threshold
50		#[must_use]
51	0	pub fn meets_threshold(&self, threshold_tok_s: f64) -> bool {
52	0	self.tokens_per_second >= threshold_tok_s
53	0	}
54
55		/// Compare this result to a baseline
56		#[must_use]
57	0	pub fn compare_to_baseline(&self, baseline: &AprBenchmarkResult) -> AprParityComparison {
58	0	let throughput_ratio = if baseline.tokens_per_second > 0.0 {
59	0	self.tokens_per_second / baseline.tokens_per_second
60		} else {
61	0	1.0
62		};
63
64	0	let memory_ratio = if baseline.peak_memory_mb > 0.0 {
65	0	self.peak_memory_mb / baseline.peak_memory_mb
66		} else {
67	0	1.0
68		};
69
70	0	AprParityComparison {
71	0	throughput_ratio,
72	0	memory_ratio,
73	0	parity_threshold_pct: APR_PARITY_THRESHOLD_PCT,
74	0	}
75	0	}
76		}
77
78		/// Result of prefill benchmark
79		#[derive(Debug, Clone, Default)]
80		pub struct AprPrefillResult {
81		/// Number of prompt tokens processed
82		pub prompt_tokens: usize,
83		/// Prefill time in milliseconds
84		pub prefill_time_ms: f64,
85		/// Prefill throughput in tokens per second
86		pub prefill_tok_s: f64,
87		}
88
89		/// Result of load time benchmark
90		#[derive(Debug, Clone, Default)]
91		pub struct AprLoadResult {
92		/// Load time in milliseconds
93		pub load_time_ms: f64,
94		}
95
96		/// Comparison of APR benchmark to baseline (for parity validation)
97		#[derive(Debug, Clone)]
98		pub struct AprParityComparison {
99		/// Ratio of APR throughput to baseline
100		pub throughput_ratio: f64,
101		/// Ratio of APR memory to baseline
102		pub memory_ratio: f64,
103		/// Parity threshold percentage
104		pub parity_threshold_pct: f64,
105		}
106
107		impl AprParityComparison {
108		/// Check if APR achieves parity with baseline
109		#[must_use]
110	0	pub fn is_parity(&self) -> bool {
111	0	self.throughput_ratio >= (self.parity_threshold_pct / 100.0)
112	0	}
113		}
114
115		/// Benchmark runner for APR transformers (Y6)
116		///
117		/// Provides standardized benchmarking following the benchmark spec:
118		/// - Dynamic CV-based sampling
119		/// - Statistical metrics (p50, p99, std_dev)
120		/// - Throughput and memory measurement
121		#[derive(Debug)]
122		pub struct AprBenchmarkRunner {
123		/// The transformer to benchmark
124		transformer: AprTransformer,
125		/// Number of warmup iterations
126		warmup_iterations: usize,
127		/// Number of measurement iterations
128		measure_iterations: usize,
129		}
130
131		impl AprBenchmarkRunner {
132		/// Create a new benchmark runner for the given transformer
133		#[must_use]
134	0	pub fn new(transformer: AprTransformer) -> Self {
135	0	Self {
136	0	transformer,
137	0	warmup_iterations: 3,
138	0	measure_iterations: 10,
139	0	}
140	0	}
141
142		/// Get warmup iterations
143		#[must_use]
144	0	pub fn warmup_iterations(&self) -> usize {
145	0	self.warmup_iterations
146	0	}
147
148		/// Get measure iterations
149		#[must_use]
150	0	pub fn measure_iterations(&self) -> usize {
151	0	self.measure_iterations
152	0	}
153
154		/// Set warmup iterations
155	0	pub fn set_warmup_iterations(&mut self, n: usize) {
156	0	self.warmup_iterations = n;
157	0	}
158
159		/// Set measure iterations
160	0	pub fn set_measure_iterations(&mut self, n: usize) {
161	0	self.measure_iterations = n.max(1);
162	0	}
163
164		/// Benchmark decode throughput
165		///
166		/// # Arguments
167		///
168		/// * `prompt` - Initial token IDs
169		/// * `num_tokens` - Number of tokens to generate
170		///
171		/// # Returns
172		///
173		/// Benchmark result with throughput metrics
174	0	pub fn benchmark_decode(
175	0	&mut self,
176	0	prompt: &[u32],
177	0	num_tokens: usize,
178	0	) -> Result<AprBenchmarkResult> {
179		use std::time::Instant;
180
181		// Warmup
182	0	for _ in 0..self.warmup_iterations {
183	0	let gen_config = GenerateConfig {
184	0	max_tokens: num_tokens.min(5),
185	0	temperature: 0.0,
186	0	..Default::default()
187	0	};
188	0	let _ = self.transformer.generate_with_cache(prompt, &gen_config)?;
189		}
190
191		// Measurement runs
192	0	let mut throughputs = Vec::with_capacity(self.measure_iterations);
193	0	let mut total_tokens = 0usize;
194	0	let mut total_time_ms = 0.0f64;
195
196	0	for _ in 0..self.measure_iterations {
197	0	let gen_config = GenerateConfig {
198	0	max_tokens: num_tokens,
199	0	temperature: 0.0,
200	0	..Default::default()
201	0	};
202
203	0	let start = Instant::now();
204	0	let output = self.transformer.generate_with_cache(prompt, &gen_config)?;
205	0	let elapsed = start.elapsed();
206
207	0	let generated = output.len().saturating_sub(prompt.len());
208	0	let time_ms = elapsed.as_secs_f64() * 1000.0;
209	0	let throughput = if time_ms > 0.0 {
210	0	(generated as f64) / (time_ms / 1000.0)
211		} else {
212	0	0.0
213		};
214
215	0	throughputs.push(throughput);
216	0	total_tokens += generated;
217	0	total_time_ms += time_ms;
218		}
219
220		// Calculate statistics
221	0	let mean_throughput = if !throughputs.is_empty() {
222	0	throughputs.iter().sum::<f64>() / throughputs.len() as f64
223		} else {
224	0	0.0
225		};
226
227	0	let mut sorted = throughputs.clone();
228	0	sorted.sort_by(\|a, b\| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
229
230	0	let p50 = if !sorted.is_empty() {
231	0	sorted[sorted.len() / 2]
232		} else {
233	0	0.0
234		};
235
236	0	let p99_idx =
237	0	((sorted.len() as f64 * 0.01).floor() as usize).min(sorted.len().saturating_sub(1));
238	0	let p99 = if !sorted.is_empty() {
239	0	sorted[p99_idx]
240		} else {
241	0	0.0
242		};
243
244	0	let std_dev = if throughputs.len() > 1 {
245	0	let variance = throughputs
246	0	.iter()
247	0	.map(\|t\| (t - mean_throughput).powi(2))
248	0	.sum::<f64>()
249	0	/ (throughputs.len() - 1) as f64;
250	0	variance.sqrt()
251		} else {
252	0	0.0
253		};
254
255		// Memory estimation
256	0	let model_memory_mb = (self.transformer.memory_size() as f64) / (1024.0 * 1024.0);
257
258	0	Ok(AprBenchmarkResult {
259	0	tokens_generated: total_tokens / self.measure_iterations.max(1),
260	0	total_time_ms: total_time_ms / self.measure_iterations.max(1) as f64,
261	0	tokens_per_second: mean_throughput,
262	0	throughput_p50: p50,
263	0	throughput_p99: p99,
264	0	throughput_std_dev: std_dev,
265	0	peak_memory_mb: model_memory_mb * 1.5, // Estimate: model + KV cache
266	0	model_memory_mb,
267	0	})
268	0	}
269
270		/// Benchmark prefill throughput
271		///
272		/// # Arguments
273		///
274		/// * `prompt` - Tokens to prefill
275		///
276		/// # Returns
277		///
278		/// Prefill benchmark result
279	0	pub fn benchmark_prefill(&mut self, prompt: &[u32]) -> Result<AprPrefillResult> {
280		use std::time::Instant;
281
282		// Warmup
283	0	for _ in 0..self.warmup_iterations {
284	0	let _ = self.transformer.forward(prompt)?;
285		}
286
287		// Measurement runs
288	0	let mut prefill_times_ms = Vec::with_capacity(self.measure_iterations);
289
290	0	for _ in 0..self.measure_iterations {
291	0	let start = Instant::now();
292	0	let _ = self.transformer.forward(prompt)?;
293	0	let elapsed = start.elapsed();
294	0	prefill_times_ms.push(elapsed.as_secs_f64() * 1000.0);
295		}
296
297	0	let mean_time_ms = if !prefill_times_ms.is_empty() {
298	0	prefill_times_ms.iter().sum::<f64>() / prefill_times_ms.len() as f64
299		} else {
300	0	0.0
301		};
302
303	0	let prefill_tok_s = if mean_time_ms > 0.0 {
304	0	(prompt.len() as f64) / (mean_time_ms / 1000.0)
305		} else {
306	0	0.0
307		};
308
309	0	Ok(AprPrefillResult {
310	0	prompt_tokens: prompt.len(),
311	0	prefill_time_ms: mean_time_ms,
312	0	prefill_tok_s,
313	0	})
314	0	}
315
316		/// Benchmark model load time
317		///
318		/// # Arguments
319		///
320		/// * `loader` - Closure that creates the transformer
321		///
322		/// # Returns
323		///
324		/// Load time result
325	0	pub fn benchmark_load<F>(loader: F) -> Result<AprLoadResult>
326	0	where
327	0	F: Fn() -> AprTransformer,
328		{
329		use std::time::Instant;
330
331		// Single measurement (load is typically done once)
332	0	let start = Instant::now();
333	0	let _transformer = loader();
334	0	let elapsed = start.elapsed();
335
336	0	Ok(AprLoadResult {
337	0	load_time_ms: elapsed.as_secs_f64() * 1000.0,
338	0	})
339	0	}
340		}