## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/gguf/inference_types.rs

    Line
    Count
    Source
    1
    //! Inference support types for quantized model execution
    2
    //!
    3
    //! This module contains pre-allocated buffers and caches for zero-allocation inference.
    4
    //!
    5
    //! Types included:
    6
    //! - `InferenceScratchBuffer`: Scratch buffers for forward passes
    7
    //! - `OwnedInferenceScratchBuffer`: Owned variant with Q8K support
    8
    //! - `ContiguousKVCache`: Cache-aligned KV cache for efficient attention
    9
    //! - `DispatchMetrics`: Thread-safe CPU/GPU dispatch metrics
    10
    11
    use super::config::GGUFConfig;
    12
    13
    /// Cache line size in bytes (typical x86-64)
    14
    const CACHE_LINE_BYTES: usize = 64;
    15
    16
    /// Number of f32 elements per cache line (64 bytes / 4 bytes per f32)
    17
    const FLOATS_PER_CACHE_LINE: usize = CACHE_LINE_BYTES / std::mem::size_of::<f32>();
    18
    19
    // ============================================================================
    20
    // InferenceScratchBuffer
    21
    // ============================================================================
    22
    23
    /// Pre-allocated scratch buffers for zero-allocation forward passes
    24
    ///
    25
    /// ## Buffer Reuse Pattern
    26
    ///
    27
    /// - First use: hidden → normed → qkv → q/k/v → attn_out
    28
    /// - FFN pass: normed → ffn_up/ffn_gate → ffn_down → hidden
    29
    ///
    30
    /// PAR-126: Added Q8K scratch buffers for VNNI-accelerated Q4K×Q8K matmul path.
    31
    #[derive(Debug)]
    32
    pub struct InferenceScratchBuffer {
    33
        /// Hidden state buffer [hidden_dim]
    34
        pub hidden: Vec<f32>,
    35
        /// Normalized hidden state [hidden_dim]
    36
        pub normed: Vec<f32>,
    37
        /// Combined QKV projection [q_dim + k_dim + v_dim]
    38
        pub qkv: Vec<f32>,
    39
        /// Query projection [q_dim]
    40
        pub q: Vec<f32>,
    41
        /// Key projection [k_dim]
    42
        pub k: Vec<f32>,
    43
        /// Value projection [v_dim]
    44
        pub v: Vec<f32>,
    45
        /// Attention output [hidden_dim]
    46
        pub attn_out: Vec<f32>,
    47
        /// Attention projection output [hidden_dim]
    48
        pub attn_proj: Vec<f32>,
    49
        /// FFN up projection [intermediate_dim]
    50
        pub ffn_up: Vec<f32>,
    51
        /// FFN gate projection [intermediate_dim] (for SwiGLU)
    52
        pub ffn_gate: Vec<f32>,
    53
        /// FFN down projection [hidden_dim]
    54
        pub ffn_down: Vec<f32>,
    55
        /// Output logits [vocab_size]
    56
        pub logits: Vec<f32>,
    57
        // PAR-126: Q8K scratch buffers for VNNI-accelerated matmul
    58
        /// Q8K scales for hidden-dim activations [hidden_dim/256]
    59
        pub q8k_hidden_scales: Vec<f32>,
    60
        /// Q8K quants for hidden-dim activations [hidden_dim]
    61
        pub q8k_hidden_quants: Vec<i8>,
    62
        /// Q8K scales for intermediate-dim activations [intermediate_dim/256]
    63
        pub q8k_inter_scales: Vec<f32>,
    64
        /// Q8K quants for intermediate-dim activations [intermediate_dim]
    65
        pub q8k_inter_quants: Vec<i8>,
    66
    }
    67
    68
    impl InferenceScratchBuffer {
    69
        /// Create scratch buffer from model config
    70
        ///
    71
        /// Pre-allocates all buffers to their maximum required size.
    72
        /// Total memory: ~2.5MB for TinyLlama-1.1B, ~10MB for 7B models.
    73
        #[must_use]
    74
    0
        pub fn from_config(config: &GGUFConfig) -> Self {
    75
    0
            let hidden_dim = config.hidden_dim;
    76
    0
            let intermediate_dim = config.intermediate_dim;
    77
    0
            let vocab_size = config.vocab_size;
    78
    0
            let qkv_dim = hidden_dim * 3; // Max for fused QKV
    79
    80
            // PAR-126: Q8K uses 256-element super-blocks for VNNI path
    81
            const QK_K: usize = 256;
    82
    0
            let q8k_hidden_padded = hidden_dim.div_ceil(QK_K) * QK_K;
    83
    0
            let q8k_inter_padded = intermediate_dim.div_ceil(QK_K) * QK_K;
    84
    85
    0
            Self {
    86
    0
                hidden: vec![0.0; hidden_dim],
    87
    0
                normed: vec![0.0; hidden_dim],
    88
    0
                qkv: vec![0.0; qkv_dim],
    89
    0
                q: vec![0.0; hidden_dim],
    90
    0
                k: vec![0.0; hidden_dim],
    91
    0
                v: vec![0.0; hidden_dim],
    92
    0
                attn_out: vec![0.0; hidden_dim],
    93
    0
                attn_proj: vec![0.0; hidden_dim],
    94
    0
                ffn_up: vec![0.0; intermediate_dim],
    95
    0
                ffn_gate: vec![0.0; intermediate_dim],
    96
    0
                ffn_down: vec![0.0; hidden_dim],
    97
    0
                logits: vec![0.0; vocab_size],
    98
    0
                q8k_hidden_scales: vec![0.0f32; q8k_hidden_padded / QK_K],
    99
    0
                q8k_hidden_quants: vec![0i8; q8k_hidden_padded],
    100
    0
                q8k_inter_scales: vec![0.0f32; q8k_inter_padded / QK_K],
    101
    0
                q8k_inter_quants: vec![0i8; q8k_inter_padded],
    102
    0
            }
    103
    0
        }
    104
    105
        /// Reset all buffers to zero for a new forward pass
    106
        #[inline]
    107
    0
        pub fn reset(&mut self) {
    108
    0
            self.hidden.iter_mut().for_each(|x| *x = 0.0);
    109
    0
            self.normed.iter_mut().for_each(|x| *x = 0.0);
    110
    0
        }
    111
    }
    112
    113
    // ============================================================================
    114
    // OwnedInferenceScratchBuffer
    115
    // ============================================================================
    116
    117
    /// Pre-allocated scratch buffers for OwnedQuantizedModel forward passes
    118
    ///
    119
    /// Eliminates per-token allocations by reusing buffers across forward passes.
    120
    /// For Qwen2.5-0.5B with intermediate_dim=4864, this saves ~40KB per token.
    121
    ///
    122
    /// PAR-126: Added Q8K scratch buffers for fused Q4K×Q8K matmul path.
    123
    #[derive(Debug)]
    124
    pub struct OwnedInferenceScratchBuffer {
    125
        /// QKV output buffer [hidden_dim + 2*kv_dim]
    126
        pub qkv: Vec<f32>,
    127
        /// Attention output buffer [hidden_dim]
    128
        pub attn_out: Vec<f32>,
    129
        /// FFN up projection buffer [intermediate_dim]
    130
        pub ffn_up: Vec<f32>,
    131
        /// FFN gate projection buffer [intermediate_dim]
    132
        pub ffn_gate: Vec<f32>,
    133
        /// FFN down output buffer [hidden_dim]
    134
        pub ffn_down: Vec<f32>,
    135
        /// Expanded V buffer for first token GQA [hidden_dim]
    136
        pub expanded_v: Vec<f32>,
    137
        /// Logits buffer [vocab_size]
    138
        pub logits: Vec<f32>,
    139
        /// Q8 quantization scales scratch [num_blocks]
    140
        pub q8_scales: Vec<f32>,
    141
        /// Q8 quantization values scratch [num_blocks * 32]
    142
        pub q8_quants: Vec<i8>,
    143
        // PAR-126: Q8K scratch buffers for VNNI-accelerated matmul
    144
        /// Q8K scales for hidden-dim activations [hidden_dim/256]
    145
        pub q8k_hidden_scales: Vec<f32>,
    146
        /// Q8K quants for hidden-dim activations [hidden_dim]
    147
        pub q8k_hidden_quants: Vec<i8>,
    148
        /// Q8K scales for intermediate-dim activations [intermediate_dim/256]
    149
        pub q8k_inter_scales: Vec<f32>,
    150
        /// Q8K quants for intermediate-dim activations [intermediate_dim]
    151
        pub q8k_inter_quants: Vec<i8>,
    152
    }
    153
    154
    impl OwnedInferenceScratchBuffer {
    155
        /// Create scratch buffer from model config
    156
        #[must_use]
    157
    0
        pub fn from_config(config: &GGUFConfig) -> Self {
    158
    0
            let hidden_dim = config.hidden_dim;
    159
    0
            let num_kv_heads = config.num_kv_heads;
    160
    0
            let head_dim = hidden_dim / config.num_heads;
    161
    0
            let kv_dim = num_kv_heads * head_dim;
    162
    0
            let qkv_dim = hidden_dim + 2 * kv_dim;
    163
    0
            let intermediate_dim = hidden_dim * 6; // Conservative estimate
    164
    0
            let num_blocks = hidden_dim.div_ceil(32);
    165
    166
            const QK_K: usize = 256;
    167
    0
            let q8k_hidden_padded = hidden_dim.div_ceil(QK_K) * QK_K;
    168
    0
            let q8k_inter_padded = intermediate_dim.div_ceil(QK_K) * QK_K;
    169
    170
    0
            Self {
    171
    0
                qkv: vec![0.0f32; qkv_dim],
    172
    0
                attn_out: vec![0.0f32; hidden_dim],
    173
    0
                ffn_up: vec![0.0f32; intermediate_dim],
    174
    0
                ffn_gate: vec![0.0f32; intermediate_dim],
    175
    0
                ffn_down: vec![0.0f32; hidden_dim],
    176
    0
                expanded_v: vec![0.0f32; hidden_dim],
    177
    0
                logits: vec![0.0f32; config.vocab_size],
    178
    0
                q8_scales: vec![0.0f32; num_blocks],
    179
    0
                q8_quants: vec![0i8; num_blocks * 32],
    180
    0
                q8k_hidden_scales: vec![0.0f32; q8k_hidden_padded / QK_K],
    181
    0
                q8k_hidden_quants: vec![0i8; q8k_hidden_padded],
    182
    0
                q8k_inter_scales: vec![0.0f32; q8k_inter_padded / QK_K],
    183
    0
                q8k_inter_quants: vec![0i8; q8k_inter_padded],
    184
    0
            }
    185
    0
        }
    186
    187
        /// Reset all buffers (clear without deallocating)
    188
    0
        pub fn reset(&mut self) {
    189
    0
            self.qkv.clear();
    190
    0
            self.attn_out.clear();
    191
    0
            self.ffn_up.clear();
    192
    0
            self.ffn_gate.clear();
    193
    0
            self.ffn_down.clear();
    194
    0
            self.expanded_v.clear();
    195
    0
            self.logits.clear();
    196
    0
            self.q8_scales.clear();
    197
    0
            self.q8_quants.clear();
    198
    0
            self.q8k_hidden_scales.clear();
    199
    0
            self.q8k_hidden_quants.clear();
    200
    0
            self.q8k_inter_scales.clear();
    201
    0
            self.q8k_inter_quants.clear();
    202
    0
        }
    203
    }
    204
    205
    // ============================================================================
    206
    // ContiguousKVCache (PARITY-005)
    207
    // ============================================================================
    208
    209
    /// Contiguous KV cache with 64-byte cache line alignment (PARITY-005)
    210
    ///
    211
    /// This cache uses a single contiguous allocation for all K and V data,
    212
    /// aligned to 64-byte cache lines for optimal L2 cache performance.
    213
    ///
    214
    /// ## Memory Layout
    215
    ///
    216
    /// ```text
    217
    /// K cache: [layer_0][layer_1]...[layer_n] (all contiguous)
    218
    /// V cache: [layer_0][layer_1]...[layer_n] (all contiguous)
    219
    ///
    220
    /// Each layer: [pos_0][pos_1]...[pos_max_seq] where each pos is [hidden_dim]
    221
    /// ```
    222
    #[derive(Debug)]
    223
    pub struct ContiguousKVCache {
    224
        num_layers: usize,
    225
        hidden_dim: usize,
    226
        max_seq_len: usize,
    227
        seq_len: usize,
    228
        layer_stride: usize,
    229
        k_data: Vec<f32>,
    230
        v_data: Vec<f32>,
    231
    }
    232
    233
    impl ContiguousKVCache {
    234
        /// Create a new contiguous KV cache
    235
        #[must_use]
    236
    109
        pub fn new(num_layers: usize, hidden_dim: usize, max_seq_len: usize) -> Self {
    237
    109
            let raw_layer_size = max_seq_len * hidden_dim;
    238
    109
            let layer_stride = Self::align_to_cache_line(raw_layer_size);
    239
    109
            let total_size = num_layers * layer_stride;
    240
    241
    109
            Self {
    242
    109
                num_layers,
    243
    109
                hidden_dim,
    244
    109
                max_seq_len,
    245
    109
                seq_len: 0,
    246
    109
                layer_stride,
    247
    109
                k_data: vec![0.0f32; total_size],
    248
    109
                v_data: vec![0.0f32; total_size],
    249
    109
            }
    250
    109
        }
    251
    252
        #[inline]
    253
    109
        fn align_to_cache_line(size: usize) -> usize {
    254
    109
            let remainder = size % FLOATS_PER_CACHE_LINE;
    255
    109
            if remainder == 0 { size } else { size + FLOATS_PER_CACHE_LINE - remainder0 }
    256
    109
        }
    257
    258
        /// Create cache from model configuration
    259
        #[must_use]
    260
    0
        pub fn from_config(config: &GGUFConfig, max_seq_len: usize) -> Self {
    261
    0
            Self::new(config.num_layers, config.hidden_dim, max_seq_len)
    262
    0
        }
    263
    264
        /// Check if this cache has contiguous layout
    265
        #[must_use]
    266
    1
        pub const fn is_contiguous(&self) -> bool { true }
    267
    268
        /// Check if data is cache-line aligned
    269
        #[must_use]
    270
    5
        pub fn is_cache_aligned(&self) -> bool {
    271
    5
            self.layer_stride % FLOATS_PER_CACHE_LINE == 0
    272
    5
        }
    273
    274
        /// Get the layer stride
    275
        #[must_use]
    276
    5
        pub fn layer_stride(&self) -> usize { self.layer_stride }
    277
    278
        #[inline]
    279
    25.6k
        fn layer_offset(&self, layer: usize) -> usize { layer * self.layer_stride }
    280
    281
        /// Append K and V vectors for a single position to a layer's cache
    282
    25.6k
        pub fn append(&mut self, layer: usize, k: &[f32], v: &[f32]) {
    283
    25.6k
            if layer >= self.num_layers || self.seq_len >= self.max_seq_len { return0; }
    284
    25.6k
            let start = self.layer_offset(layer) + self.seq_len * self.hidden_dim;
    285
    25.6k
            let end = start + self.hidden_dim;
    286
    25.6k
            if end <= self.k_data.len() {
    287
    25.6k
                self.k_data[start..end].copy_from_slice(k);
    288
    25.6k
                self.v_data[start..end].copy_from_slice(v);
    289
    25.6k
            }0
    290
    25.6k
        }
    291
    292
        /// Advance the sequence position
    293
    6.40k
        pub fn advance(&mut self) {
    294
    6.40k
            if self.seq_len < self.max_seq_len { self.seq_len += 1; }0
    295
    6.40k
        }
    296
    297
        /// Get cached keys for a layer
    298
        #[must_use]
    299
    3
        pub fn get_k(&self, layer: usize) -> &[f32] {
    300
    3
            if layer >= self.num_layers { return &[]0; }
    301
    3
            let start = self.layer_offset(layer);
    302
    3
            &self.k_data[start..start + self.seq_len * self.hidden_dim]
    303
    3
        }
    304
    305
        /// Get cached values for a layer
    306
        #[must_use]
    307
    0
        pub fn get_v(&self, layer: usize) -> &[f32] {
    308
    0
            if layer >= self.num_layers { return &[]; }
    309
    0
            let start = self.layer_offset(layer);
    310
    0
            &self.v_data[start..start + self.seq_len * self.hidden_dim]
    311
    0
        }
    312
    313
        /// Get mutable cached keys for a layer
    314
    0
        pub fn get_k_mut(&mut self, layer: usize) -> &mut [f32] {
    315
    0
            if layer >= self.num_layers { return &mut []; }
    316
    0
            let start = self.layer_offset(layer);
    317
    0
            let len = self.seq_len * self.hidden_dim;
    318
    0
            &mut self.k_data[start..start + len]
    319
    0
        }
    320
    321
        /// Get mutable cached values for a layer
    322
    0
        pub fn get_v_mut(&mut self, layer: usize) -> &mut [f32] {
    323
    0
            if layer >= self.num_layers { return &mut []; }
    324
    0
            let start = self.layer_offset(layer);
    325
    0
            let len = self.seq_len * self.hidden_dim;
    326
    0
            &mut self.v_data[start..start + len]
    327
    0
        }
    328
    329
        /// Current sequence length
    330
        #[must_use]
    331
    4
        pub fn len(&self) -> usize { self.seq_len }
    332
    333
        /// Check if cache is empty
    334
        #[must_use]
    335
    0
        pub fn is_empty(&self) -> bool { self.seq_len == 0 }
    336
    337
        /// Reset cache for new generation
    338
    1
        pub fn reset(&mut self) { self.seq_len = 0; }
    339
    340
        /// Reset cache and zero all data
    341
    0
        pub fn reset_and_zero(&mut self) {
    342
    0
            self.seq_len = 0;
    343
    0
            self.k_data.fill(0.0);
    344
    0
            self.v_data.fill(0.0);
    345
    0
        }
    346
    347
        /// Get maximum sequence length
    348
        #[must_use]
    349
    0
        pub fn max_len(&self) -> usize { self.max_seq_len }
    350
    351
        /// Get total memory usage in bytes
    352
        #[must_use]
    353
    1
        pub fn memory_bytes(&self) -> usize {
    354
    1
            (self.k_data.len() + self.v_data.len()) * std::mem::size_of::<f32>()
    355
    1
        }
    356
    357
        /// Prefetch K cache for a layer
    358
        #[inline]
    359
    0
        pub fn prefetch_k(&self, layer: usize) {
    360
    0
            if layer < self.num_layers {
    361
    0
                let _ = self.k_data.get(self.layer_offset(layer));
    362
    0
            }
    363
    0
        }
    364
    365
        /// Prefetch V cache for a layer
    366
        #[inline]
    367
    0
        pub fn prefetch_v(&self, layer: usize) {
    368
    0
            if layer < self.num_layers {
    369
    0
                let _ = self.v_data.get(self.layer_offset(layer));
    370
    0
            }
    371
    0
        }
    372
    }
    373
    374
    // ============================================================================
    375
    // DispatchMetrics (IMP-123)
    376
    // ============================================================================
    377
    378
    /// Thread-safe metrics for tracking CPU vs GPU dispatch decisions
    379
    ///
    380
    /// Tracks dispatch counts and latency histograms for performance analysis.
    381
    #[derive(Debug)]
    382
    pub struct DispatchMetrics {
    383
        cpu_dispatches: std::sync::atomic::AtomicUsize,
    384
        gpu_dispatches: std::sync::atomic::AtomicUsize,
    385
        cpu_latency_count: std::sync::atomic::AtomicUsize,
    386
        cpu_latency_sum_us: std::sync::atomic::AtomicU64,
    387
        gpu_latency_count: std::sync::atomic::AtomicUsize,
    388
        gpu_latency_sum_us: std::sync::atomic::AtomicU64,
    389
        cpu_latency_buckets: [std::sync::atomic::AtomicUsize; 5],
    390
        gpu_latency_buckets: [std::sync::atomic::AtomicUsize; 5],
    391
        cpu_latency_min_us: std::sync::atomic::AtomicU64,
    392
        cpu_latency_max_us: std::sync::atomic::AtomicU64,
    393
        gpu_latency_min_us: std::sync::atomic::AtomicU64,
    394
        gpu_latency_max_us: std::sync::atomic::AtomicU64,
    395
        cpu_latency_sum_sq_us: std::sync::atomic::AtomicU64,
    396
        gpu_latency_sum_sq_us: std::sync::atomic::AtomicU64,
    397
        start_time_ms: std::sync::atomic::AtomicU64,
    398
    }
    399
    400
    impl DispatchMetrics {
    401
        /// Histogram bucket boundaries in microseconds
    402
        pub const BUCKET_BOUNDARIES: [u64; 4] = [100, 500, 1000, 5000];
    403
    404
        /// Create new metrics tracker
    405
        #[must_use]
    406
    78
        pub fn new() -> Self {
    407
            Self {
    408
    78
                cpu_dispatches: std::sync::atomic::AtomicUsize::new(0),
    409
    78
                gpu_dispatches: std::sync::atomic::AtomicUsize::new(0),
    410
    78
                cpu_latency_count: std::sync::atomic::AtomicUsize::new(0),
    411
    78
                cpu_latency_sum_us: std::sync::atomic::AtomicU64::new(0),
    412
    78
                gpu_latency_count: std::sync::atomic::AtomicUsize::new(0),
    413
    78
                gpu_latency_sum_us: std::sync::atomic::AtomicU64::new(0),
    414
    78
                cpu_latency_buckets: [
    415
    78
                    std::sync::atomic::AtomicUsize::new(0),
    416
    78
                    std::sync::atomic::AtomicUsize::new(0),
    417
    78
                    std::sync::atomic::AtomicUsize::new(0),
    418
    78
                    std::sync::atomic::AtomicUsize::new(0),
    419
    78
                    std::sync::atomic::AtomicUsize::new(0),
    420
    78
                ],
    421
    78
                gpu_latency_buckets: [
    422
    78
                    std::sync::atomic::AtomicUsize::new(0),
    423
    78
                    std::sync::atomic::AtomicUsize::new(0),
    424
    78
                    std::sync::atomic::AtomicUsize::new(0),
    425
    78
                    std::sync::atomic::AtomicUsize::new(0),
    426
    78
                    std::sync::atomic::AtomicUsize::new(0),
    427
    78
                ],
    428
    78
                cpu_latency_min_us: std::sync::atomic::AtomicU64::new(u64::MAX),
    429
    78
                cpu_latency_max_us: std::sync::atomic::AtomicU64::new(0),
    430
    78
                gpu_latency_min_us: std::sync::atomic::AtomicU64::new(u64::MAX),
    431
    78
                gpu_latency_max_us: std::sync::atomic::AtomicU64::new(0),
    432
    78
                cpu_latency_sum_sq_us: std::sync::atomic::AtomicU64::new(0),
    433
    78
                gpu_latency_sum_sq_us: std::sync::atomic::AtomicU64::new(0),
    434
    78
                start_time_ms: std::sync::atomic::AtomicU64::new(
    435
    78
                    std::time::SystemTime::now()
    436
    78
                        .duration_since(std::time::UNIX_EPOCH)
    437
    78
                        .map(|d| d.as_millis() as u64)
    438
    78
                        .unwrap_or(0),
    439
                ),
    440
            }
    441
    78
        }
    442
    443
    938
        fn bucket_index(latency_us: u64) -> usize {
    444
    1.61k
            for (i, &boundary) in Self::BUCKET_BOUNDARIES938.iter938().enumerate938() {
    445
    1.61k
                if latency_us < boundary { return i874; }743
    446
            }
    447
    64
            4
    448
    938
        }
    449
    450
        /// Record a CPU dispatch
    451
    629
        pub fn record_cpu_dispatch(&self) {
    452
    629
            self.cpu_dispatches.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
    453
    629
        }
    454
    455
        /// Record a GPU dispatch
    456
    272
        pub fn record_gpu_dispatch(&self) {
    457
    272
            self.gpu_dispatches.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
    458
    272
        }
    459
    460
        /// Record CPU dispatch latency
    461
    653
        pub fn record_cpu_latency(&self, latency: std::time::Duration) {
    462
    653
            let latency_us = latency.as_micros() as u64;
    463
    653
            self.cpu_latency_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
    464
    653
            self.cpu_latency_sum_us.fetch_add(latency_us, std::sync::atomic::Ordering::Relaxed);
    465
    653
            self.cpu_latency_buckets[Self::bucket_index(latency_us)].fetch_add(1, std::sync::atomic::Ordering::Relaxed);
    466
    653
            self.cpu_latency_min_us.fetch_min(latency_us, std::sync::atomic::Ordering::Relaxed);
    467
    653
            self.cpu_latency_max_us.fetch_max(latency_us, std::sync::atomic::Ordering::Relaxed);
    468
    653
            self.cpu_latency_sum_sq_us.fetch_add(latency_us * latency_us, std::sync::atomic::Ordering::Relaxed);
    469
    653
        }
    470
    471
        /// Record GPU dispatch latency
    472
    285
        pub fn record_gpu_latency(&self, latency: std::time::Duration) {
    473
    285
            let latency_us = latency.as_micros() as u64;
    474
    285
            self.gpu_latency_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
    475
    285
            self.gpu_latency_sum_us.fetch_add(latency_us, std::sync::atomic::Ordering::Relaxed);
    476
    285
            self.gpu_latency_buckets[Self::bucket_index(latency_us)].fetch_add(1, std::sync::atomic::Ordering::Relaxed);
    477
    285
            self.gpu_latency_min_us.fetch_min(latency_us, std::sync::atomic::Ordering::Relaxed);
    478
    285
            self.gpu_latency_max_us.fetch_max(latency_us, std::sync::atomic::Ordering::Relaxed);
    479
    285
            self.gpu_latency_sum_sq_us.fetch_add(latency_us * latency_us, std::sync::atomic::Ordering::Relaxed);
    480
    285
        }
    481
    482
        /// Get CPU dispatch count
    483
        #[must_use]
    484
    78
        pub fn cpu_dispatches(&self) -> usize {
    485
    78
            self.cpu_dispatches.load(std::sync::atomic::Ordering::Relaxed)
    486
    78
        }
    487
    488
        /// Get GPU dispatch count
    489
        #[must_use]
    490
    84
        pub fn gpu_dispatches(&self) -> usize {
    491
    84
            self.gpu_dispatches.load(std::sync::atomic::Ordering::Relaxed)
    492
    84
        }
    493
    494
        /// Get total dispatches
    495
        #[must_use]
    496
    46
        pub fn total_dispatches(&self) -> usize {
    497
    46
            self.cpu_dispatches() + self.gpu_dispatches()
    498
    46
        }
    499
    500
        /// Get GPU dispatch ratio
    501
        #[must_use]
    502
    21
        pub fn gpu_ratio(&self) -> f64 {
    503
    21
            let total = self.total_dispatches();
    504
    21
            if total == 0 { 0.015 } else { self.gpu_dispatches() as f64 / total as f646 }
    505
    21
        }
    506
    507
        /// Get CPU latency count
    508
        #[must_use]
    509
    70
        pub fn cpu_latency_count(&self) -> usize {
    510
    70
            self.cpu_latency_count.load(std::sync::atomic::Ordering::Relaxed)
    511
    70
        }
    512
    513
        /// Get GPU latency count
    514
        #[must_use]
    515
    58
        pub fn gpu_latency_count(&self) -> usize {
    516
    58
            self.gpu_latency_count.load(std::sync::atomic::Ordering::Relaxed)
    517
    58
        }
    518
    519
        /// Get mean CPU latency in microseconds
    520
        #[must_use]
    521
    16
        pub fn cpu_latency_mean_us(&self) -> f64 {
    522
    16
            let count = self.cpu_latency_count();
    523
    16
            if count == 0 { 0.08 } else {
    524
    8
                self.cpu_latency_sum_us.load(std::sync::atomic::Ordering::Relaxed) as f64 / count as f64
    525
            }
    526
    16
        }
    527
    528
        /// Get mean GPU latency in microseconds
    529
        #[must_use]
    530
    14
        pub fn gpu_latency_mean_us(&self) -> f64 {
    531
    14
            let count = self.gpu_latency_count();
    532
    14
            if count == 0 { 0.09 } else {
    533
    5
                self.gpu_latency_sum_us.load(std::sync::atomic::Ordering::Relaxed) as f64 / count as f64
    534
            }
    535
    14
        }
    536
    537
        /// Get CPU latency sum
    538
        #[must_use]
    539
    10
        pub fn cpu_latency_sum_us(&self) -> u64 {
    540
    10
            self.cpu_latency_sum_us.load(std::sync::atomic::Ordering::Relaxed)
    541
    10
        }
    542
    543
        /// Get GPU latency sum
    544
        #[must_use]
    545
    10
        pub fn gpu_latency_sum_us(&self) -> u64 {
    546
    10
            self.gpu_latency_sum_us.load(std::sync::atomic::Ordering::Relaxed)
    547
    10
        }
    548
    549
        /// Get CPU latency min
    550
        #[must_use]
    551
    12
        pub fn cpu_latency_min_us(&self) -> u64 {
    552
    12
            if self.cpu_latency_count() == 0 { 07 } else {
    553
    5
                self.cpu_latency_min_us.load(std::sync::atomic::Ordering::Relaxed)
    554
            }
    555
    12
        }
    556
    557
        /// Get CPU latency max
    558
        #[must_use]
    559
    12
        pub fn cpu_latency_max_us(&self) -> u64 {
    560
    12
            self.cpu_latency_max_us.load(std::sync::atomic::Ordering::Relaxed)
    561
    12
        }
    562
    563
        /// Get GPU latency min
    564
        #[must_use]
    565
    10
        pub fn gpu_latency_min_us(&self) -> u64 {
    566
    10
            if self.gpu_latency_count() == 0 { 08 } else {
    567
    2
                self.gpu_latency_min_us.load(std::sync::atomic::Ordering::Relaxed)
    568
            }
    569
    10
        }
    570
    571
        /// Get GPU latency max
    572
        #[must_use]
    573
    10
        pub fn gpu_latency_max_us(&self) -> u64 {
    574
    10
            self.gpu_latency_max_us.load(std::sync::atomic::Ordering::Relaxed)
    575
    10
        }
    576
    577
        /// Get CPU latency variance
    578
        #[must_use]
    579
    25
        pub fn cpu_latency_variance_us(&self) -> f64 {
    580
    25
            let count = self.cpu_latency_count();
    581
    25
            if count < 2 { return 0.018; }7
    582
    7
            let sum = self.cpu_latency_sum_us.load(std::sync::atomic::Ordering::Relaxed) as f64;
    583
    7
            let sum_sq = self.cpu_latency_sum_sq_us.load(std::sync::atomic::Ordering::Relaxed) as f64;
    584
    7
            let n = count as f64;
    585
    7
            (sum_sq / n) - (sum / n).powi(2)
    586
    25
        }
    587
    588
        /// Get CPU latency stddev
    589
        #[must_use]
    590
    13
        pub fn cpu_latency_stddev_us(&self) -> f64 { self.cpu_latency_variance_us().sqrt() }
    591
    592
        /// Get GPU latency variance
    593
        #[must_use]
    594
    19
        pub fn gpu_latency_variance_us(&self) -> f64 {
    595
    19
            let count = self.gpu_latency_count();
    596
    19
            if count < 2 { return 0.016; }3
    597
    3
            let sum = self.gpu_latency_sum_us.load(std::sync::atomic::Ordering::Relaxed) as f64;
    598
    3
            let sum_sq = self.gpu_latency_sum_sq_us.load(std::sync::atomic::Ordering::Relaxed) as f64;
    599
    3
            let n = count as f64;
    600
    3
            (sum_sq / n) - (sum / n).powi(2)
    601
    19
        }
    602
    603
        /// Get GPU latency stddev
    604
        #[must_use]
    605
    10
        pub fn gpu_latency_stddev_us(&self) -> f64 { self.gpu_latency_variance_us().sqrt() }
    606
    607
        /// Get CPU latency histogram buckets
    608
        #[must_use]
    609
    50
        pub fn cpu_latency_buckets(&self) -> [usize; 5] {
    610
    50
            [
    611
    50
                self.cpu_latency_buckets[0].load(std::sync::atomic::Ordering::Relaxed),
    612
    50
                self.cpu_latency_buckets[1].load(std::sync::atomic::Ordering::Relaxed),
    613
    50
                self.cpu_latency_buckets[2].load(std::sync::atomic::Ordering::Relaxed),
    614
    50
                self.cpu_latency_buckets[3].load(std::sync::atomic::Ordering::Relaxed),
    615
    50
                self.cpu_latency_buckets[4].load(std::sync::atomic::Ordering::Relaxed),
    616
    50
            ]
    617
    50
        }
    618
    619
        /// Get GPU latency histogram buckets
    620
        #[must_use]
    621
    43
        pub fn gpu_latency_buckets(&self) -> [usize; 5] {
    622
    43
            [
    623
    43
                self.gpu_latency_buckets[0].load(std::sync::atomic::Ordering::Relaxed),
    624
    43
                self.gpu_latency_buckets[1].load(std::sync::atomic::Ordering::Relaxed),
    625
    43
                self.gpu_latency_buckets[2].load(std::sync::atomic::Ordering::Relaxed),
    626
    43
                self.gpu_latency_buckets[3].load(std::sync::atomic::Ordering::Relaxed),
    627
    43
                self.gpu_latency_buckets[4].load(std::sync::atomic::Ordering::Relaxed),
    628
    43
            ]
    629
    43
        }
    630
    631
    54
        fn estimate_percentile_from_buckets(buckets: &[usize; 5], percentile: f64) -> f64 {
    632
            const BUCKET_UPPER_BOUNDS: [f64; 5] = [100.0, 500.0, 1000.0, 5000.0, 10000.0];
    633
            const BUCKET_LOWER_BOUNDS: [f64; 5] = [0.0, 100.0, 500.0, 1000.0, 5000.0];
    634
    54
            let total: usize = buckets.iter().sum();
    635
    54
            if total == 0 { return 0.037; }17
    636
    17
            let target_rank = (percentile / 100.0) * total as f64;
    637
    17
            let mut cumulative: f64 = 0.0;
    638
    35
            for (i, &count) in buckets17.iter17().enumerate17() {
    639
    35
                let prev_cumulative = cumulative;
    640
    35
                cumulative += count as f64;
    641
    35
                if cumulative >= target_rank {
    642
    17
                    if count == 0 { return BUCKET_LOWER_BOUNDS[i]0; }
    643
    17
                    let fraction = (target_rank - prev_cumulative) / count as f64;
    644
    17
                    return BUCKET_LOWER_BOUNDS[i] + fraction * (BUCKET_UPPER_BOUNDS[i] - BUCKET_LOWER_BOUNDS[i]);
    645
    18
                }
    646
            }
    647
    0
            BUCKET_UPPER_BOUNDS[4]
    648
    54
        }
    649
    650
        /// Get CPU p50 latency
    651
        #[must_use]
    652
    10
        pub fn cpu_latency_p50_us(&self) -> f64 { Self::estimate_percentile_from_buckets(&self.cpu_latency_buckets(), 50.0) }
    653
    654
        /// Get CPU p95 latency
    655
        #[must_use]
    656
    10
        pub fn cpu_latency_p95_us(&self) -> f64 { Self::estimate_percentile_from_buckets(&self.cpu_latency_buckets(), 95.0) }
    657
    658
        /// Get CPU p99 latency
    659
        #[must_use]
    660
    9
        pub fn cpu_latency_p99_us(&self) -> f64 { Self::estimate_percentile_from_buckets(&self.cpu_latency_buckets(), 99.0) }
    661
    662
        /// Get GPU p50 latency
    663
        #[must_use]
    664
    9
        pub fn gpu_latency_p50_us(&self) -> f64 { Self::estimate_percentile_from_buckets(&self.gpu_latency_buckets(), 50.0) }
    665
    666
        /// Get GPU p95 latency
    667
        #[must_use]
    668
    8
        pub fn gpu_latency_p95_us(&self) -> f64 { Self::estimate_percentile_from_buckets(&self.gpu_latency_buckets(), 95.0) }
    669
    670
        /// Get GPU p99 latency
    671
        #[must_use]
    672
    8
        pub fn gpu_latency_p99_us(&self) -> f64 { Self::estimate_percentile_from_buckets(&self.gpu_latency_buckets(), 99.0) }
    673
    674
        /// Get bucket boundaries as strings
    675
        #[must_use]
    676
    9
        pub fn bucket_boundaries_us(&self) -> Vec<String> {
    677
    9
            vec![
    678
    9
                format!("0-{}", Self::BUCKET_BOUNDARIES[0]),
    679
    9
                format!("{}-{}", Self::BUCKET_BOUNDARIES[0], Self::BUCKET_BOUNDARIES[1]),
    680
    9
                format!("{}-{}", Self::BUCKET_BOUNDARIES[1], Self::BUCKET_BOUNDARIES[2]),
    681
    9
                format!("{}-{}", Self::BUCKET_BOUNDARIES[2], Self::BUCKET_BOUNDARIES[3]),
    682
    9
                format!("{}+", Self::BUCKET_BOUNDARIES[3]),
    683
            ]
    684
    9
        }
    685
    686
        /// Get start time
    687
        #[must_use]
    688
    37
        pub fn start_time_ms(&self) -> u64 {
    689
    37
            self.start_time_ms.load(std::sync::atomic::Ordering::Relaxed)
    690
    37
        }
    691
    692
        /// Get elapsed seconds
    693
        #[must_use]
    694
    36
        pub fn elapsed_seconds(&self) -> f64 {
    695
    36
            let start = self.start_time_ms();
    696
    36
            let now = std::time::SystemTime::now()
    697
    36
                .duration_since(std::time::UNIX_EPOCH)
    698
    36
                .map(|d| d.as_millis() as u64)
    699
    36
                .unwrap_or(0);
    700
    36
            (now.saturating_sub(start)) as f64 / 1000.0
    701
    36
        }
    702
    703
        /// Get throughput
    704
        #[must_use]
    705
    18
        pub fn throughput_rps(&self) -> f64 {
    706
    18
            let elapsed = self.elapsed_seconds();
    707
    18
            if elapsed < 0.001 { 0.011 } else { self.total_dispatches() as f64 / elapsed7 }
    708
    18
        }
    709
    710
        /// Get CPU latency CV
    711
        #[must_use]
    712
    1
        pub fn cpu_latency_cv(&self) -> f64 {
    713
    1
            let mean = self.cpu_latency_mean_us();
    714
    1
            if mean < 0.001 { 0.00 } else { (self.cpu_latency_stddev_us() / mean) * 100.0 }
    715
    1
        }
    716
    717
        /// Get GPU latency CV
    718
        #[must_use]
    719
    1
        pub fn gpu_latency_cv(&self) -> f64 {
    720
    1
            let mean = self.gpu_latency_mean_us();
    721
    1
            if mean < 0.001 { 0.00 } else { (self.gpu_latency_stddev_us() / mean) * 100.0 }
    722
    1
        }
    723
    724
        /// Get CPU/GPU speedup
    725
        #[must_use]
    726
    2
        pub fn cpu_gpu_speedup(&self) -> f64 {
    727
    2
            let gpu_mean = self.gpu_latency_mean_us();
    728
    2
            if gpu_mean < 0.001 { 0.01 } else { self1.cpu_latency_mean_us() / gpu_mean }
    729
    2
        }
    730
    731
        /// Reset all metrics
    732
    5
        pub fn reset(&self) {
    733
    5
            self.cpu_dispatches.store(0, std::sync::atomic::Ordering::Relaxed);
    734
    5
            self.gpu_dispatches.store(0, std::sync::atomic::Ordering::Relaxed);
    735
    5
            self.cpu_latency_count.store(0, std::sync::atomic::Ordering::Relaxed);
    736
    5
            self.cpu_latency_sum_us.store(0, std::sync::atomic::Ordering::Relaxed);
    737
    5
            self.gpu_latency_count.store(0, std::sync::atomic::Ordering::Relaxed);
    738
    5
            self.gpu_latency_sum_us.store(0, std::sync::atomic::Ordering::Relaxed);
    739
    5
            self.cpu_latency_min_us.store(u64::MAX, std::sync::atomic::Ordering::Relaxed);
    740
    5
            self.cpu_latency_max_us.store(0, std::sync::atomic::Ordering::Relaxed);
    741
    5
            self.gpu_latency_min_us.store(u64::MAX, std::sync::atomic::Ordering::Relaxed);
    742
    5
            self.gpu_latency_max_us.store(0, std::sync::atomic::Ordering::Relaxed);
    743
    5
            self.cpu_latency_sum_sq_us.store(0, std::sync::atomic::Ordering::Relaxed);
    744
    5
            self.gpu_latency_sum_sq_us.store(0, std::sync::atomic::Ordering::Relaxed);
    745
    30
            for bucket25 in &self.cpu_latency_buckets {
    746
    25
                bucket.store(0, std::sync::atomic::Ordering::Relaxed);
    747
    25
            }
    748
    30
            for bucket25 in &self.gpu_latency_buckets {
    749
    25
                bucket.store(0, std::sync::atomic::Ordering::Relaxed);
    750
    25
            }
    751
    5
            let now = std::time::SystemTime::now()
    752
    5
                .duration_since(std::time::UNIX_EPOCH)
    753
    5
                .map(|d| d.as_millis() as u64)
    754
    5
                .unwrap_or(0);
    755
    5
            self.start_time_ms.store(now, std::sync::atomic::Ordering::Relaxed);
    756
    5
        }
    757
    }
    758
    759
    impl Default for DispatchMetrics {
    760
    0
        fn default() -> Self { Self::new() }
    761
    }