## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/gguf/runtime.rs

    Line
    Count
    Source
    1
    //! Runtime types for GGUF model inference
    2
    //!
    3
    //! This module contains types used during inference runtime:
    4
    //!
    5
    //! - `QuantizedGenerateConfig`: Configuration for text generation
    6
    //! - `OwnedQuantizedKVCache`: KV cache for incremental decoding
    7
    //!
    8
    //! These are "leaf nodes" in the dependency graph - they don't depend
    9
    //! on other complex types, making them easy to extract.
    10
    11
    use super::config::GGUFConfig;
    12
    13
    // ============================================================================
    14
    // QuantizedGenerateConfig - Generation parameters
    15
    // ============================================================================
    16
    17
    /// Configuration for quantized generation
    18
    ///
    19
    /// Per benchmark-model-runners-spec.md "What's Remaining" item 1:
    20
    /// End-to-end Q4_K inference with generation config.
    21
    #[derive(Debug, Clone)]
    22
    pub struct QuantizedGenerateConfig {
    23
        /// Maximum tokens to generate
    24
        pub max_tokens: usize,
    25
        /// Sampling temperature (0.0 = greedy)
    26
        pub temperature: f32,
    27
        /// Top-k sampling (1 = greedy)
    28
        pub top_k: usize,
    29
        /// Stop token IDs
    30
        pub stop_tokens: Vec<u32>,
    31
    }
    32
    33
    impl Default for QuantizedGenerateConfig {
    34
    3
        fn default() -> Self {
    35
    3
            Self {
    36
    3
                max_tokens: 64,
    37
    3
                temperature: 0.0,
    38
    3
                top_k: 1,
    39
    3
                stop_tokens: Vec::new(),
    40
    3
            }
    41
    3
        }
    42
    }
    43
    44
    impl QuantizedGenerateConfig {
    45
        /// Create config for deterministic (greedy) generation
    46
        #[must_use]
    47
    2
        pub fn deterministic(max_tokens: usize) -> Self {
    48
    2
            Self {
    49
    2
                max_tokens,
    50
    2
                temperature: 0.0,
    51
    2
                top_k: 1,
    52
    2
                stop_tokens: Vec::new(),
    53
    2
            }
    54
    2
        }
    55
    56
        /// Builder method to set max tokens
    57
        #[must_use]
    58
    1
        pub fn with_max_tokens(mut self, max_tokens: usize) -> Self {
    59
    1
            self.max_tokens = max_tokens;
    60
    1
            self
    61
    1
        }
    62
    63
        /// Builder method to set temperature
    64
        #[must_use]
    65
    1
        pub fn with_temperature(mut self, temperature: f32) -> Self {
    66
    1
            self.temperature = temperature;
    67
    1
            self
    68
    1
        }
    69
    70
        /// Builder method to set top_k
    71
        #[must_use]
    72
    1
        pub fn with_top_k(mut self, top_k: usize) -> Self {
    73
    1
            self.top_k = top_k;
    74
    1
            self
    75
    1
        }
    76
    77
        /// Builder method to set stop tokens
    78
        #[must_use]
    79
    1
        pub fn with_stop_tokens(mut self, stop_tokens: Vec<u32>) -> Self {
    80
    1
            self.stop_tokens = stop_tokens;
    81
    1
            self
    82
    1
        }
    83
    }
    84
    85
    // ============================================================================
    86
    // OwnedQuantizedKVCache - KV cache for incremental decoding
    87
    // ============================================================================
    88
    89
    /// KV Cache for OwnedQuantizedModel incremental decoding (IMP-101c)
    90
    ///
    91
    /// Stores Key and Value projections for all layers to enable O(n) per-token
    92
    /// decoding instead of O(n²). Reference: Spec Section 5.4 "Continuous Flow".
    93
    ///
    94
    /// Memory layout: [num_layers, seq_len, hidden_dim]
    95
    #[derive(Debug, Clone)]
    96
    pub struct OwnedQuantizedKVCache {
    97
        /// Number of transformer layers
    98
        num_layers: usize,
    99
        /// Hidden dimension (stored for future use)
    100
        _hidden_dim: usize,
    101
        /// Maximum sequence length
    102
        max_seq_len: usize,
    103
        /// Current sequence length (tokens processed)
    104
        seq_len: usize,
    105
        /// Key cache: [num_layers][seq_len][hidden_dim]
    106
        k_cache: Vec<Vec<f32>>,
    107
        /// Value cache: [num_layers][seq_len][hidden_dim]
    108
        v_cache: Vec<Vec<f32>>,
    109
    }
    110
    111
    /// PARITY-096: Default impl for std::mem::take optimization in batch_generate_gpu
    112
    impl Default for OwnedQuantizedKVCache {
    113
    1
        fn default() -> Self {
    114
    1
            Self {
    115
    1
                num_layers: 0,
    116
    1
                _hidden_dim: 0,
    117
    1
                max_seq_len: 0,
    118
    1
                seq_len: 0,
    119
    1
                k_cache: Vec::new(),
    120
    1
                v_cache: Vec::new(),
    121
    1
            }
    122
    1
        }
    123
    }
    124
    125
    impl OwnedQuantizedKVCache {
    126
        /// Create a new KV cache for the given model configuration
    127
        ///
    128
        /// # Arguments
    129
        /// * `num_layers` - Number of transformer layers
    130
        /// * `hidden_dim` - Hidden dimension (num_heads * head_dim)
    131
        /// * `max_seq_len` - Maximum sequence length to cache
    132
        #[must_use]
    133
    198
        pub fn new(num_layers: usize, hidden_dim: usize, max_seq_len: usize) -> Self {
    134
    198
            Self {
    135
    198
                num_layers,
    136
    198
                _hidden_dim: hidden_dim,
    137
    198
                max_seq_len,
    138
    198
                seq_len: 0,
    139
    198
                k_cache: vec![Vec::with_capacity(max_seq_len * hidden_dim); num_layers],
    140
    198
                v_cache: vec![Vec::with_capacity(max_seq_len * hidden_dim); num_layers],
    141
    198
            }
    142
    198
        }
    143
    144
        /// Create cache from model configuration
    145
        #[must_use]
    146
    37
        pub fn from_config(config: &GGUFConfig, max_seq_len: usize) -> Self {
    147
    37
            Self::new(config.num_layers, config.hidden_dim, max_seq_len)
    148
    37
        }
    149
    150
        /// Append K and V vectors for a single position to a layer's cache
    151
        ///
    152
        /// # Arguments
    153
        /// * `layer` - Layer index
    154
        /// * `k` - Key vector [hidden_dim]
    155
        /// * `v` - Value vector [hidden_dim]
    156
    719
        pub fn append(&mut self, layer: usize, k: &[f32], v: &[f32]) {
    157
    719
            if layer < self.num_layers && self.seq_len < self.max_seq_len {
    158
    719
                self.k_cache[layer].extend_from_slice(k);
    159
    719
                self.v_cache[layer].extend_from_slice(v);
    160
    719
            }0
    161
    719
        }
    162
    163
        /// Advance the sequence position after processing a token
    164
    704
        pub fn advance(&mut self) {
    165
    704
            if self.seq_len < self.max_seq_len {
    166
    704
                self.seq_len += 1;
    167
    704
            }0
    168
    704
        }
    169
    170
        /// PAR-097: Append multiple K/V entries to a layer's cache (for speculative decode)
    171
        ///
    172
        /// # Arguments
    173
        /// * `layer` - Layer index
    174
        /// * `k_all` - Key vectors for batch_size positions [batch_size × kv_dim]
    175
        /// * `v_all` - Value vectors for batch_size positions [batch_size × kv_dim]
    176
    25.6k
        pub fn append_kv(&mut self, layer: usize, k_all: &[f32], v_all: &[f32]) {
    177
    25.6k
            if layer < self.num_layers {
    178
    25.6k
                self.k_cache[layer].extend_from_slice(k_all);
    179
    25.6k
                self.v_cache[layer].extend_from_slice(v_all);
    180
    25.6k
            }0
    181
    25.6k
        }
    182
    183
        /// PAR-097: Advance sequence position by n tokens (for speculative decode)
    184
    0
        pub fn advance_by(&mut self, n: usize) {
    185
    0
            self.seq_len = (self.seq_len + n).min(self.max_seq_len);
    186
    0
        }
    187
    188
        /// PAR-098: Rollback cache to a previous position (for speculative decode rejection)
    189
        ///
    190
        /// When draft tokens are rejected, we need to remove their K/V entries.
    191
        /// This truncates each layer's cache to keep only the first `new_len` positions.
    192
        ///
    193
        /// # Arguments
    194
        /// * `new_len` - The new sequence length (must be <= current length)
    195
        /// * `kv_dim` - The dimension of each K/V entry (num_kv_heads * head_dim)
    196
    1
        pub fn rollback_to(&mut self, new_len: usize, kv_dim: usize) {
    197
    1
            if new_len >= self.seq_len {
    198
    0
                return; // Nothing to rollback
    199
    1
            }
    200
    1
            let target_size = new_len * kv_dim;
    201
    2
            for layer_k1 in &mut self.k_cache {
    202
    1
                layer_k.truncate(target_size);
    203
    1
            }
    204
    2
            for layer_v1 in &mut self.v_cache {
    205
    1
                layer_v.truncate(target_size);
    206
    1
            }
    207
    1
            self.seq_len = new_len;
    208
    1
        }
    209
    210
        /// PAR-098: Get a snapshot of current cache lengths for rollback
    211
        #[must_use]
    212
    0
        pub fn snapshot_len(&self) -> usize {
    213
    0
            self.seq_len
    214
    0
        }
    215
    216
        /// Get cached keys for a layer
    217
        ///
    218
        /// Returns slice of [seq_len, hidden_dim]
    219
        #[must_use]
    220
    636
        pub fn get_k(&self, layer: usize) -> &[f32] {
    221
    636
            if layer < self.num_layers {
    222
    636
                &self.k_cache[layer]
    223
            } else {
    224
    0
                &[]
    225
            }
    226
    636
        }
    227
    228
        /// Get cached values for a layer
    229
        ///
    230
        /// Returns slice of [seq_len, hidden_dim]
    231
        #[must_use]
    232
    631
        pub fn get_v(&self, layer: usize) -> &[f32] {
    233
    631
            if layer < self.num_layers {
    234
    631
                &self.v_cache[layer]
    235
            } else {
    236
    0
                &[]
    237
            }
    238
    631
        }
    239
    240
        /// Current sequence length
    241
        #[must_use]
    242
    14
        pub fn len(&self) -> usize {
    243
    14
            self.seq_len
    244
    14
        }
    245
    246
        /// Check if cache is empty
    247
        #[must_use]
    248
    6
        pub fn is_empty(&self) -> bool {
    249
    6
            self.seq_len == 0
    250
    6
        }
    251
    252
        /// Reset cache for new generation
    253
    3
        pub fn reset(&mut self) {
    254
    3
            self.seq_len = 0;
    255
    39
            for layer_k36 in &mut self.k_cache {
    256
    36
                layer_k.clear();
    257
    36
            }
    258
    39
            for layer_v36 in &mut self.v_cache {
    259
    36
                layer_v.clear();
    260
    36
            }
    261
    3
        }
    262
    263
        /// Get maximum sequence length
    264
        #[must_use]
    265
    5
        pub fn max_len(&self) -> usize {
    266
    5
            self.max_seq_len
    267
    5
        }
    268
    }
    269
    270
    #[cfg(test)]
    271
    mod tests {
    272
        use super::*;
    273
    274
        #[test]
    275
    1
        fn test_generate_config_default() {
    276
    1
            let config = QuantizedGenerateConfig::default();
    277
    1
            assert_eq!(config.max_tokens, 64);
    278
    1
            assert!((config.temperature - 0.0).abs() < f32::EPSILON);
    279
    1
            assert_eq!(config.top_k, 1);
    280
    1
            assert!(config.stop_tokens.is_empty());
    281
    1
        }
    282
    283
        #[test]
    284
    1
        fn test_generate_config_deterministic() {
    285
    1
            let config = QuantizedGenerateConfig::deterministic(128);
    286
    1
            assert_eq!(config.max_tokens, 128);
    287
    1
            assert!((config.temperature - 0.0).abs() < f32::EPSILON);
    288
    1
            assert_eq!(config.top_k, 1);
    289
    1
        }
    290
    291
        #[test]
    292
    1
        fn test_kv_cache_new() {
    293
    1
            let cache = OwnedQuantizedKVCache::new(4, 256, 512);
    294
    1
            assert_eq!(cache.len(), 0);
    295
    1
            assert!(cache.is_empty());
    296
    1
            assert_eq!(cache.max_len(), 512);
    297
    1
        }
    298
    299
        #[test]
    300
    1
        fn test_kv_cache_append_advance() {
    301
    1
            let mut cache = OwnedQuantizedKVCache::new(2, 4, 10);
    302
    303
    1
            let k = vec![1.0, 2.0, 3.0, 4.0];
    304
    1
            let v = vec![5.0, 6.0, 7.0, 8.0];
    305
    306
    1
            cache.append(0, &k, &v);
    307
    1
            cache.advance();
    308
    309
    1
            assert_eq!(cache.len(), 1);
    310
    1
            assert!(!cache.is_empty());
    311
    1
            assert_eq!(cache.get_k(0), &[1.0, 2.0, 3.0, 4.0]);
    312
    1
            assert_eq!(cache.get_v(0), &[5.0, 6.0, 7.0, 8.0]);
    313
    1
        }
    314
    315
        #[test]
    316
    1
        fn test_kv_cache_reset() {
    317
    1
            let mut cache = OwnedQuantizedKVCache::new(2, 4, 10);
    318
    319
    1
            let k = vec![1.0, 2.0, 3.0, 4.0];
    320
    1
            let v = vec![5.0, 6.0, 7.0, 8.0];
    321
    322
    1
            cache.append(0, &k, &v);
    323
    1
            cache.advance();
    324
    325
    1
            cache.reset();
    326
    327
    1
            assert_eq!(cache.len(), 0);
    328
    1
            assert!(cache.is_empty());
    329
    1
            assert!(cache.get_k(0).is_empty());
    330
    1
        }
    331
    332
        #[test]
    333
    1
        fn test_kv_cache_rollback() {
    334
    1
            let mut cache = OwnedQuantizedKVCache::new(1, 2, 10);
    335
    336
            // Append 3 tokens
    337
    4
            for i3 in 0..3 {
    338
    3
                let k = vec![i as f32, i as f32 + 0.5];
    339
    3
                let v = vec![i as f32 + 1.0, i as f32 + 1.5];
    340
    3
                cache.append(0, &k, &v);
    341
    3
                cache.advance();
    342
    3
            }
    343
    344
    1
            assert_eq!(cache.len(), 3);
    345
    1
            assert_eq!(cache.get_k(0).len(), 6); // 3 tokens * 2 dims
    346
    347
            // Rollback to position 1
    348
    1
            cache.rollback_to(1, 2);
    349
    350
    1
            assert_eq!(cache.len(), 1);
    351
    1
            assert_eq!(cache.get_k(0).len(), 2); // 1 token * 2 dims
    352
    1
        }
    353
    354
        #[test]
    355
    1
        fn test_kv_cache_from_config() {
    356
    1
            let config = GGUFConfig {
    357
    1
                architecture: "llama".to_string(),
    358
    1
                hidden_dim: 256,
    359
    1
                num_layers: 4,
    360
    1
                num_heads: 4,
    361
    1
                num_kv_heads: 4,
    362
    1
                vocab_size: 1000,
    363
    1
                intermediate_dim: 512,
    364
    1
                context_length: 2048,
    365
    1
                rope_theta: 10000.0,
    366
    1
                eps: 1e-5,
    367
    1
                rope_type: 0,
    368
    1
            };
    369
    370
    1
            let cache = OwnedQuantizedKVCache::from_config(&config, 512);
    371
    1
            assert_eq!(cache.max_len(), 512);
    372
    1
        }
    373
    374
        #[test]
    375
    1
        fn test_kv_cache_default() {
    376
    1
            let cache = OwnedQuantizedKVCache::default();
    377
    1
            assert_eq!(cache.len(), 0);
    378
    1
            assert_eq!(cache.max_len(), 0);
    379
    1
            assert!(cache.is_empty());
    380
    1
        }
    381
    }