## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/gguf/model.rs

    Line
    Count
    Source
    1
    //! GGUF model types
    2
    //!
    3
    //! This module contains the core model structures for GGUF inference:
    4
    //!
    5
    //! - `MappedGGUFModel`: Memory-mapped GGUF file with zero-copy access
    6
    //! - `GGUFTransformer`: F32 transformer weights (dequantized from GGUF)
    7
    //! - `GGUFTransformerLayer`: Per-layer transformer weights
    8
    //! - `OwnedQuantizedModel`: Quantized model with owned weight data
    9
    //!
    10
    //! ## Design Philosophy
    11
    //!
    12
    //! Per Wulf & McKee (1995) "Hitting the Memory Wall", memory bandwidth is the
    13
    //! bottleneck for LLM inference. These types support:
    14
    //! - Zero-copy mmap loading (`MappedGGUFModel`)
    15
    //! - Quantized weights for 8x bandwidth reduction (`OwnedQuantizedModel`)
    16
    //! - Lazy dequantization during computation
    17
    18
    use std::fs::File;
    19
    use std::path::Path;
    20
    21
    use memmap2::Mmap;
    22
    23
    use super::config::GGUFConfig;
    24
    use super::quantized::{OwnedQuantizedLayer, OwnedQuantizedTensor};
    25
    use super::types::GGUFModel;
    26
    use crate::error::{RealizarError, Result};
    27
    28
    // ============================================================================
    29
    // MappedGGUFModel - Zero-copy memory-mapped model
    30
    // ============================================================================
    31
    32
    /// Memory-mapped GGUF model for zero-copy tensor access
    33
    ///
    34
    /// Uses `memmap2` for efficient large model loading without copying
    35
    /// entire file contents into heap memory.
    36
    ///
    37
    /// # Example
    38
    ///
    39
    /// ```rust,ignore
    40
    /// let model = MappedGGUFModel::from_path("phi-2-q4_k_m.gguf")?;
    41
    /// println!("Loaded {} tensors", model.model.tensors.len());
    42
    /// ```
    43
    pub struct MappedGGUFModel {
    44
        /// Parsed model metadata (header, tensors, etc.)
    45
        pub model: GGUFModel,
    46
        /// Memory-mapped file contents
    47
        pub(crate) mmap: Mmap,
    48
    }
    49
    50
    impl MappedGGUFModel {
    51
        /// Load GGUF model via memory mapping (zero-copy)
    52
        ///
    53
        /// # Arguments
    54
        ///
    55
        /// * `path` - Path to GGUF model file
    56
        ///
    57
        /// # Errors
    58
        ///
    59
        /// Returns error if:
    60
        /// - File cannot be opened
    61
        /// - Memory mapping fails
    62
        /// - GGUF parsing fails (invalid format)
    63
        ///
    64
        /// # Performance
    65
        ///
    66
        /// Memory-mapped loading is faster than `std::fs::read` for large models:
    67
        /// - No file content copy to heap memory
    68
        /// - Kernel handles page management
    69
        /// - Model remains accessible even if larger than RAM (via swap)
    70
        ///
    71
        /// # Examples
    72
        ///
    73
        /// ```rust,ignore
    74
        /// let model = MappedGGUFModel::from_path("phi-2-q4_k_m.gguf")?;
    75
        /// println!("Loaded {} tensors", model.model.tensors.len());
    76
        /// ```
    77
    6
        pub fn from_path<P: AsRef<Path>>(path: P) -> Result<Self> {
    78
    6
            let file2 = File::open(path.as_ref()).map_err(|e| RealizarError::UnsupportedOperation {
    79
    4
                operation: "open_model_file".to_string(),
    80
    4
                reason: format!("Failed to open {}: {}", path.as_ref().display(), e),
    81
    4
            })?;
    82
    83
            // SAFETY: Memory mapping is safe as long as the file isn't modified
    84
            // while mapped. We only read from the mapping, never write.
    85
    2
            let mmap = unsafe {
    86
    2
                Mmap::map(&file).map_err(|e| RealizarError::UnsupportedOperation {
    87
    0
                    operation: "mmap_model_file".to_string(),
    88
    0
                    reason: format!("Failed to mmap {}: {}", path.as_ref().display(), e),
    89
    0
                })?
    90
            };
    91
    92
            // Parse the memory-mapped data
    93
    2
            let model = GGUFModel::from_bytes(&mmap)?0;
    94
    95
    2
            Ok(Self { model, mmap })
    96
    6
        }
    97
    98
        /// Get the raw memory-mapped file data
    99
        ///
    100
        /// This provides direct access to the file contents without copying.
    101
        /// Use this with tensor offsets to read quantized weights directly.
    102
        #[must_use]
    103
    4
        pub fn data(&self) -> &[u8] {
    104
    4
            &self.mmap
    105
    4
        }
    106
    107
        /// Get tensor data slice by offset and size
    108
        ///
    109
        /// Returns a slice pointing directly into the memory-mapped file.
    110
        /// No data is copied.
    111
        ///
    112
        /// # Arguments
    113
        ///
    114
        /// * `offset` - Byte offset from start of file
    115
        /// * `size` - Size in bytes
    116
        ///
    117
        /// # Returns
    118
        ///
    119
        /// Slice of tensor data, or None if out of bounds
    120
        #[must_use]
    121
    0
        pub fn tensor_slice(&self, offset: usize, size: usize) -> Option<&[u8]> {
    122
    0
            let end = offset.checked_add(size)?;
    123
    0
            if end <= self.mmap.len() {
    124
    0
                Some(&self.mmap[offset..end])
    125
            } else {
    126
    0
                None
    127
            }
    128
    0
        }
    129
    130
        /// Get the size of the memory-mapped file
    131
        #[must_use]
    132
    0
        pub fn file_size(&self) -> usize {
    133
    0
            self.mmap.len()
    134
    0
        }
    135
    136
        /// Advise kernel to prefetch model data sequentially
    137
        ///
    138
        /// Per llama.cpp: Use madvise(MADV_SEQUENTIAL) to hint that the model
    139
        /// will be read sequentially during loading. This improves prefetching.
    140
        #[cfg(unix)]
    141
    0
        pub fn advise_sequential(&self) {
    142
            // SAFETY: Memory safety ensured by mmap lifetime
    143
    0
            unsafe {
    144
    0
                libc::madvise(
    145
    0
                    self.mmap.as_ptr().cast_mut().cast::<libc::c_void>(),
    146
    0
                    self.mmap.len(),
    147
    0
                    libc::MADV_SEQUENTIAL,
    148
    0
                );
    149
    0
            }
    150
    0
        }
    151
    152
        /// Advise kernel for random access pattern during inference
    153
        ///
    154
        /// Per llama.cpp: Use madvise(MADV_RANDOM) during inference when
    155
        /// accessing weights non-sequentially.
    156
        #[cfg(unix)]
    157
    0
        pub fn advise_random(&self) {
    158
            // SAFETY: Memory safety ensured by mmap lifetime
    159
    0
            unsafe {
    160
    0
                libc::madvise(
    161
    0
                    self.mmap.as_ptr().cast_mut().cast::<libc::c_void>(),
    162
    0
                    self.mmap.len(),
    163
    0
                    libc::MADV_RANDOM,
    164
    0
                );
    165
    0
            }
    166
    0
        }
    167
    168
        /// Advise kernel to keep model in memory (reduce swap pressure)
    169
        ///
    170
        /// Per llama.cpp: Use madvise(MADV_WILLNEED) to hint that the model
    171
        /// will be needed soon, triggering prefetch.
    172
        #[cfg(unix)]
    173
    0
        pub fn advise_willneed(&self) {
    174
            // SAFETY: Memory safety ensured by mmap lifetime
    175
    0
            unsafe {
    176
    0
                libc::madvise(
    177
    0
                    self.mmap.as_ptr().cast_mut().cast::<libc::c_void>(),
    178
    0
                    self.mmap.len(),
    179
    0
                    libc::MADV_WILLNEED,
    180
    0
                );
    181
    0
            }
    182
    0
        }
    183
    184
        /// Lock model in memory to prevent swapping (requires privileges)
    185
        ///
    186
        /// Per llama.cpp: Use mlock() to ensure model stays in RAM.
    187
        /// Returns true if successful, false if failed (often due to ulimit).
    188
        #[cfg(unix)]
    189
    0
        pub fn lock_memory(&self) -> bool {
    190
            // SAFETY: Memory safety ensured by mmap lifetime
    191
    0
            unsafe { libc::mlock(self.mmap.as_ptr().cast::<libc::c_void>(), self.mmap.len()) == 0 }
    192
    0
        }
    193
    }
    194
    195
    // ============================================================================
    196
    // GGUFTransformer - F32 transformer weights
    197
    // ============================================================================
    198
    199
    /// F32 transformer weights loaded from GGUF
    200
    ///
    201
    /// This struct holds dequantized weights in F32 format.
    202
    /// Used for reference implementations and debugging.
    203
    /// For production inference, use `OwnedQuantizedModel` instead.
    204
    pub struct GGUFTransformer {
    205
        /// Model configuration
    206
        pub config: GGUFConfig,
    207
        /// Token embedding weights [vocab_size, hidden_dim]
    208
        pub token_embedding: Vec<f32>,
    209
        /// Attention weights per layer
    210
        pub layers: Vec<GGUFTransformerLayer>,
    211
        /// Output norm weight
    212
        pub output_norm_weight: Vec<f32>,
    213
        /// Output norm bias (optional)
    214
        pub output_norm_bias: Option<Vec<f32>>,
    215
        /// LM head / output projection weight
    216
        pub lm_head_weight: Vec<f32>,
    217
        /// LM head bias (optional)
    218
        pub lm_head_bias: Option<Vec<f32>>,
    219
    }
    220
    221
    // ============================================================================
    222
    // GGUFTransformerLayer - Per-layer F32 weights
    223
    // ============================================================================
    224
    225
    /// Weights for a single transformer layer
    226
    pub struct GGUFTransformerLayer {
    227
        /// Attention norm weight
    228
        pub attn_norm_weight: Vec<f32>,
    229
        /// Attention norm bias
    230
        pub attn_norm_bias: Option<Vec<f32>>,
    231
        /// QKV projection weights (combined for phi-2, concatenated Q+K+V for llama)
    232
        pub qkv_weight: Vec<f32>,
    233
        /// QKV bias (phi-2 has bias, llama doesn't)
    234
        pub qkv_bias: Option<Vec<f32>>,
    235
        /// Attention output projection weight
    236
        pub attn_output_weight: Vec<f32>,
    237
        /// Attention output projection bias
    238
        pub attn_output_bias: Option<Vec<f32>>,
    239
        /// FFN gate projection weight (SwiGLU models like llama)
    240
        pub ffn_gate_weight: Option<Vec<f32>>,
    241
        /// FFN gate projection bias
    242
        pub ffn_gate_bias: Option<Vec<f32>>,
    243
        /// FFN up projection weight
    244
        pub ffn_up_weight: Vec<f32>,
    245
        /// FFN up projection bias
    246
        pub ffn_up_bias: Option<Vec<f32>>,
    247
        /// FFN down projection weight
    248
        pub ffn_down_weight: Vec<f32>,
    249
        /// FFN down projection bias
    250
        pub ffn_down_bias: Option<Vec<f32>>,
    251
        /// FFN norm weight (for models with separate FFN normalization)
    252
        pub ffn_norm_weight: Option<Vec<f32>>,
    253
        /// FFN norm bias
    254
        pub ffn_norm_bias: Option<Vec<f32>>,
    255
    }
    256
    257
    // ============================================================================
    258
    // OwnedQuantizedModel - Quantized model with owned data
    259
    // ============================================================================
    260
    261
    /// Owned quantized model with all weight data
    262
    ///
    263
    /// IMP-100: This struct owns all quantized weight data, allowing storage
    264
    /// in `Arc` without lifetime parameters. Essential for async handlers.
    265
    ///
    266
    /// # Memory Layout
    267
    ///
    268
    /// - Token embedding: F32 for fast lookup
    269
    /// - Layer weights: Quantized (Q4_K, Q6_K, etc.)
    270
    /// - Output norm: F32 (small)
    271
    /// - LM head: Quantized
    272
    ///
    273
    /// # GPU Acceleration
    274
    ///
    275
    /// When the `cuda` feature is enabled, this struct includes a
    276
    /// `CudaExecutor` for GPU-accelerated matmul operations.
    277
    pub struct OwnedQuantizedModel {
    278
        /// Model configuration
    279
        pub config: GGUFConfig,
    280
        /// Token embedding (f32 for fast lookup)
    281
        pub token_embedding: Vec<f32>,
    282
        /// Owned quantized layers
    283
        pub layers: Vec<OwnedQuantizedLayer>,
    284
        /// Output norm weight (f32)
    285
        pub output_norm_weight: Vec<f32>,
    286
        /// Output norm bias (optional)
    287
        pub output_norm_bias: Option<Vec<f32>>,
    288
        /// LM head weight (owned quantized)
    289
        pub lm_head_weight: OwnedQuantizedTensor,
    290
        /// LM head bias (optional, f32)
    291
        pub lm_head_bias: Option<Vec<f32>>,
    292
        /// PARITY-113: Optional CUDA executor for GPU acceleration
    293
        /// When present, fused_matmul routes to CUDA GEMM kernels
    294
        /// Uses Mutex for thread-safety in async handlers
    295
        #[cfg(feature = "cuda")]
    296
        pub(crate) cuda_executor: Option<std::sync::Mutex<crate::cuda::CudaExecutor>>,
    297
        /// Track CUDA kernel execution count for metrics
    298
        /// Uses AtomicU64 for thread-safe counting
    299
        #[cfg(feature = "cuda")]
    300
        pub(crate) cuda_kernel_count: std::sync::atomic::AtomicU64,
    301
        /// PARITY-003: Set of weight names that have been cached on GPU
    302
        /// Used to avoid repeated dequantization for the same weight
    303
        #[cfg(feature = "cuda")]
    304
        pub(crate) cached_weight_names: std::sync::Mutex<std::collections::HashSet<String>>,
    305
    }
    306
    307
    // Manual Debug implementation (skip CUDA executor which doesn't impl Debug)
    308
    impl std::fmt::Debug for OwnedQuantizedModel {
    309
    1
        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
    310
    1
            let mut s = f.debug_struct("OwnedQuantizedModel");
    311
    1
            s.field("config", &self.config)
    312
    1
                .field("token_embedding_len", &self.token_embedding.len())
    313
    1
                .field("layers_count", &self.layers.len())
    314
    1
                .field("output_norm_weight_len", &self.output_norm_weight.len())
    315
    1
                .field("has_output_norm_bias", &self.output_norm_bias.is_some())
    316
    1
                .field("lm_head_weight", &self.lm_head_weight)
    317
    1
                .field("has_lm_head_bias", &self.lm_head_bias.is_some());
    318
    319
            #[cfg(feature = "cuda")]
    320
            s.field("cuda_enabled", &self.cuda_executor.is_some())
    321
                .field(
    322
                    "cuda_kernel_count",
    323
                    &self
    324
                        .cuda_kernel_count
    325
                        .load(std::sync::atomic::Ordering::Relaxed),
    326
                )
    327
                .field(
    328
                    "cached_weight_count",
    329
                    &self
    330
                        .cached_weight_names
    331
                        .lock()
    332
                        .map(|g| g.len())
    333
                        .unwrap_or(0),
    334
                );
    335
    336
    1
            s.finish()
    337
    1
        }
    338
    }
    339
    340
    // Manual Clone implementation due to Mutex
    341
    impl Clone for OwnedQuantizedModel {
    342
    4
        fn clone(&self) -> Self {
    343
    4
            Self {
    344
    4
                config: self.config.clone(),
    345
    4
                token_embedding: self.token_embedding.clone(),
    346
    4
                layers: self.layers.clone(),
    347
    4
                output_norm_weight: self.output_norm_weight.clone(),
    348
    4
                output_norm_bias: self.output_norm_bias.clone(),
    349
    4
                lm_head_weight: self.lm_head_weight.clone(),
    350
    4
                lm_head_bias: self.lm_head_bias.clone(),
    351
    4
                // CUDA executor is not cloned - new instance must enable CUDA separately
    352
    4
                #[cfg(feature = "cuda")]
    353
    4
                cuda_executor: None,
    354
    4
                #[cfg(feature = "cuda")]
    355
    4
                cuda_kernel_count: std::sync::atomic::AtomicU64::new(0),
    356
    4
                #[cfg(feature = "cuda")]
    357
    4
                cached_weight_names: std::sync::Mutex::new(std::collections::HashSet::new()),
    358
    4
            }
    359
    4
        }
    360
    }
    361
    362
    #[cfg(test)]
    363
    mod tests {
    364
        use super::*;
    365
    366
        #[test]
    367
    1
        fn test_gguf_transformer_struct() {
    368
    1
            let config = GGUFConfig {
    369
    1
                architecture: "llama".to_string(),
    370
    1
                hidden_dim: 256,
    371
    1
                num_layers: 2,
    372
    1
                num_heads: 4,
    373
    1
                num_kv_heads: 4,
    374
    1
                vocab_size: 1000,
    375
    1
                intermediate_dim: 512,
    376
    1
                context_length: 512,
    377
    1
                rope_theta: 10000.0,
    378
    1
                eps: 1e-5,
    379
    1
                rope_type: 0,
    380
    1
            };
    381
    382
    1
            let transformer = GGUFTransformer {
    383
    1
                config,
    384
    1
                token_embedding: vec![0.0; 256000], // 1000 vocab * 256 hidden
    385
    1
                layers: vec![],
    386
    1
                output_norm_weight: vec![1.0; 256],
    387
    1
                output_norm_bias: None,
    388
    1
                lm_head_weight: vec![0.0; 256000],
    389
    1
                lm_head_bias: None,
    390
    1
            };
    391
    392
    1
            assert_eq!(transformer.config.architecture, "llama");
    393
    1
            assert_eq!(transformer.token_embedding.len(), 256000);
    394
    1
            assert!(transformer.layers.is_empty());
    395
    1
        }
    396
    397
        #[test]
    398
    1
        fn test_gguf_transformer_layer_struct() {
    399
    1
            let layer = GGUFTransformerLayer {
    400
    1
                attn_norm_weight: vec![1.0; 256],
    401
    1
                attn_norm_bias: None,
    402
    1
                qkv_weight: vec![0.0; 256 * 768], // 3 * hidden_dim
    403
    1
                qkv_bias: None,
    404
    1
                attn_output_weight: vec![0.0; 256 * 256],
    405
    1
                attn_output_bias: None,
    406
    1
                ffn_gate_weight: Some(vec![0.0; 256 * 512]),
    407
    1
                ffn_gate_bias: None,
    408
    1
                ffn_up_weight: vec![0.0; 256 * 512],
    409
    1
                ffn_up_bias: None,
    410
    1
                ffn_down_weight: vec![0.0; 512 * 256],
    411
    1
                ffn_down_bias: None,
    412
    1
                ffn_norm_weight: Some(vec![1.0; 256]),
    413
    1
                ffn_norm_bias: None,
    414
    1
            };
    415
    416
    1
            assert_eq!(layer.attn_norm_weight.len(), 256);
    417
    1
            assert!(layer.ffn_gate_weight.is_some());
    418
    1
            assert!(layer.ffn_norm_weight.is_some());
    419
    1
        }
    420
    421
        #[test]
    422
    1
        fn test_owned_quantized_model_clone() {
    423
            use super::super::quantized::OwnedQuantizedTensor;
    424
            use super::super::types::GGUF_TYPE_Q4_K;
    425
    426
    1
            let config = GGUFConfig {
    427
    1
                architecture: "test".to_string(),
    428
    1
                hidden_dim: 64,
    429
    1
                num_layers: 1,
    430
    1
                num_heads: 2,
    431
    1
                num_kv_heads: 2,
    432
    1
                vocab_size: 100,
    433
    1
                intermediate_dim: 128,
    434
    1
                context_length: 256,
    435
    1
                rope_theta: 10000.0,
    436
    1
                eps: 1e-5,
    437
    1
                rope_type: 0,
    438
    1
            };
    439
    440
    1
            let model = OwnedQuantizedModel {
    441
    1
                config,
    442
    1
                token_embedding: vec![0.1; 6400],
    443
    1
                layers: vec![],
    444
    1
                output_norm_weight: vec![1.0; 64],
    445
    1
                output_norm_bias: None,
    446
    1
                lm_head_weight: OwnedQuantizedTensor {
    447
    1
                    data: vec![0u8; 128],
    448
    1
                    in_dim: 64,
    449
    1
                    out_dim: 100,
    450
    1
                    qtype: GGUF_TYPE_Q4_K,
    451
    1
                },
    452
    1
                lm_head_bias: None,
    453
    1
                #[cfg(feature = "cuda")]
    454
    1
                cuda_executor: None,
    455
    1
                #[cfg(feature = "cuda")]
    456
    1
                cuda_kernel_count: std::sync::atomic::AtomicU64::new(5),
    457
    1
                #[cfg(feature = "cuda")]
    458
    1
                cached_weight_names: std::sync::Mutex::new(std::collections::HashSet::new()),
    459
    1
            };
    460
    461
    1
            let cloned = model.clone();
    462
    1
            assert_eq!(cloned.config.architecture, "test");
    463
    1
            assert_eq!(cloned.token_embedding.len(), 6400);
    464
    465
            // CUDA executor is not cloned
    466
            #[cfg(feature = "cuda")]
    467
            {
    468
                assert!(cloned.cuda_executor.is_none());
    469
                assert_eq!(
    470
                    cloned
    471
                        .cuda_kernel_count
    472
                        .load(std::sync::atomic::Ordering::Relaxed),
    473
                    0
    474
                );
    475
            }
    476
    1
        }
    477
    478
        #[test]
    479
    1
        fn test_owned_quantized_model_debug() {
    480
            use super::super::quantized::OwnedQuantizedTensor;
    481
            use super::super::types::GGUF_TYPE_Q4_K;
    482
    483
    1
            let config = GGUFConfig {
    484
    1
                architecture: "debug_test".to_string(),
    485
    1
                hidden_dim: 32,
    486
    1
                num_layers: 1,
    487
    1
                num_heads: 1,
    488
    1
                num_kv_heads: 1,
    489
    1
                vocab_size: 50,
    490
    1
                intermediate_dim: 64,
    491
    1
                context_length: 128,
    492
    1
                rope_theta: 10000.0,
    493
    1
                eps: 1e-5,
    494
    1
                rope_type: 0,
    495
    1
            };
    496
    497
    1
            let model = OwnedQuantizedModel {
    498
    1
                config,
    499
    1
                token_embedding: vec![0.0; 1600],
    500
    1
                layers: vec![],
    501
    1
                output_norm_weight: vec![1.0; 32],
    502
    1
                output_norm_bias: Some(vec![0.0; 32]),
    503
    1
                lm_head_weight: OwnedQuantizedTensor {
    504
    1
                    data: vec![],
    505
    1
                    in_dim: 32,
    506
    1
                    out_dim: 50,
    507
    1
                    qtype: GGUF_TYPE_Q4_K,
    508
    1
                },
    509
    1
                lm_head_bias: None,
    510
    1
                #[cfg(feature = "cuda")]
    511
    1
                cuda_executor: None,
    512
    1
                #[cfg(feature = "cuda")]
    513
    1
                cuda_kernel_count: std::sync::atomic::AtomicU64::new(0),
    514
    1
                #[cfg(feature = "cuda")]
    515
    1
                cached_weight_names: std::sync::Mutex::new(std::collections::HashSet::new()),
    516
    1
            };
    517
    518
    1
            let debug_str = format!("{:?}", model);
    519
    1
            assert!(debug_str.contains("debug_test"));
    520
    1
            assert!(debug_str.contains("token_embedding_len"));
    521
    1
            assert!(debug_str.contains("1600"));
    522
    1
        }
    523
    }