## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/gguf/transformer.rs

    Line
    Count
    Source
    1
    //! Quantized GGUF transformer types
    2
    //!
    3
    //! This module contains the quantized transformer layer and model structures
    4
    //! that enable fused dequantization operations for memory-efficient inference.
    5
    6
    use crate::error::{RealizarError, Result};
    7
    use crate::quantize::QK_K;
    8
    9
    use super::config::GGUFConfig;
    10
    use super::quantized::{QKVWeights, QuantizedTensorRef};
    11
    use super::types::{
    12
        GGUFModel, GGUF_TYPE_F32, GGUF_TYPE_Q2_K, GGUF_TYPE_Q4_0, GGUF_TYPE_Q4_1, GGUF_TYPE_Q4_K,
    13
        GGUF_TYPE_Q5_0, GGUF_TYPE_Q5_K, GGUF_TYPE_Q6_K, GGUF_TYPE_Q8_0,
    14
    };
    15
    16
    /// Quantized transformer layer weights (stored as byte references)
    17
    ///
    18
    /// Unlike `GGUFTransformerLayer` which stores dequantized Vec<f32>,
    19
    /// this stores references to quantized data for fused operations.
    20
    pub struct QuantizedGGUFTransformerLayer {
    21
        /// Attention norm weight (kept as f32 - small, read once per token)
    22
        pub attn_norm_weight: Vec<f32>,
    23
        /// Attention norm bias (optional)
    24
        pub attn_norm_bias: Option<Vec<f32>>,
    25
        /// QKV projection weights (quantized) - supports fused or separate
    26
        pub qkv_weight: QKVWeights,
    27
        /// QKV bias (optional, f32)
    28
        pub qkv_bias: Option<Vec<f32>>,
    29
        /// Attention output projection (quantized)
    30
        pub attn_output_weight: QuantizedTensorRef,
    31
        /// Attention output bias (optional, f32)
    32
        pub attn_output_bias: Option<Vec<f32>>,
    33
        /// FFN up projection (quantized)
    34
        pub ffn_up_weight: QuantizedTensorRef,
    35
        /// FFN up bias (optional, f32)
    36
        pub ffn_up_bias: Option<Vec<f32>>,
    37
        /// FFN down projection (quantized)
    38
        pub ffn_down_weight: QuantizedTensorRef,
    39
        /// FFN down bias (optional, f32)
    40
        pub ffn_down_bias: Option<Vec<f32>>,
    41
        /// FFN gate projection (quantized, SwiGLU models like LLaMA)
    42
        pub ffn_gate_weight: Option<QuantizedTensorRef>,
    43
        /// FFN gate bias (optional, f32)
    44
        pub ffn_gate_bias: Option<Vec<f32>>,
    45
        /// FFN norm weight (pre-FFN layer norm, LLaMA-style)
    46
        pub ffn_norm_weight: Option<Vec<f32>>,
    47
        /// FFN norm bias (optional, f32)
    48
        pub ffn_norm_bias: Option<Vec<f32>>,
    49
    }
    50
    51
    /// Quantized GGUF Transformer for fused inference
    52
    ///
    53
    /// Per Williams et al. (2009) roofline model, LLM inference is memory-bound.
    54
    /// This transformer stores weights in quantized form and uses fused
    55
    /// dequant+dot operations to minimize memory bandwidth.
    56
    ///
    57
    /// # Performance Benefits
    58
    ///
    59
    /// - **8x bandwidth reduction** for Q4_K vs f32 (144 bytes vs 1024 bytes per 256 values)
    60
    /// - **Zero intermediate buffers** - dequantization happens inline with dot product
    61
    /// - **SIMD acceleration** - AVX2/FMA fused operations when available
    62
    /// - **Zero-copy loading** - weights stay in memory-mapped file
    63
    ///
    64
    /// # Architecture
    65
    ///
    66
    /// ```text
    67
    /// [Memory-mapped Q4_K bytes] → [fused_q4k_dot_simd] → [f32 result]
    68
    ///                               ↑
    69
    ///                         No intermediate Vec<f32>!
    70
    /// ```
    71
    pub struct QuantizedGGUFTransformer<'a> {
    72
        /// Model configuration
    73
        pub config: GGUFConfig,
    74
        /// Reference to memory-mapped file data
    75
        pub data: &'a [u8],
    76
        /// Token embedding (kept as f32 for lookup)
    77
        pub token_embedding: Vec<f32>,
    78
        /// Quantized layer weights
    79
        pub layers: Vec<QuantizedGGUFTransformerLayer>,
    80
        /// Output norm weight (f32)
    81
        pub output_norm_weight: Vec<f32>,
    82
        /// Output norm bias (optional)
    83
        pub output_norm_bias: Option<Vec<f32>>,
    84
        /// LM head weight (quantized for large vocab)
    85
        pub lm_head_weight: QuantizedTensorRef,
    86
        /// LM head bias (optional, f32)
    87
        pub lm_head_bias: Option<Vec<f32>>,
    88
    }
    89
    90
    impl<'a> QuantizedGGUFTransformer<'a> {
    91
        /// Load quantized transformer from memory-mapped GGUF model
    92
        ///
    93
        /// # Arguments
    94
        ///
    95
        /// * `model` - Parsed GGUF model metadata
    96
        /// * `data` - Memory-mapped file data (zero-copy)
    97
        ///
    98
        /// # Errors
    99
        ///
    100
        /// Returns error if required tensors are missing or have unsupported format
    101
    12
        pub fn from_gguf(model: &GGUFModel, data: &'a [u8]) -> Result<Self> {
    102
    12
            let config10 = GGUFConfig::from_gguf(model)?2;
    103
    104
            // Token embedding - keep as f32 for efficient lookup
    105
    10
            let token_embedding9 = model.get_tensor_f32("token_embd.weight", data)?1;
    106
    107
            // Load layers with quantized weight references
    108
    9
            let mut layers = Vec::with_capacity(config.num_layers);
    109
    10
            for layer_idx in 0..config.num_layers9 {
    110
    10
                let layer = Self::load_quantized_layer(model, data, layer_idx)?0;
    111
    10
                layers.push(layer);
    112
            }
    113
    114
            // Output norm - small, keep as f32
    115
    9
            let output_norm_weight = model.get_tensor_f32("output_norm.weight", data)?0;
    116
    9
            let output_norm_bias = model.get_tensor_f32("output_norm.bias", data).ok();
    117
    118
            // LM head - large, keep quantized
    119
            // Fall back to token_embd.weight for tied embeddings (Qwen2, some LLaMA variants)
    120
    9
            let lm_head_weight = Self::get_tensor_ref(model, data, "output.weight")
    121
    9
                .or_else(|_| Self::get_tensor_ref(model, data, "token_embd.weight"))?0;
    122
    9
            let lm_head_bias = model.get_tensor_f32("output.bias", data).ok();
    123
    124
    9
            Ok(Self {
    125
    9
                config,
    126
    9
                data,
    127
    9
                token_embedding,
    128
    9
                layers,
    129
    9
                output_norm_weight,
    130
    9
                output_norm_bias,
    131
    9
                lm_head_weight,
    132
    9
                lm_head_bias,
    133
    9
            })
    134
    12
        }
    135
    136
        /// Get tensor reference (offset + size + qtype) without dequantization
    137
    95
        fn get_tensor_ref(model: &GGUFModel, data: &[u8], name: &str) -> Result<QuantizedTensorRef> {
    138
    95
            let tensor76 = model
    139
    95
                .tensors
    140
    95
                .iter()
    141
    727
                .find95(|t| t.name == name)
    142
    95
                .ok_or_else(|| RealizarError::InvalidShape {
    143
    19
                    reason: format!("Tensor '{}' not found", name),
    144
    19
                })?;
    145
    146
    152
            let num_elements76: usize76 = tensor.dims.iter()76.map76(|&d| d as usize).product76();
    147
    76
            let offset = model.tensor_data_start + tensor.offset as usize;
    148
    149
            // Calculate byte size based on quantization type
    150
    76
            let byte_size = match tensor.qtype {
    151
    9
                GGUF_TYPE_F32 => num_elements * 4,
    152
                GGUF_TYPE_Q4_0 => {
    153
                    // Q4_0: 32 elements per block
    154
                    // Layout: 1×f16 scale (2 bytes) + 16 bytes (32×4-bit values) = 18 bytes
    155
                    const BLOCK_SIZE: usize = 32;
    156
                    const BLOCK_BYTES: usize = 18;
    157
    7
                    let num_blocks = num_elements.div_ceil(BLOCK_SIZE);
    158
    7
                    num_blocks * BLOCK_BYTES
    159
                },
    160
                GGUF_TYPE_Q8_0 => {
    161
                    const BLOCK_SIZE: usize = 32;
    162
                    const BLOCK_BYTES: usize = 34; // 2 (f16 scale) + 32 (i8 quants)
    163
    7
                    let num_blocks = num_elements.div_ceil(BLOCK_SIZE);
    164
    7
                    num_blocks * BLOCK_BYTES
    165
                },
    166
                GGUF_TYPE_Q2_K => {
    167
                    // Q2_K: 256 elements per super-block
    168
                    // Layout: 16 bytes scales + 64 bytes quants + 2 bytes d + 2 bytes dmin = 84 bytes
    169
                    const SUPER_BLOCK_BYTES: usize = 84;
    170
    0
                    let num_super_blocks = num_elements.div_ceil(QK_K);
    171
    0
                    num_super_blocks * SUPER_BLOCK_BYTES
    172
                },
    173
                GGUF_TYPE_Q4_1 => {
    174
                    // Q4_1: 32 elements per block
    175
                    // Layout: 1×f16 scale (2 bytes) + 1×f16 min (2 bytes) + 16 bytes (32×4-bit values) = 20 bytes
    176
                    const BLOCK_SIZE: usize = 32;
    177
                    const BLOCK_BYTES: usize = 20;
    178
    0
                    let num_blocks = num_elements.div_ceil(BLOCK_SIZE);
    179
    0
                    num_blocks * BLOCK_BYTES
    180
                },
    181
                GGUF_TYPE_Q5_0 => {
    182
                    // Q5_0: 32 elements per block
    183
                    // Layout: 1×f16 scale (2 bytes) + 4 bytes high bits + 16 bytes quants = 22 bytes
    184
                    const BLOCK_SIZE: usize = 32;
    185
                    const BLOCK_BYTES: usize = 22;
    186
    0
                    let num_blocks = num_elements.div_ceil(BLOCK_SIZE);
    187
    0
                    num_blocks * BLOCK_BYTES
    188
                },
    189
                GGUF_TYPE_Q4_K => {
    190
                    const SUPER_BLOCK_BYTES: usize = 144;
    191
    39
                    let num_super_blocks = num_elements.div_ceil(QK_K);
    192
    39
                    num_super_blocks * SUPER_BLOCK_BYTES
    193
                },
    194
                GGUF_TYPE_Q5_K => {
    195
                    const SUPER_BLOCK_BYTES: usize = 176;
    196
    7
                    let num_super_blocks = num_elements.div_ceil(QK_K);
    197
    7
                    num_super_blocks * SUPER_BLOCK_BYTES
    198
                },
    199
                GGUF_TYPE_Q6_K => {
    200
                    const SUPER_BLOCK_BYTES: usize = 210;
    201
    7
                    let num_super_blocks = num_elements.div_ceil(QK_K);
    202
    7
                    num_super_blocks * SUPER_BLOCK_BYTES
    203
                },
    204
                _ => {
    205
    0
                    return Err(RealizarError::UnsupportedOperation {
    206
    0
                        operation: "get_tensor_ref".to_string(),
    207
    0
                        reason: format!("Unsupported quantization type: {}", tensor.qtype),
    208
    0
                    });
    209
                },
    210
            };
    211
    212
            // PAR-058-RESOLVED: Validate byte size and auto-correct qtype if mismatch detected
    213
            // Some GGUF files have incorrect qtype in header (e.g., Q5_0 header but Q4_0 data)
    214
            // Detect this by checking if the calculated byte_size would exceed file bounds,
    215
            // and try alternative qtypes that match the actual data size.
    216
    76
            let (byte_size, actual_qtype) = {
    217
                // Try the claimed qtype first
    218
    76
                if offset + byte_size <= data.len() {
    219
    76
                    (byte_size, tensor.qtype)
    220
                } else {
    221
                    // Mismatch! Try to infer correct qtype from available data
    222
                    // This happens when GGUF header has wrong qtype (e.g., qwen2.5-coder-0.5b)
    223
    0
                    let avail = data.len().saturating_sub(offset);
    224
    225
                    // Try Q4_0 (18 bytes per 32 elements)
    226
    0
                    let q4_0_size = {
    227
                        const BLOCK_SIZE: usize = 32;
    228
                        const BLOCK_BYTES: usize = 18;
    229
    0
                        num_elements.div_ceil(BLOCK_SIZE) * BLOCK_BYTES
    230
                    };
    231
    0
                    if q4_0_size <= avail && q4_0_size > 0 {
    232
    0
                        eprintln!(
    233
    0
                            "[PAR-058-RESOLVED] Tensor '{}' qtype mismatch: header says {} but byte size suggests Q4_0. Using Q4_0.",
    234
                            name, tensor.qtype
    235
                        );
    236
    0
                        (q4_0_size, GGUF_TYPE_Q4_0)
    237
                    } else {
    238
                        // Try Q8_0 (34 bytes per 32 elements)
    239
    0
                        let q8_0_size = {
    240
                            const BLOCK_SIZE: usize = 32;
    241
                            const BLOCK_BYTES: usize = 34;
    242
    0
                            num_elements.div_ceil(BLOCK_SIZE) * BLOCK_BYTES
    243
                        };
    244
    0
                        if q8_0_size <= avail && q8_0_size > 0 {
    245
    0
                            eprintln!(
    246
    0
                                "[PAR-058-RESOLVED] Tensor '{}' qtype mismatch: header says {} but byte size suggests Q8_0. Using Q8_0.",
    247
                                name, tensor.qtype
    248
                            );
    249
    0
                            (q8_0_size, GGUF_TYPE_Q8_0)
    250
                        } else {
    251
                            // Fallback to original (will fail bounds check below)
    252
    0
                            (byte_size, tensor.qtype)
    253
                        }
    254
                    }
    255
                }
    256
            };
    257
    258
            // Validate bounds
    259
    76
            if offset + byte_size > data.len() {
    260
    0
                return Err(RealizarError::InvalidShape {
    261
    0
                    reason: format!(
    262
    0
                        "Tensor '{}' data range [{}, {}) exceeds file size {}",
    263
    0
                        name,
    264
    0
                        offset,
    265
    0
                        offset + byte_size,
    266
    0
                        data.len()
    267
    0
                    ),
    268
    0
                });
    269
    76
            }
    270
    271
    76
            Ok(QuantizedTensorRef {
    272
    76
                offset,
    273
    76
                byte_size,
    274
    76
                num_elements,
    275
    76
                qtype: actual_qtype, // PAR-058-RESOLVED: Use auto-corrected qtype
    276
    76
            })
    277
    95
        }
    278
    279
        /// Load a single quantized transformer layer
    280
    10
        fn load_quantized_layer(
    281
    10
            model: &GGUFModel,
    282
    10
            data: &[u8],
    283
    10
            layer_idx: usize,
    284
    10
        ) -> Result<QuantizedGGUFTransformerLayer> {
    285
    10
            let prefix = format!("blk.{}", layer_idx);
    286
    287
            // Attention norm - small, keep as f32
    288
    10
            let attn_norm_weight =
    289
    10
                model.get_tensor_f32(&format!("{}.attn_norm.weight", prefix), data)?0;
    290
    10
            let attn_norm_bias = model
    291
    10
                .get_tensor_f32(&format!("{}.attn_norm.bias", prefix), data)
    292
    10
                .ok();
    293
    294
            // QKV - large, keep quantized
    295
            // Try fused first (phi-2 style), fall back to separate (llama style)
    296
    10
            let (qkv_weight, qkv_bias) = if let Ok(fused1) =
    297
    10
                Self::get_tensor_ref(model, data, &format!("{}.attn_qkv.weight", prefix))
    298
            {
    299
                // phi-2 style: fused QKV tensor
    300
    1
                let bias = model
    301
    1
                    .get_tensor_f32(&format!("{}.attn_qkv.bias", prefix), data)
    302
    1
                    .ok();
    303
    1
                (QKVWeights::Fused(fused), bias)
    304
            } else {
    305
                // llama style: separate Q, K, V tensors
    306
    9
                let q = Self::get_tensor_ref(model, data, &format!("{}.attn_q.weight", prefix))?0;
    307
    9
                let k = Self::get_tensor_ref(model, data, &format!("{}.attn_k.weight", prefix))?0;
    308
    9
                let v = Self::get_tensor_ref(model, data, &format!("{}.attn_v.weight", prefix))?0;
    309
    310
                // Try to get biases (llama usually doesn't have them)
    311
    9
                let q_bias = model
    312
    9
                    .get_tensor_f32(&format!("{}.attn_q.bias", prefix), data)
    313
    9
                    .ok();
    314
    9
                let k_bias = model
    315
    9
                    .get_tensor_f32(&format!("{}.attn_k.bias", prefix), data)
    316
    9
                    .ok();
    317
    9
                let v_bias = model
    318
    9
                    .get_tensor_f32(&format!("{}.attn_v.bias", prefix), data)
    319
    9
                    .ok();
    320
    321
    9
                let bias = match (q_bias, k_bias, v_bias) {
    322
    0
                    (Some(qb), Some(kb), Some(vb)) => {
    323
    0
                        let mut combined = Vec::with_capacity(qb.len() + kb.len() + vb.len());
    324
    0
                        combined.extend_from_slice(&qb);
    325
    0
                        combined.extend_from_slice(&kb);
    326
    0
                        combined.extend_from_slice(&vb);
    327
    0
                        Some(combined)
    328
                    },
    329
    9
                    _ => None,
    330
                };
    331
    332
    9
                (QKVWeights::Separate { q, k, v }, bias)
    333
            };
    334
    335
            // Attention output - large, keep quantized
    336
    10
            let attn_output_weight =
    337
    10
                Self::get_tensor_ref(model, data, &format!("{}.attn_output.weight", prefix))?0;
    338
    10
            let attn_output_bias = model
    339
    10
                .get_tensor_f32(&format!("{}.attn_output.bias", prefix), data)
    340
    10
                .ok();
    341
    342
            // FFN - large, keep quantized
    343
    10
            let ffn_up_weight =
    344
    10
                Self::get_tensor_ref(model, data, &format!("{}.ffn_up.weight", prefix))?0;
    345
    10
            let ffn_up_bias = model
    346
    10
                .get_tensor_f32(&format!("{}.ffn_up.bias", prefix), data)
    347
    10
                .ok();
    348
    10
            let ffn_down_weight =
    349
    10
                Self::get_tensor_ref(model, data, &format!("{}.ffn_down.weight", prefix))?0;
    350
    10
            let ffn_down_bias = model
    351
    10
                .get_tensor_f32(&format!("{}.ffn_down.bias", prefix), data)
    352
    10
                .ok();
    353
    354
            // FFN gate - SwiGLU models like LLaMA have this
    355
    10
            let ffn_gate_weight =
    356
    10
                Self::get_tensor_ref(model, data, &format!("{}.ffn_gate.weight", prefix)).ok();
    357
    10
            let ffn_gate_bias = model
    358
    10
                .get_tensor_f32(&format!("{}.ffn_gate.bias", prefix), data)
    359
    10
                .ok();
    360
    361
            // FFN norm - LLaMA-style pre-FFN layer norm
    362
    10
            let ffn_norm_weight = model
    363
    10
                .get_tensor_f32(&format!("{}.ffn_norm.weight", prefix), data)
    364
    10
                .ok();
    365
    10
            let ffn_norm_bias = model
    366
    10
                .get_tensor_f32(&format!("{}.ffn_norm.bias", prefix), data)
    367
    10
                .ok();
    368
    369
    10
            Ok(QuantizedGGUFTransformerLayer {
    370
    10
                attn_norm_weight,
    371
    10
                attn_norm_bias,
    372
    10
                qkv_weight,
    373
    10
                qkv_bias,
    374
    10
                attn_output_weight,
    375
    10
                attn_output_bias,
    376
    10
                ffn_up_weight,
    377
    10
                ffn_up_bias,
    378
    10
                ffn_down_weight,
    379
    10
                ffn_down_bias,
    380
    10
                ffn_gate_weight,
    381
    10
                ffn_gate_bias,
    382
    10
                ffn_norm_weight,
    383
    10
                ffn_norm_bias,
    384
    10
            })
    385
    10
        }
    386
    }