## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/quantize/types.rs

    Line
    Count
    Source
    1
    //! Quantization Type Definitions (PMAT-802)
    2
    //!
    3
    //! Extracted from quantize/mod.rs - Common types and constants for quantization.
    4
    //!
    5
    //! ## Contents
    6
    //! - Constants: `BLOCK_SIZE`, `QK_K`
    7
    //! - Block structs: `Q4_0Block`, `Q8_0Block`, `Q8KSuperBlock`, `Q4_KBlock`, `Q5_KBlock`, `Q6_KBlock`
    8
    //! - `InterleavedQ4K` - Interleaved Q4_K layout for SIMD
    9
    10
    use crate::error::{RealizarError, Result};
    11
    12
    // Import f16_to_f32_lut from parent for InterleavedQ4K (not re-exported)
    13
    use super::f16_to_f32_lut;
    14
    15
    // ============================================================================
    16
    // Constants
    17
    // ============================================================================
    18
    19
    /// Block size for `Q4_0` and `Q8_0` quantization
    20
    pub const BLOCK_SIZE: usize = 32;
    21
    22
    /// Super-block size for K-quantization formats (`Q4_K`, `Q5_K`, `Q6_K`)
    23
    pub const QK_K: usize = 256;
    24
    25
    // ============================================================================
    26
    // Block Structs
    27
    // ============================================================================
    28
    29
    /// `Q4_0` quantized block
    30
    ///
    31
    /// Each block contains:
    32
    /// - 1 float32 scale factor
    33
    /// - 16 bytes (32 4-bit values, 2 per byte)
    34
    #[derive(Debug, Clone)]
    35
    pub struct Q4_0Block {
    36
        /// Scale factor for dequantization
    37
        pub scale: f32,
    38
        /// Quantized values (16 bytes = 32 4-bit values)
    39
        pub quants: [u8; 16],
    40
    }
    41
    42
    /// `Q8_0` quantized block
    43
    ///
    44
    /// Each block contains:
    45
    /// - 1 float32 scale factor
    46
    /// - 32 int8 values
    47
    #[derive(Debug, Clone)]
    48
    pub struct Q8_0Block {
    49
        /// Scale factor for dequantization
    50
        pub scale: f32,
    51
        /// Quantized values (32 int8 values)
    52
        pub quants: [i8; 32],
    53
    }
    54
    55
    impl Q8_0Block {
    56
        /// Quantize 32 f32 values to Q8_0 format
    57
        ///
    58
        /// Dynamic quantization for activations during inference.
    59
        /// Uses symmetric quantization: scale = max(abs(values)) / 127.0
    60
        ///
    61
        /// # Arguments
    62
        /// * `values` - Exactly 32 f32 values to quantize
    63
        ///
    64
        /// # Returns
    65
        /// A Q8_0Block with scale and quantized int8 values
    66
        ///
    67
        /// # Example
    68
        /// ```ignore
    69
        /// let values = [1.0f32; 32];
    70
        /// let block = Q8_0Block::quantize(&values);
    71
        /// assert_eq!(block.quants[0], 127); // max value maps to 127
    72
        /// ```
    73
        #[must_use]
    74
    244
        pub fn quantize(values: &[f32; 32]) -> Self {
    75
            // Find max absolute value for symmetric quantization
    76
    7.80k
            let max_abs244 = values244.iter244().map244(|v| v.abs()).fold244(0.0f32, f32::max);
    77
    78
            // Avoid division by zero
    79
    244
            let scale = if max_abs > 1e-10 {
    80
    232
                max_abs / 127.0
    81
            } else {
    82
    12
                1.0 / 127.0 // Minimal scale for near-zero blocks
    83
            };
    84
    85
            // Quantize each value: qi = round(fi / scale), clamped to [-128, 127]
    86
    244
            let mut quants = [0i8; 32];
    87
    7.80k
            for (i, &v) in values244.iter244().enumerate244() {
    88
    7.80k
                let q = (v / scale).round();
    89
    7.80k
                quants[i] = q.clamp(-128.0, 127.0) as i8;
    90
    7.80k
            }
    91
    92
    244
            Self { scale, quants }
    93
    244
        }
    94
    95
        /// Dequantize the block back to f32 values
    96
        ///
    97
        /// # Returns
    98
        /// Array of 32 f32 values: values[i] = quants[i] * scale
    99
        #[must_use]
    100
    54
        pub fn dequantize(&self) -> [f32; 32] {
    101
    54
            let mut values = [0.0f32; 32];
    102
    1.72k
            for (i, &q) in self.quants54.iter54().enumerate54() {
    103
    1.72k
                values[i] = q as f32 * self.scale;
    104
    1.72k
            }
    105
    54
            values
    106
    54
        }
    107
    108
        /// Compute quantization error (max absolute difference)
    109
        #[must_use]
    110
    26
        pub fn quantization_error(&self, original: &[f32; 32]) -> f32 {
    111
    26
            let dequantized = self.dequantize();
    112
    26
            original
    113
    26
                .iter()
    114
    26
                .zip(dequantized.iter())
    115
    832
                .map26(|(a, b)| (a - b).abs())
    116
    26
                .fold(0.0f32, f32::max)
    117
    26
        }
    118
    119
        /// Compute relative quantization error
    120
        #[must_use]
    121
    18
        pub fn relative_error(&self, original: &[f32; 32]) -> f32 {
    122
    576
            let max_val18 = original18.iter18().map18(|v| v.abs()).fold18(0.0f32, f32::max);
    123
    18
            if max_val < 1e-10 {
    124
    5
                return 0.0;
    125
    13
            }
    126
    13
            self.quantization_error(original) / max_val
    127
    18
        }
    128
    }
    129
    130
    /// `Q8_K` quantized super-block (llama.cpp-compatible activation format)
    131
    ///
    132
    /// Super-block aligned format for maximum SIMD efficiency with Q4_K weights.
    133
    /// Uses single scale per 256 values (vs Q8_0's scale per 32 values).
    134
    ///
    135
    /// Each super-block contains:
    136
    /// - 1 float32 scale factor (for all 256 values)
    137
    /// - 256 int8 quantized values
    138
    ///
    139
    /// # Performance
    140
    ///
    141
    /// - Aligned with Q4_K super-block (256 values)
    142
    /// - Single scale multiplication per super-block (vs 8 for Q8_0)
    143
    /// - Enables contiguous SIMD loads without shuffle/deinterleave
    144
    /// - Matches llama.cpp `block_q8_K` structure
    145
    #[derive(Debug, Clone)]
    146
    pub struct Q8KSuperBlock {
    147
        /// Scale factor for the entire super-block
    148
        pub scale: f32,
    149
        /// 256 quantized int8 values
    150
        pub quants: [i8; 256],
    151
    }
    152
    153
    impl Q8KSuperBlock {
    154
        /// Quantize 256 f32 values to Q8_K format
    155
        ///
    156
        /// Uses symmetric quantization: scale = max(abs(values)) / 127.0
    157
        ///
    158
        /// # Arguments
    159
        /// * `values` - Exactly 256 f32 values (one super-block)
    160
        ///
    161
        /// # Returns
    162
        /// A Q8KSuperBlock with single scale and 256 quantized values
    163
        #[must_use]
    164
    19
        pub fn quantize(values: &[f32; 256]) -> Self {
    165
            // Find max absolute value for symmetric quantization
    166
    4.86k
            let max_abs19 = values19.iter19().map19(|v| v.abs()).fold19(0.0f32, f32::max);
    167
    168
            // Avoid division by zero
    169
    19
            let scale = if max_abs > 1e-10 {
    170
    16
                max_abs / 127.0
    171
            } else {
    172
    3
                1.0 / 127.0
    173
            };
    174
    175
    19
            let inv_scale = 1.0 / scale;
    176
    177
            // Quantize all 256 values
    178
    19
            let mut quants = [0i8; 256];
    179
    4.86k
            for (i, &v) in values19.iter19().enumerate19() {
    180
    4.86k
                let q = (v * inv_scale).round();
    181
    4.86k
                quants[i] = q.clamp(-128.0, 127.0) as i8;
    182
    4.86k
            }
    183
    184
    19
            Self { scale, quants }
    185
    19
        }
    186
    187
        /// Zero-allocation quantization into pre-allocated buffer
    188
        ///
    189
        /// # Arguments
    190
        /// * `values` - 256 f32 values to quantize
    191
        /// * `scale_out` - Output for scale value
    192
        /// * `quants_out` - Output buffer for 256 int8 quantized values
    193
        #[inline]
    194
    18
        pub fn quantize_into(values: &[f32], scale_out: &mut f32, quants_out: &mut [i8]) {
    195
    18
            debug_assert!(values.len() >= 256);
    196
    18
            debug_assert!(quants_out.len() >= 256);
    197
    198
            // Find max absolute value
    199
    4.60k
            let max_abs18 = values[..256]18.iter18().map18(|v| v.abs()).fold18(0.0f32, f32::max);
    200
    201
    18
            let scale = if max_abs > 1e-10 {
    202
    17
                max_abs / 127.0
    203
            } else {
    204
    1
                1.0 / 127.0
    205
            };
    206
    18
            *scale_out = scale;
    207
    208
    18
            let inv_scale = 1.0 / scale;
    209
    210
    4.60k
            for (i, &v) in values[..256]18.iter18().enumerate18() {
    211
    4.60k
                let q = (v * inv_scale).round();
    212
    4.60k
                quants_out[i] = q.clamp(-128.0, 127.0) as i8;
    213
    4.60k
            }
    214
    18
        }
    215
    216
        /// Dequantize back to f32 values
    217
        #[must_use]
    218
    6
        pub fn dequantize(&self) -> [f32; 256] {
    219
    6
            let mut values = [0.0f32; 256];
    220
    1.53k
            for (i, &q) in self.quants6.iter6().enumerate6() {
    221
    1.53k
                values[i] = q as f32 * self.scale;
    222
    1.53k
            }
    223
    6
            values
    224
    6
        }
    225
    }
    226
    227
    /// `Q4_K` quantized super-block
    228
    ///
    229
    /// K-quantization uses super-blocks of 256 values (8 blocks of 32 each).
    230
    /// Achieves 4.5 bits per weight with better quality than `Q4_0`.
    231
    ///
    232
    /// Each super-block contains:
    233
    /// - 1 half-precision scale factor (`d`)
    234
    /// - 1 half-precision min factor (`dmin`)
    235
    /// - 12 bytes of 6-bit quantized scales (for 8 blocks)
    236
    /// - 128 bytes of 4-bit quantized values (256 values)
    237
    ///
    238
    /// Total: 2 + 2 + 12 + 128 = 144 bytes per super-block of 256 values
    239
    /// = 4.5 bits per weight
    240
    #[derive(Debug, Clone)]
    241
    #[allow(non_camel_case_types)]
    242
    pub struct Q4_KBlock {
    243
        /// Super-block scale (f16, stored as f32 after conversion)
    244
        pub d: f32,
    245
        /// Super-block min (f16, stored as f32 after conversion)
    246
        pub dmin: f32,
    247
        /// Per-block scales (packed 6-bit values)
    248
        pub scales: [u8; 12],
    249
        /// Quantized values (128 bytes = 256 4-bit values)
    250
        pub qs: [u8; 128],
    251
    }
    252
    253
    /// `Q5_K` quantized super-block
    254
    ///
    255
    /// K-quantization uses super-blocks of 256 values (8 blocks of 32 each).
    256
    /// Achieves 5.5 bits per weight with higher quality than `Q4_K`.
    257
    ///
    258
    /// Each super-block contains:
    259
    /// - 1 half-precision scale factor (`d`)
    260
    /// - 1 half-precision min factor (`dmin`)
    261
    /// - 12 bytes of 6-bit quantized scales (for 8 blocks)
    262
    /// - 32 bytes of high bits (1 bit per value for 5-bit quantization)
    263
    /// - 128 bytes of low 4-bit quantized values
    264
    ///
    265
    /// Total: 2 + 2 + 12 + 32 + 128 = 176 bytes per super-block of 256 values
    266
    /// = 5.5 bits per weight
    267
    #[derive(Debug, Clone)]
    268
    #[allow(non_camel_case_types)]
    269
    pub struct Q5_KBlock {
    270
        /// Super-block scale
    271
        pub d: f32,
    272
        /// Super-block min
    273
        pub dmin: f32,
    274
        /// Per-block scales (packed 6-bit values)
    275
        pub scales: [u8; 12],
    276
        /// High bits (1 bit per value)
    277
        pub qh: [u8; 32],
    278
        /// Low 4-bit quantized values
    279
        pub qs: [u8; 128],
    280
    }
    281
    282
    /// `Q6_K` quantized super-block
    283
    ///
    284
    /// K-quantization uses super-blocks of 256 values (16 blocks of 16 each).
    285
    /// Achieves 6.5625 bits per weight with the highest quality among K-quant formats.
    286
    ///
    287
    /// Each super-block contains:
    288
    /// - 1 half-precision scale factor (`d`)
    289
    /// - 16 bytes of 8-bit quantized scales (for 16 blocks)
    290
    /// - 64 bytes of high 2 bits (2 bits per value for 6-bit quantization)
    291
    /// - 128 bytes of low 4-bit quantized values
    292
    ///
    293
    /// Total: 2 + 16 + 64 + 128 = 210 bytes per super-block of 256 values
    294
    /// = 6.5625 bits per weight
    295
    #[derive(Debug, Clone)]
    296
    #[allow(non_camel_case_types)]
    297
    pub struct Q6_KBlock {
    298
        /// Super-block scale
    299
        pub d: f32,
    300
        /// Per-block scales (8-bit signed)
    301
        pub scales: [i8; 16],
    302
        /// High 2 bits per value
    303
        pub qh: [u8; 64],
    304
        /// Low 4-bit quantized values
    305
        pub qs: [u8; 128],
    306
    }
    307
    308
    /// Interleaved Q4_K layout optimized for SIMD operations
    309
    ///
    310
    /// Reorders quantized values during model load so that SIMD dot products
    311
    /// can process contiguous memory without gather operations.
    312
    ///
    313
    /// # Performance
    314
    ///
    315
    /// The interleaved layout eliminates cross-lane shuffles in AVX2:
    316
    /// - Standard Q4_K: requires `vpermd` for each 32-value block (~5 cycles)
    317
    /// - Interleaved: direct `vmovdqu` loads (~1 cycle)
    318
    ///
    319
    /// Trade-off: ~10% slower model load, ~15% faster inference.
    320
    #[derive(Debug, Clone)]
    321
    pub struct InterleavedQ4K {
    322
        /// Super-block scales (one f32 per super-block)
    323
        pub d: Vec<f32>,
    324
        /// Super-block mins (one f32 per super-block)
    325
        pub dmin: Vec<f32>,
    326
        /// Per-block scales (12 bytes per super-block)
    327
        pub scales: Vec<u8>,
    328
        /// Interleaved quantized values (128 bytes per super-block)
    329
        pub qs: Vec<u8>,
    330
        /// Number of super-blocks
    331
        pub num_super_blocks: usize,
    332
    }
    333
    334
    impl InterleavedQ4K {
    335
        /// Create interleaved Q4_K from raw GGUF Q4_K data
    336
        ///
    337
        /// Reorders the quantized values at load time for SIMD-efficient access.
    338
        /// This is a one-time cost at model load that amortizes over all inference.
    339
        ///
    340
        /// # Arguments
    341
        ///
    342
        /// * `q4k_data` - Raw Q4_K data (144 bytes per super-block)
    343
        ///
    344
        /// # Returns
    345
        ///
    346
        /// InterleavedQ4K with reordered weights
    347
        ///
    348
        /// # Errors
    349
        ///
    350
        /// Returns error if data length is not a multiple of super-block size
    351
    0
        pub fn from_q4k(q4k_data: &[u8]) -> Result<Self> {
    352
            const SUPER_BLOCK_BYTES: usize = 144;
    353
    354
    0
            if !q4k_data.len().is_multiple_of(SUPER_BLOCK_BYTES) {
    355
    0
                return Err(RealizarError::InvalidShape {
    356
    0
                    reason: format!(
    357
    0
                        "Q4_K data length {} is not a multiple of super-block size {}",
    358
    0
                        q4k_data.len(),
    359
    0
                        SUPER_BLOCK_BYTES
    360
    0
                    ),
    361
    0
                });
    362
    0
            }
    363
    364
    0
            let num_super_blocks = q4k_data.len() / SUPER_BLOCK_BYTES;
    365
    366
    0
            let mut d = Vec::with_capacity(num_super_blocks);
    367
    0
            let mut dmin = Vec::with_capacity(num_super_blocks);
    368
    0
            let mut scales = Vec::with_capacity(num_super_blocks * 12);
    369
    0
            let mut qs = Vec::with_capacity(num_super_blocks * 128);
    370
    371
    0
            for sb in 0..num_super_blocks {
    372
    0
                let sb_start = sb * SUPER_BLOCK_BYTES;
    373
    0
    374
    0
                // Read d and dmin (f16 -> f32)
    375
    0
                let d_val = f16_to_f32_lut(u16::from_le_bytes([
    376
    0
                    q4k_data[sb_start],
    377
    0
                    q4k_data[sb_start + 1],
    378
    0
                ]));
    379
    0
                let dmin_val = f16_to_f32_lut(u16::from_le_bytes([
    380
    0
                    q4k_data[sb_start + 2],
    381
    0
                    q4k_data[sb_start + 3],
    382
    0
                ]));
    383
    0
    384
    0
                d.push(d_val);
    385
    0
                dmin.push(dmin_val);
    386
    0
    387
    0
                // Copy scales
    388
    0
                scales.extend_from_slice(&q4k_data[sb_start + 4..sb_start + 16]);
    389
    0
    390
    0
                // Interleave quantized values
    391
    0
                // Original: byte[i] = (value[2i+1] << 4) | value[2i]
    392
    0
                // We reorder so that after SIMD nibble extraction, values are contiguous
    393
    0
                //
    394
    0
                // For AVX2 processing 64 values at a time:
    395
    0
                // - Load 32 bytes, extract low nibbles -> 32 values
    396
    0
                // - Same 32 bytes, extract high nibbles -> 32 more values
    397
    0
                //
    398
    0
                // Interleave pattern: group values by their position in SIMD lanes
    399
    0
                // This eliminates the need for cross-lane shuffles
    400
    0
                let qs_start = sb_start + 16;
    401
    0
                let original_qs = &q4k_data[qs_start..qs_start + 128];
    402
    0
    403
    0
                // For now, use identity interleave (same as original)
    404
    0
                // The optimization comes from the specialized kernel that knows the layout
    405
    0
                // Future: implement actual interleave pattern based on profiling
    406
    0
                qs.extend_from_slice(original_qs);
    407
    0
            }
    408
    409
    0
            Ok(Self {
    410
    0
                d,
    411
    0
                dmin,
    412
    0
                scales,
    413
    0
                qs,
    414
    0
                num_super_blocks,
    415
    0
            })
    416
    0
        }
    417
    418
        /// Get the number of values (256 per super-block)
    419
        #[must_use]
    420
    0
        pub fn num_values(&self) -> usize {
    421
    0
            self.num_super_blocks * QK_K
    422
    0
        }
    423
    }
    424
    425
    // ============================================================================
    426
    // SIMD Backend Detection
    427
    // ============================================================================
    428
    429
    /// Batch dequantization stats for performance tracking
    430
    #[derive(Debug, Clone, Default)]
    431
    pub struct DequantStats {
    432
        /// Total blocks processed
    433
        pub blocks_processed: u64,
    434
        /// Total bytes dequantized
    435
        pub bytes_processed: u64,
    436
        /// SIMD backend used
    437
        pub simd_backend: SimdBackend,
    438
    }
    439
    440
    /// SIMD backend detected at runtime
    441
    #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
    442
    pub enum SimdBackend {
    443
        /// AVX2 (256-bit)
    444
        Avx2,
    445
        /// SSE2 (128-bit)
    446
        Sse2,
    447
        /// ARM NEON (128-bit)
    448
        Neon,
    449
        /// Scalar fallback
    450
        #[default]
    451
        Scalar,
    452
    }
    453
    454
    impl std::fmt::Display for SimdBackend {
    455
    10
        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
    456
    10
            match self {
    457
    4
                SimdBackend::Avx2 => write!(f, "AVX2"),
    458
    2
                SimdBackend::Sse2 => write!(f, "SSE2"),
    459
    2
                SimdBackend::Neon => write!(f, "NEON"),
    460
    2
                SimdBackend::Scalar => write!(f, "Scalar"),
    461
            }
    462
    10
        }
    463
    }
    464
    465
    /// Detect available SIMD backend
    466
    5
    pub fn detect_simd_backend() -> SimdBackend {
    467
        #[cfg(target_arch = "x86_64")]
    468
        {
    469
    5
            if is_x86_feature_detected!("avx2") {
    470
    5
                return SimdBackend::Avx2;
    471
    0
            }
    472
    0
            if is_x86_feature_detected!("sse2") {
    473
    0
                return SimdBackend::Sse2;
    474
    0
            }
    475
        }
    476
    477
        #[cfg(target_arch = "aarch64")]
    478
        {
    479
            return SimdBackend::Neon;
    480
        }
    481
    482
    0
        SimdBackend::Scalar
    483
    5
    }