## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/quantize/fused_q5k_q6k.rs

    Line
    Count
    Source
    1
    //! Fused Q5K and Q6K dot product operations (PMAT-802)
    2
    //!
    3
    //! Implements fused dequant+dot operations for Q5_K and Q6_K formats:
    4
    //! - `fused_q6k_dot`, `fused_q6k_dot_simd` - Q6_K dot products
    5
    //! - `fused_q5k_dot`, `fused_q5k_dot_simd` - Q5_K dot products
    6
    7
    use crate::error::{RealizarError, Result};
    8
    use super::dequant::read_f16;
    9
    use super::simd::extract_scale_min;
    10
    use super::types::{QK_K, Q8_0Block};
    11
    12
    /// Fused Q6_K dequantize + dot product
    13
    ///
    14
    /// Computes the dot product of Q6_K quantized weights with f32 activations.
    15
    8
    pub fn fused_q6k_dot(q6k_data: &[u8], activations: &[f32]) -> Result<f32> {
    16
        const SUPER_BLOCK_BYTES: usize = 210;
    17
    18
        // Validate Q6_K data length
    19
    8
        if !q6k_data.len().is_multiple_of(SUPER_BLOCK_BYTES) {
    20
    1
            return Err(RealizarError::InvalidShape {
    21
    1
                reason: format!(
    22
    1
                    "Q6_K data length {} is not a multiple of super-block size {}",
    23
    1
                    q6k_data.len(),
    24
    1
                    SUPER_BLOCK_BYTES
    25
    1
                ),
    26
    1
            });
    27
    7
        }
    28
    29
    7
        let num_super_blocks = q6k_data.len() / SUPER_BLOCK_BYTES;
    30
    7
        let expected_values = num_super_blocks * QK_K;
    31
    32
        // Validate activation length matches
    33
    7
        if activations.len() != expected_values {
    34
    3
            return Err(RealizarError::InvalidShape {
    35
    3
                reason: format!(
    36
    3
                    "Activation length {} doesn't match Q6_K values count {}",
    37
    3
                    activations.len(),
    38
    3
                    expected_values
    39
    3
                ),
    40
    3
            });
    41
    4
        }
    42
    43
        // Accumulator for dot product result
    44
    4
        let mut acc = 0.0f32;
    45
    46
    7
        for sb_idx in 0..num_super_blocks4 {
    47
    7
            let sb_start = sb_idx * SUPER_BLOCK_BYTES;
    48
    7
            let act_start = sb_idx * QK_K;
    49
    50
            // Q6_K layout: ql (128) + qh (64) + scales (16) + d (2)
    51
    7
            let ql = &q6k_data[sb_start..sb_start + 128];
    52
    7
            let qh = &q6k_data[sb_start + 128..sb_start + 192];
    53
    54
            // Read scales (16 bytes, i8)
    55
    7
            let mut scales = [0i8; 16];
    56
    112
            for (i, scale) in scales7.iter_mut7().enumerate7() {
    57
                #[allow(clippy::cast_possible_wrap)]
    58
    112
                {
    59
    112
                    *scale = q6k_data[sb_start + 192 + i] as i8;
    60
    112
                }
    61
            }
    62
    63
            // Read d (f16 -> f32) at offset 208
    64
    7
            let d = read_f16(&q6k_data[sb_start + 208..sb_start + 210]);
    65
    66
            // Fused dequant+dot following candle's exact layout
    67
            // Process 128 values at a time (n=0, n=128)
    68
    14
            for n in (0..QK_K)7.step_by7(128) {
    69
    14
                let idx = n / 128;
    70
    14
                let sc = &scales[8 * idx..];
    71
    14
                let ql_slice = &ql[64 * idx..];
    72
    14
                let qh_slice = &qh[32 * idx..];
    73
    74
    462
                for l448 in 0..32 {
    75
    448
                    let is = l / 16; // Scale index selector
    76
    448
    77
    448
                    // Extract 4 values per iteration (at positions l, l+32, l+64, l+96)
    78
    448
                    let q1 = ((ql_slice[l] & 0xF) | ((qh_slice[l] & 3) << 4)) as i32 - 32;
    79
    448
                    let q2 = ((ql_slice[l + 32] & 0xF) | (((qh_slice[l] >> 2) & 3) << 4)) as i32 - 32;
    80
    448
                    let q3 = ((ql_slice[l] >> 4) | (((qh_slice[l] >> 4) & 3) << 4)) as i32 - 32;
    81
    448
                    let q4 = ((ql_slice[l + 32] >> 4) | (((qh_slice[l] >> 6) & 3) << 4)) as i32 - 32;
    82
    448
    83
    448
                    // Dequantize and accumulate dot product
    84
    448
                    let v1 = d * (sc[is] as f32) * (q1 as f32);
    85
    448
                    let v2 = d * (sc[is + 2] as f32) * (q2 as f32);
    86
    448
                    let v3 = d * (sc[is + 4] as f32) * (q3 as f32);
    87
    448
                    let v4 = d * (sc[is + 6] as f32) * (q4 as f32);
    88
    448
    89
    448
                    acc += v1 * activations[act_start + n + l];
    90
    448
                    acc += v2 * activations[act_start + n + l + 32];
    91
    448
                    acc += v3 * activations[act_start + n + l + 64];
    92
    448
                    acc += v4 * activations[act_start + n + l + 96];
    93
    448
                }
    94
            }
    95
        }
    96
    97
    4
        Ok(acc)
    98
    8
    }
    99
    100
    /// SIMD-accelerated fused Q6_K dequant+dot (with scalar fallback)
    101
    ///
    102
    /// Per Williams et al. (2009) roofline model, memory bandwidth is the bottleneck.
    103
    /// This function provides a unified interface with runtime feature detection.
    104
    /// Currently uses scalar implementation; SIMD Q6_K optimization can be added later.
    105
    ///
    106
    /// # Arguments
    107
    ///
    108
    /// * `q6k_data` - Raw Q6_K quantized data (210 bytes per super-block)
    109
    /// * `activations` - Input activations (256 values per super-block)
    110
    ///
    111
    /// # Returns
    112
    ///
    113
    /// Dot product result as f32
    114
    ///
    115
    /// # Errors
    116
    ///
    117
    /// Returns error if data sizes don't match or are malformed
    118
    2.18k
    pub fn fused_q6k_dot_simd(q6k_data: &[u8], activations: &[f32]) -> Result<f32> {
    119
        // PAR-126: AVX2 SIMD implementation for Q6_K
    120
        // Critical optimization: Q6_K scalar was 9x slower than Q4_K SIMD
    121
        #[cfg(target_arch = "x86_64")]
    122
        {
    123
    2.18k
            if is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma") {
    124
                // SAFETY: We've verified AVX2 and FMA are available at runtime
    125
    2.18k
                return unsafe { fused_q6k_dot_avx2(q6k_data, activations) };
    126
    0
            }
    127
        }
    128
        // Fallback to scalar implementation
    129
    0
        fused_q6k_dot(q6k_data, activations)
    130
    2.18k
    }
    131
    132
    /// PAR-126: AVX2 SIMD implementation for Q6_K dot product
    133
    ///
    134
    /// Uses AVX2 + FMA to achieve ~8x speedup over scalar.
    135
    /// Q6_K layout: ql (128) + qh (64) + scales (16) + d (2) = 210 bytes
    136
    ///
    137
    /// # Safety
    138
    /// Requires AVX2 and FMA instruction sets
    139
    #[cfg(target_arch = "x86_64")]
    140
    #[target_feature(enable = "avx2", enable = "fma")]
    141
    #[allow(unsafe_op_in_unsafe_fn)]
    142
    2.18k
    unsafe fn fused_q6k_dot_avx2(q6k_data: &[u8], activations: &[f32]) -> Result<f32> {
    143
        #[allow(clippy::wildcard_imports)]
    144
        use std::arch::x86_64::*;
    145
    146
        const SUPER_BLOCK_BYTES: usize = 210;
    147
    148
    2.18k
        if !q6k_data.len().is_multiple_of(SUPER_BLOCK_BYTES) {
    149
    1
            return Err(RealizarError::InvalidShape {
    150
    1
                reason: format!(
    151
    1
                    "Q6_K data length {} is not a multiple of super-block size {}",
    152
    1
                    q6k_data.len(),
    153
    1
                    SUPER_BLOCK_BYTES
    154
    1
                ),
    155
    1
            });
    156
    2.18k
        }
    157
    158
    2.18k
        let num_super_blocks = q6k_data.len() / SUPER_BLOCK_BYTES;
    159
    2.18k
        let expected_values = num_super_blocks * QK_K;
    160
    161
    2.18k
        if activations.len() != expected_values {
    162
    1
            return Err(RealizarError::InvalidShape {
    163
    1
                reason: format!(
    164
    1
                    "Activation length {} doesn't match Q6_K values count {}",
    165
    1
                    activations.len(),
    166
    1
                    expected_values
    167
    1
                ),
    168
    1
            });
    169
    2.18k
        }
    170
    171
        // 4 independent accumulators to hide FMA latency
    172
    2.18k
        let mut acc0 = _mm256_setzero_ps();
    173
    2.18k
        let mut acc1 = _mm256_setzero_ps();
    174
    2.18k
        let mut acc2 = _mm256_setzero_ps();
    175
    2.18k
        let mut acc3 = _mm256_setzero_ps();
    176
    177
        // Masks for 6-bit extraction (reserved for optimized path)
    178
    2.18k
        let _mask_0f = _mm256_set1_epi8(0x0F_i8);
    179
    2.18k
        let _mask_03 = _mm256_set1_epi8(0x03_i8);
    180
    2.18k
        let offset_32 = _mm256_set1_epi32(32);
    181
    182
    2.18k
        for sb_idx in 0..num_super_blocks {
    183
    2.18k
            let sb_start = sb_idx * SUPER_BLOCK_BYTES;
    184
    2.18k
            let act_start = sb_idx * QK_K;
    185
    186
            // Prefetch next super-block
    187
    2.18k
            if sb_idx + 1 < num_super_blocks {
    188
    0
                let next_sb = (sb_idx + 1) * SUPER_BLOCK_BYTES;
    189
    0
                _mm_prefetch(q6k_data.as_ptr().add(next_sb).cast::<i8>(), _MM_HINT_T0);
    190
    2.18k
            }
    191
    192
            // Q6_K layout: ql (128) + qh (64) + scales (16) + d (2)
    193
    2.18k
            let ql_ptr = q6k_data.as_ptr().add(sb_start);
    194
    2.18k
            let qh_ptr = q6k_data.as_ptr().add(sb_start + 128);
    195
    2.18k
            let scales_ptr = q6k_data.as_ptr().add(sb_start + 192);
    196
    197
            // Read d (f16 -> f32)
    198
    2.18k
            let d = read_f16(&q6k_data[sb_start + 208..sb_start + 210]);
    199
    2.18k
            let d_vec = _mm256_set1_ps(d);
    200
    201
            // Read all 16 scales (as i8, will use in inner loop)
    202
    2.18k
            let mut scales = [0i8; 16];
    203
    2.18k
            std::ptr::copy_nonoverlapping(scales_ptr, scales.as_mut_ptr().cast::<u8>(), 16);
    204
    205
            // Process 128 values at a time (n=0, n=128)
    206
    4.36k
            for n in (0..QK_K)2.18k.step_by2.18k(128) {
    207
    4.36k
                let idx = n / 128;
    208
    4.36k
                let sc = &scales[8 * idx..];
    209
    4.36k
                let ql_slice = ql_ptr.add(64 * idx);
    210
    4.36k
                let qh_slice = qh_ptr.add(32 * idx);
    211
    4.36k
                let act_base = activations.as_ptr().add(act_start + n);
    212
    213
                // Process 32 values at a time using AVX2
    214
                // Each iteration handles l=0..8, extracting 4 values each (32 total)
    215
    17.4k
                for l_base in (0..32)4.36k.step_by4.36k(8) {
    216
    17.4k
                    // Load 8 bytes of ql[l], ql[l+32], qh[l]
    217
    17.4k
                    let ql_lo_64 = std::ptr::read_unaligned(ql_slice.add(l_base).cast::<u64>());
    218
    17.4k
                    let ql_hi_64 = std::ptr::read_unaligned(ql_slice.add(l_base + 32).cast::<u64>());
    219
    17.4k
                    let qh_64 = std::ptr::read_unaligned(qh_slice.add(l_base).cast::<u64>());
    220
    17.4k
    221
    17.4k
                    // Convert to SIMD vectors (expand u8 to i32 for arithmetic)
    222
    17.4k
                    let ql_lo = _mm256_cvtepu8_epi32(_mm_set_epi64x(0, ql_lo_64 as i64));
    223
    17.4k
                    let ql_hi = _mm256_cvtepu8_epi32(_mm_set_epi64x(0, ql_hi_64 as i64));
    224
    17.4k
                    let qh = _mm256_cvtepu8_epi32(_mm_set_epi64x(0, qh_64 as i64));
    225
    17.4k
    226
    17.4k
                    // Extract 6-bit values (4 values per input byte)
    227
    17.4k
                    // q1 = (ql[l] & 0xF) | ((qh[l] & 3) << 4) - 32
    228
    17.4k
                    let q1_lo = _mm256_and_si256(ql_lo, _mm256_set1_epi32(0x0F));
    229
    17.4k
                    let q1_hi = _mm256_slli_epi32(_mm256_and_si256(qh, _mm256_set1_epi32(0x03)), 4);
    230
    17.4k
                    let q1 = _mm256_sub_epi32(_mm256_or_si256(q1_lo, q1_hi), offset_32);
    231
    17.4k
    232
    17.4k
                    // q2 = (ql[l+32] & 0xF) | (((qh[l] >> 2) & 3) << 4) - 32
    233
    17.4k
                    let q2_lo = _mm256_and_si256(ql_hi, _mm256_set1_epi32(0x0F));
    234
    17.4k
                    let q2_hi = _mm256_slli_epi32(
    235
    17.4k
                        _mm256_and_si256(_mm256_srli_epi32(qh, 2), _mm256_set1_epi32(0x03)),
    236
    17.4k
                        4,
    237
    17.4k
                    );
    238
    17.4k
                    let q2 = _mm256_sub_epi32(_mm256_or_si256(q2_lo, q2_hi), offset_32);
    239
    17.4k
    240
    17.4k
                    // q3 = (ql[l] >> 4) | (((qh[l] >> 4) & 3) << 4) - 32
    241
    17.4k
                    let q3_lo = _mm256_srli_epi32(ql_lo, 4);
    242
    17.4k
                    let q3_hi = _mm256_slli_epi32(
    243
    17.4k
                        _mm256_and_si256(_mm256_srli_epi32(qh, 4), _mm256_set1_epi32(0x03)),
    244
    17.4k
                        4,
    245
    17.4k
                    );
    246
    17.4k
                    let q3 = _mm256_sub_epi32(_mm256_or_si256(q3_lo, q3_hi), offset_32);
    247
    17.4k
    248
    17.4k
                    // q4 = (ql[l+32] >> 4) | (((qh[l] >> 6) & 3) << 4) - 32
    249
    17.4k
                    let q4_lo = _mm256_srli_epi32(ql_hi, 4);
    250
    17.4k
                    let q4_hi = _mm256_slli_epi32(_mm256_srli_epi32(qh, 6), 4);
    251
    17.4k
                    let q4 = _mm256_sub_epi32(_mm256_or_si256(q4_lo, q4_hi), offset_32);
    252
    17.4k
    253
    17.4k
                    // Determine scale index: is = l / 16 (0 for l<16, 1 for l>=16)
    254
    17.4k
                    let is = l_base / 16;
    255
    17.4k
    256
    17.4k
                    // Get scales for each of the 4 output values
    257
    17.4k
                    let sc1 = sc[is] as f32;
    258
    17.4k
                    let sc2 = sc[is + 2] as f32;
    259
    17.4k
                    let sc3 = sc[is + 4] as f32;
    260
    17.4k
                    let sc4 = sc[is + 6] as f32;
    261
    17.4k
    262
    17.4k
                    // Convert quantized values to f32 and multiply by d*scale
    263
    17.4k
                    let q1_f32 = _mm256_cvtepi32_ps(q1);
    264
    17.4k
                    let q2_f32 = _mm256_cvtepi32_ps(q2);
    265
    17.4k
                    let q3_f32 = _mm256_cvtepi32_ps(q3);
    266
    17.4k
                    let q4_f32 = _mm256_cvtepi32_ps(q4);
    267
    17.4k
    268
    17.4k
                    let dequant1 = _mm256_mul_ps(_mm256_mul_ps(d_vec, _mm256_set1_ps(sc1)), q1_f32);
    269
    17.4k
                    let dequant2 = _mm256_mul_ps(_mm256_mul_ps(d_vec, _mm256_set1_ps(sc2)), q2_f32);
    270
    17.4k
                    let dequant3 = _mm256_mul_ps(_mm256_mul_ps(d_vec, _mm256_set1_ps(sc3)), q3_f32);
    271
    17.4k
                    let dequant4 = _mm256_mul_ps(_mm256_mul_ps(d_vec, _mm256_set1_ps(sc4)), q4_f32);
    272
    17.4k
    273
    17.4k
                    // Load activations
    274
    17.4k
                    let act1 = _mm256_loadu_ps(act_base.add(l_base));
    275
    17.4k
                    let act2 = _mm256_loadu_ps(act_base.add(l_base + 32));
    276
    17.4k
                    let act3 = _mm256_loadu_ps(act_base.add(l_base + 64));
    277
    17.4k
                    let act4 = _mm256_loadu_ps(act_base.add(l_base + 96));
    278
    17.4k
    279
    17.4k
                    // FMA: acc += dequant * act
    280
    17.4k
                    acc0 = _mm256_fmadd_ps(dequant1, act1, acc0);
    281
    17.4k
                    acc1 = _mm256_fmadd_ps(dequant2, act2, acc1);
    282
    17.4k
                    acc2 = _mm256_fmadd_ps(dequant3, act3, acc2);
    283
    17.4k
                    acc3 = _mm256_fmadd_ps(dequant4, act4, acc3);
    284
    17.4k
                }
    285
            }
    286
        }
    287
    288
        // Combine 4 accumulators
    289
    2.18k
        let acc_01 = _mm256_add_ps(acc0, acc1);
    290
    2.18k
        let acc_23 = _mm256_add_ps(acc2, acc3);
    291
    2.18k
        let acc = _mm256_add_ps(acc_01, acc_23);
    292
    293
        // Horizontal sum
    294
    2.18k
        let sum_halves = _mm_add_ps(_mm256_castps256_ps128(acc), _mm256_extractf128_ps(acc, 1));
    295
    2.18k
        let temp = _mm_add_ps(sum_halves, _mm_movehl_ps(sum_halves, sum_halves));
    296
    2.18k
        let temp = _mm_add_ss(temp, _mm_shuffle_ps(temp, temp, 1));
    297
    2.18k
        let result = _mm_cvtss_f32(temp);
    298
    299
    2.18k
        Ok(result)
    300
    2.18k
    }
    301
    302
    /// Fused Q5_K dequantize + dot product
    303
    ///
    304
    /// Computes the dot product of Q5_K quantized weights with f32 activations
    305
    /// WITHOUT allocating an intermediate f32 buffer. Dequantization happens
    306
    /// inline, accumulating directly into a register.
    307
    ///
    308
    /// # Arguments
    309
    ///
    310
    /// * `q5k_data` - Raw Q5_K quantized data (super-blocks of 176 bytes)
    311
    /// * `activations` - f32 activation values (must match dequantized length)
    312
    ///
    313
    /// # Returns
    314
    ///
    315
    /// The dot product as f32
    316
    ///
    317
    /// # Errors
    318
    ///
    319
    /// Returns error if:
    320
    /// - `q5k_data` length is not a multiple of 176 bytes (super-block size)
    321
    /// - `activations` length doesn't match the number of quantized values
    322
    ///
    323
    /// # Examples
    324
    ///
    325
    /// ```rust,ignore
    326
    /// let weights_q5k = load_q5k_weights();
    327
    /// let activations = get_layer_activations();
    328
    /// let result = fused_q5k_dot(&weights_q5k, &activations)?;
    329
    /// ```
    330
    #[allow(clippy::similar_names)]
    331
    2.15k
    pub fn fused_q5k_dot(q5k_data: &[u8], activations: &[f32]) -> Result<f32> {
    332
        const SUPER_BLOCK_BYTES: usize = 176;
    333
    334
        // Validate Q5_K data length
    335
    2.15k
        if !q5k_data.len().is_multiple_of(SUPER_BLOCK_BYTES) {
    336
    3
            return Err(RealizarError::InvalidShape {
    337
    3
                reason: format!(
    338
    3
                    "Q5_K data length {} is not a multiple of super-block size {}",
    339
    3
                    q5k_data.len(),
    340
    3
                    SUPER_BLOCK_BYTES
    341
    3
                ),
    342
    3
            });
    343
    2.15k
        }
    344
    345
    2.15k
        let num_super_blocks = q5k_data.len() / SUPER_BLOCK_BYTES;
    346
    2.15k
        let expected_values = num_super_blocks * QK_K;
    347
    348
        // Validate activation length matches
    349
    2.15k
        if activations.len() != expected_values {
    350
    1
            return Err(RealizarError::InvalidShape {
    351
    1
                reason: format!(
    352
    1
                    "Activation length {} doesn't match Q5_K values count {}",
    353
    1
                    activations.len(),
    354
    1
                    expected_values
    355
    1
                ),
    356
    1
            });
    357
    2.15k
        }
    358
    359
        // Accumulator for dot product result
    360
    2.15k
        let mut acc = 0.0f32;
    361
    2.15k
        let mut activation_idx = 0;
    362
    363
    2.15k
        for sb_idx in 0..num_super_blocks {
    364
    2.15k
            let sb_start = sb_idx * SUPER_BLOCK_BYTES;
    365
    366
            // Read d (f16 -> f32)
    367
    2.15k
            let d = read_f16(&q5k_data[sb_start..sb_start + 2]);
    368
    369
            // Read dmin (f16 -> f32)
    370
    2.15k
            let dmin = read_f16(&q5k_data[sb_start + 2..sb_start + 4]);
    371
    372
            // Read scales (12 bytes)
    373
    2.15k
            let mut scales = [0u8; 12];
    374
    2.15k
            scales.copy_from_slice(&q5k_data[sb_start + 4..sb_start + 16]);
    375
    376
            // Read qh - high bits (32 bytes)
    377
    2.15k
            let qh_start = sb_start + 16;
    378
    2.15k
            let qh = &q5k_data[qh_start..qh_start + 32];
    379
    380
            // Read qs - low 4 bits (128 bytes)
    381
    2.15k
            let qs_start = sb_start + 48;
    382
    2.15k
            let qs = &q5k_data[qs_start..qs_start + 128];
    383
    384
            // Fused dequant+dot for 8 blocks of 32 values each
    385
    19.3k
            for block_idx17.2k in 0..8 {
    386
                // Extract 6-bit scale and min for this block
    387
    17.2k
                let (scale, min) = extract_scale_min(&scales, block_idx);
    388
    389
                // Process 32 values
    390
    17.2k
                let block_start = block_idx * 16;
    391
    17.2k
                let qh_block_start = block_idx * 4;
    392
    393
    292k
                for byte_idx275k in 0..16 {
    394
    275k
                    let qs_byte = qs[block_start + byte_idx];
    395
    275k
                    let high_bits_byte = qh[qh_block_start + byte_idx / 4];
    396
    275k
                    let bit_offset = (byte_idx % 4) * 2;
    397
    275k
    398
    275k
                    // Low value: dequantize and accumulate
    399
    275k
                    let q_low_4bit = qs_byte & 0x0F;
    400
    275k
                    let q_low_high_bit = (high_bits_byte >> bit_offset) & 0x01;
    401
    275k
                    #[allow(clippy::cast_possible_wrap)]
    402
    275k
                    let q_low = ((q_low_high_bit << 4) | q_low_4bit) as i8;
    403
    275k
                    let value_low = d * scale * f32::from(q_low) - dmin * min;
    404
    275k
                    acc += value_low * activations[activation_idx];
    405
    275k
                    activation_idx += 1;
    406
    275k
    407
    275k
                    // High value: dequantize and accumulate
    408
    275k
                    let q_high_4bit = (qs_byte >> 4) & 0x0F;
    409
    275k
                    let q_high_high_bit = (high_bits_byte >> (bit_offset + 1)) & 0x01;
    410
    275k
                    #[allow(clippy::cast_possible_wrap)]
    411
    275k
                    let q_high = ((q_high_high_bit << 4) | q_high_4bit) as i8;
    412
    275k
                    let value_high = d * scale * f32::from(q_high) - dmin * min;
    413
    275k
                    acc += value_high * activations[activation_idx];
    414
    275k
                    activation_idx += 1;
    415
    275k
                }
    416
            }
    417
        }
    418
    419
    2.15k
        Ok(acc)
    420
    2.15k
    }
    421
    422
    /// SIMD-accelerated fused Q5_K dequant+dot (with scalar fallback)
    423
    ///
    424
    /// Provides unified interface with runtime feature detection.
    425
    /// Currently uses scalar implementation; SIMD Q5_K can be added later.
    426
    ///
    427
    /// # Errors
    428
    ///
    429
    /// Returns error if data sizes don't match or are malformed.
    430
    /// See [`fused_q5k_dot`] for details.
    431
    2.15k
    pub fn fused_q5k_dot_simd(q5k_data: &[u8], activations: &[f32]) -> Result<f32> {
    432
        // Q5_K SIMD optimization deferred to Phase 2
    433
    2.15k
        fused_q5k_dot(q5k_data, activations)
    434
    2.15k
    }
    435
    436
    /// Fused Q4_K with Q8 blocks dot product
    437
    ///
    438
    /// Computes the dot product of Q4_K quantized weights with Q8_0 quantized activations.
    439
    1.02k
    pub fn fused_q4k_q8_dot(q4k_data: &[u8], q8_blocks: &[Q8_0Block]) -> Result<f32> {
    440
        const SUPER_BLOCK_BYTES: usize = 144;
    441
    442
        // Validate Q4_K data length
    443
    1.02k
        if !q4k_data.len().is_multiple_of(SUPER_BLOCK_BYTES) {
    444
    4
            return Err(RealizarError::InvalidShape {
    445
    4
                reason: format!(
    446
    4
                    "Q4_K data length {} is not a multiple of super-block size {}",
    447
    4
                    q4k_data.len(),
    448
    4
                    SUPER_BLOCK_BYTES
    449
    4
                ),
    450
    4
            });
    451
    1.01k
        }
    452
    453
    1.01k
        let num_super_blocks = q4k_data.len() / SUPER_BLOCK_BYTES;
    454
    1.01k
        let expected_values = num_super_blocks * QK_K; // 256 values per super-block
    455
    1.01k
        let expected_q8_blocks = expected_values / 32;
    456
    457
        // Validate Q8 block count matches
    458
    1.01k
        if q8_blocks.len() != expected_q8_blocks {
    459
    3
            return Err(RealizarError::InvalidShape {
    460
    3
                reason: format!(
    461
    3
                    "Q8_0 block count {} doesn't match expected {} (for {} Q4_K values)",
    462
    3
                    q8_blocks.len(),
    463
    3
                    expected_q8_blocks,
    464
    3
                    expected_values
    465
    3
                ),
    466
    3
            });
    467
    1.01k
        }
    468
    469
        // Accumulator for dot product result
    470
    1.01k
        let mut acc = 0.0f32;
    471
    1.01k
        let mut q8_block_idx = 0;
    472
    473
    16.1k
        for sb_idx in 0..num_super_blocks1.01k {
    474
    16.1k
            let sb_start = sb_idx * SUPER_BLOCK_BYTES;
    475
    476
            // Read d (f16 -> f32) - super-block scale
    477
    16.1k
            let d = read_f16(&q4k_data[sb_start..sb_start + 2]);
    478
    479
            // Read dmin (f16 -> f32) - super-block min
    480
    16.1k
            let dmin = read_f16(&q4k_data[sb_start + 2..sb_start + 4]);
    481
    482
            // Read scales (12 bytes) - packed 6-bit scales for 8 blocks
    483
    16.1k
            let mut scales = [0u8; 12];
    484
    16.1k
            scales.copy_from_slice(&q4k_data[sb_start + 4..sb_start + 16]);
    485
    486
            // Read qs (128 bytes) - 256 4-bit quantized values
    487
    16.1k
            let qs_start = sb_start + 16;
    488
    16.1k
            let qs = &q4k_data[qs_start..qs_start + 128];
    489
    490
            // Process 8 blocks of 32 values each
    491
    145k
            for block_idx129k in 0..8 {
    492
                // Extract 6-bit scale and min for this block
    493
    129k
                let (scale, min) = extract_scale_min(&scales, block_idx);
    494
    495
                // Get the Q8 block for this 32-value chunk
    496
    129k
                let q8_block = &q8_blocks[q8_block_idx];
    497
    129k
                let q8_scale = q8_block.scale;
    498
    129k
                q8_block_idx += 1;
    499
    500
                // Process 32 values (16 bytes, 2 4-bit values per byte)
    501
    129k
                let block_start = block_idx * 16;
    502
    2.19M
                for byte_idx2.06M in 0..16 {
    503
    2.06M
                    let byte = qs[block_start + byte_idx];
    504
    2.06M
                    let q8_idx = byte_idx * 2;
    505
    2.06M
    506
    2.06M
                    // Low 4 bits: fused dequant and accumulate
    507
    2.06M
                    #[allow(clippy::cast_possible_wrap)]
    508
    2.06M
                    let q4_low = (byte & 0x0F) as i8;
    509
    2.06M
                    let w_low = d * scale * f32::from(q4_low) - dmin * min;
    510
    2.06M
                    let a_low = q8_scale * f32::from(q8_block.quants[q8_idx]);
    511
    2.06M
                    acc += w_low * a_low;
    512
    2.06M
    513
    2.06M
                    // High 4 bits: fused dequant and accumulate
    514
    2.06M
                    #[allow(clippy::cast_possible_wrap)]
    515
    2.06M
                    let q4_high = ((byte >> 4) & 0x0F) as i8;
    516
    2.06M
                    let w_high = d * scale * f32::from(q4_high) - dmin * min;
    517
    2.06M
                    let a_high = q8_scale * f32::from(q8_block.quants[q8_idx + 1]);
    518
    2.06M
                    acc += w_high * a_high;
    519
    2.06M
                }
    520
            }
    521
        }
    522
    523
    1.01k
        Ok(acc)
    524
    1.02k
    }
    525