## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/trueno/src/backends/q6k/mod.rs

    Line
    Count
    Source
    1
    //! Fused Q6_K Matrix-Vector Multiply
    2
    //!
    3
    //! Q6_K format (210 bytes per 256 elements):
    4
    //! - `ql`: 128 bytes (lower 4 bits of each value)
    5
    //! - `qh`: 64 bytes (upper 2 bits, packed 4 values per byte)
    6
    //! - `scales`: 16 bytes (8-bit scales for 16 groups of 16 values)
    7
    //! - `d`: 2 bytes (f16 global scale)
    8
    9
    #![allow(dead_code)]
    10
    11
    // Sub-modules
    12
    mod colmajor;
    13
    mod gemv;
    14
    15
    // Re-exports
    16
    pub use colmajor::{matmul_q6k_f32_colmajor, matmul_q6k_f32_colmajor_dispatch};
    17
    pub use gemv::{matmul_q6k_f32, matmul_q6k_f32_dispatch, matmul_q6k_f32_scalar};
    18
    19
    // Constants (pub(crate) for submodule access)
    20
    pub(crate) const SUPER_BLOCK_SIZE: usize = 256;
    21
    pub(crate) const SUPER_BLOCK_BYTES: usize = 210;
    22
    23
    /// Convert f16 bits to f32
    24
    #[inline(always)]
    25
    0
    pub(crate) fn f16_to_f32(bits: u16) -> f32 {
    26
    0
        let sign = ((bits & 0x8000) as u32) << 16;
    27
    0
        let exp = (bits >> 10) & 0x1F;
    28
    0
        let mantissa = (bits & 0x3FF) as u32;
    29
    30
    0
        if exp == 0 {
    31
    0
            if mantissa == 0 {
    32
    0
                f32::from_bits(sign)
    33
            } else {
    34
                // Subnormal
    35
    0
                let mut m = mantissa;
    36
    0
                let mut e = 0i32;
    37
    0
                while (m & 0x400) == 0 {
    38
    0
                    m <<= 1;
    39
    0
                    e -= 1;
    40
    0
                }
    41
    0
                let new_exp = ((127 - 15 + 1 + e) as u32) << 23;
    42
    0
                let new_mantissa = (m & 0x3FF) << 13;
    43
    0
                f32::from_bits(sign | new_exp | new_mantissa)
    44
            }
    45
    0
        } else if exp == 31 {
    46
    0
            f32::from_bits(sign | (0xFF << 23) | (mantissa << 13))
    47
        } else {
    48
    0
            let new_exp = ((exp as i32 - 15 + 127) as u32) << 23;
    49
    0
            f32::from_bits(sign | new_exp | (mantissa << 13))
    50
        }
    51
    0
    }
    52
    53
    #[cfg(test)]
    54
    mod tests {
    55
        use super::gemv::compute_chunk_scalar;
    56
        use super::*;
    57
    58
        #[test]
    59
        fn test_q6k_basic() {
    60
            let in_dim = 256;
    61
            let out_dim = 2;
    62
    63
            // Create Q6K test data (210 bytes per block)
    64
            let mut q6k_data = Vec::new();
    65
            for _ in 0..out_dim {
    66
                // ql: 128 bytes (all zeros = q4 part is 0)
    67
                q6k_data.extend_from_slice(&[0x55u8; 128]); // 5 in each nibble
    68
                // qh: 64 bytes (all zeros = q2 part is 0)
    69
                q6k_data.extend_from_slice(&[0x00u8; 64]);
    70
                // scales: 16 bytes (all ones)
    71
                q6k_data.extend_from_slice(&[0x01u8; 16]);
    72
                // d: f16 = 1.0
    73
                q6k_data.extend_from_slice(&[0x00, 0x3C]);
    74
            }
    75
    76
            let input: Vec<f32> = (0..in_dim).map(|i| (i as f32) * 0.01).collect();
    77
            let output = matmul_q6k_f32(&q6k_data, &input, out_dim, in_dim);
    78
    79
            assert_eq!(output.len(), out_dim);
    80
            for val in &output {
    81
                assert!(val.is_finite(), "Output contains non-finite value: {}", val);
    82
            }
    83
        }
    84
    85
        #[cfg(target_arch = "x86_64")]
    86
        #[test]
    87
        fn test_q6k_avx2_vs_scalar() {
    88
            if !is_x86_feature_detected!("avx2") || !is_x86_feature_detected!("fma") {
    89
                return;
    90
            }
    91
    92
            let in_dim = 512;
    93
            let out_dim = 4;
    94
    95
            let mut q6k_data = Vec::new();
    96
            for row in 0..out_dim {
    97
                for _ in 0..2 {
    98
                    // 2 blocks per row
    99
                    q6k_data.extend_from_slice(&[(row as u8 * 17).wrapping_add(0x33); 128]);
    100
                    q6k_data.extend_from_slice(&[(row as u8).wrapping_add(0x11); 64]);
    101
                    q6k_data.extend_from_slice(&[0x02u8; 16]);
    102
                    q6k_data.extend_from_slice(&[0x66, 0x2E]); // d ~ 0.1
    103
                }
    104
            }
    105
    106
            let input: Vec<f32> = (0..in_dim).map(|i| (i as f32) * 0.002 - 0.5).collect();
    107
    108
            let scalar = matmul_q6k_f32_scalar(&q6k_data, &input, out_dim, in_dim);
    109
            let dispatch = matmul_q6k_f32_dispatch(&q6k_data, &input, out_dim, in_dim);
    110
    111
            for (i, (s, d)) in scalar.iter().zip(dispatch.iter()).enumerate() {
    112
                let diff = (s - d).abs();
    113
                assert!(
    114
                    diff < 1e-4,
    115
                    "Row {}: scalar {} vs dispatch {} (diff {})",
    116
                    i, s, d, diff
    117
                );
    118
            }
    119
        }
    120
    121
        #[test]
    122
        fn test_f16_to_f32_normal() {
    123
            // Normal f16 value: 1.0 = 0x3C00
    124
            let result = f16_to_f32(0x3C00);
    125
            assert!((result - 1.0).abs() < 1e-6, "Expected 1.0, got {}", result);
    126
    127
            // 2.0 = 0x4000
    128
            let result = f16_to_f32(0x4000);
    129
            assert!((result - 2.0).abs() < 1e-6, "Expected 2.0, got {}", result);
    130
    131
            // -1.0 = 0xBC00
    132
            let result = f16_to_f32(0xBC00);
    133
            assert!((result + 1.0).abs() < 1e-6, "Expected -1.0, got {}", result);
    134
        }
    135
    136
        #[test]
    137
        fn test_f16_to_f32_zero() {
    138
            // Positive zero
    139
            let result = f16_to_f32(0x0000);
    140
            assert_eq!(result, 0.0, "Expected +0.0");
    141
            assert!(result.is_sign_positive());
    142
    143
            // Negative zero
    144
            let result = f16_to_f32(0x8000);
    145
            assert_eq!(result, 0.0, "Expected -0.0");
    146
            assert!(result.is_sign_negative());
    147
        }
    148
    149
        #[test]
    150
        fn test_f16_to_f32_infinity() {
    151
            // Positive infinity = 0x7C00
    152
            let result = f16_to_f32(0x7C00);
    153
            assert!(result.is_infinite() && result.is_sign_positive());
    154
    155
            // Negative infinity = 0xFC00
    156
            let result = f16_to_f32(0xFC00);
    157
            assert!(result.is_infinite() && result.is_sign_negative());
    158
        }
    159
    160
        #[test]
    161
        fn test_f16_to_f32_subnormal() {
    162
            // Smallest subnormal: 0x0001 ≈ 5.96e-8
    163
            let result = f16_to_f32(0x0001);
    164
            assert!(result > 0.0 && result < 1e-6, "Expected small subnormal, got {}", result);
    165
    166
            // Larger subnormal: 0x03FF (largest subnormal)
    167
            let result = f16_to_f32(0x03FF);
    168
            assert!(result > 0.0 && result < 1e-4, "Expected subnormal, got {}", result);
    169
        }
    170
    171
        #[test]
    172
        fn test_q6k_colmajor_basic() {
    173
            let in_dim = 256;
    174
            let out_dim = 2;
    175
    176
            // Create Q6K test data
    177
            let mut q6k_data = Vec::new();
    178
            for _ in 0..out_dim {
    179
                q6k_data.extend_from_slice(&[0x33u8; 128]); // ql
    180
                q6k_data.extend_from_slice(&[0x00u8; 64]);  // qh
    181
                q6k_data.extend_from_slice(&[0x01u8; 16]); // scales
    182
                q6k_data.extend_from_slice(&[0x00, 0x3C]); // d = 1.0
    183
            }
    184
    185
            let input: Vec<f32> = (0..in_dim).map(|i| (i as f32) * 0.01).collect();
    186
            let output = matmul_q6k_f32_colmajor(&q6k_data, &input, out_dim, in_dim);
    187
    188
            assert_eq!(output.len(), out_dim);
    189
            for val in &output {
    190
                assert!(val.is_finite(), "Output contains non-finite value: {}", val);
    191
            }
    192
        }
    193
    194
        #[test]
    195
        fn test_q6k_colmajor_dispatch() {
    196
            let in_dim = 256;
    197
            let out_dim = 4;
    198
    199
            let mut q6k_data = Vec::new();
    200
            for row in 0..out_dim {
    201
                q6k_data.extend_from_slice(&[(row as u8).wrapping_add(0x22); 128]);
    202
                q6k_data.extend_from_slice(&[(row as u8).wrapping_add(0x11); 64]);
    203
                q6k_data.extend_from_slice(&[0x02u8; 16]);
    204
                q6k_data.extend_from_slice(&[0x00, 0x3C]); // d = 1.0
    205
            }
    206
    207
            let input: Vec<f32> = (0..in_dim).map(|i| (i as f32) * 0.01 - 1.0).collect();
    208
    209
            let result = matmul_q6k_f32_colmajor_dispatch(&q6k_data, &input, out_dim, in_dim);
    210
            assert_eq!(result.len(), out_dim);
    211
            for val in &result {
    212
                assert!(val.is_finite());
    213
            }
    214
        }
    215
    216
        #[test]
    217
        fn test_q6k_unaligned_dimensions() {
    218
            // Test with dimensions not aligned to block size (256)
    219
            let in_dim = 300; // Not a multiple of 256
    220
            let out_dim = 3;
    221
            let num_blocks = (in_dim + 255) / 256; // = 2 blocks
    222
    223
            let mut q6k_data = Vec::new();
    224
            for _ in 0..out_dim {
    225
                for _ in 0..num_blocks {
    226
                    q6k_data.extend_from_slice(&[0x11u8; 128]);
    227
                    q6k_data.extend_from_slice(&[0x00u8; 64]);
    228
                    q6k_data.extend_from_slice(&[0x01u8; 16]);
    229
                    q6k_data.extend_from_slice(&[0x00, 0x3C]);
    230
                }
    231
            }
    232
    233
            let input: Vec<f32> = (0..in_dim).map(|i| (i as f32) * 0.01).collect();
    234
            let output = matmul_q6k_f32(&q6k_data, &input, out_dim, in_dim);
    235
    236
            assert_eq!(output.len(), out_dim);
    237
            for val in &output {
    238
                assert!(val.is_finite());
    239
            }
    240
        }
    241
    242
        #[test]
    243
        fn test_q6k_single_row() {
    244
            let in_dim = 256;
    245
            let out_dim = 1;
    246
    247
            let mut q6k_data = Vec::new();
    248
            q6k_data.extend_from_slice(&[0xAAu8; 128]); // ql
    249
            q6k_data.extend_from_slice(&[0x55u8; 64]);  // qh (alternating bits)
    250
            q6k_data.extend_from_slice(&[0x01u8; 16]); // scales
    251
            q6k_data.extend_from_slice(&[0x00, 0x3C]); // d = 1.0
    252
    253
            let input: Vec<f32> = vec![1.0; in_dim];
    254
            let output = matmul_q6k_f32_scalar(&q6k_data, &input, out_dim, in_dim);
    255
    256
            assert_eq!(output.len(), 1);
    257
            assert!(output[0].is_finite());
    258
        }
    259
    260
        #[test]
    261
        fn test_q6k_large_dimensions() {
    262
            let in_dim = 1024;
    263
            let out_dim = 8;
    264
            let num_blocks = in_dim / 256;
    265
    266
            let mut q6k_data = Vec::new();
    267
            for row in 0..out_dim {
    268
                for blk in 0..num_blocks {
    269
                    let val = ((row * num_blocks + blk) as u8).wrapping_mul(17);
    270
                    q6k_data.extend_from_slice(&[val; 128]);
    271
                    q6k_data.extend_from_slice(&[val.wrapping_add(1); 64]);
    272
                    q6k_data.extend_from_slice(&[0x02u8; 16]);
    273
                    q6k_data.extend_from_slice(&[0x66, 0x2E]); // d ~ 0.1
    274
                }
    275
            }
    276
    277
            let input: Vec<f32> = (0..in_dim).map(|i| ((i % 100) as f32) * 0.01).collect();
    278
            let output = matmul_q6k_f32(&q6k_data, &input, out_dim, in_dim);
    279
    280
            assert_eq!(output.len(), out_dim);
    281
            for val in &output {
    282
                assert!(val.is_finite());
    283
            }
    284
        }
    285
    286
        #[test]
    287
        fn test_q6k_zero_input() {
    288
            let in_dim = 256;
    289
            let out_dim = 2;
    290
    291
            let mut q6k_data = Vec::new();
    292
            for _ in 0..out_dim {
    293
                q6k_data.extend_from_slice(&[0xFFu8; 128]);
    294
                q6k_data.extend_from_slice(&[0xFFu8; 64]);
    295
                q6k_data.extend_from_slice(&[0x7Fu8; 16]); // max positive scale
    296
                q6k_data.extend_from_slice(&[0x00, 0x3C]); // d = 1.0
    297
            }
    298
    299
            let input: Vec<f32> = vec![0.0; in_dim];
    300
            let output = matmul_q6k_f32(&q6k_data, &input, out_dim, in_dim);
    301
    302
            assert_eq!(output.len(), out_dim);
    303
            for val in &output {
    304
                assert_eq!(*val, 0.0, "Output should be zero when input is zero");
    305
            }
    306
        }
    307
    308
        #[test]
    309
        fn test_q6k_negative_scales() {
    310
            let in_dim = 256;
    311
            let out_dim = 1;
    312
    313
            let mut q6k_data = Vec::new();
    314
            q6k_data.extend_from_slice(&[0x00u8; 128]); // ql = 0
    315
            q6k_data.extend_from_slice(&[0x00u8; 64]);  // qh = 0
    316
            q6k_data.extend_from_slice(&[0x80u8; 16]); // scales = -128 (negative)
    317
            q6k_data.extend_from_slice(&[0x00, 0x3C]); // d = 1.0
    318
    319
            let input: Vec<f32> = vec![1.0; in_dim];
    320
            let output = matmul_q6k_f32_scalar(&q6k_data, &input, out_dim, in_dim);
    321
    322
            assert_eq!(output.len(), 1);
    323
            assert!(output[0].is_finite());
    324
            // With negative scales and quant=0-32=-32, result should be positive
    325
        }
    326
    327
        // =========================================================================
    328
        // Golden Vector Tests: Q6K scalar reference vs dispatch/SIMD paths
    329
        // =========================================================================
    330
    331
        /// Golden Test: Q6K scalar == dispatch for random input
    332
        #[test]
    333
        fn test_golden_q6k_scalar_vs_dispatch() {
    334
            // Realistic LLM dimensions
    335
            let in_dim = 512; // 2 super-blocks
    336
            let out_dim = 8;
    337
    338
            let mut q6k_data = Vec::new();
    339
            for row in 0..out_dim {
    340
                for sb in 0..(in_dim / 256) {
    341
                    // ql: varied 4-bit low values
    342
                    for i in 0..128 {
    343
                        let low = ((row + sb + i) % 16) as u8;
    344
                        let high = ((row + sb + i + 3) % 16) as u8;
    345
                        q6k_data.push(low | (high << 4));
    346
                    }
    347
                    // qh: varied 2-bit high values
    348
                    for i in 0..64 {
    349
                        let vals = [
    350
                            ((row + i) % 4) as u8,
    351
                            ((row + i + 1) % 4) as u8,
    352
                            ((row + i + 2) % 4) as u8,
    353
                            ((row + i + 3) % 4) as u8,
    354
                        ];
    355
                        q6k_data.push(vals[0] | (vals[1] << 2) | (vals[2] << 4) | (vals[3] << 6));
    356
                    }
    357
                    // scales: varied signed 8-bit
    358
                    for i in 0..16 {
    359
                        q6k_data.push(((row * 7 + sb * 3 + i) % 64) as u8);
    360
                    }
    361
                    // d ~ 0.1
    362
                    q6k_data.extend_from_slice(&[0x66, 0x2E]);
    363
                }
    364
            }
    365
    366
            // Sinusoidal input
    367
            let input: Vec<f32> = (0..in_dim)
    368
                .map(|i| ((i as f32) * 0.019).sin() * 0.4)
    369
                .collect();
    370
    371
            let scalar_output = matmul_q6k_f32_scalar(&q6k_data, &input, out_dim, in_dim);
    372
            let dispatch_output = matmul_q6k_f32_dispatch(&q6k_data, &input, out_dim, in_dim);
    373
    374
            assert_eq!(scalar_output.len(), dispatch_output.len());
    375
            let mut max_abs_error = 0.0f32;
    376
    377
            for (i, (scalar, dispatch)) in scalar_output.iter().zip(dispatch_output.iter()).enumerate() {
    378
                let abs_error = (scalar - dispatch).abs();
    379
                max_abs_error = max_abs_error.max(abs_error);
    380
    381
                // Scalar and dispatch should match closely (minor FMA ordering differences)
    382
                assert!(
    383
                    abs_error < 2e-4,
    384
                    "Row {}: scalar={}, dispatch={}, diff={}",
    385
                    i, scalar, dispatch, abs_error
    386
                );
    387
            }
    388
    389
            eprintln!("[Golden Q6K Test] max_abs_error={:.6}", max_abs_error);
    390
        }
    391
    392
        /// Golden Test: Q6K colmajor path consistency
    393
        #[test]
    394
        fn test_golden_q6k_colmajor_consistency() {
    395
            let in_dim = 512;
    396
            let out_dim = 4;
    397
    398
            let mut q6k_data = Vec::new();
    399
            for row in 0..out_dim {
    400
                for sb in 0..2 {
    401
                    // ql
    402
                    for i in 0..128 {
    403
                        q6k_data.push(((row * 5 + sb * 13 + i) % 256) as u8);
    404
                    }
    405
                    // qh
    406
                    for i in 0..64 {
    407
                        q6k_data.push(((row * 7 + sb * 11 + i * 2) % 256) as u8);
    408
                    }
    409
                    // scales
    410
                    for i in 0..16 {
    411
                        q6k_data.push(((row + sb + i) % 128) as u8);
    412
                    }
    413
                    // d ~ 0.5
    414
                    q6k_data.extend_from_slice(&[0x00, 0x38]);
    415
                }
    416
            }
    417
    418
            let input: Vec<f32> = (0..in_dim)
    419
                .map(|i| ((i as f32) * 0.011 + 0.3).cos() * 0.5)
    420
                .collect();
    421
    422
            let colmajor_output = matmul_q6k_f32_colmajor(&q6k_data, &input, out_dim, in_dim);
    423
            let colmajor_dispatch = matmul_q6k_f32_colmajor_dispatch(&q6k_data, &input, out_dim, in_dim);
    424
    425
            assert_eq!(colmajor_output.len(), colmajor_dispatch.len());
    426
            for (i, (base, dispatch)) in colmajor_output.iter().zip(colmajor_dispatch.iter()).enumerate() {
    427
                let diff = (base - dispatch).abs();
    428
                assert!(
    429
                    diff < 1e-4,
    430
                    "Row {}: colmajor base={}, dispatch={}, diff={}",
    431
                    i, base, dispatch, diff
    432
                );
    433
            }
    434
        }
    435
    436
        /// Edge case: maximum 6-bit values (63)
    437
        #[test]
    438
        fn test_golden_q6k_max_quant_values() {
    439
            let in_dim = 256;
    440
            let out_dim = 2;
    441
    442
            let mut q6k_data = Vec::new();
    443
            for _ in 0..out_dim {
    444
                // ql: all 0xF (low nibble = 15)
    445
                q6k_data.extend_from_slice(&[0xFFu8; 128]);
    446
                // qh: all 0xFF (all 2-bit high = 3), so value = 15 + 3*16 = 63
    447
                q6k_data.extend_from_slice(&[0xFFu8; 64]);
    448
                // scales: positive
    449
                q6k_data.extend_from_slice(&[0x3Fu8; 16]); // scale = 63
    450
                // d = 1.0
    451
                q6k_data.extend_from_slice(&[0x00, 0x3C]);
    452
            }
    453
    454
            let input = vec![1.0f32; in_dim];
    455
            let scalar_output = matmul_q6k_f32_scalar(&q6k_data, &input, out_dim, in_dim);
    456
            let dispatch_output = matmul_q6k_f32_dispatch(&q6k_data, &input, out_dim, in_dim);
    457
    458
            for (i, (scalar, dispatch)) in scalar_output.iter().zip(dispatch_output.iter()).enumerate() {
    459
                assert!(
    460
                    scalar.is_finite() && dispatch.is_finite(),
    461
                    "Row {}: max values should produce finite output",
    462
                    i
    463
                );
    464
                let diff = (scalar - dispatch).abs();
    465
                assert!(
    466
                    diff < 1e-4,
    467
                    "Row {}: max quant scalar={}, dispatch={}, diff={}",
    468
                    i, scalar, dispatch, diff
    469
                );
    470
            }
    471
        }
    472
    473
        /// Edge case: alternating positive/negative scales
    474
        #[test]
    475
        fn test_golden_q6k_alternating_scales() {
    476
            let in_dim = 256;
    477
            let out_dim = 2;
    478
    479
            let mut q6k_data = Vec::new();
    480
            for _ in 0..out_dim {
    481
                // ql: mid-range values
    482
                q6k_data.extend_from_slice(&[0x77u8; 128]); // 7, 7 repeated
    483
                // qh: zeros (full value = 7)
    484
                q6k_data.extend_from_slice(&[0x00u8; 64]);
    485
                // scales: alternating +32, -32
    486
                for i in 0..16 {
    487
                    if i % 2 == 0 {
    488
                        q6k_data.push(0x20); // +32
    489
                    } else {
    490
                        q6k_data.push(0xE0); // -32 (as signed i8)
    491
                    }
    492
                }
    493
                // d = 0.5
    494
                q6k_data.extend_from_slice(&[0x00, 0x38]);
    495
            }
    496
    497
            let input = vec![1.0f32; in_dim];
    498
            let scalar_output = matmul_q6k_f32_scalar(&q6k_data, &input, out_dim, in_dim);
    499
            let dispatch_output = matmul_q6k_f32_dispatch(&q6k_data, &input, out_dim, in_dim);
    500
    501
            for (i, (scalar, dispatch)) in scalar_output.iter().zip(dispatch_output.iter()).enumerate() {
    502
                let diff = (scalar - dispatch).abs();
    503
                assert!(
    504
                    diff < 1e-4,
    505
                    "Row {}: alternating scales scalar={}, dispatch={}, diff={}",
    506
                    i, scalar, dispatch, diff
    507
                );
    508
            }
    509
        }
    510
    511
        /// Large scale test for SIMD path coverage
    512
        #[cfg(target_arch = "x86_64")]
    513
        #[test]
    514
        fn test_golden_q6k_large_simd() {
    515
            if !is_x86_feature_detected!("avx2") || !is_x86_feature_detected!("fma") {
    516
                eprintln!("Skipping Q6K SIMD test - no AVX2+FMA");
    517
                return;
    518
            }
    519
    520
            let in_dim = 2048; // 8 super-blocks
    521
            let out_dim = 32;
    522
    523
            let mut q6k_data = Vec::new();
    524
            for row in 0..out_dim {
    525
                for sb in 0..(in_dim / 256) {
    526
                    for i in 0..128 {
    527
                        let val = ((row * 3 + sb * 7 + i) % 256) as u8;
    528
                        q6k_data.push(val);
    529
                    }
    530
                    for i in 0..64 {
    531
                        let val = ((row * 5 + sb * 11 + i * 2) % 256) as u8;
    532
                        q6k_data.push(val);
    533
                    }
    534
                    for i in 0..16 {
    535
                        q6k_data.push(((row + sb + i) % 64) as u8);
    536
                    }
    537
                    q6k_data.extend_from_slice(&[0x66, 0x2E]);
    538
                }
    539
            }
    540
    541
            let input: Vec<f32> = (0..in_dim)
    542
                .map(|i| ((i as f32) * 0.007 - 1.0).tanh())
    543
                .collect();
    544
    545
            let scalar_output = matmul_q6k_f32_scalar(&q6k_data, &input, out_dim, in_dim);
    546
            let dispatch_output = matmul_q6k_f32_dispatch(&q6k_data, &input, out_dim, in_dim);
    547
    548
            let mut max_rel_error = 0.0f32;
    549
            for (i, (scalar, dispatch)) in scalar_output.iter().zip(dispatch_output.iter()).enumerate() {
    550
                let abs_error = (scalar - dispatch).abs();
    551
                let rel_error = if scalar.abs() > 1e-6 {
    552
                    abs_error / scalar.abs()
    553
                } else {
    554
                    abs_error
    555
                };
    556
                max_rel_error = max_rel_error.max(rel_error);
    557
    558
                assert!(
    559
                    rel_error < 1e-4 || abs_error < 1e-4,
    560
                    "Row {}: large SIMD scalar={}, dispatch={}, rel_err={:.6}",
    561
                    i, scalar, dispatch, rel_error
    562
                );
    563
            }
    564
    565
            eprintln!("[Golden Q6K Large SIMD] max_rel_error={:.6}", max_rel_error);
    566
        }
    567
    568
        #[test]
    569
        fn test_parallel_dispatch_large_matrix() {
    570
            // Test parallel path: total_work >= 8_000_000
    571
            // Use 4096 x 2048 = 8_388_608 ops (triggers parallel)
    572
            let out_dim = 4096;
    573
            let in_dim = 2048; // Must be multiple of 256 (SUPER_BLOCK_SIZE)
    574
            let total_work = out_dim * in_dim;
    575
            assert!(
    576
                total_work >= 8_000_000,
    577
                "Test must trigger parallel path"
    578
            );
    579
    580
            let num_superblocks_per_row = (in_dim + SUPER_BLOCK_SIZE - 1) / SUPER_BLOCK_SIZE;
    581
            let row_bytes = num_superblocks_per_row * SUPER_BLOCK_BYTES;
    582
            let total_bytes = out_dim * row_bytes;
    583
    584
            // Create deterministic test data
    585
            let mut q6k_data = vec![0u8; total_bytes];
    586
            for row in 0..out_dim {
    587
                for sb in 0..num_superblocks_per_row {
    588
                    let offset = row * row_bytes + sb * SUPER_BLOCK_BYTES;
    589
                    // d = 1.0 as f16
    590
                    q6k_data[offset] = 0x00;
    591
                    q6k_data[offset + 1] = 0x3C;
    592
                    // ql: 6-bit low parts
    593
                    for i in 0..128 {
    594
                        q6k_data[offset + 2 + i] = ((row + sb + i) % 64) as u8;
    595
                    }
    596
                    // qh: 2-bit high parts
    597
                    for i in 0..64 {
    598
                        q6k_data[offset + 130 + i] = ((row ^ sb ^ i) % 4) as u8;
    599
                    }
    600
                    // scales
    601
                    for i in 0..16 {
    602
                        q6k_data[offset + 194 + i] = 0x10;
    603
                    }
    604
                }
    605
            }
    606
    607
            let input: Vec<f32> = (0..in_dim).map(|i| (i % 10) as f32 * 0.1).collect();
    608
    609
            // Call dispatch - should use parallel path
    610
            let result = matmul_q6k_f32_dispatch(&q6k_data, &input, out_dim, in_dim);
    611
    612
            // Verify dimensions and finiteness
    613
            assert_eq!(result.len(), out_dim);
    614
            for (i, &val) in result.iter().enumerate() {
    615
                assert!(
    616
                    val.is_finite(),
    617
                    "Result[{}] is not finite: {}",
    618
                    i,
    619
                    val
    620
                );
    621
            }
    622
    623
            // Compare a few rows against scalar for consistency
    624
            let scalar_result = matmul_q6k_f32_scalar(&q6k_data, &input, out_dim, in_dim);
    625
            for i in (0..out_dim).step_by(512) {
    626
                let diff = (result[i] - scalar_result[i]).abs();
    627
                let tol = scalar_result[i].abs() * 0.01 + 1e-4;
    628
                assert!(
    629
                    diff < tol,
    630
                    "Parallel vs scalar mismatch at row {}: parallel={}, scalar={}, diff={}",
    631
                    i,
    632
                    result[i],
    633
                    scalar_result[i],
    634
                    diff
    635
                );
    636
            }
    637
        }
    638
    639
        #[test]
    640
        fn test_parallel_colmajor_large_matrix() {
    641
            // Test colmajor path
    642
            // ne0 = output dimension, ne1 = input dimension
    643
            let ne0 = 2048; // output dimension, must be multiple of 256
    644
            let ne1 = 4096; // input dimension
    645
    646
            let blocks_per_col = (ne0 + SUPER_BLOCK_SIZE - 1) / SUPER_BLOCK_SIZE;
    647
            let col_bytes = blocks_per_col * SUPER_BLOCK_BYTES;
    648
            let total_bytes = ne1 * col_bytes;
    649
    650
            let mut q6k_data = vec![0u8; total_bytes];
    651
            for col in 0..ne1 {
    652
                for sb in 0..blocks_per_col {
    653
                    let offset = col * col_bytes + sb * SUPER_BLOCK_BYTES;
    654
                    // d = 0.5 as f16
    655
                    q6k_data[offset] = 0x00;
    656
                    q6k_data[offset + 1] = 0x38;
    657
                    // ql
    658
                    for i in 0..128 {
    659
                        q6k_data[offset + 2 + i] = ((col ^ sb ^ i) % 64) as u8;
    660
                    }
    661
                    // qh
    662
                    for i in 0..64 {
    663
                        q6k_data[offset + 130 + i] = ((col + sb) % 4) as u8;
    664
                    }
    665
                    // scales
    666
                    for i in 0..16 {
    667
                        q6k_data[offset + 194 + i] = 0x20;
    668
                    }
    669
                }
    670
            }
    671
    672
            // Input must have length ne1
    673
            let input: Vec<f32> = (0..ne1).map(|i| ((i % 7) as f32 - 3.0) * 0.1).collect();
    674
    675
            // Use colmajor dispatch
    676
            let result = matmul_q6k_f32_colmajor_dispatch(&q6k_data, &input, ne0, ne1);
    677
    678
            // Output has ne0 elements
    679
            assert_eq!(result.len(), ne0);
    680
            for (i, &val) in result.iter().enumerate() {
    681
                assert!(val.is_finite(), "Result[{}] is not finite: {}", i, val);
    682
            }
    683
        }
    684
    685
        #[test]
    686
        fn test_compute_chunk_scalar_small() {
    687
            // Directly test compute_chunk_scalar
    688
            let in_dim = 256;
    689
            let out_dim = 4;
    690
            let num_blocks_per_row = 1;
    691
            let row_bytes = SUPER_BLOCK_BYTES;
    692
    693
            let mut q6k_data = vec![0u8; out_dim * row_bytes];
    694
            for row in 0..out_dim {
    695
                let offset = row * row_bytes;
    696
                // d = 1.0 as f16
    697
                q6k_data[offset] = 0x00;
    698
                q6k_data[offset + 1] = 0x3C;
    699
                // ql = all zeros
    700
                for i in 0..128 {
    701
                    q6k_data[offset + 2 + i] = 0x00;
    702
                }
    703
                // qh = all zeros
    704
                for i in 0..64 {
    705
                    q6k_data[offset + 130 + i] = 0x00;
    706
                }
    707
                // scales = 1
    708
                for i in 0..16 {
    709
                    q6k_data[offset + 194 + i] = 0x01;
    710
                }
    711
            }
    712
    713
            let input = vec![1.0f32; in_dim];
    714
            let mut chunk = vec![0.0f32; out_dim];
    715
    716
            compute_chunk_scalar(
    717
                &q6k_data,
    718
                &input,
    719
                &mut chunk,
    720
                0,
    721
                out_dim,
    722
                in_dim,
    723
                num_blocks_per_row,
    724
                row_bytes,
    725
            );
    726
    727
            // Verify results are finite
    728
            for (i, &val) in chunk.iter().enumerate() {
    729
                assert!(
    730
                    val.is_finite(),
    731
                    "Chunk[{}] is not finite: {}",
    732
                    i,
    733
                    val
    734
                );
    735
            }
    736
        }
    737
    }