## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/apr_transformer/helpers.rs

    Line
    Count
    Source
    1
    //! APR Transformer Helper Functions (PMAT-802)
    2
    //!
    3
    //! Row-major matmul wrappers and SIMD primitives for APR inference.
    4
    5
    use crate::quantize::{fused_q4k_parallel_matvec, fused_q6k_parallel_matvec};
    6
    7
    8
    /// Row-major Q4K matmul wrapper (LAYOUT-001)
    9
    ///
    10
    /// Wraps `fused_q4k_parallel_matvec` with dimension order matching the old API.
    11
    /// OLD API: `matmul_q4k_rowmajor(bytes, input, out_dim, in_dim)` - column-major, WRONG
    12
    /// NEW API: `matmul_q4k_rowmajor(bytes, input, out_dim, in_dim)` - row-major, CORRECT
    13
    ///
    14
    /// FORBIDDEN: Never use `trueno::backends::q4k::matmul_q4k_f32_colmajor*` for GGUF/APR.
    15
    #[inline]
    16
    0
    pub(crate) fn matmul_q4k_rowmajor(q4k_bytes: &[u8], input: &[f32], out_dim: usize, in_dim: usize) -> Vec<f32> {
    17
        // fused_q4k_parallel_matvec expects (bytes, input, in_dim, out_dim) - swap order!
    18
    0
        fused_q4k_parallel_matvec(q4k_bytes, input, in_dim, out_dim)
    19
    0
            .expect("Q4K matmul failed - check tensor dimensions")
    20
    0
    }
    21
    22
    /// Row-major Q6K matmul wrapper (LAYOUT-001)
    23
    #[inline]
    24
    0
    pub(crate) fn matmul_q6k_rowmajor(q6k_bytes: &[u8], input: &[f32], out_dim: usize, in_dim: usize) -> Vec<f32> {
    25
    0
        fused_q6k_parallel_matvec(q6k_bytes, input, in_dim, out_dim)
    26
    0
            .expect("Q6K matmul failed - check tensor dimensions")
    27
    0
    }
    28
    29
    // ============================================================================
    30
    // PMAT-103: SIMD Attention Primitives for 5.0+ tok/s target
    31
    // ============================================================================
    32
    33
    /// SIMD dot product with AVX2 acceleration (PMAT-103)
    34
    ///
    35
    /// Computes the dot product of two f32 slices using AVX2 when available.
    36
    /// Falls back to scalar when AVX2 is not supported or slices are small.
    37
    #[inline]
    38
    4
    pub(crate) fn simd_dot_f32(a: &[f32], b: &[f32]) -> f32 {
    39
    4
        debug_assert_eq!(a.len(), b.len(), "SIMD dot: length mismatch"0);
    40
    41
        #[cfg(target_arch = "x86_64")]
    42
        {
    43
    4
            if is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma") && a.len() >= 8 {
    44
    2
                return unsafe { simd_dot_f32_avx2(a, b) };
    45
    2
            }
    46
        }
    47
    48
        // Scalar fallback
    49
    3
        a2.iter2().zip2(b2.iter2()).map2(|(x, y)| x * y).sum2()
    50
    4
    }
    51
    52
    /// AVX2 dot product implementation (PMAT-103)
    53
    #[cfg(target_arch = "x86_64")]
    54
    #[target_feature(enable = "avx2", enable = "fma")]
    55
    2
    unsafe fn simd_dot_f32_avx2(a: &[f32], b: &[f32]) -> f32 {
    56
        // SAFETY: Memory safety ensured by bounds checking before SIMD operations
    57
        unsafe {
    58
            use std::arch::x86_64::{
    59
                _mm256_castps256_ps128, _mm256_extractf128_ps, _mm256_fmadd_ps, _mm256_loadu_ps,
    60
                _mm256_setzero_ps, _mm_add_ps, _mm_cvtss_f32, _mm_hadd_ps,
    61
            };
    62
    63
    2
            let n = a.len();
    64
    2
            let mut acc = _mm256_setzero_ps();
    65
    66
            // Process 8 elements at a time
    67
    2
            let chunks = n / 8;
    68
    13
            for i in 0..chunks2 {
    69
    13
                let offset = i * 8;
    70
    13
                let va = _mm256_loadu_ps(a.as_ptr().add(offset));
    71
    13
                let vb = _mm256_loadu_ps(b.as_ptr().add(offset));
    72
    13
                acc = _mm256_fmadd_ps(va, vb, acc);
    73
    13
            }
    74
    75
            // Horizontal sum of 8 floats
    76
    2
            let hi = _mm256_extractf128_ps(acc, 1);
    77
    2
            let lo = _mm256_castps256_ps128(acc);
    78
    2
            let sum128 = _mm_add_ps(lo, hi);
    79
    2
            let sum128 = _mm_hadd_ps(sum128, sum128);
    80
    2
            let sum128 = _mm_hadd_ps(sum128, sum128);
    81
    2
            let mut result = _mm_cvtss_f32(sum128);
    82
    83
            // Handle remaining elements
    84
    2
            let remainder = n % 8;
    85
    2
            if remainder > 0 {
    86
    1
                let start = chunks * 8;
    87
    4
                for i in start1..n1 {
    88
    4
                    result += a[i] * b[i];
    89
    4
                }
    90
    1
            }
    91
    92
    2
            result
    93
        }
    94
    2
    }
    95
    96
    /// SIMD weighted accumulation: out[i] += weight * val[i] (PMAT-103)
    97
    ///
    98
    /// Uses AVX2 FMA for efficient multiply-accumulate operations.
    99
    #[inline]
    100
    5
    pub(crate) fn simd_add_weighted(out: &mut [f32], val: &[f32], weight: f32) {
    101
    5
        debug_assert_eq!(out.len(), val.len(), "SIMD add_weighted: length mismatch"0);
    102
    103
        #[cfg(target_arch = "x86_64")]
    104
        {
    105
    5
            if is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma") && out.len() >= 8 {
    106
    2
                unsafe { simd_add_weighted_avx2(out, val, weight) };
    107
    2
                return;
    108
    3
            }
    109
        }
    110
    111
        // Scalar fallback
    112
    7
        for (o, v) in out3.iter_mut3().zip3(val3.iter3()) {
    113
    7
            *o += weight * v;
    114
    7
        }
    115
    5
    }
    116
    117
    /// AVX2 weighted accumulation implementation (PMAT-103)
    118
    #[cfg(target_arch = "x86_64")]
    119
    #[target_feature(enable = "avx2", enable = "fma")]
    120
    2
    unsafe fn simd_add_weighted_avx2(out: &mut [f32], val: &[f32], weight: f32) {
    121
        // SAFETY: Memory safety ensured by bounds checking before SIMD operations
    122
        unsafe {
    123
            use std::arch::x86_64::{_mm256_fmadd_ps, _mm256_loadu_ps, _mm256_set1_ps, _mm256_storeu_ps};
    124
    125
    2
            let n = out.len();
    126
    2
            let w = _mm256_set1_ps(weight);
    127
    128
            // Process 8 elements at a time
    129
    2
            let chunks = n / 8;
    130
    13
            for i in 0..chunks2 {
    131
    13
                let offset = i * 8;
    132
    13
                let v_out = _mm256_loadu_ps(out.as_ptr().add(offset));
    133
    13
                let v_val = _mm256_loadu_ps(val.as_ptr().add(offset));
    134
    13
                let result = _mm256_fmadd_ps(w, v_val, v_out);
    135
    13
                _mm256_storeu_ps(out.as_mut_ptr().add(offset), result);
    136
    13
            }
    137
    138
            // Handle remaining elements
    139
    2
            let remainder = n % 8;
    140
    2
            if remainder > 0 {
    141
    1
                let start = chunks * 8;
    142
    4
                for i in start1..n1 {
    143
    4
                    out[i] += weight * val[i];
    144
    4
                }
    145
    1
            }
    146
        }
    147
    2
    }