## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/trueno/src/backends/avx2/ops/reductions.rs

    Line
    Count
    Source
    1
    //! AVX2 reduction operations (dot, sum, max, min, argmax, argmin).
    2
    3
    #[cfg(target_arch = "x86_64")]
    4
    use std::arch::x86_64::*;
    5
    6
    use crate::backends::VectorBackend;
    7
    8
    /// AVX2 dot product with 4-accumulator unrolling for ILP.
    9
    #[inline]
    10
    #[target_feature(enable = "avx2,fma")]
    11
    229k
    pub unsafe fn dot(a: &[f32], b: &[f32]) -> f32 {
    12
    229k
        let len = a.len();
    13
    229k
        let mut i = 0;
    14
    15
    229k
        let mut acc0 = _mm256_setzero_ps();
    16
    229k
        let mut acc1 = _mm256_setzero_ps();
    17
    229k
        let mut acc2 = _mm256_setzero_ps();
    18
    229k
        let mut acc3 = _mm256_setzero_ps();
    19
    20
    269k
        while i + 32 <= len {
    21
    39.9k
            let va0 = _mm256_loadu_ps(a.as_ptr().add(i));
    22
    39.9k
            let vb0 = _mm256_loadu_ps(b.as_ptr().add(i));
    23
    39.9k
            let va1 = _mm256_loadu_ps(a.as_ptr().add(i + 8));
    24
    39.9k
            let vb1 = _mm256_loadu_ps(b.as_ptr().add(i + 8));
    25
    39.9k
            let va2 = _mm256_loadu_ps(a.as_ptr().add(i + 16));
    26
    39.9k
            let vb2 = _mm256_loadu_ps(b.as_ptr().add(i + 16));
    27
    39.9k
            let va3 = _mm256_loadu_ps(a.as_ptr().add(i + 24));
    28
    39.9k
            let vb3 = _mm256_loadu_ps(b.as_ptr().add(i + 24));
    29
    39.9k
    30
    39.9k
            acc0 = _mm256_fmadd_ps(va0, vb0, acc0);
    31
    39.9k
            acc1 = _mm256_fmadd_ps(va1, vb1, acc1);
    32
    39.9k
            acc2 = _mm256_fmadd_ps(va2, vb2, acc2);
    33
    39.9k
            acc3 = _mm256_fmadd_ps(va3, vb3, acc3);
    34
    39.9k
    35
    39.9k
            i += 32;
    36
    39.9k
        }
    37
    38
    680k
        while i + 8 <= len {
    39
    451k
            let va = _mm256_loadu_ps(a.as_ptr().add(i));
    40
    451k
            let vb = _mm256_loadu_ps(b.as_ptr().add(i));
    41
    451k
            acc0 = _mm256_fmadd_ps(va, vb, acc0);
    42
    451k
            i += 8;
    43
    451k
        }
    44
    45
    229k
        let acc01 = _mm256_add_ps(acc0, acc1);
    46
    229k
        let acc23 = _mm256_add_ps(acc2, acc3);
    47
    229k
        let acc = _mm256_add_ps(acc01, acc23);
    48
    49
    229k
        let mut result = {
    50
    229k
            let sum_halves = _mm_add_ps(_mm256_castps256_ps128(acc), _mm256_extractf128_ps(acc, 1));
    51
    229k
            let temp = _mm_add_ps(sum_halves, _mm_movehl_ps(sum_halves, sum_halves));
    52
    229k
            let temp = _mm_add_ss(temp, _mm_shuffle_ps(temp, temp, 1));
    53
    229k
            _mm_cvtss_f32(temp)
    54
        };
    55
    56
    229k
        result += a[i..].iter().zip(&b[i..]).map(|(x, y)| x206 * y206).sum::<f32>();
    57
    229k
        result
    58
    229k
    }
    59
    60
    /// AVX2 vector sum.
    61
    #[inline]
    62
    #[target_feature(enable = "avx2")]
    63
    118
    pub unsafe fn sum(a: &[f32]) -> f32 {
    64
    118
        let len = a.len();
    65
    118
        let mut i = 0;
    66
    118
        let mut acc = _mm256_setzero_ps();
    67
    68
    14.2k
        while i + 8 <= len {
    69
    14.0k
            let va = _mm256_loadu_ps(a.as_ptr().add(i));
    70
    14.0k
            acc = _mm256_add_ps(acc, va);
    71
    14.0k
            i += 8;
    72
    14.0k
        }
    73
    74
    118
        let mut result = {
    75
    118
            let sum_halves = _mm_add_ps(_mm256_castps256_ps128(acc), _mm256_extractf128_ps(acc, 1));
    76
    118
            let temp = _mm_add_ps(sum_halves, _mm_movehl_ps(sum_halves, sum_halves));
    77
    118
            let temp = _mm_add_ss(temp, _mm_shuffle_ps(temp, temp, 1));
    78
    118
            _mm_cvtss_f32(temp)
    79
        };
    80
    81
    118
        result += a[i..].iter().sum::<f32>();
    82
    118
        result
    83
    118
    }
    84
    85
    /// AVX2 vector max.
    86
    #[inline]
    87
    #[target_feature(enable = "avx2")]
    88
    10.4k
    pub unsafe fn max(a: &[f32]) -> f32 {
    89
    10.4k
        let len = a.len();
    90
    10.4k
        let mut i = 0;
    91
    10.4k
        let mut vmax = _mm256_set1_ps(a[0]);
    92
    93
    34.1k
        while i + 8 <= len {
    94
    23.7k
            let va = _mm256_loadu_ps(a.as_ptr().add(i));
    95
    23.7k
            vmax = _mm256_max_ps(vmax, va);
    96
    23.7k
            i += 8;
    97
    23.7k
        }
    98
    99
    10.4k
        let mut result = {
    100
    10.4k
            let max_halves = _mm_max_ps(_mm256_castps256_ps128(vmax), _mm256_extractf128_ps(vmax, 1));
    101
    10.4k
            let temp = _mm_max_ps(max_halves, _mm_movehl_ps(max_halves, max_halves));
    102
    10.4k
            let temp = _mm_max_ss(temp, _mm_shuffle_ps(temp, temp, 1));
    103
    10.4k
            _mm_cvtss_f32(temp)
    104
        };
    105
    106
    37.9k
        for &val in &a[i..]10.4k {
    107
    37.9k
            if val > result {
    108
    19.3k
                result = val;
    109
    19.3k
            }18.6k
    110
        }
    111
    10.4k
        result
    112
    10.4k
    }
    113
    114
    /// AVX2 vector min.
    115
    #[inline]
    116
    #[target_feature(enable = "avx2")]
    117
    0
    pub unsafe fn min(a: &[f32]) -> f32 {
    118
    0
        let len = a.len();
    119
    0
        let mut i = 0;
    120
    0
        let mut vmin = _mm256_set1_ps(a[0]);
    121
    122
    0
        while i + 8 <= len {
    123
    0
            let va = _mm256_loadu_ps(a.as_ptr().add(i));
    124
    0
            vmin = _mm256_min_ps(vmin, va);
    125
    0
            i += 8;
    126
    0
        }
    127
    128
    0
        let mut result = {
    129
    0
            let min_halves = _mm_min_ps(_mm256_castps256_ps128(vmin), _mm256_extractf128_ps(vmin, 1));
    130
    0
            let temp = _mm_min_ps(min_halves, _mm_movehl_ps(min_halves, min_halves));
    131
    0
            let temp = _mm_min_ss(temp, _mm_shuffle_ps(temp, temp, 1));
    132
    0
            _mm_cvtss_f32(temp)
    133
        };
    134
    135
    0
        for &val in &a[i..] {
    136
    0
            if val < result {
    137
    0
                result = val;
    138
    0
            }
    139
        }
    140
    0
        result
    141
    0
    }
    142
    143
    /// AVX2 argmax.
    144
    #[inline]
    145
    #[target_feature(enable = "avx2")]
    146
    0
    pub unsafe fn argmax(a: &[f32]) -> usize {
    147
    0
        let len = a.len();
    148
    0
        let mut max_idx: usize = 0;
    149
    0
        let mut max_val = a[0];
    150
    0
        let mut i = 0;
    151
    152
    0
        let mut vmax = _mm256_set1_ps(a[0]);
    153
    0
        let mut vidx_max = _mm256_setzero_ps();
    154
    0
        let vidx_inc = _mm256_set1_ps(8.0);
    155
    0
        let mut vcurrent_idx = _mm256_set_ps(7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0);
    156
    157
    0
        while i + 8 <= len {
    158
    0
            let va = _mm256_loadu_ps(a.as_ptr().add(i));
    159
    0
            let mask = _mm256_cmp_ps(va, vmax, _CMP_GT_OQ);
    160
    0
            vmax = _mm256_blendv_ps(vmax, va, mask);
    161
    0
            vidx_max = _mm256_blendv_ps(vidx_max, vcurrent_idx, mask);
    162
    0
            vcurrent_idx = _mm256_add_ps(vcurrent_idx, vidx_inc);
    163
    0
            i += 8;
    164
    0
        }
    165
    166
        // Extract max from vector
    167
    0
        let mut vals = [0.0f32; 8];
    168
    0
        let mut idxs = [0.0f32; 8];
    169
    0
        _mm256_storeu_ps(vals.as_mut_ptr(), vmax);
    170
    0
        _mm256_storeu_ps(idxs.as_mut_ptr(), vidx_max);
    171
    172
    0
        for j in 0..8 {
    173
    0
            if vals[j] > max_val {
    174
    0
                max_val = vals[j];
    175
    0
                max_idx = idxs[j] as usize;
    176
    0
            }
    177
        }
    178
    179
        // Check remaining elements
    180
    0
        for (j, &val) in a[i..].iter().enumerate() {
    181
    0
            if val > max_val {
    182
    0
                max_val = val;
    183
    0
                max_idx = i + j;
    184
    0
            }
    185
        }
    186
    187
    0
        max_idx
    188
    0
    }
    189
    190
    /// AVX2 argmin.
    191
    #[inline]
    192
    #[target_feature(enable = "avx2")]
    193
    0
    pub unsafe fn argmin(a: &[f32]) -> usize {
    194
    0
        let len = a.len();
    195
    0
        let mut min_idx: usize = 0;
    196
    0
        let mut min_val = a[0];
    197
    0
        let mut i = 0;
    198
    199
    0
        let mut vmin = _mm256_set1_ps(a[0]);
    200
    0
        let mut vidx_min = _mm256_setzero_ps();
    201
    0
        let vidx_inc = _mm256_set1_ps(8.0);
    202
    0
        let mut vcurrent_idx = _mm256_set_ps(7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0);
    203
    204
    0
        while i + 8 <= len {
    205
    0
            let va = _mm256_loadu_ps(a.as_ptr().add(i));
    206
    0
            let mask = _mm256_cmp_ps(va, vmin, _CMP_LT_OQ);
    207
    0
            vmin = _mm256_blendv_ps(vmin, va, mask);
    208
    0
            vidx_min = _mm256_blendv_ps(vidx_min, vcurrent_idx, mask);
    209
    0
            vcurrent_idx = _mm256_add_ps(vcurrent_idx, vidx_inc);
    210
    0
            i += 8;
    211
    0
        }
    212
    213
    0
        let mut vals = [0.0f32; 8];
    214
    0
        let mut idxs = [0.0f32; 8];
    215
    0
        _mm256_storeu_ps(vals.as_mut_ptr(), vmin);
    216
    0
        _mm256_storeu_ps(idxs.as_mut_ptr(), vidx_min);
    217
    218
    0
        for j in 0..8 {
    219
    0
            if vals[j] < min_val {
    220
    0
                min_val = vals[j];
    221
    0
                min_idx = idxs[j] as usize;
    222
    0
            }
    223
        }
    224
    225
    0
        for (j, &val) in a[i..].iter().enumerate() {
    226
    0
            if val < min_val {
    227
    0
                min_val = val;
    228
    0
                min_idx = i + j;
    229
    0
            }
    230
        }
    231
    232
    0
        min_idx
    233
    0
    }
    234
    235
    /// Kahan sum for numerical stability (delegates to scalar).
    236
    #[inline]
    237
    0
    pub unsafe fn sum_kahan(a: &[f32]) -> f32 {
    238
    0
        crate::backends::scalar::ScalarBackend::sum_kahan(a)
    239
    0
    }