## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/trueno/src/backends/avx2/mod.rs

    Line
    Count
    Source
    1
    //! AVX2 backend implementation (x86_64 advanced SIMD)
    2
    //!
    3
    //! This backend uses AVX2 intrinsics for 256-bit SIMD operations with FMA.
    4
    //! AVX2 is available on Intel Haswell (2013+) and AMD Excavator (2015+) CPUs.
    5
    6
    #[cfg(target_arch = "x86_64")]
    7
    use std::arch::x86_64::*;
    8
    9
    use super::VectorBackend;
    10
    11
    mod ops;
    12
    13
    /// AVX2 backend (256-bit SIMD for x86_64)
    14
    pub struct Avx2Backend;
    15
    16
    impl VectorBackend for Avx2Backend {
    17
        #[inline]
    18
        #[target_feature(enable = "avx2")]
    19
    15
        unsafe fn add(a: &[f32], b: &[f32], result: &mut [f32]) {
    20
    15
            ops::arithmetic::add(a, b, result)
    21
    15
        }
    22
    23
        #[inline]
    24
        #[target_feature(enable = "avx2")]
    25
    15
        unsafe fn sub(a: &[f32], b: &[f32], result: &mut [f32]) {
    26
    15
            ops::arithmetic::sub(a, b, result)
    27
    15
        }
    28
    29
        #[inline]
    30
        #[target_feature(enable = "avx2")]
    31
    61
        unsafe fn mul(a: &[f32], b: &[f32], result: &mut [f32]) {
    32
    61
            ops::arithmetic::mul(a, b, result)
    33
    61
        }
    34
    35
        #[inline]
    36
        #[target_feature(enable = "avx2")]
    37
    0
        unsafe fn div(a: &[f32], b: &[f32], result: &mut [f32]) {
    38
    0
            ops::arithmetic::div(a, b, result)
    39
    0
        }
    40
    41
        #[inline]
    42
        #[target_feature(enable = "avx2,fma")]
    43
    229k
        unsafe fn dot(a: &[f32], b: &[f32]) -> f32 {
    44
    229k
            ops::reductions::dot(a, b)
    45
    229k
        }
    46
    47
        #[inline]
    48
        #[target_feature(enable = "avx2")]
    49
    118
        unsafe fn sum(a: &[f32]) -> f32 {
    50
    118
            ops::reductions::sum(a)
    51
    118
        }
    52
    53
        #[inline]
    54
        #[target_feature(enable = "avx2")]
    55
    10.4k
        unsafe fn max(a: &[f32]) -> f32 {
    56
    10.4k
            ops::reductions::max(a)
    57
    10.4k
        }
    58
    59
        #[inline]
    60
        #[target_feature(enable = "avx2")]
    61
    0
        unsafe fn min(a: &[f32]) -> f32 {
    62
    0
            ops::reductions::min(a)
    63
    0
        }
    64
    65
        #[inline]
    66
        #[target_feature(enable = "avx2")]
    67
    0
        unsafe fn argmax(a: &[f32]) -> usize {
    68
    0
            ops::reductions::argmax(a)
    69
    0
        }
    70
    71
        #[inline]
    72
        #[target_feature(enable = "avx2")]
    73
    0
        unsafe fn argmin(a: &[f32]) -> usize {
    74
    0
            ops::reductions::argmin(a)
    75
    0
        }
    76
    77
        #[inline]
    78
    0
        unsafe fn sum_kahan(a: &[f32]) -> f32 {
    79
    0
            ops::reductions::sum_kahan(a)
    80
    0
        }
    81
    82
        #[inline]
    83
        #[target_feature(enable = "avx2,fma")]
    84
    0
        unsafe fn norm_l2(a: &[f32]) -> f32 {
    85
    0
            if a.is_empty() { return 0.0; }
    86
    0
            Self::dot(a, a).sqrt()
    87
    0
        }
    88
    89
        #[inline]
    90
        #[target_feature(enable = "avx2")]
    91
    0
        unsafe fn norm_l1(a: &[f32]) -> f32 {
    92
    0
            if a.is_empty() { return 0.0; }
    93
    0
            let len = a.len();
    94
    0
            let mut i = 0;
    95
    0
            let mut acc = _mm256_setzero_ps();
    96
    0
            let sign_mask = _mm256_set1_ps(f32::from_bits(0x7FFF_FFFF));
    97
    98
    0
            while i + 8 <= len {
    99
    0
                let va = _mm256_loadu_ps(a.as_ptr().add(i));
    100
    0
                let abs_va = _mm256_and_ps(va, sign_mask);
    101
    0
                acc = _mm256_add_ps(acc, abs_va);
    102
    0
                i += 8;
    103
    0
            }
    104
    105
    0
            let mut result = {
    106
    0
                let sum_halves = _mm_add_ps(_mm256_castps256_ps128(acc), _mm256_extractf128_ps(acc, 1));
    107
    0
                let temp = _mm_add_ps(sum_halves, _mm_movehl_ps(sum_halves, sum_halves));
    108
    0
                let temp = _mm_add_ss(temp, _mm_shuffle_ps(temp, temp, 1));
    109
    0
                _mm_cvtss_f32(temp)
    110
            };
    111
    112
    0
            for &val in &a[i..] { result += val.abs(); }
    113
    0
            result
    114
    0
        }
    115
    116
        #[inline]
    117
        #[target_feature(enable = "avx2")]
    118
    0
        unsafe fn norm_linf(a: &[f32]) -> f32 {
    119
    0
            if a.is_empty() { return 0.0; }
    120
    0
            let len = a.len();
    121
    0
            let mut i = 0;
    122
    0
            let mut max_vec = _mm256_setzero_ps();
    123
    0
            let sign_mask = _mm256_set1_ps(f32::from_bits(0x7FFF_FFFF));
    124
    125
    0
            while i + 8 <= len {
    126
    0
                let va = _mm256_loadu_ps(a.as_ptr().add(i));
    127
    0
                let abs_va = _mm256_and_ps(va, sign_mask);
    128
    0
                max_vec = _mm256_max_ps(max_vec, abs_va);
    129
    0
                i += 8;
    130
    0
            }
    131
    132
    0
            let mut result = {
    133
    0
                let max_halves = _mm_max_ps(_mm256_castps256_ps128(max_vec), _mm256_extractf128_ps(max_vec, 1));
    134
    0
                let temp = _mm_max_ps(max_halves, _mm_movehl_ps(max_halves, max_halves));
    135
    0
                let temp = _mm_max_ss(temp, _mm_shuffle_ps(temp, temp, 1));
    136
    0
                _mm_cvtss_f32(temp)
    137
            };
    138
    139
    0
            for &val in &a[i..] {
    140
    0
                let abs_val = val.abs();
    141
    0
                if abs_val > result { result = abs_val; }
    142
            }
    143
    0
            result
    144
    0
        }
    145
    146
        #[inline]
    147
        #[target_feature(enable = "avx2")]
    148
    1
        unsafe fn scale(a: &[f32], scalar: f32, result: &mut [f32]) {
    149
    1
            let len = a.len();
    150
    1
            let mut i = 0;
    151
    1
            let scalar_vec = _mm256_set1_ps(scalar);
    152
    153
    1
            while i + 8 <= len {
    154
    0
                let va = _mm256_loadu_ps(a.as_ptr().add(i));
    155
    0
                let vresult = _mm256_mul_ps(va, scalar_vec);
    156
    0
                _mm256_storeu_ps(result.as_mut_ptr().add(i), vresult);
    157
    0
                i += 8;
    158
    0
            }
    159
    160
    5
            while i < len {
    161
    4
                result[i] = a[i] * scalar;
    162
    4
                i += 1;
    163
    4
            }
    164
    1
        }
    165
    166
        #[inline]
    167
        #[target_feature(enable = "avx2")]
    168
    0
        unsafe fn abs(a: &[f32], result: &mut [f32]) {
    169
    0
            let len = a.len();
    170
    0
            let mut i = 0;
    171
    0
            let sign_mask = _mm256_set1_ps(f32::from_bits(0x7FFF_FFFF));
    172
    173
    0
            while i + 8 <= len {
    174
    0
                let va = _mm256_loadu_ps(a.as_ptr().add(i));
    175
    0
                let vresult = _mm256_and_ps(va, sign_mask);
    176
    0
                _mm256_storeu_ps(result.as_mut_ptr().add(i), vresult);
    177
    0
                i += 8;
    178
    0
            }
    179
    180
    0
            for j in i..len { result[j] = a[j].abs(); }
    181
    0
        }
    182
    183
        #[inline]
    184
        #[target_feature(enable = "avx2")]
    185
    0
        unsafe fn clamp(a: &[f32], min_val: f32, max_val: f32, result: &mut [f32]) {
    186
    0
            let len = a.len();
    187
    0
            let mut i = 0;
    188
    0
            let vmin = _mm256_set1_ps(min_val);
    189
    0
            let vmax = _mm256_set1_ps(max_val);
    190
    191
    0
            while i + 8 <= len {
    192
    0
                let va = _mm256_loadu_ps(a.as_ptr().add(i));
    193
    0
                let vresult = _mm256_min_ps(_mm256_max_ps(va, vmin), vmax);
    194
    0
                _mm256_storeu_ps(result.as_mut_ptr().add(i), vresult);
    195
    0
                i += 8;
    196
    0
            }
    197
    198
    0
            for j in i..len { result[j] = a[j].clamp(min_val, max_val); }
    199
    0
        }
    200
    201
        #[inline]
    202
        #[target_feature(enable = "avx2,fma")]
    203
    0
        unsafe fn lerp(a: &[f32], b: &[f32], t: f32, result: &mut [f32]) {
    204
    0
            let len = a.len();
    205
    0
            let mut i = 0;
    206
    0
            let vt = _mm256_set1_ps(t);
    207
    0
            let v1_minus_t = _mm256_set1_ps(1.0 - t);
    208
    209
    0
            while i + 8 <= len {
    210
    0
                let va = _mm256_loadu_ps(a.as_ptr().add(i));
    211
    0
                let vb = _mm256_loadu_ps(b.as_ptr().add(i));
    212
    0
                let vresult = _mm256_fmadd_ps(vb, vt, _mm256_mul_ps(va, v1_minus_t));
    213
    0
                _mm256_storeu_ps(result.as_mut_ptr().add(i), vresult);
    214
    0
                i += 8;
    215
    0
            }
    216
    217
    0
            for j in i..len { result[j] = a[j] * (1.0 - t) + b[j] * t; }
    218
    0
        }
    219
    220
        #[inline]
    221
        #[target_feature(enable = "avx2,fma")]
    222
    0
        unsafe fn fma(a: &[f32], b: &[f32], c: &[f32], result: &mut [f32]) {
    223
    0
            let len = a.len();
    224
    0
            let mut i = 0;
    225
    226
    0
            while i + 8 <= len {
    227
    0
                let va = _mm256_loadu_ps(a.as_ptr().add(i));
    228
    0
                let vb = _mm256_loadu_ps(b.as_ptr().add(i));
    229
    0
                let vc = _mm256_loadu_ps(c.as_ptr().add(i));
    230
    0
                let vresult = _mm256_fmadd_ps(va, vb, vc);
    231
    0
                _mm256_storeu_ps(result.as_mut_ptr().add(i), vresult);
    232
    0
                i += 8;
    233
    0
            }
    234
    235
    0
            for j in i..len { result[j] = a[j] * b[j] + c[j]; }
    236
    0
        }
    237
    238
        #[inline]
    239
        #[target_feature(enable = "avx2")]
    240
    0
        unsafe fn relu(a: &[f32], result: &mut [f32]) {
    241
    0
            let len = a.len();
    242
    0
            let mut i = 0;
    243
    0
            let vzero = _mm256_setzero_ps();
    244
    245
    0
            while i + 8 <= len {
    246
    0
                let va = _mm256_loadu_ps(a.as_ptr().add(i));
    247
    0
                let vresult = _mm256_max_ps(va, vzero);
    248
    0
                _mm256_storeu_ps(result.as_mut_ptr().add(i), vresult);
    249
    0
                i += 8;
    250
    0
            }
    251
    252
    0
            for j in i..len { result[j] = a[j].max(0.0); }
    253
    0
        }
    254
    255
        // Delegate transcendental functions to scalar backend
    256
        #[inline]
    257
    0
        unsafe fn exp(a: &[f32], result: &mut [f32]) {
    258
    0
            super::scalar::ScalarBackend::exp(a, result)
    259
    0
        }
    260
    261
        #[inline]
    262
    0
        unsafe fn sigmoid(a: &[f32], result: &mut [f32]) {
    263
    0
            super::scalar::ScalarBackend::sigmoid(a, result)
    264
    0
        }
    265
    266
        #[inline]
    267
    0
        unsafe fn gelu(a: &[f32], result: &mut [f32]) {
    268
    0
            super::scalar::ScalarBackend::gelu(a, result)
    269
    0
        }
    270
    271
        #[inline]
    272
    0
        unsafe fn swish(a: &[f32], result: &mut [f32]) {
    273
    0
            super::scalar::ScalarBackend::swish(a, result)
    274
    0
        }
    275
    276
        #[inline]
    277
    0
        unsafe fn tanh(a: &[f32], result: &mut [f32]) {
    278
    0
            super::scalar::ScalarBackend::tanh(a, result)
    279
    0
        }
    280
    281
        #[inline]
    282
        #[target_feature(enable = "avx2")]
    283
    0
        unsafe fn sqrt(a: &[f32], result: &mut [f32]) {
    284
    0
            let len = a.len();
    285
    0
            let mut i = 0;
    286
    287
    0
            while i + 8 <= len {
    288
    0
                let va = _mm256_loadu_ps(a.as_ptr().add(i));
    289
    0
                let vresult = _mm256_sqrt_ps(va);
    290
    0
                _mm256_storeu_ps(result.as_mut_ptr().add(i), vresult);
    291
    0
                i += 8;
    292
    0
            }
    293
    294
    0
            for j in i..len { result[j] = a[j].sqrt(); }
    295
    0
        }
    296
    297
        #[inline]
    298
        #[target_feature(enable = "avx2")]
    299
    0
        unsafe fn recip(a: &[f32], result: &mut [f32]) {
    300
    0
            let len = a.len();
    301
    0
            let mut i = 0;
    302
    0
            let vone = _mm256_set1_ps(1.0);
    303
    304
    0
            while i + 8 <= len {
    305
    0
                let va = _mm256_loadu_ps(a.as_ptr().add(i));
    306
    0
                let vresult = _mm256_div_ps(vone, va);
    307
    0
                _mm256_storeu_ps(result.as_mut_ptr().add(i), vresult);
    308
    0
                i += 8;
    309
    0
            }
    310
    311
    0
            for j in i..len { result[j] = 1.0 / a[j]; }
    312
    0
        }
    313
    314
        // Delegate log functions to scalar backend
    315
        #[inline]
    316
    0
        unsafe fn ln(a: &[f32], result: &mut [f32]) {
    317
    0
            super::scalar::ScalarBackend::ln(a, result)
    318
    0
        }
    319
    320
        #[inline]
    321
    0
        unsafe fn log2(a: &[f32], result: &mut [f32]) {
    322
    0
            super::scalar::ScalarBackend::log2(a, result)
    323
    0
        }
    324
    325
        #[inline]
    326
    0
        unsafe fn log10(a: &[f32], result: &mut [f32]) {
    327
    0
            super::scalar::ScalarBackend::log10(a, result)
    328
    0
        }
    329
    330
        // Delegate trig functions to scalar backend
    331
        #[inline]
    332
    0
        unsafe fn sin(a: &[f32], result: &mut [f32]) {
    333
    0
            super::scalar::ScalarBackend::sin(a, result)
    334
    0
        }
    335
    336
        #[inline]
    337
    0
        unsafe fn cos(a: &[f32], result: &mut [f32]) {
    338
    0
            super::scalar::ScalarBackend::cos(a, result)
    339
    0
        }
    340
    341
        #[inline]
    342
    0
        unsafe fn tan(a: &[f32], result: &mut [f32]) {
    343
    0
            super::scalar::ScalarBackend::tan(a, result)
    344
    0
        }
    345
    346
        #[inline]
    347
        #[target_feature(enable = "avx2")]
    348
    0
        unsafe fn floor(a: &[f32], result: &mut [f32]) {
    349
    0
            let len = a.len();
    350
    0
            let mut i = 0;
    351
    352
    0
            while i + 8 <= len {
    353
    0
                let va = _mm256_loadu_ps(a.as_ptr().add(i));
    354
    0
                let vresult = _mm256_floor_ps(va);
    355
    0
                _mm256_storeu_ps(result.as_mut_ptr().add(i), vresult);
    356
    0
                i += 8;
    357
    0
            }
    358
    359
    0
            for j in i..len { result[j] = a[j].floor(); }
    360
    0
        }
    361
    362
        #[inline]
    363
        #[target_feature(enable = "avx2")]
    364
    0
        unsafe fn ceil(a: &[f32], result: &mut [f32]) {
    365
    0
            let len = a.len();
    366
    0
            let mut i = 0;
    367
    368
    0
            while i + 8 <= len {
    369
    0
                let va = _mm256_loadu_ps(a.as_ptr().add(i));
    370
    0
                let vresult = _mm256_ceil_ps(va);
    371
    0
                _mm256_storeu_ps(result.as_mut_ptr().add(i), vresult);
    372
    0
                i += 8;
    373
    0
            }
    374
    375
    0
            for j in i..len { result[j] = a[j].ceil(); }
    376
    0
        }
    377
    378
        #[inline]
    379
        #[target_feature(enable = "avx2")]
    380
    0
        unsafe fn round(a: &[f32], result: &mut [f32]) {
    381
    0
            let len = a.len();
    382
    0
            let mut i = 0;
    383
    384
            // Round ties away from zero to match Rust's f32::round()
    385
    0
            let half = _mm256_set1_ps(0.5);
    386
    0
            let sign_mask = _mm256_set1_ps(f32::from_bits(0x8000_0000));
    387
    0
            let abs_mask = _mm256_set1_ps(f32::from_bits(0x7FFF_FFFF));
    388
    389
    0
            while i + 8 <= len {
    390
    0
                let va = _mm256_loadu_ps(a.as_ptr().add(i));
    391
    0
                let sign = _mm256_and_ps(va, sign_mask);
    392
    0
                let abs_val = _mm256_and_ps(va, abs_mask);
    393
    0
                let shifted = _mm256_add_ps(abs_val, half);
    394
    0
                let rounded_abs = _mm256_floor_ps(shifted);
    395
    0
                let vresult = _mm256_or_ps(rounded_abs, sign);
    396
    0
                _mm256_storeu_ps(result.as_mut_ptr().add(i), vresult);
    397
    0
                i += 8;
    398
    0
            }
    399
    400
    0
            for j in i..len { result[j] = a[j].round(); }
    401
    0
        }
    402
    }