Coverage Report

Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/home/noah/src/trueno/src/backends/avx2/mod.rs
Line
Count
Source
1
//! AVX2 backend implementation (x86_64 advanced SIMD)
2
//!
3
//! This backend uses AVX2 intrinsics for 256-bit SIMD operations with FMA.
4
//! AVX2 is available on Intel Haswell (2013+) and AMD Excavator (2015+) CPUs.
5
6
#[cfg(target_arch = "x86_64")]
7
use std::arch::x86_64::*;
8
9
use super::VectorBackend;
10
11
mod ops;
12
13
/// AVX2 backend (256-bit SIMD for x86_64)
14
pub struct Avx2Backend;
15
16
impl VectorBackend for Avx2Backend {
17
    #[inline]
18
    #[target_feature(enable = "avx2")]
19
15
    unsafe fn add(a: &[f32], b: &[f32], result: &mut [f32]) {
20
15
        ops::arithmetic::add(a, b, result)
21
15
    }
22
23
    #[inline]
24
    #[target_feature(enable = "avx2")]
25
15
    unsafe fn sub(a: &[f32], b: &[f32], result: &mut [f32]) {
26
15
        ops::arithmetic::sub(a, b, result)
27
15
    }
28
29
    #[inline]
30
    #[target_feature(enable = "avx2")]
31
61
    unsafe fn mul(a: &[f32], b: &[f32], result: &mut [f32]) {
32
61
        ops::arithmetic::mul(a, b, result)
33
61
    }
34
35
    #[inline]
36
    #[target_feature(enable = "avx2")]
37
0
    unsafe fn div(a: &[f32], b: &[f32], result: &mut [f32]) {
38
0
        ops::arithmetic::div(a, b, result)
39
0
    }
40
41
    #[inline]
42
    #[target_feature(enable = "avx2,fma")]
43
229k
    unsafe fn dot(a: &[f32], b: &[f32]) -> f32 {
44
229k
        ops::reductions::dot(a, b)
45
229k
    }
46
47
    #[inline]
48
    #[target_feature(enable = "avx2")]
49
118
    unsafe fn sum(a: &[f32]) -> f32 {
50
118
        ops::reductions::sum(a)
51
118
    }
52
53
    #[inline]
54
    #[target_feature(enable = "avx2")]
55
10.4k
    unsafe fn max(a: &[f32]) -> f32 {
56
10.4k
        ops::reductions::max(a)
57
10.4k
    }
58
59
    #[inline]
60
    #[target_feature(enable = "avx2")]
61
0
    unsafe fn min(a: &[f32]) -> f32 {
62
0
        ops::reductions::min(a)
63
0
    }
64
65
    #[inline]
66
    #[target_feature(enable = "avx2")]
67
0
    unsafe fn argmax(a: &[f32]) -> usize {
68
0
        ops::reductions::argmax(a)
69
0
    }
70
71
    #[inline]
72
    #[target_feature(enable = "avx2")]
73
0
    unsafe fn argmin(a: &[f32]) -> usize {
74
0
        ops::reductions::argmin(a)
75
0
    }
76
77
    #[inline]
78
0
    unsafe fn sum_kahan(a: &[f32]) -> f32 {
79
0
        ops::reductions::sum_kahan(a)
80
0
    }
81
82
    #[inline]
83
    #[target_feature(enable = "avx2,fma")]
84
0
    unsafe fn norm_l2(a: &[f32]) -> f32 {
85
0
        if a.is_empty() { return 0.0; }
86
0
        Self::dot(a, a).sqrt()
87
0
    }
88
89
    #[inline]
90
    #[target_feature(enable = "avx2")]
91
0
    unsafe fn norm_l1(a: &[f32]) -> f32 {
92
0
        if a.is_empty() { return 0.0; }
93
0
        let len = a.len();
94
0
        let mut i = 0;
95
0
        let mut acc = _mm256_setzero_ps();
96
0
        let sign_mask = _mm256_set1_ps(f32::from_bits(0x7FFF_FFFF));
97
98
0
        while i + 8 <= len {
99
0
            let va = _mm256_loadu_ps(a.as_ptr().add(i));
100
0
            let abs_va = _mm256_and_ps(va, sign_mask);
101
0
            acc = _mm256_add_ps(acc, abs_va);
102
0
            i += 8;
103
0
        }
104
105
0
        let mut result = {
106
0
            let sum_halves = _mm_add_ps(_mm256_castps256_ps128(acc), _mm256_extractf128_ps(acc, 1));
107
0
            let temp = _mm_add_ps(sum_halves, _mm_movehl_ps(sum_halves, sum_halves));
108
0
            let temp = _mm_add_ss(temp, _mm_shuffle_ps(temp, temp, 1));
109
0
            _mm_cvtss_f32(temp)
110
        };
111
112
0
        for &val in &a[i..] { result += val.abs(); }
113
0
        result
114
0
    }
115
116
    #[inline]
117
    #[target_feature(enable = "avx2")]
118
0
    unsafe fn norm_linf(a: &[f32]) -> f32 {
119
0
        if a.is_empty() { return 0.0; }
120
0
        let len = a.len();
121
0
        let mut i = 0;
122
0
        let mut max_vec = _mm256_setzero_ps();
123
0
        let sign_mask = _mm256_set1_ps(f32::from_bits(0x7FFF_FFFF));
124
125
0
        while i + 8 <= len {
126
0
            let va = _mm256_loadu_ps(a.as_ptr().add(i));
127
0
            let abs_va = _mm256_and_ps(va, sign_mask);
128
0
            max_vec = _mm256_max_ps(max_vec, abs_va);
129
0
            i += 8;
130
0
        }
131
132
0
        let mut result = {
133
0
            let max_halves = _mm_max_ps(_mm256_castps256_ps128(max_vec), _mm256_extractf128_ps(max_vec, 1));
134
0
            let temp = _mm_max_ps(max_halves, _mm_movehl_ps(max_halves, max_halves));
135
0
            let temp = _mm_max_ss(temp, _mm_shuffle_ps(temp, temp, 1));
136
0
            _mm_cvtss_f32(temp)
137
        };
138
139
0
        for &val in &a[i..] {
140
0
            let abs_val = val.abs();
141
0
            if abs_val > result { result = abs_val; }
142
        }
143
0
        result
144
0
    }
145
146
    #[inline]
147
    #[target_feature(enable = "avx2")]
148
1
    unsafe fn scale(a: &[f32], scalar: f32, result: &mut [f32]) {
149
1
        let len = a.len();
150
1
        let mut i = 0;
151
1
        let scalar_vec = _mm256_set1_ps(scalar);
152
153
1
        while i + 8 <= len {
154
0
            let va = _mm256_loadu_ps(a.as_ptr().add(i));
155
0
            let vresult = _mm256_mul_ps(va, scalar_vec);
156
0
            _mm256_storeu_ps(result.as_mut_ptr().add(i), vresult);
157
0
            i += 8;
158
0
        }
159
160
5
        while i < len {
161
4
            result[i] = a[i] * scalar;
162
4
            i += 1;
163
4
        }
164
1
    }
165
166
    #[inline]
167
    #[target_feature(enable = "avx2")]
168
0
    unsafe fn abs(a: &[f32], result: &mut [f32]) {
169
0
        let len = a.len();
170
0
        let mut i = 0;
171
0
        let sign_mask = _mm256_set1_ps(f32::from_bits(0x7FFF_FFFF));
172
173
0
        while i + 8 <= len {
174
0
            let va = _mm256_loadu_ps(a.as_ptr().add(i));
175
0
            let vresult = _mm256_and_ps(va, sign_mask);
176
0
            _mm256_storeu_ps(result.as_mut_ptr().add(i), vresult);
177
0
            i += 8;
178
0
        }
179
180
0
        for j in i..len { result[j] = a[j].abs(); }
181
0
    }
182
183
    #[inline]
184
    #[target_feature(enable = "avx2")]
185
0
    unsafe fn clamp(a: &[f32], min_val: f32, max_val: f32, result: &mut [f32]) {
186
0
        let len = a.len();
187
0
        let mut i = 0;
188
0
        let vmin = _mm256_set1_ps(min_val);
189
0
        let vmax = _mm256_set1_ps(max_val);
190
191
0
        while i + 8 <= len {
192
0
            let va = _mm256_loadu_ps(a.as_ptr().add(i));
193
0
            let vresult = _mm256_min_ps(_mm256_max_ps(va, vmin), vmax);
194
0
            _mm256_storeu_ps(result.as_mut_ptr().add(i), vresult);
195
0
            i += 8;
196
0
        }
197
198
0
        for j in i..len { result[j] = a[j].clamp(min_val, max_val); }
199
0
    }
200
201
    #[inline]
202
    #[target_feature(enable = "avx2,fma")]
203
0
    unsafe fn lerp(a: &[f32], b: &[f32], t: f32, result: &mut [f32]) {
204
0
        let len = a.len();
205
0
        let mut i = 0;
206
0
        let vt = _mm256_set1_ps(t);
207
0
        let v1_minus_t = _mm256_set1_ps(1.0 - t);
208
209
0
        while i + 8 <= len {
210
0
            let va = _mm256_loadu_ps(a.as_ptr().add(i));
211
0
            let vb = _mm256_loadu_ps(b.as_ptr().add(i));
212
0
            let vresult = _mm256_fmadd_ps(vb, vt, _mm256_mul_ps(va, v1_minus_t));
213
0
            _mm256_storeu_ps(result.as_mut_ptr().add(i), vresult);
214
0
            i += 8;
215
0
        }
216
217
0
        for j in i..len { result[j] = a[j] * (1.0 - t) + b[j] * t; }
218
0
    }
219
220
    #[inline]
221
    #[target_feature(enable = "avx2,fma")]
222
0
    unsafe fn fma(a: &[f32], b: &[f32], c: &[f32], result: &mut [f32]) {
223
0
        let len = a.len();
224
0
        let mut i = 0;
225
226
0
        while i + 8 <= len {
227
0
            let va = _mm256_loadu_ps(a.as_ptr().add(i));
228
0
            let vb = _mm256_loadu_ps(b.as_ptr().add(i));
229
0
            let vc = _mm256_loadu_ps(c.as_ptr().add(i));
230
0
            let vresult = _mm256_fmadd_ps(va, vb, vc);
231
0
            _mm256_storeu_ps(result.as_mut_ptr().add(i), vresult);
232
0
            i += 8;
233
0
        }
234
235
0
        for j in i..len { result[j] = a[j] * b[j] + c[j]; }
236
0
    }
237
238
    #[inline]
239
    #[target_feature(enable = "avx2")]
240
0
    unsafe fn relu(a: &[f32], result: &mut [f32]) {
241
0
        let len = a.len();
242
0
        let mut i = 0;
243
0
        let vzero = _mm256_setzero_ps();
244
245
0
        while i + 8 <= len {
246
0
            let va = _mm256_loadu_ps(a.as_ptr().add(i));
247
0
            let vresult = _mm256_max_ps(va, vzero);
248
0
            _mm256_storeu_ps(result.as_mut_ptr().add(i), vresult);
249
0
            i += 8;
250
0
        }
251
252
0
        for j in i..len { result[j] = a[j].max(0.0); }
253
0
    }
254
255
    // Delegate transcendental functions to scalar backend
256
    #[inline]
257
0
    unsafe fn exp(a: &[f32], result: &mut [f32]) {
258
0
        super::scalar::ScalarBackend::exp(a, result)
259
0
    }
260
261
    #[inline]
262
0
    unsafe fn sigmoid(a: &[f32], result: &mut [f32]) {
263
0
        super::scalar::ScalarBackend::sigmoid(a, result)
264
0
    }
265
266
    #[inline]
267
0
    unsafe fn gelu(a: &[f32], result: &mut [f32]) {
268
0
        super::scalar::ScalarBackend::gelu(a, result)
269
0
    }
270
271
    #[inline]
272
0
    unsafe fn swish(a: &[f32], result: &mut [f32]) {
273
0
        super::scalar::ScalarBackend::swish(a, result)
274
0
    }
275
276
    #[inline]
277
0
    unsafe fn tanh(a: &[f32], result: &mut [f32]) {
278
0
        super::scalar::ScalarBackend::tanh(a, result)
279
0
    }
280
281
    #[inline]
282
    #[target_feature(enable = "avx2")]
283
0
    unsafe fn sqrt(a: &[f32], result: &mut [f32]) {
284
0
        let len = a.len();
285
0
        let mut i = 0;
286
287
0
        while i + 8 <= len {
288
0
            let va = _mm256_loadu_ps(a.as_ptr().add(i));
289
0
            let vresult = _mm256_sqrt_ps(va);
290
0
            _mm256_storeu_ps(result.as_mut_ptr().add(i), vresult);
291
0
            i += 8;
292
0
        }
293
294
0
        for j in i..len { result[j] = a[j].sqrt(); }
295
0
    }
296
297
    #[inline]
298
    #[target_feature(enable = "avx2")]
299
0
    unsafe fn recip(a: &[f32], result: &mut [f32]) {
300
0
        let len = a.len();
301
0
        let mut i = 0;
302
0
        let vone = _mm256_set1_ps(1.0);
303
304
0
        while i + 8 <= len {
305
0
            let va = _mm256_loadu_ps(a.as_ptr().add(i));
306
0
            let vresult = _mm256_div_ps(vone, va);
307
0
            _mm256_storeu_ps(result.as_mut_ptr().add(i), vresult);
308
0
            i += 8;
309
0
        }
310
311
0
        for j in i..len { result[j] = 1.0 / a[j]; }
312
0
    }
313
314
    // Delegate log functions to scalar backend
315
    #[inline]
316
0
    unsafe fn ln(a: &[f32], result: &mut [f32]) {
317
0
        super::scalar::ScalarBackend::ln(a, result)
318
0
    }
319
320
    #[inline]
321
0
    unsafe fn log2(a: &[f32], result: &mut [f32]) {
322
0
        super::scalar::ScalarBackend::log2(a, result)
323
0
    }
324
325
    #[inline]
326
0
    unsafe fn log10(a: &[f32], result: &mut [f32]) {
327
0
        super::scalar::ScalarBackend::log10(a, result)
328
0
    }
329
330
    // Delegate trig functions to scalar backend
331
    #[inline]
332
0
    unsafe fn sin(a: &[f32], result: &mut [f32]) {
333
0
        super::scalar::ScalarBackend::sin(a, result)
334
0
    }
335
336
    #[inline]
337
0
    unsafe fn cos(a: &[f32], result: &mut [f32]) {
338
0
        super::scalar::ScalarBackend::cos(a, result)
339
0
    }
340
341
    #[inline]
342
0
    unsafe fn tan(a: &[f32], result: &mut [f32]) {
343
0
        super::scalar::ScalarBackend::tan(a, result)
344
0
    }
345
346
    #[inline]
347
    #[target_feature(enable = "avx2")]
348
0
    unsafe fn floor(a: &[f32], result: &mut [f32]) {
349
0
        let len = a.len();
350
0
        let mut i = 0;
351
352
0
        while i + 8 <= len {
353
0
            let va = _mm256_loadu_ps(a.as_ptr().add(i));
354
0
            let vresult = _mm256_floor_ps(va);
355
0
            _mm256_storeu_ps(result.as_mut_ptr().add(i), vresult);
356
0
            i += 8;
357
0
        }
358
359
0
        for j in i..len { result[j] = a[j].floor(); }
360
0
    }
361
362
    #[inline]
363
    #[target_feature(enable = "avx2")]
364
0
    unsafe fn ceil(a: &[f32], result: &mut [f32]) {
365
0
        let len = a.len();
366
0
        let mut i = 0;
367
368
0
        while i + 8 <= len {
369
0
            let va = _mm256_loadu_ps(a.as_ptr().add(i));
370
0
            let vresult = _mm256_ceil_ps(va);
371
0
            _mm256_storeu_ps(result.as_mut_ptr().add(i), vresult);
372
0
            i += 8;
373
0
        }
374
375
0
        for j in i..len { result[j] = a[j].ceil(); }
376
0
    }
377
378
    #[inline]
379
    #[target_feature(enable = "avx2")]
380
0
    unsafe fn round(a: &[f32], result: &mut [f32]) {
381
0
        let len = a.len();
382
0
        let mut i = 0;
383
384
        // Round ties away from zero to match Rust's f32::round()
385
0
        let half = _mm256_set1_ps(0.5);
386
0
        let sign_mask = _mm256_set1_ps(f32::from_bits(0x8000_0000));
387
0
        let abs_mask = _mm256_set1_ps(f32::from_bits(0x7FFF_FFFF));
388
389
0
        while i + 8 <= len {
390
0
            let va = _mm256_loadu_ps(a.as_ptr().add(i));
391
0
            let sign = _mm256_and_ps(va, sign_mask);
392
0
            let abs_val = _mm256_and_ps(va, abs_mask);
393
0
            let shifted = _mm256_add_ps(abs_val, half);
394
0
            let rounded_abs = _mm256_floor_ps(shifted);
395
0
            let vresult = _mm256_or_ps(rounded_abs, sign);
396
0
            _mm256_storeu_ps(result.as_mut_ptr().add(i), vresult);
397
0
            i += 8;
398
0
        }
399
400
0
        for j in i..len { result[j] = a[j].round(); }
401
0
    }
402
}