Coverage Report

Created: 2026-01-23 22:55

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/home/noah/src/trueno/src/backends/avx512.rs
Line
Count
Source
1
//! AVX-512 backend implementation (x86_64 advanced SIMD)
2
//!
3
//! This backend uses AVX-512 intrinsics for 512-bit SIMD operations.
4
//! AVX-512 is available on Intel Skylake-X/Sapphire Rapids (2017+) and AMD Zen 4 (2022+) CPUs.
5
//!
6
//! # Performance
7
//!
8
//! Expected speedup: 16x for operations on f32 vectors (16 elements per register)
9
//! This provides 2x improvement over AVX2 (8 elements) and ~16x over scalar.
10
//!
11
//! # Safety
12
//!
13
//! All AVX-512 intrinsics are marked `unsafe` by Rust. This module carefully isolates
14
//! all unsafe code and verifies correctness through comprehensive testing.
15
16
#[cfg(target_arch = "x86_64")]
17
use std::arch::x86_64::*;
18
19
use super::VectorBackend;
20
21
/// AVX-512 backend (512-bit SIMD for x86_64)
22
pub struct Avx512Backend;
23
24
impl VectorBackend for Avx512Backend {
25
    #[inline]
26
    #[target_feature(enable = "avx512f")]
27
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
28
    // 1. Loop bounds ensure `i + N <= len` before calling `.add(i)` (N=16 for AVX-512)
29
    // 2. All pointers derived from valid slice references with sufficient backing storage
30
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
31
    // 4. Unaligned loads/stores used (_mm512_loadu_ps/_mm512_storeu_ps) - no alignment requirement
32
0
    unsafe fn add(a: &[f32], b: &[f32], result: &mut [f32]) {
33
0
        let len = a.len();
34
0
        let mut i = 0;
35
36
        // Process 16 elements at a time using AVX-512 (512-bit = 16 x f32)
37
0
        while i + 16 <= len {
38
0
            // Load 16 floats from a and b
39
0
            let va = _mm512_loadu_ps(a.as_ptr().add(i));
40
0
            let vb = _mm512_loadu_ps(b.as_ptr().add(i));
41
0
42
0
            // Add them
43
0
            let vresult = _mm512_add_ps(va, vb);
44
0
45
0
            // Store result
46
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), vresult);
47
0
48
0
            i += 16;
49
0
        }
50
51
        // Handle remaining elements with scalar code
52
0
        for j in i..len {
53
0
            result[j] = a[j] + b[j];
54
0
        }
55
0
    }
56
57
    // Stub implementations for remaining methods - will implement in future phases
58
    #[inline]
59
    #[target_feature(enable = "avx512f")]
60
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
61
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
62
    // 2. All pointers derived from valid slice references
63
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
64
    // 4. Unaligned loads/stores used - no alignment requirement
65
0
    unsafe fn sub(a: &[f32], b: &[f32], result: &mut [f32]) {
66
0
        let len = a.len();
67
0
        let mut i = 0;
68
69
        // Process 16 elements at a time (512-bit = 16 x f32)
70
0
        while i + 16 <= len {
71
0
            let va = _mm512_loadu_ps(a.as_ptr().add(i));
72
0
            let vb = _mm512_loadu_ps(b.as_ptr().add(i));
73
0
            let vresult = _mm512_sub_ps(va, vb);
74
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), vresult);
75
0
            i += 16;
76
0
        }
77
78
        // Handle remaining elements with scalar code
79
0
        while i < len {
80
0
            result[i] = a[i] - b[i];
81
0
            i += 1;
82
0
        }
83
0
    }
84
85
    #[inline]
86
    #[target_feature(enable = "avx512f")]
87
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
88
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
89
    // 2. All pointers derived from valid slice references
90
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
91
    // 4. Unaligned loads/stores used - no alignment requirement
92
0
    unsafe fn mul(a: &[f32], b: &[f32], result: &mut [f32]) {
93
0
        let len = a.len();
94
0
        let mut i = 0;
95
96
        // Process 16 elements at a time (512-bit = 16 x f32)
97
0
        while i + 16 <= len {
98
0
            let va = _mm512_loadu_ps(a.as_ptr().add(i));
99
0
            let vb = _mm512_loadu_ps(b.as_ptr().add(i));
100
0
            let vresult = _mm512_mul_ps(va, vb);
101
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), vresult);
102
0
            i += 16;
103
0
        }
104
105
        // Handle remaining elements with scalar code
106
0
        while i < len {
107
0
            result[i] = a[i] * b[i];
108
0
            i += 1;
109
0
        }
110
0
    }
111
112
    #[inline]
113
    #[target_feature(enable = "avx512f")]
114
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
115
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
116
    // 2. All pointers derived from valid slice references
117
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
118
    // 4. Unaligned loads/stores used - no alignment requirement
119
0
    unsafe fn div(a: &[f32], b: &[f32], result: &mut [f32]) {
120
0
        let len = a.len();
121
0
        let mut i = 0;
122
123
        // Process 16 elements at a time (512-bit = 16 x f32)
124
0
        while i + 16 <= len {
125
0
            let va = _mm512_loadu_ps(a.as_ptr().add(i));
126
0
            let vb = _mm512_loadu_ps(b.as_ptr().add(i));
127
0
            let vresult = _mm512_div_ps(va, vb);
128
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), vresult);
129
0
            i += 16;
130
0
        }
131
132
        // Handle remaining elements with scalar code
133
0
        while i < len {
134
0
            result[i] = a[i] / b[i];
135
0
            i += 1;
136
0
        }
137
0
    }
138
139
    #[inline]
140
    #[target_feature(enable = "avx512f")]
141
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
142
    // 1. Loop bounds ensure `i + N <= len` before calling `.add(i)` (N=32 for unrolled, 16 for remainder)
143
    // 2. All pointers derived from valid slice references with sufficient backing storage
144
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
145
    // 4. Unaligned loads used (_mm512_loadu_ps) - no alignment requirement
146
    // 5. FMA intrinsic (_mm512_fmadd_ps) provides better performance and numerical accuracy
147
    //
148
    // OPTIMIZATION: 2-accumulator unrolling for better ILP (Instruction Level Parallelism)
149
    // FMA has ~4 cycle latency on modern CPUs - 2 independent accumulators help hide latency.
150
    // This follows the cuda-tile pattern for improved throughput (spec: cuda-tile-behavior.md).
151
0
    unsafe fn dot(a: &[f32], b: &[f32]) -> f32 {
152
0
        let len = a.len();
153
0
        let mut i = 0;
154
155
        // 2 independent accumulators for better ILP (cuda-tile inspired optimization)
156
0
        let mut acc0 = _mm512_setzero_ps();
157
0
        let mut acc1 = _mm512_setzero_ps();
158
159
        // Process 32 elements at a time (2 × 16) with 2 independent FMA chains
160
0
        while i + 32 <= len {
161
0
            let va0 = _mm512_loadu_ps(a.as_ptr().add(i));
162
0
            let vb0 = _mm512_loadu_ps(b.as_ptr().add(i));
163
0
            let va1 = _mm512_loadu_ps(a.as_ptr().add(i + 16));
164
0
            let vb1 = _mm512_loadu_ps(b.as_ptr().add(i + 16));
165
0
166
0
            // 2 independent FMA operations - no dependency chain between them
167
0
            acc0 = _mm512_fmadd_ps(va0, vb0, acc0);
168
0
            acc1 = _mm512_fmadd_ps(va1, vb1, acc1);
169
0
170
0
            i += 32;
171
0
        }
172
173
        // Handle 16-element chunks that don't fit in 32-element blocks
174
0
        while i + 16 <= len {
175
0
            let va = _mm512_loadu_ps(a.as_ptr().add(i));
176
0
            let vb = _mm512_loadu_ps(b.as_ptr().add(i));
177
0
            acc0 = _mm512_fmadd_ps(va, vb, acc0);
178
0
            i += 16;
179
0
        }
180
181
        // Combine both accumulators
182
0
        let acc = _mm512_add_ps(acc0, acc1);
183
184
        // Horizontal sum: reduce 16 lanes to single value
185
        // AVX-512 provides a convenient intrinsic for this
186
0
        let mut result = _mm512_reduce_add_ps(acc);
187
188
        // Handle remaining elements with scalar code
189
0
        result += a[i..].iter().zip(&b[i..]).map(|(x, y)| x * y).sum::<f32>();
190
191
0
        result
192
0
    }
193
194
    #[inline]
195
    #[target_feature(enable = "avx512f")]
196
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
197
    // 1. Loop bounds ensure `i + N <= len` before calling `.add(i)` (N=16 for AVX-512)
198
    // 2. All pointers derived from valid slice references with sufficient backing storage
199
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
200
    // 4. Unaligned loads used (_mm512_loadu_ps) - no alignment requirement
201
0
    unsafe fn sum(a: &[f32]) -> f32 {
202
0
        let len = a.len();
203
0
        let mut i = 0;
204
205
        // Accumulator for 16-way parallel accumulation
206
0
        let mut acc = _mm512_setzero_ps();
207
208
        // Process 16 elements at a time
209
0
        while i + 16 <= len {
210
0
            let va = _mm512_loadu_ps(a.as_ptr().add(i));
211
0
            acc = _mm512_add_ps(acc, va);
212
0
            i += 16;
213
0
        }
214
215
        // Horizontal sum: reduce 16 lanes to single value
216
        // AVX-512 provides a convenient intrinsic for this
217
0
        let mut result = _mm512_reduce_add_ps(acc);
218
219
        // Handle remaining elements with scalar code
220
0
        result += a[i..].iter().sum::<f32>();
221
222
0
        result
223
0
    }
224
225
    #[inline]
226
    #[target_feature(enable = "avx512f")]
227
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
228
    // 1. Loop bounds ensure `i + N <= len` before calling `.add(i)` (N=16 for AVX-512)
229
    // 2. All pointers derived from valid slice references with sufficient backing storage
230
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
231
    // 4. Unaligned loads used (_mm512_loadu_ps) - no alignment requirement
232
0
    unsafe fn max(a: &[f32]) -> f32 {
233
0
        let len = a.len();
234
0
        let mut i = 0;
235
236
        // Start with first element broadcast to all 16 lanes
237
0
        let mut vmax = _mm512_set1_ps(a[0]);
238
239
        // Process 16 elements at a time
240
0
        while i + 16 <= len {
241
0
            let va = _mm512_loadu_ps(a.as_ptr().add(i));
242
0
            vmax = _mm512_max_ps(vmax, va);
243
0
            i += 16;
244
0
        }
245
246
        // Horizontal max: find maximum across all 16 lanes
247
        // AVX-512 provides a convenient intrinsic for this
248
0
        let mut result = _mm512_reduce_max_ps(vmax);
249
250
        // Check remaining elements
251
0
        for &val in &a[i..] {
252
0
            if val > result {
253
0
                result = val;
254
0
            }
255
        }
256
257
0
        result
258
0
    }
259
260
    #[inline]
261
    #[target_feature(enable = "avx512f")]
262
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
263
    // 1. Loop bounds ensure `i + N <= len` before calling `.add(i)` (N=16 for AVX-512)
264
    // 2. All pointers derived from valid slice references with sufficient backing storage
265
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
266
    // 4. Unaligned loads used (_mm512_loadu_ps) - no alignment requirement
267
0
    unsafe fn min(a: &[f32]) -> f32 {
268
0
        let len = a.len();
269
0
        let mut i = 0;
270
271
        // Start with first element broadcast to all 16 lanes
272
0
        let mut vmin = _mm512_set1_ps(a[0]);
273
274
        // Process 16 elements at a time
275
0
        while i + 16 <= len {
276
0
            let va = _mm512_loadu_ps(a.as_ptr().add(i));
277
0
            vmin = _mm512_min_ps(vmin, va);
278
0
            i += 16;
279
0
        }
280
281
        // Horizontal min: find minimum across all 16 lanes
282
        // AVX-512 provides a convenient intrinsic for this
283
0
        let mut result = _mm512_reduce_min_ps(vmin);
284
285
        // Check remaining elements
286
0
        for &val in &a[i..] {
287
0
            if val < result {
288
0
                result = val;
289
0
            }
290
        }
291
292
0
        result
293
0
    }
294
295
    #[inline]
296
    #[target_feature(enable = "avx512f")]
297
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
298
    // 1. Loop bounds ensure proper array access
299
    // 2. All pointers derived from valid slice references
300
    // 3. AVX-512 intrinsics marked with #[target_feature]
301
    // 4. Unaligned loads/stores handle unaligned data correctly
302
0
    unsafe fn argmax(a: &[f32]) -> usize {
303
0
        if a.is_empty() {
304
0
            return 0;
305
0
        }
306
307
0
        let len = a.len();
308
0
        let mut i = 0;
309
310
        // Start with first element broadcast to all 16 lanes
311
0
        let mut vmax = _mm512_set1_ps(a[0]);
312
313
        // Process 16 elements at a time to find max value
314
0
        while i + 16 <= len {
315
0
            let va = _mm512_loadu_ps(a.as_ptr().add(i));
316
0
            vmax = _mm512_max_ps(vmax, va);
317
0
            i += 16;
318
0
        }
319
320
        // Horizontal max: find maximum value across all 16 lanes
321
0
        let mut max_val = _mm512_reduce_max_ps(vmax);
322
323
        // Check remaining elements
324
0
        for &val in &a[i..] {
325
0
            if val > max_val {
326
0
                max_val = val;
327
0
            }
328
        }
329
330
        // Find the index of the first occurrence of max_val
331
        // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
332
        // 1. Loop bounds ensure proper array access
333
        // 2. All pointers derived from valid slice references
334
        // 3. AVX-512 intrinsics marked with #[target_feature]
335
        // 4. Unaligned loads/stores handle unaligned data correctly
336
0
        a.iter().position(|&x| x == max_val).unwrap_or(0)
337
0
    }
338
339
    #[inline]
340
    #[target_feature(enable = "avx512f")]
341
0
    unsafe fn argmin(a: &[f32]) -> usize {
342
0
        if a.is_empty() {
343
0
            return 0;
344
0
        }
345
346
0
        let len = a.len();
347
0
        let mut i = 0;
348
349
        // Start with first element broadcast to all 16 lanes
350
0
        let mut vmin = _mm512_set1_ps(a[0]);
351
352
        // Process 16 elements at a time to find min value
353
0
        while i + 16 <= len {
354
0
            let va = _mm512_loadu_ps(a.as_ptr().add(i));
355
0
            vmin = _mm512_min_ps(vmin, va);
356
0
            i += 16;
357
0
        }
358
359
        // Horizontal min: find minimum value across all 16 lanes
360
0
        let mut min_val = _mm512_reduce_min_ps(vmin);
361
362
        // Check remaining elements
363
0
        for &val in &a[i..] {
364
0
            if val < min_val {
365
0
                // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
366
0
                // 1. Loop bounds ensure proper array access
367
0
                // 2. All pointers derived from valid slice references
368
0
                // 3. AVX-512 intrinsics marked with #[target_feature]
369
0
                // 4. Unaligned loads/stores handle unaligned data correctly
370
0
                min_val = val;
371
0
            }
372
        }
373
374
        // Find the index of the first occurrence of min_val
375
0
        a.iter().position(|&x| x == min_val).unwrap_or(0)
376
0
    }
377
378
    #[inline]
379
    #[target_feature(enable = "avx512f")]
380
    // SAFETY: Uses scalar implementation, no unsafe operations
381
0
    unsafe fn sum_kahan(a: &[f32]) -> f32 {
382
        // Scalar fallback (AVX-512 optimization pending)
383
0
        let mut sum = 0.0;
384
0
        let mut c = 0.0;
385
0
        for &x in a {
386
0
            let y = x - c;
387
0
            let t = sum + y;
388
0
            c = (t - sum) - y;
389
0
            sum = t;
390
0
        }
391
0
        sum
392
0
    }
393
394
    #[inline]
395
    #[target_feature(enable = "avx512f")]
396
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
397
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
398
    // 2. All pointers derived from valid slice references
399
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
400
    // 4. Unaligned loads used (_mm512_loadu_ps) - no alignment requirement
401
0
    unsafe fn norm_l2(a: &[f32]) -> f32 {
402
0
        if a.is_empty() {
403
0
            return 0.0;
404
0
        }
405
406
0
        let len = a.len();
407
0
        let mut i = 0;
408
409
        // Accumulator for sum of squares
410
0
        let mut acc = _mm512_setzero_ps();
411
412
        // Process 16 elements at a time: compute x^2 and accumulate
413
0
        while i + 16 <= len {
414
0
            let va = _mm512_loadu_ps(a.as_ptr().add(i));
415
0
            let squared = _mm512_mul_ps(va, va);
416
0
            acc = _mm512_add_ps(acc, squared);
417
0
            i += 16;
418
0
        }
419
420
        // Horizontal sum: reduce 16 lanes to single value
421
0
        let mut sum_of_squares = _mm512_reduce_add_ps(acc);
422
423
        // Handle remaining elements with scalar code
424
0
        for &val in &a[i..] {
425
0
            sum_of_squares += val * val;
426
0
        }
427
428
0
        sum_of_squares.sqrt()
429
0
    }
430
431
    #[inline]
432
    #[target_feature(enable = "avx512f")]
433
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
434
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
435
    // 2. All pointers derived from valid slice references
436
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
437
    // 4. Unaligned loads used - no alignment requirement
438
0
    unsafe fn norm_l1(a: &[f32]) -> f32 {
439
0
        let len = a.len();
440
0
        let mut i = 0;
441
442
        // Sign bit mask for abs
443
0
        let sign_mask = _mm512_set1_ps(f32::from_bits(0x7FFF_FFFF));
444
445
        // Accumulator for sum
446
0
        let mut acc = _mm512_setzero_ps();
447
448
        // Process 16 elements at a time
449
        // norm_l1 = sum(|x|)
450
0
        while i + 16 <= len {
451
0
            let va = _mm512_loadu_ps(a.as_ptr().add(i));
452
0
            let vabs = _mm512_and_ps(va, sign_mask);
453
0
            acc = _mm512_add_ps(acc, vabs);
454
0
            i += 16;
455
0
        }
456
457
        // Horizontal sum: reduce 16 lanes to single value
458
0
        let mut result = _mm512_reduce_add_ps(acc);
459
460
        // Handle remaining elements with scalar code
461
0
        for &val in &a[i..] {
462
0
            result += val.abs();
463
0
        }
464
465
0
        result
466
0
    }
467
468
    #[inline]
469
    #[target_feature(enable = "avx512f")]
470
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
471
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
472
    // 2. All pointers derived from valid slice references
473
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
474
    // 4. Unaligned loads used - no alignment requirement
475
0
    unsafe fn norm_linf(a: &[f32]) -> f32 {
476
0
        let len = a.len();
477
0
        let mut i = 0;
478
479
        // Sign bit mask for abs
480
0
        let sign_mask = _mm512_set1_ps(f32::from_bits(0x7FFF_FFFF));
481
482
        // Accumulator for max
483
0
        let mut acc = _mm512_setzero_ps();
484
485
        // Process 16 elements at a time
486
        // norm_linf = max(|x|)
487
0
        while i + 16 <= len {
488
0
            let va = _mm512_loadu_ps(a.as_ptr().add(i));
489
0
            let vabs = _mm512_and_ps(va, sign_mask);
490
0
            acc = _mm512_max_ps(acc, vabs);
491
0
            i += 16;
492
0
        }
493
494
        // Horizontal max: reduce 16 lanes to single value
495
0
        let mut result = _mm512_reduce_max_ps(acc);
496
497
        // Handle remaining elements with scalar code
498
0
        for &val in &a[i..] {
499
0
            result = result.max(val.abs());
500
0
        }
501
502
0
        result
503
0
    }
504
505
    #[inline]
506
    #[target_feature(enable = "avx512f")]
507
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
508
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
509
    // 2. All pointers derived from valid slice references
510
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
511
    // 4. Unaligned loads/stores used (_mm512_loadu_ps/_mm512_storeu_ps) - no alignment requirement
512
0
    unsafe fn scale(a: &[f32], scalar: f32, result: &mut [f32]) {
513
0
        let len = a.len();
514
0
        let mut i = 0;
515
516
        // Broadcast scalar to all 16 lanes
517
0
        let scalar_vec = _mm512_set1_ps(scalar);
518
519
        // Process 16 elements at a time (512-bit = 16 x f32)
520
0
        while i + 16 <= len {
521
0
            let va = _mm512_loadu_ps(a.as_ptr().add(i));
522
0
            let vresult = _mm512_mul_ps(va, scalar_vec);
523
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), vresult);
524
0
            i += 16;
525
0
        }
526
527
        // Handle remaining elements with scalar code
528
0
        while i < len {
529
0
            result[i] = a[i] * scalar;
530
0
            i += 1;
531
0
        }
532
0
    }
533
534
    #[inline]
535
    #[target_feature(enable = "avx512f")]
536
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
537
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
538
    // 2. All pointers derived from valid slice references
539
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
540
    // 4. Unaligned loads/stores used - no alignment requirement
541
0
    unsafe fn abs(a: &[f32], result: &mut [f32]) {
542
0
        let len = a.len();
543
0
        let mut i = 0;
544
545
        // Sign bit mask: 0x7FFFFFFF clears sign bit (keeps magnitude)
546
0
        let sign_mask = _mm512_set1_ps(f32::from_bits(0x7FFF_FFFF));
547
548
        // Process 16 elements at a time
549
0
        while i + 16 <= len {
550
0
            let va = _mm512_loadu_ps(a.as_ptr().add(i));
551
0
            let vabs = _mm512_and_ps(va, sign_mask);
552
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), vabs);
553
0
            i += 16;
554
0
        }
555
556
        // Handle remaining elements with scalar code
557
0
        while i < len {
558
0
            result[i] = a[i].abs();
559
0
            i += 1;
560
0
        }
561
0
    }
562
563
    #[inline]
564
    #[target_feature(enable = "avx512f")]
565
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
566
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
567
    // 2. All pointers derived from valid slice references
568
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
569
    // 4. Unaligned loads/stores used - no alignment requirement
570
0
    unsafe fn clamp(a: &[f32], min_val: f32, max_val: f32, result: &mut [f32]) {
571
0
        let len = a.len();
572
0
        let mut i = 0;
573
574
        // Broadcast min/max to all 16 lanes
575
0
        let vmin = _mm512_set1_ps(min_val);
576
0
        let vmax = _mm512_set1_ps(max_val);
577
578
        // Process 16 elements at a time
579
0
        while i + 16 <= len {
580
0
            let va = _mm512_loadu_ps(a.as_ptr().add(i));
581
0
            // clamp(x, min, max) = min(max(x, min), max)
582
0
            let vclamped = _mm512_min_ps(_mm512_max_ps(va, vmin), vmax);
583
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), vclamped);
584
0
            i += 16;
585
0
        }
586
587
        // Handle remaining elements with scalar code
588
0
        while i < len {
589
0
            result[i] = a[i].clamp(min_val, max_val);
590
0
            i += 1;
591
0
        }
592
0
    }
593
594
    #[inline]
595
    #[target_feature(enable = "avx512f")]
596
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
597
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
598
    // 2. All pointers derived from valid slice references
599
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
600
    // 4. Unaligned loads/stores used - no alignment requirement
601
0
    unsafe fn lerp(a: &[f32], b: &[f32], t: f32, result: &mut [f32]) {
602
0
        let len = a.len();
603
0
        let mut i = 0;
604
605
        // Broadcast t to all 16 lanes
606
0
        let t_vec = _mm512_set1_ps(t);
607
608
        // Process 16 elements at a time
609
        // lerp(a, b, t) = a + t * (b - a)
610
0
        while i + 16 <= len {
611
0
            let va = _mm512_loadu_ps(a.as_ptr().add(i));
612
0
            let vb = _mm512_loadu_ps(b.as_ptr().add(i));
613
0
614
0
            // Compute: b - a
615
0
            let vdiff = _mm512_sub_ps(vb, va);
616
0
617
0
            // Compute: t * (b - a)
618
0
            let vscaled = _mm512_mul_ps(t_vec, vdiff);
619
0
620
0
            // Compute: a + t * (b - a)
621
0
            let vresult = _mm512_add_ps(va, vscaled);
622
0
623
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), vresult);
624
0
            i += 16;
625
0
        }
626
627
        // Handle remaining elements with scalar code
628
0
        while i < len {
629
0
            result[i] = a[i] + t * (b[i] - a[i]);
630
0
            i += 1;
631
0
        }
632
0
    }
633
634
    #[inline]
635
    #[target_feature(enable = "avx512f")]
636
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
637
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
638
    // 2. All pointers derived from valid slice references
639
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
640
    // 4. Unaligned loads/stores used - no alignment requirement
641
    // 5. FMA instruction is part of AVX-512F (foundation)
642
0
    unsafe fn fma(a: &[f32], b: &[f32], c: &[f32], result: &mut [f32]) {
643
0
        let len = a.len();
644
0
        let mut i = 0;
645
646
        // Process 16 elements at a time
647
        // fma(a, b, c) = a * b + c
648
0
        while i + 16 <= len {
649
0
            let va = _mm512_loadu_ps(a.as_ptr().add(i));
650
0
            let vb = _mm512_loadu_ps(b.as_ptr().add(i));
651
0
            let vc = _mm512_loadu_ps(c.as_ptr().add(i));
652
0
653
0
            // Single fused multiply-add instruction (higher precision + faster)
654
0
            let vresult = _mm512_fmadd_ps(va, vb, vc);
655
0
656
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), vresult);
657
0
            i += 16;
658
0
        }
659
660
        // Handle remaining elements with scalar code
661
0
        while i < len {
662
0
            result[i] = a[i].mul_add(b[i], c[i]);
663
0
            i += 1;
664
0
        }
665
0
    }
666
667
    #[inline]
668
    #[target_feature(enable = "avx512f")]
669
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
670
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
671
    // 2. All pointers derived from valid slice references
672
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
673
    // 4. Unaligned loads/stores used - no alignment requirement
674
0
    unsafe fn relu(a: &[f32], result: &mut [f32]) {
675
0
        let len = a.len();
676
0
        let mut i = 0;
677
678
        // Create zero vector for comparison
679
0
        let zero = _mm512_setzero_ps();
680
681
        // Process 16 elements at a time
682
        // relu(x) = max(x, 0)
683
0
        while i + 16 <= len {
684
0
            let va = _mm512_loadu_ps(a.as_ptr().add(i));
685
0
            let vresult = _mm512_max_ps(va, zero);
686
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), vresult);
687
0
            i += 16;
688
0
        }
689
690
        // Handle remaining elements with scalar code
691
0
        while i < len {
692
0
            result[i] = a[i].max(0.0);
693
0
            i += 1;
694
0
        }
695
0
    }
696
697
    #[inline]
698
    #[target_feature(enable = "avx512f")]
699
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
700
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
701
    // 2. All pointers derived from valid slice references
702
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
703
    // 4. Unaligned loads/stores used - no alignment requirement
704
0
    unsafe fn exp(a: &[f32], result: &mut [f32]) {
705
0
        let len = a.len();
706
0
        let mut i = 0;
707
708
        // Constants for range reduction: exp(x) = 2^(x * log2(e)) = 2^k * 2^r
709
0
        let log2e = _mm512_set1_ps(std::f32::consts::LOG2_E); // 1.442695...
710
0
        let ln2 = _mm512_set1_ps(std::f32::consts::LN_2); // 0.693147...
711
0
        let half = _mm512_set1_ps(0.5);
712
0
        let one = _mm512_set1_ps(1.0);
713
714
        // Polynomial coefficients for e^r approximation (Remez minimax on [-ln(2)/2, ln(2)/2])
715
        // e^r ≈ 1 + c1*r + c2*r^2 + c3*r^3 + c4*r^4 + c5*r^5 + c6*r^6
716
0
        let c1 = _mm512_set1_ps(1.0);
717
0
        let c2 = _mm512_set1_ps(0.5);
718
0
        let c3 = _mm512_set1_ps(0.166_666_67); // 1/6
719
0
        let c4 = _mm512_set1_ps(0.041_666_668); // 1/24
720
0
        let c5 = _mm512_set1_ps(0.008_333_334); // 1/120
721
0
        let c6 = _mm512_set1_ps(0.001_388_889); // 1/720
722
723
        // Limits for overflow/underflow handling
724
0
        let exp_hi = _mm512_set1_ps(88.376_26); // ln(FLT_MAX)
725
0
        let exp_lo = _mm512_set1_ps(-87.336_55); // ln(FLT_MIN) approximately
726
727
        // Process 16 elements at a time
728
0
        while i + 16 <= len {
729
0
            let x = _mm512_loadu_ps(a.as_ptr().add(i));
730
0
731
0
            // Clamp x to avoid overflow/underflow
732
0
            let x = _mm512_max_ps(_mm512_min_ps(x, exp_hi), exp_lo);
733
0
734
0
            // Range reduction: x' = x * log2(e), then k = round(x'), r = x' - k
735
0
            let x_scaled = _mm512_mul_ps(x, log2e);
736
0
737
0
            // k = round(x_scaled) = floor(x_scaled + 0.5)
738
0
            // AVX512 uses roundscale instead of floor: mode 0x09 = floor
739
0
            let k = _mm512_roundscale_ps(_mm512_add_ps(x_scaled, half), 0x09);
740
0
741
0
            // r = x - k * ln(2) (in original base e space)
742
0
            let r = _mm512_sub_ps(x, _mm512_mul_ps(k, ln2));
743
0
744
0
            // Polynomial approximation: e^r ≈ 1 + c1*r + c2*r^2 + c3*r^3 + c4*r^4 + c5*r^5 + c6*r^6
745
0
            // Use Horner's method: ((((((c6*r + c5)*r + c4)*r + c3)*r + c2)*r + c1)*r + 1)
746
0
            let mut p = c6;
747
0
            p = _mm512_fmadd_ps(p, r, c5);
748
0
            p = _mm512_fmadd_ps(p, r, c4);
749
0
            p = _mm512_fmadd_ps(p, r, c3);
750
0
            p = _mm512_fmadd_ps(p, r, c2);
751
0
            p = _mm512_fmadd_ps(p, r, c1);
752
0
            p = _mm512_fmadd_ps(p, r, one);
753
0
754
0
            // Scale by 2^k using IEEE754 exponent manipulation
755
0
            // 2^k is computed by adding k to the exponent bits
756
0
            let k_int = _mm512_cvtps_epi32(k);
757
0
            let k_shifted = _mm512_slli_epi32(k_int, 23); // shift to exponent position
758
0
            let scale = _mm512_castsi512_ps(_mm512_add_epi32(_mm512_castps_si512(one), k_shifted));
759
0
760
0
            // Final result: e^x = e^r * 2^k
761
0
            let vresult = _mm512_mul_ps(p, scale);
762
0
763
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), vresult);
764
0
            i += 16;
765
0
        }
766
767
        // Handle remaining elements with scalar code
768
0
        while i < len {
769
0
            result[i] = a[i].exp();
770
0
            i += 1;
771
0
        }
772
0
    }
773
774
    #[inline]
775
    #[target_feature(enable = "avx512f")]
776
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
777
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
778
    // 2. All pointers derived from valid slice references
779
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
780
    // 4. Unaligned loads/stores used - no alignment requirement
781
0
    unsafe fn sigmoid(a: &[f32], result: &mut [f32]) {
782
        // sigmoid(x) = 1 / (1 + exp(-x))
783
        // Use SIMD exp approximation with range reduction
784
0
        let len = a.len();
785
0
        let mut i = 0;
786
787
        // Constants for exp(-x) computation
788
0
        let log2e = _mm512_set1_ps(std::f32::consts::LOG2_E);
789
0
        let ln2 = _mm512_set1_ps(std::f32::consts::LN_2);
790
0
        let half = _mm512_set1_ps(0.5);
791
0
        let one = _mm512_set1_ps(1.0);
792
793
        // Polynomial coefficients for e^r
794
0
        let c1 = _mm512_set1_ps(1.0);
795
0
        let c2 = _mm512_set1_ps(0.5);
796
0
        let c3 = _mm512_set1_ps(0.166_666_67);
797
0
        let c4 = _mm512_set1_ps(0.041_666_668);
798
0
        let c5 = _mm512_set1_ps(0.008_333_334);
799
0
        let c6 = _mm512_set1_ps(0.001_388_889);
800
801
        // Limits for overflow/underflow
802
0
        let exp_hi = _mm512_set1_ps(88.376_26);
803
0
        let exp_lo = _mm512_set1_ps(-87.336_55);
804
805
        // Process 16 elements at a time
806
0
        while i + 16 <= len {
807
0
            let x = _mm512_loadu_ps(a.as_ptr().add(i));
808
0
809
0
            // Compute -x for exp(-x)
810
0
            let neg_x = _mm512_sub_ps(_mm512_setzero_ps(), x);
811
0
812
0
            // Clamp to avoid overflow/underflow
813
0
            let neg_x = _mm512_max_ps(_mm512_min_ps(neg_x, exp_hi), exp_lo);
814
0
815
0
            // Range reduction: exp(-x) computation
816
0
            let x_scaled = _mm512_mul_ps(neg_x, log2e);
817
0
            let k = _mm512_roundscale_ps(_mm512_add_ps(x_scaled, half), 0x09);
818
0
            let r = _mm512_sub_ps(neg_x, _mm512_mul_ps(k, ln2));
819
0
820
0
            // Polynomial approximation using Horner's method with FMA
821
0
            let mut p = c6;
822
0
            p = _mm512_fmadd_ps(p, r, c5);
823
0
            p = _mm512_fmadd_ps(p, r, c4);
824
0
            p = _mm512_fmadd_ps(p, r, c3);
825
0
            p = _mm512_fmadd_ps(p, r, c2);
826
0
            p = _mm512_fmadd_ps(p, r, c1);
827
0
            p = _mm512_fmadd_ps(p, r, one);
828
0
829
0
            // Scale by 2^k
830
0
            let k_int = _mm512_cvtps_epi32(k);
831
0
            let k_shifted = _mm512_slli_epi32(k_int, 23);
832
0
            let scale = _mm512_castsi512_ps(_mm512_add_epi32(_mm512_castps_si512(one), k_shifted));
833
0
            let exp_neg_x = _mm512_mul_ps(p, scale);
834
0
835
0
            // sigmoid = 1 / (1 + exp(-x))
836
0
            let denom = _mm512_add_ps(one, exp_neg_x);
837
0
            let sigmoid_result = _mm512_div_ps(one, denom);
838
0
839
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), sigmoid_result);
840
0
            i += 16;
841
0
        }
842
843
        // Handle remaining elements with scalar code
844
0
        while i < len {
845
0
            let val = a[i];
846
0
            result[i] = if val < -50.0 {
847
0
                0.0
848
0
            } else if val > 50.0 {
849
0
                1.0
850
            } else {
851
0
                1.0 / (1.0 + (-val).exp())
852
            };
853
0
            i += 1;
854
        }
855
0
    }
856
857
    #[inline]
858
    #[target_feature(enable = "avx512f")]
859
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
860
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
861
    // 2. All pointers derived from valid slice references
862
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
863
    // 4. Unaligned loads/stores used - no alignment requirement
864
0
    unsafe fn gelu(a: &[f32], result: &mut [f32]) {
865
        // gelu(x) = 0.5 * x * (1 + tanh(sqrt(2/π) * (x + 0.044715 * x³)))
866
0
        let len = a.len();
867
0
        let mut i = 0;
868
869
0
        let sqrt_2_over_pi = _mm512_set1_ps(0.797_884_6);
870
0
        let coeff = _mm512_set1_ps(0.044715);
871
0
        let half = _mm512_set1_ps(0.5);
872
0
        let one = _mm512_set1_ps(1.0);
873
0
        let two = _mm512_set1_ps(2.0);
874
875
0
        let log2e = _mm512_set1_ps(std::f32::consts::LOG2_E);
876
0
        let ln2 = _mm512_set1_ps(std::f32::consts::LN_2);
877
878
0
        let c1 = _mm512_set1_ps(1.0);
879
0
        let c2 = _mm512_set1_ps(0.5);
880
0
        let c3 = _mm512_set1_ps(0.166_666_67);
881
0
        let c4 = _mm512_set1_ps(0.041_666_668);
882
0
        let c5 = _mm512_set1_ps(0.008_333_334);
883
0
        let c6 = _mm512_set1_ps(0.001_388_889);
884
885
0
        let exp_hi = _mm512_set1_ps(88.376_26);
886
0
        let exp_lo = _mm512_set1_ps(-87.336_55);
887
888
0
        while i + 16 <= len {
889
0
            let x = _mm512_loadu_ps(a.as_ptr().add(i));
890
0
891
0
            // Compute inner = sqrt(2/π) * (x + 0.044715 * x³)
892
0
            let x2 = _mm512_mul_ps(x, x);
893
0
            let x3 = _mm512_mul_ps(x2, x);
894
0
            let inner_sum = _mm512_fmadd_ps(coeff, x3, x);
895
0
            let inner = _mm512_mul_ps(sqrt_2_over_pi, inner_sum);
896
0
897
0
            // Compute tanh(inner) = (exp(2*inner) - 1) / (exp(2*inner) + 1)
898
0
            let two_inner = _mm512_mul_ps(two, inner);
899
0
            let two_inner = _mm512_max_ps(_mm512_min_ps(two_inner, exp_hi), exp_lo);
900
0
901
0
            let x_scaled = _mm512_mul_ps(two_inner, log2e);
902
0
            let k = _mm512_roundscale_ps(_mm512_add_ps(x_scaled, half), 0x09);
903
0
            let r = _mm512_sub_ps(two_inner, _mm512_mul_ps(k, ln2));
904
0
905
0
            let mut p = c6;
906
0
            p = _mm512_fmadd_ps(p, r, c5);
907
0
            p = _mm512_fmadd_ps(p, r, c4);
908
0
            p = _mm512_fmadd_ps(p, r, c3);
909
0
            p = _mm512_fmadd_ps(p, r, c2);
910
0
            p = _mm512_fmadd_ps(p, r, c1);
911
0
            p = _mm512_fmadd_ps(p, r, one);
912
0
913
0
            let k_int = _mm512_cvtps_epi32(k);
914
0
            let k_shifted = _mm512_slli_epi32(k_int, 23);
915
0
            let scale = _mm512_castsi512_ps(_mm512_add_epi32(_mm512_castps_si512(one), k_shifted));
916
0
            let exp_2inner = _mm512_mul_ps(p, scale);
917
0
918
0
            // tanh = (exp(2x) - 1) / (exp(2x) + 1)
919
0
            let tanh_numer = _mm512_sub_ps(exp_2inner, one);
920
0
            let tanh_denom = _mm512_add_ps(exp_2inner, one);
921
0
            let tanh_result = _mm512_div_ps(tanh_numer, tanh_denom);
922
0
923
0
            // gelu = 0.5 * x * (1 + tanh)
924
0
            let one_plus_tanh = _mm512_add_ps(one, tanh_result);
925
0
            let gelu_result = _mm512_mul_ps(half, _mm512_mul_ps(x, one_plus_tanh));
926
0
927
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), gelu_result);
928
0
            i += 16;
929
0
        }
930
931
        const SQRT_2_OVER_PI: f32 = 0.797_884_6;
932
        const COEFF: f32 = 0.044715;
933
934
0
        while i < len {
935
0
            let x = a[i];
936
0
            let x3 = x * x * x;
937
0
            let inner = SQRT_2_OVER_PI * (x + COEFF * x3);
938
0
            result[i] = 0.5 * x * (1.0 + inner.tanh());
939
0
            i += 1;
940
0
        }
941
0
    }
942
943
    #[inline]
944
    #[target_feature(enable = "avx512f")]
945
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
946
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
947
    // 2. All pointers derived from valid slice references
948
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
949
    // 4. Unaligned loads/stores used - no alignment requirement
950
0
    unsafe fn swish(a: &[f32], result: &mut [f32]) {
951
        // swish(x) = x * sigmoid(x) = x / (1 + exp(-x))
952
0
        let len = a.len();
953
0
        let mut i = 0;
954
955
0
        let log2e = _mm512_set1_ps(std::f32::consts::LOG2_E);
956
0
        let ln2 = _mm512_set1_ps(std::f32::consts::LN_2);
957
0
        let half = _mm512_set1_ps(0.5);
958
0
        let one = _mm512_set1_ps(1.0);
959
960
0
        let c1 = _mm512_set1_ps(1.0);
961
0
        let c2 = _mm512_set1_ps(0.5);
962
0
        let c3 = _mm512_set1_ps(0.166_666_67);
963
0
        let c4 = _mm512_set1_ps(0.041_666_668);
964
0
        let c5 = _mm512_set1_ps(0.008_333_334);
965
0
        let c6 = _mm512_set1_ps(0.001_388_889);
966
967
0
        let exp_hi = _mm512_set1_ps(88.376_26);
968
0
        let exp_lo = _mm512_set1_ps(-87.336_55);
969
970
0
        while i + 16 <= len {
971
0
            let x = _mm512_loadu_ps(a.as_ptr().add(i));
972
0
            let neg_x = _mm512_sub_ps(_mm512_setzero_ps(), x);
973
0
            let neg_x = _mm512_max_ps(_mm512_min_ps(neg_x, exp_hi), exp_lo);
974
0
975
0
            let x_scaled = _mm512_mul_ps(neg_x, log2e);
976
0
            let k = _mm512_roundscale_ps(_mm512_add_ps(x_scaled, half), 0x09);
977
0
            let r = _mm512_sub_ps(neg_x, _mm512_mul_ps(k, ln2));
978
0
979
0
            let mut p = c6;
980
0
            p = _mm512_fmadd_ps(p, r, c5);
981
0
            p = _mm512_fmadd_ps(p, r, c4);
982
0
            p = _mm512_fmadd_ps(p, r, c3);
983
0
            p = _mm512_fmadd_ps(p, r, c2);
984
0
            p = _mm512_fmadd_ps(p, r, c1);
985
0
            p = _mm512_fmadd_ps(p, r, one);
986
0
987
0
            let k_int = _mm512_cvtps_epi32(k);
988
0
            let k_shifted = _mm512_slli_epi32(k_int, 23);
989
0
            let scale = _mm512_castsi512_ps(_mm512_add_epi32(_mm512_castps_si512(one), k_shifted));
990
0
            let exp_neg_x = _mm512_mul_ps(p, scale);
991
0
992
0
            // swish = x / (1 + exp(-x))
993
0
            let denom = _mm512_add_ps(one, exp_neg_x);
994
0
            let swish_result = _mm512_div_ps(x, denom);
995
0
996
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), swish_result);
997
0
            i += 16;
998
0
        }
999
1000
0
        while i < len {
1001
0
            let x = a[i];
1002
0
            result[i] = if x < -50.0 {
1003
0
                0.0
1004
0
            } else if x > 50.0 {
1005
0
                x
1006
            } else {
1007
0
                x / (1.0 + (-x).exp())
1008
            };
1009
0
            i += 1;
1010
        }
1011
0
    }
1012
1013
    #[inline]
1014
    #[target_feature(enable = "avx512f")]
1015
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
1016
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
1017
    // 2. All pointers derived from valid slice references
1018
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
1019
    // 4. Unaligned loads/stores used - no alignment requirement
1020
0
    unsafe fn tanh(a: &[f32], result: &mut [f32]) {
1021
        // tanh(x) = (exp(2x) - 1) / (exp(2x) + 1)
1022
0
        let len = a.len();
1023
0
        let mut i = 0;
1024
1025
0
        let log2e = _mm512_set1_ps(std::f32::consts::LOG2_E);
1026
0
        let ln2 = _mm512_set1_ps(std::f32::consts::LN_2);
1027
0
        let half = _mm512_set1_ps(0.5);
1028
0
        let one = _mm512_set1_ps(1.0);
1029
0
        let two = _mm512_set1_ps(2.0);
1030
1031
0
        let c1 = _mm512_set1_ps(1.0);
1032
0
        let c2 = _mm512_set1_ps(0.5);
1033
0
        let c3 = _mm512_set1_ps(0.166_666_67);
1034
0
        let c4 = _mm512_set1_ps(0.041_666_668);
1035
0
        let c5 = _mm512_set1_ps(0.008_333_334);
1036
0
        let c6 = _mm512_set1_ps(0.001_388_889);
1037
1038
0
        let exp_hi = _mm512_set1_ps(88.376_26);
1039
0
        let exp_lo = _mm512_set1_ps(-87.336_55);
1040
1041
0
        while i + 16 <= len {
1042
0
            let x = _mm512_loadu_ps(a.as_ptr().add(i));
1043
0
            let two_x = _mm512_mul_ps(two, x);
1044
0
            let two_x = _mm512_max_ps(_mm512_min_ps(two_x, exp_hi), exp_lo);
1045
0
1046
0
            let x_scaled = _mm512_mul_ps(two_x, log2e);
1047
0
            let k = _mm512_roundscale_ps(_mm512_add_ps(x_scaled, half), 0x09);
1048
0
            let r = _mm512_sub_ps(two_x, _mm512_mul_ps(k, ln2));
1049
0
1050
0
            let mut p = c6;
1051
0
            p = _mm512_fmadd_ps(p, r, c5);
1052
0
            p = _mm512_fmadd_ps(p, r, c4);
1053
0
            p = _mm512_fmadd_ps(p, r, c3);
1054
0
            p = _mm512_fmadd_ps(p, r, c2);
1055
0
            p = _mm512_fmadd_ps(p, r, c1);
1056
0
            p = _mm512_fmadd_ps(p, r, one);
1057
0
1058
0
            let k_int = _mm512_cvtps_epi32(k);
1059
0
            let k_shifted = _mm512_slli_epi32(k_int, 23);
1060
0
            let scale = _mm512_castsi512_ps(_mm512_add_epi32(_mm512_castps_si512(one), k_shifted));
1061
0
            let exp_2x = _mm512_mul_ps(p, scale);
1062
0
1063
0
            let tanh_numer = _mm512_sub_ps(exp_2x, one);
1064
0
            // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
1065
0
            // 1. Loop bounds ensure proper array access
1066
0
            // 2. All pointers derived from valid slice references
1067
0
            // 3. AVX-512 intrinsics marked with #[target_feature]
1068
0
            // 4. Unaligned loads/stores handle unaligned data correctly
1069
0
            let tanh_denom = _mm512_add_ps(exp_2x, one);
1070
0
            let tanh_result = _mm512_div_ps(tanh_numer, tanh_denom);
1071
0
1072
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), tanh_result);
1073
0
            i += 16;
1074
0
        }
1075
1076
0
        while i < len {
1077
0
            result[i] = a[i].tanh();
1078
0
            i += 1;
1079
0
        }
1080
0
    }
1081
1082
    #[inline]
1083
    #[target_feature(enable = "avx512f")]
1084
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
1085
    // 1. Loop bounds ensure proper array access
1086
    // 2. All pointers derived from valid slice references
1087
    // 3. AVX-512 intrinsics marked with #[target_feature]
1088
    // 4. Unaligned loads/stores handle unaligned data correctly
1089
0
    unsafe fn sqrt(a: &[f32], result: &mut [f32]) {
1090
0
        let len = a.len();
1091
0
        let mut i = 0;
1092
1093
        // Process 16 elements at a time with AVX-512
1094
0
        while i + 16 <= len {
1095
0
            let vec = _mm512_loadu_ps(a.as_ptr().add(i));
1096
0
            let sqrt_vec = _mm512_sqrt_ps(vec);
1097
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), sqrt_vec);
1098
0
            i += 16;
1099
0
        }
1100
1101
0
        while i < len {
1102
0
            result[i] = a[i].sqrt();
1103
0
            i += 1;
1104
0
        }
1105
0
    }
1106
1107
    #[inline]
1108
    #[target_feature(enable = "avx512f")]
1109
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
1110
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
1111
    // 2. All pointers derived from valid slice references
1112
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
1113
    // 4. Unaligned loads/stores handle unaligned data correctly
1114
0
    unsafe fn recip(a: &[f32], result: &mut [f32]) {
1115
0
        let len = a.len();
1116
0
        let mut i = 0;
1117
1118
0
        let one = _mm512_set1_ps(1.0);
1119
0
        while i + 16 <= len {
1120
0
            let vec = _mm512_loadu_ps(a.as_ptr().add(i));
1121
0
            let recip_vec = _mm512_div_ps(one, vec);
1122
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), recip_vec);
1123
0
            i += 16;
1124
0
        }
1125
1126
0
        while i < len {
1127
0
            result[i] = a[i].recip();
1128
0
            i += 1;
1129
0
        }
1130
0
    }
1131
1132
    #[target_feature(enable = "avx512f")]
1133
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
1134
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
1135
    // 2. All pointers derived from valid slice references
1136
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
1137
    // 4. Unaligned loads/stores used - no alignment requirement
1138
    //
1139
    // Natural logarithm implementation using range reduction:
1140
    // For x = 2^k * m where m ∈ [1, 2):
1141
    //   ln(x) = k*ln(2) + ln(m)
1142
    //   ln(m) approximated using 7th-degree polynomial
1143
    #[inline]
1144
    #[target_feature(enable = "avx512f,fma")]
1145
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
1146
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
1147
    // 2. All pointers derived from valid slice references
1148
    // 3. AVX-512 and FMA intrinsics marked with #[target_feature]
1149
    // 4. Unaligned loads/stores handle unaligned data correctly
1150
0
    unsafe fn ln(a: &[f32], result: &mut [f32]) {
1151
0
        let len = a.len();
1152
0
        let mut i = 0;
1153
1154
        // Constants for ln calculation
1155
0
        let ln2 = _mm512_set1_ps(std::f32::consts::LN_2); // 0.693147...
1156
0
        let one = _mm512_set1_ps(1.0);
1157
1158
        // Use atanh transformation for better accuracy
1159
0
        let two = _mm512_set1_ps(2.0);
1160
0
        let c1 = _mm512_set1_ps(1.0);
1161
0
        let c3 = _mm512_set1_ps(1.0 / 3.0);
1162
0
        let c5 = _mm512_set1_ps(1.0 / 5.0);
1163
0
        let c7 = _mm512_set1_ps(1.0 / 7.0);
1164
0
        let c9 = _mm512_set1_ps(1.0 / 9.0);
1165
0
        let c11 = _mm512_set1_ps(1.0 / 11.0);
1166
1167
0
        let mantissa_mask = _mm512_set1_epi32(0x007F_FFFF_u32 as i32);
1168
0
        let exponent_127 = _mm512_set1_epi32(127 << 23);
1169
1170
0
        while i + 16 <= len {
1171
0
            let x = _mm512_loadu_ps(a.as_ptr().add(i));
1172
0
            let x_int = _mm512_castps_si512(x);
1173
0
1174
0
            let exp_biased = _mm512_srli_epi32(x_int, 23);
1175
0
            let exp_biased_masked = _mm512_and_si512(exp_biased, _mm512_set1_epi32(0xFF));
1176
0
            let k_int = _mm512_sub_epi32(exp_biased_masked, _mm512_set1_epi32(127));
1177
0
            let k = _mm512_cvtepi32_ps(k_int);
1178
0
1179
0
            let mantissa_bits = _mm512_and_si512(x_int, mantissa_mask);
1180
0
            let m_int = _mm512_or_si512(mantissa_bits, exponent_127);
1181
0
            let m = _mm512_castsi512_ps(m_int);
1182
0
1183
0
            // atanh transformation
1184
0
            let m_minus_1 = _mm512_sub_ps(m, one);
1185
0
            let m_plus_1 = _mm512_add_ps(m, one);
1186
0
            let u = _mm512_div_ps(m_minus_1, m_plus_1);
1187
0
            let u2 = _mm512_mul_ps(u, u);
1188
0
1189
0
            let p = _mm512_fmadd_ps(c11, u2, c9);
1190
0
            let p = _mm512_fmadd_ps(p, u2, c7);
1191
0
            let p = _mm512_fmadd_ps(p, u2, c5);
1192
0
            let p = _mm512_fmadd_ps(p, u2, c3);
1193
0
            let p = _mm512_fmadd_ps(p, u2, c1);
1194
0
1195
0
            let ln_m = _mm512_mul_ps(two, _mm512_mul_ps(u, p));
1196
0
1197
0
            // ln(x) = k*ln(2) + ln(m)
1198
0
            let result_vec = _mm512_fmadd_ps(k, ln2, ln_m);
1199
0
1200
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), result_vec);
1201
0
            i += 16;
1202
0
        }
1203
1204
        // Handle remaining elements with scalar code
1205
0
        while i < len {
1206
0
            result[i] = a[i].ln();
1207
0
            i += 1;
1208
0
        }
1209
0
    }
1210
1211
    #[target_feature(enable = "avx512f")]
1212
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
1213
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
1214
    // 2. All pointers derived from valid slice references
1215
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
1216
    // 4. Unaligned loads/stores used - no alignment requirement
1217
    //
1218
    // Base-2 logarithm implementation using range reduction:
1219
    // For x = 2^k * m where m ∈ [1, 2):
1220
    //   log2(x) = k + log2(m)
1221
    //   log2(m) = ln(m) / ln(2)
1222
    #[inline]
1223
    #[target_feature(enable = "avx512f,fma")]
1224
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
1225
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
1226
    // 2. All pointers derived from valid slice references
1227
    // 3. AVX-512 and FMA intrinsics marked with #[target_feature]
1228
    // 4. Unaligned loads/stores handle unaligned data correctly
1229
0
    unsafe fn log2(a: &[f32], result: &mut [f32]) {
1230
0
        let len = a.len();
1231
0
        let mut i = 0;
1232
1233
0
        let inv_ln2 = _mm512_set1_ps(std::f32::consts::LOG2_E);
1234
0
        let one = _mm512_set1_ps(1.0);
1235
1236
0
        let two = _mm512_set1_ps(2.0);
1237
0
        let c1 = _mm512_set1_ps(1.0);
1238
0
        let c3 = _mm512_set1_ps(1.0 / 3.0);
1239
0
        let c5 = _mm512_set1_ps(1.0 / 5.0);
1240
0
        let c7 = _mm512_set1_ps(1.0 / 7.0);
1241
0
        let c9 = _mm512_set1_ps(1.0 / 9.0);
1242
0
        let c11 = _mm512_set1_ps(1.0 / 11.0);
1243
1244
0
        let mantissa_mask = _mm512_set1_epi32(0x007F_FFFF_u32 as i32);
1245
0
        let exponent_127 = _mm512_set1_epi32(127 << 23);
1246
1247
0
        while i + 16 <= len {
1248
0
            let x = _mm512_loadu_ps(a.as_ptr().add(i));
1249
0
            let x_int = _mm512_castps_si512(x);
1250
0
1251
0
            // Extract exponent k
1252
0
            let exp_biased = _mm512_srli_epi32(x_int, 23);
1253
0
            let exp_biased_masked = _mm512_and_si512(exp_biased, _mm512_set1_epi32(0xFF));
1254
0
            let k_int = _mm512_sub_epi32(exp_biased_masked, _mm512_set1_epi32(127));
1255
0
            let k = _mm512_cvtepi32_ps(k_int);
1256
0
1257
0
            let mantissa_bits = _mm512_and_si512(x_int, mantissa_mask);
1258
0
            let m_int = _mm512_or_si512(mantissa_bits, exponent_127);
1259
0
            let m = _mm512_castsi512_ps(m_int);
1260
0
1261
0
            // atanh transformation
1262
0
            let m_minus_1 = _mm512_sub_ps(m, one);
1263
0
            let m_plus_1 = _mm512_add_ps(m, one);
1264
0
            let u = _mm512_div_ps(m_minus_1, m_plus_1);
1265
0
            let u2 = _mm512_mul_ps(u, u);
1266
0
1267
0
            let p = _mm512_fmadd_ps(c11, u2, c9);
1268
0
            let p = _mm512_fmadd_ps(p, u2, c7);
1269
0
            let p = _mm512_fmadd_ps(p, u2, c5);
1270
0
            let p = _mm512_fmadd_ps(p, u2, c3);
1271
0
            let p = _mm512_fmadd_ps(p, u2, c1);
1272
0
1273
0
            let ln_m = _mm512_mul_ps(two, _mm512_mul_ps(u, p));
1274
0
1275
0
            let log2_m = _mm512_mul_ps(ln_m, inv_ln2);
1276
0
1277
0
            // log2(x) = k + log2(m)
1278
0
            let result_vec = _mm512_add_ps(k, log2_m);
1279
0
1280
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), result_vec);
1281
0
            i += 16;
1282
0
        }
1283
1284
0
        while i < len {
1285
0
            result[i] = a[i].log2();
1286
0
            i += 1;
1287
0
        }
1288
0
    }
1289
1290
    #[target_feature(enable = "avx512f")]
1291
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
1292
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
1293
    // 2. All pointers derived from valid slice references
1294
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
1295
    // 4. Unaligned loads/stores used - no alignment requirement
1296
    //
1297
    // Base-10 logarithm implementation using range reduction:
1298
    // For x = 2^k * m where m ∈ [1, 2):
1299
    //   log10(x) = k*log10(2) + log10(m)
1300
    //   log10(m) = ln(m) / ln(10)
1301
    #[inline]
1302
    #[target_feature(enable = "avx512f,fma")]
1303
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
1304
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
1305
    // 2. All pointers derived from valid slice references
1306
    // 3. AVX-512 and FMA intrinsics marked with #[target_feature]
1307
    // 4. Unaligned loads/stores handle unaligned data correctly
1308
0
    unsafe fn log10(a: &[f32], result: &mut [f32]) {
1309
0
        let len = a.len();
1310
0
        let mut i = 0;
1311
1312
0
        let log10_2 = _mm512_set1_ps(std::f32::consts::LOG10_2);
1313
0
        let inv_ln10 = _mm512_set1_ps(1.0 / std::f32::consts::LN_10);
1314
0
        let one = _mm512_set1_ps(1.0);
1315
1316
0
        let two = _mm512_set1_ps(2.0);
1317
0
        let c1 = _mm512_set1_ps(1.0);
1318
0
        let c3 = _mm512_set1_ps(1.0 / 3.0);
1319
0
        let c5 = _mm512_set1_ps(1.0 / 5.0);
1320
0
        let c7 = _mm512_set1_ps(1.0 / 7.0);
1321
0
        let c9 = _mm512_set1_ps(1.0 / 9.0);
1322
0
        let c11 = _mm512_set1_ps(1.0 / 11.0);
1323
1324
0
        let mantissa_mask = _mm512_set1_epi32(0x007F_FFFF_u32 as i32);
1325
0
        let exponent_127 = _mm512_set1_epi32(127 << 23);
1326
1327
0
        while i + 16 <= len {
1328
0
            let x = _mm512_loadu_ps(a.as_ptr().add(i));
1329
0
            let x_int = _mm512_castps_si512(x);
1330
0
1331
0
            // Extract exponent k
1332
0
            let exp_biased = _mm512_srli_epi32(x_int, 23);
1333
0
            let exp_biased_masked = _mm512_and_si512(exp_biased, _mm512_set1_epi32(0xFF));
1334
0
            let k_int = _mm512_sub_epi32(exp_biased_masked, _mm512_set1_epi32(127));
1335
0
            let k = _mm512_cvtepi32_ps(k_int);
1336
0
1337
0
            // Extract mantissa m ∈ [1, 2)
1338
0
            let mantissa_bits = _mm512_and_si512(x_int, mantissa_mask);
1339
0
            let m_int = _mm512_or_si512(mantissa_bits, exponent_127);
1340
0
            let m = _mm512_castsi512_ps(m_int);
1341
0
1342
0
            // atanh transformation
1343
0
            let m_minus_1 = _mm512_sub_ps(m, one);
1344
0
            let m_plus_1 = _mm512_add_ps(m, one);
1345
0
            let u = _mm512_div_ps(m_minus_1, m_plus_1);
1346
0
            let u2 = _mm512_mul_ps(u, u);
1347
0
1348
0
            let p = _mm512_fmadd_ps(c11, u2, c9);
1349
0
            let p = _mm512_fmadd_ps(p, u2, c7);
1350
0
            let p = _mm512_fmadd_ps(p, u2, c5);
1351
0
            let p = _mm512_fmadd_ps(p, u2, c3);
1352
0
            let p = _mm512_fmadd_ps(p, u2, c1);
1353
0
1354
0
            let ln_m = _mm512_mul_ps(two, _mm512_mul_ps(u, p));
1355
0
1356
0
            let log10_m = _mm512_mul_ps(ln_m, inv_ln10);
1357
0
1358
0
            // log10(x) = k*log10(2) + log10(m)
1359
0
            let result_vec = _mm512_fmadd_ps(k, log10_2, log10_m);
1360
0
1361
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), result_vec);
1362
0
            i += 16;
1363
0
        }
1364
1365
0
        while i < len {
1366
0
            result[i] = a[i].log10();
1367
0
            i += 1;
1368
0
        }
1369
0
    }
1370
1371
    // Trigonometric functions currently use scalar implementations
1372
    // Full SIMD trig functions require complex range reduction and are left for future work
1373
1374
    #[inline]
1375
    #[target_feature(enable = "avx512f")]
1376
    // SAFETY: Delegates to scalar implementation, no direct SIMD operations
1377
0
    unsafe fn sin(a: &[f32], result: &mut [f32]) {
1378
0
        super::scalar::ScalarBackend::sin(a, result);
1379
0
    }
1380
1381
    #[inline]
1382
    #[target_feature(enable = "avx512f")]
1383
    // SAFETY: Delegates to scalar implementation, no direct SIMD operations
1384
0
    unsafe fn cos(a: &[f32], result: &mut [f32]) {
1385
0
        super::scalar::ScalarBackend::cos(a, result);
1386
0
    }
1387
1388
    #[inline]
1389
    #[target_feature(enable = "avx512f")]
1390
    // SAFETY: Delegates to scalar implementation, no direct SIMD operations
1391
0
    unsafe fn tan(a: &[f32], result: &mut [f32]) {
1392
0
        super::scalar::ScalarBackend::tan(a, result);
1393
0
    }
1394
1395
    #[inline]
1396
    #[target_feature(enable = "avx512f")]
1397
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
1398
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
1399
    // 2. All pointers derived from valid slice references
1400
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
1401
    // 4. Unaligned loads/stores used - no alignment requirement
1402
0
    unsafe fn floor(a: &[f32], result: &mut [f32]) {
1403
0
        let len = a.len();
1404
0
        let mut i = 0;
1405
1406
        // Process 16 elements at a time
1407
        // Rounding mode 0x09 = round down (floor)
1408
0
        while i + 16 <= len {
1409
0
            let va = _mm512_loadu_ps(a.as_ptr().add(i));
1410
0
            let vresult = _mm512_roundscale_ps(va, 0x09);
1411
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), vresult);
1412
0
            i += 16;
1413
0
        }
1414
1415
        // Handle remaining elements with scalar code
1416
0
        while i < len {
1417
0
            result[i] = a[i].floor();
1418
0
            i += 1;
1419
0
        }
1420
0
    }
1421
1422
    #[inline]
1423
    #[target_feature(enable = "avx512f")]
1424
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
1425
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
1426
    // 2. All pointers derived from valid slice references
1427
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
1428
    // 4. Unaligned loads/stores used - no alignment requirement
1429
0
    unsafe fn ceil(a: &[f32], result: &mut [f32]) {
1430
0
        let len = a.len();
1431
0
        let mut i = 0;
1432
1433
        // Process 16 elements at a time
1434
        // Rounding mode 0x0A = round up (ceil)
1435
0
        while i + 16 <= len {
1436
0
            let va = _mm512_loadu_ps(a.as_ptr().add(i));
1437
0
            let vresult = _mm512_roundscale_ps(va, 0x0A);
1438
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), vresult);
1439
0
            i += 16;
1440
0
        }
1441
1442
        // Handle remaining elements with scalar code
1443
0
        while i < len {
1444
0
            result[i] = a[i].ceil();
1445
0
            i += 1;
1446
0
        }
1447
0
    }
1448
1449
    #[inline]
1450
    #[target_feature(enable = "avx512f")]
1451
    // SAFETY: Pointer arithmetic and SIMD intrinsics are safe because:
1452
    // 1. Loop bounds ensure `i + 16 <= len` before calling `.add(i)`
1453
    // 2. All pointers derived from valid slice references
1454
    // 3. AVX-512 intrinsics marked with #[target_feature(enable = "avx512f")]
1455
    // 4. Unaligned loads/stores used - no alignment requirement
1456
0
    unsafe fn round(a: &[f32], result: &mut [f32]) {
1457
0
        let len = a.len();
1458
0
        let mut i = 0;
1459
1460
        // Rust's .round() rounds ties away from zero, but SIMD round modes don't support this.
1461
        // Implement manually: round(x) = sign(x) * floor(abs(x) + 0.5)
1462
0
        let half = _mm512_set1_ps(0.5);
1463
0
        let sign_mask = _mm512_set1_ps(f32::from_bits(0x8000_0000)); // Sign bit only
1464
0
        let abs_mask = _mm512_set1_ps(f32::from_bits(0x7FFF_FFFF)); // All except sign bit
1465
1466
        // Process 16 elements at a time
1467
0
        while i + 16 <= len {
1468
0
            let va = _mm512_loadu_ps(a.as_ptr().add(i));
1469
0
1470
0
            // Extract sign and absolute value
1471
0
            let sign = _mm512_and_ps(va, sign_mask);
1472
0
            let abs_val = _mm512_and_ps(va, abs_mask);
1473
0
1474
0
            // Round away from zero: floor(abs(x) + 0.5) * sign(x)
1475
0
            let shifted = _mm512_add_ps(abs_val, half);
1476
0
            let rounded_abs = _mm512_roundscale_ps(shifted, 0x09); // floor
1477
0
            let vresult = _mm512_or_ps(rounded_abs, sign);
1478
0
1479
0
            _mm512_storeu_ps(result.as_mut_ptr().add(i), vresult);
1480
0
            i += 16;
1481
0
        }
1482
1483
        // Handle remaining elements with scalar code
1484
0
        while i < len {
1485
0
            result[i] = a[i].round();
1486
0
            i += 1;
1487
0
        }
1488
0
    }
1489
}
1490
1491
#[cfg(all(test, target_arch = "x86_64"))]
1492
mod tests {
1493
    use super::*;
1494
    use crate::backends::scalar::ScalarBackend;
1495
1496
    /// Helper to run AVX-512 test only on CPUs that support it
1497
    fn avx512_test<F>(test_fn: F)
1498
    where
1499
        F: FnOnce(),
1500
    {
1501
        if is_x86_feature_detected!("avx512f") {
1502
            test_fn();
1503
        } else {
1504
            // Skip test on CPUs without AVX-512 support
1505
            println!("Skipping AVX-512 test (CPU does not support avx512f)");
1506
        }
1507
    }
1508
1509
    #[test]
1510
    fn test_avx512_add_basic() {
1511
        avx512_test(|| {
1512
            let a = vec![1.0, 2.0, 3.0, 4.0];
1513
            let b = vec![5.0, 6.0, 7.0, 8.0];
1514
            let mut result = vec![0.0; 4];
1515
1516
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
1517
            unsafe {
1518
                Avx512Backend::add(&a, &b, &mut result);
1519
            }
1520
1521
            assert_eq!(result, vec![6.0, 8.0, 10.0, 12.0]);
1522
        });
1523
    }
1524
1525
    #[test]
1526
    fn test_avx512_add_aligned_16() {
1527
        avx512_test(|| {
1528
            // Test with exactly 16 elements (one AVX-512 register)
1529
            let a: Vec<f32> = (0..16).map(|i| i as f32).collect();
1530
            let b: Vec<f32> = (0..16).map(|i| (i + 10) as f32).collect();
1531
            let mut result = vec![0.0; 16];
1532
            let expected: Vec<f32> = (0..16).map(|i| (i + i + 10) as f32).collect();
1533
1534
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
1535
            unsafe {
1536
                Avx512Backend::add(&a, &b, &mut result);
1537
            }
1538
1539
            assert_eq!(result, expected);
1540
        });
1541
    }
1542
1543
    #[test]
1544
    fn test_avx512_add_non_aligned() {
1545
        avx512_test(|| {
1546
            // Test with 18 elements (16 + 2 remainder)
1547
            let a: Vec<f32> = (0..18).map(|i| i as f32).collect();
1548
            let b: Vec<f32> = (0..18).map(|i| (i * 2) as f32).collect();
1549
            let mut result = vec![0.0; 18];
1550
            let expected: Vec<f32> = (0..18).map(|i| (i + i * 2) as f32).collect();
1551
1552
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
1553
            unsafe {
1554
                Avx512Backend::add(&a, &b, &mut result);
1555
            }
1556
1557
            assert_eq!(result, expected);
1558
        });
1559
    }
1560
1561
    #[test]
1562
    fn test_avx512_add_large() {
1563
        avx512_test(|| {
1564
            // Test with 1000 elements (many AVX-512 iterations)
1565
            let a: Vec<f32> = (0..1000).map(|i| i as f32 * 0.5).collect();
1566
            let b: Vec<f32> = (0..1000).map(|i| i as f32 * 0.3).collect();
1567
            let mut result = vec![0.0; 1000];
1568
1569
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
1570
            unsafe {
1571
                Avx512Backend::add(&a, &b, &mut result);
1572
            }
1573
1574
            for (i, &value) in result.iter().enumerate().take(1000) {
1575
                let expected = i as f32 * 0.5 + i as f32 * 0.3;
1576
                assert!(
1577
                    (value - expected).abs() < 1e-5,
1578
                    "Mismatch at index {}: expected {}, got {}",
1579
                    i,
1580
                    expected,
1581
                    value
1582
                );
1583
            }
1584
        });
1585
    }
1586
1587
    #[test]
1588
    fn test_avx512_add_single_element() {
1589
        avx512_test(|| {
1590
            let a = vec![42.0];
1591
            let b = vec![13.0];
1592
            let mut result = vec![0.0];
1593
1594
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
1595
            unsafe {
1596
                Avx512Backend::add(&a, &b, &mut result);
1597
            }
1598
1599
            assert_eq!(result, vec![55.0]);
1600
        });
1601
    }
1602
1603
    #[test]
1604
    fn test_avx512_add_negative_values() {
1605
        avx512_test(|| {
1606
            let a = vec![
1607
                -1.0, -2.0, -3.0, -4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0,
1608
                15.0, 16.0,
1609
            ];
1610
            let b = vec![
1611
                10.0, 20.0, 30.0, 40.0, -50.0, -60.0, -70.0, -80.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0,
1612
                7.0, 8.0,
1613
            ];
1614
            let mut result = vec![0.0; 16];
1615
            let expected = vec![
1616
                9.0, 18.0, 27.0, 36.0, -45.0, -54.0, -63.0, -72.0, 10.0, 12.0, 14.0, 16.0, 18.0,
1617
                20.0, 22.0, 24.0,
1618
            ];
1619
1620
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
1621
            unsafe {
1622
                Avx512Backend::add(&a, &b, &mut result);
1623
            }
1624
1625
            for i in 0..16 {
1626
                assert!(
1627
                    (result[i] - expected[i]).abs() < 1e-5,
1628
                    "Mismatch at index {}: expected {}, got {}",
1629
                    i,
1630
                    expected[i],
1631
                    result[i]
1632
                );
1633
            }
1634
        });
1635
    }
1636
1637
    #[test]
1638
    fn test_avx512_add_equivalence_to_scalar() {
1639
        avx512_test(|| {
1640
            // Backend equivalence: AVX-512 should produce same results as Scalar
1641
            let sizes = vec![1, 7, 15, 16, 17, 32, 63, 100, 1000];
1642
1643
            for size in sizes {
1644
                let a: Vec<f32> = (0..size).map(|i| (i as f32 * 1.5) - 50.0).collect();
1645
                let b: Vec<f32> = (0..size).map(|i| (i as f32 * 0.7) + 20.0).collect();
1646
1647
                let mut result_avx512 = vec![0.0; size];
1648
                let mut result_scalar = vec![0.0; size];
1649
1650
                // SAFETY: CPU feature verified at runtime, slices bounds-checked
1651
                unsafe {
1652
                    Avx512Backend::add(&a, &b, &mut result_avx512);
1653
                    ScalarBackend::add(&a, &b, &mut result_scalar);
1654
                }
1655
1656
                for i in 0..size {
1657
                    assert!(
1658
                        (result_avx512[i] - result_scalar[i]).abs() < 1e-5,
1659
                        "Backend mismatch at size {} index {}: AVX512={}, Scalar={}",
1660
                        size,
1661
                        i,
1662
                        result_avx512[i],
1663
                        result_scalar[i]
1664
                    );
1665
                }
1666
            }
1667
        });
1668
    }
1669
1670
    #[test]
1671
    fn test_avx512_add_special_values() {
1672
        avx512_test(|| {
1673
            // Test with infinity, zero, and very small/large values
1674
            let a = vec![
1675
                0.0,
1676
                -0.0,
1677
                f32::INFINITY,
1678
                f32::NEG_INFINITY,
1679
                1e-20,
1680
                -1e-20,
1681
                1e20,
1682
                -1e20,
1683
                0.0,
1684
                0.0,
1685
                0.0,
1686
                0.0,
1687
                0.0,
1688
                0.0,
1689
                0.0,
1690
                0.0,
1691
            ];
1692
            let b = vec![
1693
                0.0, 0.0, 1.0, -1.0, 2e-20, -2e-20, 2e20, -2e20, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0,
1694
                8.0,
1695
            ];
1696
            let mut result = vec![0.0; 16];
1697
1698
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
1699
            unsafe {
1700
                Avx512Backend::add(&a, &b, &mut result);
1701
            }
1702
1703
            assert_eq!(result[0], 0.0);
1704
            assert_eq!(result[1], 0.0);
1705
            assert_eq!(result[2], f32::INFINITY);
1706
            assert_eq!(result[3], f32::NEG_INFINITY);
1707
            assert!((result[4] - 3e-20).abs() < 1e-25);
1708
            assert!((result[5] + 3e-20).abs() < 1e-25);
1709
        });
1710
    }
1711
1712
    #[test]
1713
    fn test_avx512_add_remainder_correctness() {
1714
        avx512_test(|| {
1715
            // Specifically test remainder handling for sizes 16+1 through 16+15
1716
            for remainder in 1..=15 {
1717
                let size = 16 + remainder;
1718
                let a: Vec<f32> = (0..size).map(|i| i as f32).collect();
1719
                let b: Vec<f32> = (0..size).map(|i| (size - i) as f32).collect();
1720
                let mut result = vec![0.0; size];
1721
1722
                // SAFETY: CPU feature verified at runtime, slices bounds-checked
1723
                unsafe {
1724
                    Avx512Backend::add(&a, &b, &mut result);
1725
                }
1726
1727
                // Verify all elements, especially the remainder portion
1728
                for (i, &value) in result.iter().enumerate().take(size) {
1729
                    let expected = i as f32 + (size - i) as f32;
1730
                    assert_eq!(
1731
                        value, expected,
1732
                        "Remainder test failed at size {} (remainder {}), index {}",
1733
                        size, remainder, i
1734
                    );
1735
                }
1736
            }
1737
        });
1738
    }
1739
1740
    // =====================
1741
    // AVX-512 dot() tests
1742
    // =====================
1743
1744
    #[test]
1745
    fn test_avx512_dot_basic() {
1746
        avx512_test(|| {
1747
            let a = vec![1.0, 2.0, 3.0, 4.0];
1748
            let b = vec![5.0, 6.0, 7.0, 8.0];
1749
            // Expected: 1*5 + 2*6 + 3*7 + 4*8 = 5 + 12 + 21 + 32 = 70
1750
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
1751
            let result = unsafe { Avx512Backend::dot(&a, &b) };
1752
            assert!(
1753
                (result - 70.0).abs() < 1e-5,
1754
                "Expected 70.0, got {}",
1755
                result
1756
            );
1757
        });
1758
    }
1759
1760
    #[test]
1761
    fn test_avx512_dot_aligned_16() {
1762
        avx512_test(|| {
1763
            // Test with exactly 16 elements (one AVX-512 register)
1764
            let a: Vec<f32> = (0..16).map(|i| i as f32).collect();
1765
            let b: Vec<f32> = (0..16).map(|i| (i + 1) as f32).collect();
1766
            // Expected: sum of i * (i + 1) for i in 0..16
1767
            // = 0*1 + 1*2 + 2*3 + ... + 15*16
1768
            let expected: f32 = (0..16).map(|i| (i * (i + 1)) as f32).sum();
1769
1770
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
1771
            let result = unsafe { Avx512Backend::dot(&a, &b) };
1772
            assert!(
1773
                (result - expected).abs() < 1e-4,
1774
                "Expected {}, got {}",
1775
                expected,
1776
                result
1777
            );
1778
        });
1779
    }
1780
1781
    #[test]
1782
    fn test_avx512_dot_non_aligned() {
1783
        avx512_test(|| {
1784
            // Test with 18 elements (16 + 2 remainder)
1785
            let a: Vec<f32> = (0..18).map(|i| (i as f32) * 1.5).collect();
1786
            let b: Vec<f32> = (0..18).map(|i| (i as f32) * 0.7).collect();
1787
            // Expected: sum of (i * 1.5) * (i * 0.7) = sum of i^2 * 1.05
1788
            let expected: f32 = (0..18).map(|i| ((i * i) as f32) * 1.05).sum();
1789
1790
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
1791
            let result = unsafe { Avx512Backend::dot(&a, &b) };
1792
            assert!(
1793
                (result - expected).abs() < 1e-3,
1794
                "Expected {}, got {}",
1795
                expected,
1796
                result
1797
            );
1798
        });
1799
    }
1800
1801
    #[test]
1802
    fn test_avx512_dot_large() {
1803
        avx512_test(|| {
1804
            // Test with 1000 elements (62 full AVX-512 registers + 8 remainder)
1805
            let size = 1000;
1806
            let a: Vec<f32> = (0..size).map(|i| (i as f32) * 0.5).collect();
1807
            let b: Vec<f32> = (0..size).map(|i| (i as f32) * 0.3).collect();
1808
            // Expected: sum of (i * 0.5) * (i * 0.3) = sum of i^2 * 0.15
1809
            let expected: f32 = (0..size).map(|i| ((i * i) as f32) * 0.15).sum();
1810
1811
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
1812
            let result = unsafe { Avx512Backend::dot(&a, &b) };
1813
            // Larger tolerance for accumulation of floating point errors
1814
            assert!(
1815
                (result - expected).abs() / expected.abs() < 1e-4,
1816
                "Expected {}, got {}, relative error: {}",
1817
                expected,
1818
                result,
1819
                ((result - expected).abs() / expected.abs())
1820
            );
1821
        });
1822
    }
1823
1824
    #[test]
1825
    fn test_avx512_dot_equivalence_to_scalar() {
1826
        avx512_test(|| {
1827
            // Backend equivalence: AVX-512 should produce same results as Scalar
1828
            let sizes = vec![1, 7, 15, 16, 17, 32, 63, 100, 1000];
1829
1830
            for size in sizes {
1831
                let a: Vec<f32> = (0..size).map(|i| (i as f32 * 1.5) - 50.0).collect();
1832
                let b: Vec<f32> = (0..size).map(|i| (i as f32 * 0.7) + 20.0).collect();
1833
1834
                // SAFETY: CPU feature verified at runtime, slices bounds-checked
1835
                let result_avx512 = unsafe { Avx512Backend::dot(&a, &b) };
1836
                // SAFETY: CPU feature verified at runtime, slices bounds-checked
1837
                let result_scalar = unsafe { ScalarBackend::dot(&a, &b) };
1838
1839
                // Use relative tolerance for larger values
1840
                let tolerance = if result_scalar.abs() > 1.0 {
1841
                    result_scalar.abs() * 1e-5
1842
                } else {
1843
                    1e-5
1844
                };
1845
1846
                assert!(
1847
                    (result_avx512 - result_scalar).abs() < tolerance,
1848
                    "Backend mismatch at size {}: AVX512={}, Scalar={}, diff={}",
1849
                    size,
1850
                    result_avx512,
1851
                    result_scalar,
1852
                    (result_avx512 - result_scalar).abs()
1853
                );
1854
            }
1855
        });
1856
    }
1857
1858
    #[test]
1859
    fn test_avx512_dot_special_values() {
1860
        avx512_test(|| {
1861
            // Test with zero, negative, small, and large values
1862
            let a = vec![
1863
                0.0, -1.0, 1.0, -5.0, 5.0, 1e-10, 1e10, -1e10, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0,
1864
                9.0,
1865
            ];
1866
            let b = vec![
1867
                10.0, 2.0, 3.0, -2.0, 4.0, 2e-10, 2e10, -2e10, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
1868
                1.0,
1869
            ];
1870
1871
            // Expected: 0*10 + (-1)*2 + 1*3 + (-5)*(-2) + 5*4 + (1e-10)*(2e-10) + (1e10)*(2e10) + (-1e10)*(-2e10) + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9
1872
            //         = 0 - 2 + 3 + 10 + 20 + 2e-20 + 2e20 + 2e20 + 44
1873
            //         = 75 + 2e-20 + 4e20
1874
            // Note: 2e-20 is negligible compared to 4e20
1875
1876
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
1877
            let result = unsafe { Avx512Backend::dot(&a, &b) };
1878
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
1879
            let expected = unsafe { ScalarBackend::dot(&a, &b) };
1880
1881
            // Use relative tolerance due to large values
1882
            let rel_error = if expected.abs() > 1.0 {
1883
                (result - expected).abs() / expected.abs()
1884
            } else {
1885
                (result - expected).abs()
1886
            };
1887
1888
            assert!(
1889
                rel_error < 1e-5,
1890
                "Expected {}, got {}, relative error: {}",
1891
                expected,
1892
                result,
1893
                rel_error
1894
            );
1895
        });
1896
    }
1897
1898
    #[test]
1899
    fn test_avx512_dot_remainder_sizes() {
1900
        avx512_test(|| {
1901
            // Test all remainder sizes from 16+1 to 16+15
1902
            for remainder in 1..=15 {
1903
                let size = 16 + remainder;
1904
                let a: Vec<f32> = (0..size).map(|i| (i as f32) + 1.0).collect();
1905
                let b: Vec<f32> = (0..size).map(|i| (i as f32) + 2.0).collect();
1906
1907
                // SAFETY: CPU feature verified at runtime, slices bounds-checked
1908
                let result_avx512 = unsafe { Avx512Backend::dot(&a, &b) };
1909
                // SAFETY: CPU feature verified at runtime, slices bounds-checked
1910
                let result_scalar = unsafe { ScalarBackend::dot(&a, &b) };
1911
1912
                let tolerance = if result_scalar.abs() > 1.0 {
1913
                    result_scalar.abs() * 1e-5
1914
                } else {
1915
                    1e-5
1916
                };
1917
1918
                assert!(
1919
                    (result_avx512 - result_scalar).abs() < tolerance,
1920
                    "Remainder test failed at size {} (remainder {}): AVX512={}, Scalar={}",
1921
                    size,
1922
                    remainder,
1923
                    result_avx512,
1924
                    result_scalar
1925
                );
1926
            }
1927
        });
1928
    }
1929
1930
    #[test]
1931
    fn test_avx512_dot_zero_vector() {
1932
        avx512_test(|| {
1933
            let a = vec![
1934
                1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0,
1935
                16.0,
1936
            ];
1937
            let b = vec![0.0; 16];
1938
1939
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
1940
            let result = unsafe { Avx512Backend::dot(&a, &b) };
1941
            assert_eq!(
1942
                result, 0.0,
1943
                "Dot product with zero vector should be 0.0, got {}",
1944
                result
1945
            );
1946
        });
1947
    }
1948
1949
    #[test]
1950
    fn test_avx512_dot_orthogonal() {
1951
        avx512_test(|| {
1952
            // Orthogonal vectors: [1, 0, 0, 0, ...] and [0, 1, 0, 0, ...]
1953
            let mut a = vec![0.0; 16];
1954
            let mut b = vec![0.0; 16];
1955
            a[0] = 1.0;
1956
            b[1] = 1.0;
1957
1958
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
1959
            let result = unsafe { Avx512Backend::dot(&a, &b) };
1960
            assert_eq!(
1961
                result, 0.0,
1962
                "Dot product of orthogonal vectors should be 0.0, got {}",
1963
                result
1964
            );
1965
        });
1966
    }
1967
1968
    // =====================
1969
    // AVX-512 sum() tests
1970
    // =====================
1971
1972
    #[test]
1973
    fn test_avx512_sum_basic() {
1974
        avx512_test(|| {
1975
            let a = vec![1.0, 2.0, 3.0, 4.0];
1976
            // Expected: 1 + 2 + 3 + 4 = 10
1977
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
1978
            let result = unsafe { Avx512Backend::sum(&a) };
1979
            assert!(
1980
                (result - 10.0).abs() < 1e-5,
1981
                "Expected 10.0, got {}",
1982
                result
1983
            );
1984
        });
1985
    }
1986
1987
    #[test]
1988
    fn test_avx512_sum_aligned_16() {
1989
        avx512_test(|| {
1990
            // Test with exactly 16 elements (one AVX-512 register)
1991
            let a: Vec<f32> = (0..16).map(|i| i as f32).collect();
1992
            // Expected: sum of 0..16 = 0+1+2+...+15 = 120
1993
            let expected: f32 = (0..16).map(|i| i as f32).sum();
1994
1995
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
1996
            let result = unsafe { Avx512Backend::sum(&a) };
1997
            assert!(
1998
                (result - expected).abs() < 1e-4,
1999
                "Expected {}, got {}",
2000
                expected,
2001
                result
2002
            );
2003
        });
2004
    }
2005
2006
    #[test]
2007
    fn test_avx512_sum_non_aligned() {
2008
        avx512_test(|| {
2009
            // Test with 18 elements (16 + 2 remainder)
2010
            let a: Vec<f32> = (0..18).map(|i| (i as f32) * 1.5).collect();
2011
            // Expected: sum of (i * 1.5) for i in 0..18
2012
            let expected: f32 = (0..18).map(|i| (i as f32) * 1.5).sum();
2013
2014
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2015
            let result = unsafe { Avx512Backend::sum(&a) };
2016
            assert!(
2017
                (result - expected).abs() < 1e-3,
2018
                "Expected {}, got {}",
2019
                expected,
2020
                result
2021
            );
2022
        });
2023
    }
2024
2025
    #[test]
2026
    fn test_avx512_sum_large() {
2027
        avx512_test(|| {
2028
            // Test with 1000 elements (62 full AVX-512 registers + 8 remainder)
2029
            let size = 1000;
2030
            let a: Vec<f32> = (0..size).map(|i| (i as f32) * 0.5).collect();
2031
            // Expected: sum of (i * 0.5) for i in 0..1000
2032
            let expected: f32 = (0..size).map(|i| (i as f32) * 0.5).sum();
2033
2034
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2035
            let result = unsafe { Avx512Backend::sum(&a) };
2036
            // Larger tolerance for accumulation of floating point errors
2037
            let rel_error = if expected.abs() > 1.0 {
2038
                (result - expected).abs() / expected.abs()
2039
            } else {
2040
                (result - expected).abs()
2041
            };
2042
            assert!(
2043
                rel_error < 1e-4,
2044
                "Expected {}, got {}, relative error: {}",
2045
                expected,
2046
                result,
2047
                rel_error
2048
            );
2049
        });
2050
    }
2051
2052
    #[test]
2053
    fn test_avx512_sum_equivalence_to_scalar() {
2054
        avx512_test(|| {
2055
            // Backend equivalence: AVX-512 should produce same results as Scalar
2056
            let sizes = vec![1, 7, 15, 16, 17, 32, 63, 100, 1000];
2057
2058
            for size in sizes {
2059
                let a: Vec<f32> = (0..size).map(|i| (i as f32 * 1.5) - 50.0).collect();
2060
2061
                // SAFETY: CPU feature verified at runtime, slices bounds-checked
2062
                let result_avx512 = unsafe { Avx512Backend::sum(&a) };
2063
                // SAFETY: CPU feature verified at runtime, slices bounds-checked
2064
                let result_scalar = unsafe { ScalarBackend::sum(&a) };
2065
2066
                // Use relative tolerance for larger values
2067
                let tolerance = if result_scalar.abs() > 1.0 {
2068
                    result_scalar.abs() * 1e-5
2069
                } else {
2070
                    1e-5
2071
                };
2072
2073
                assert!(
2074
                    (result_avx512 - result_scalar).abs() < tolerance,
2075
                    "Backend mismatch at size {}: AVX512={}, Scalar={}, diff={}",
2076
                    size,
2077
                    result_avx512,
2078
                    result_scalar,
2079
                    (result_avx512 - result_scalar).abs()
2080
                );
2081
            }
2082
        });
2083
    }
2084
2085
    #[test]
2086
    fn test_avx512_sum_negative_values() {
2087
        avx512_test(|| {
2088
            let a = vec![
2089
                -1.0, -2.0, -3.0, -4.0, 5.0, 6.0, 7.0, 8.0, -9.0, -10.0, 11.0, 12.0, -13.0, 14.0,
2090
                -15.0, 16.0,
2091
            ];
2092
            // Expected: -1 - 2 - 3 - 4 + 5 + 6 + 7 + 8 - 9 - 10 + 11 + 12 - 13 + 14 - 15 + 16 = 22
2093
            let expected = 22.0;
2094
2095
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2096
            let result = unsafe { Avx512Backend::sum(&a) };
2097
            assert!(
2098
                (result - expected).abs() < 1e-5,
2099
                "Expected {}, got {}",
2100
                expected,
2101
                result
2102
            );
2103
        });
2104
    }
2105
2106
    #[test]
2107
    fn test_avx512_sum_zero_vector() {
2108
        avx512_test(|| {
2109
            let a = vec![0.0; 16];
2110
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2111
            let result = unsafe { Avx512Backend::sum(&a) };
2112
            assert_eq!(result, 0.0, "Sum of zeros should be 0.0, got {}", result);
2113
        });
2114
    }
2115
2116
    #[test]
2117
    fn test_avx512_sum_single_element() {
2118
        avx512_test(|| {
2119
            let a = vec![42.0];
2120
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2121
            let result = unsafe { Avx512Backend::sum(&a) };
2122
            assert_eq!(
2123
                result, 42.0,
2124
                "Sum of single element should be that element, got {}",
2125
                result
2126
            );
2127
        });
2128
    }
2129
2130
    #[test]
2131
    fn test_avx512_sum_remainder_sizes() {
2132
        avx512_test(|| {
2133
            // Test all remainder sizes from 16+1 to 16+15
2134
            for remainder in 1..=15 {
2135
                let size = 16 + remainder;
2136
                let a: Vec<f32> = (0..size).map(|i| (i as f32) + 1.0).collect();
2137
2138
                // SAFETY: CPU feature verified at runtime, slices bounds-checked
2139
                let result_avx512 = unsafe { Avx512Backend::sum(&a) };
2140
                // SAFETY: CPU feature verified at runtime, slices bounds-checked
2141
                let result_scalar = unsafe { ScalarBackend::sum(&a) };
2142
2143
                let tolerance = if result_scalar.abs() > 1.0 {
2144
                    result_scalar.abs() * 1e-5
2145
                } else {
2146
                    1e-5
2147
                };
2148
2149
                assert!(
2150
                    (result_avx512 - result_scalar).abs() < tolerance,
2151
                    "Remainder test failed at size {} (remainder {}): AVX512={}, Scalar={}",
2152
                    size,
2153
                    remainder,
2154
                    result_avx512,
2155
                    result_scalar
2156
                );
2157
            }
2158
        });
2159
    }
2160
2161
    // =====================
2162
    // AVX-512 max() tests
2163
    // =====================
2164
2165
    #[test]
2166
    fn test_avx512_max_basic() {
2167
        avx512_test(|| {
2168
            let a = vec![1.0, 5.0, 3.0, 9.0, 2.0];
2169
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2170
            let result = unsafe { Avx512Backend::max(&a) };
2171
            assert_eq!(result, 9.0, "Expected 9.0, got {}", result);
2172
        });
2173
    }
2174
2175
    #[test]
2176
    fn test_avx512_max_aligned_16() {
2177
        avx512_test(|| {
2178
            let mut a: Vec<f32> = (0..16).map(|i| i as f32).collect();
2179
            a[8] = 100.0; // Max is in the middle
2180
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2181
            let result = unsafe { Avx512Backend::max(&a) };
2182
            assert_eq!(result, 100.0, "Expected 100.0, got {}", result);
2183
        });
2184
    }
2185
2186
    #[test]
2187
    fn test_avx512_max_non_aligned() {
2188
        avx512_test(|| {
2189
            let mut a: Vec<f32> = (0..18).map(|i| (i as f32) * 1.5).collect();
2190
            a[17] = 200.0; // Max is in remainder
2191
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2192
            let result = unsafe { Avx512Backend::max(&a) };
2193
            assert_eq!(result, 200.0, "Expected 200.0, got {}", result);
2194
        });
2195
    }
2196
2197
    #[test]
2198
    fn test_avx512_max_negative_values() {
2199
        avx512_test(|| {
2200
            let a = vec![-5.0, -2.0, -10.0, -1.0, -8.0];
2201
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2202
            let result = unsafe { Avx512Backend::max(&a) };
2203
            assert_eq!(result, -1.0, "Expected -1.0, got {}", result);
2204
        });
2205
    }
2206
2207
    #[test]
2208
    fn test_avx512_max_equivalence_to_scalar() {
2209
        avx512_test(|| {
2210
            let sizes = vec![1, 7, 15, 16, 17, 32, 63, 100, 1000];
2211
            for size in sizes {
2212
                let a: Vec<f32> = (0..size).map(|i| ((i * 7) % 100) as f32 - 50.0).collect();
2213
                // SAFETY: CPU feature verified at runtime, slices bounds-checked
2214
                let result_avx512 = unsafe { Avx512Backend::max(&a) };
2215
                // SAFETY: CPU feature verified at runtime, slices bounds-checked
2216
                let result_scalar = unsafe { ScalarBackend::max(&a) };
2217
                assert_eq!(
2218
                    result_avx512, result_scalar,
2219
                    "Backend mismatch at size {}: AVX512={}, Scalar={}",
2220
                    size, result_avx512, result_scalar
2221
                );
2222
            }
2223
        });
2224
    }
2225
2226
    // =====================
2227
    // AVX-512 min() tests
2228
    // =====================
2229
2230
    #[test]
2231
    fn test_avx512_min_basic() {
2232
        avx512_test(|| {
2233
            let a = vec![5.0, 1.0, 9.0, 3.0, 2.0];
2234
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2235
            let result = unsafe { Avx512Backend::min(&a) };
2236
            assert_eq!(result, 1.0, "Expected 1.0, got {}", result);
2237
        });
2238
    }
2239
2240
    #[test]
2241
    fn test_avx512_min_aligned_16() {
2242
        avx512_test(|| {
2243
            let mut a: Vec<f32> = (0..16).map(|i| (i + 10) as f32).collect();
2244
            a[8] = -100.0; // Min is in the middle
2245
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2246
            let result = unsafe { Avx512Backend::min(&a) };
2247
            assert_eq!(result, -100.0, "Expected -100.0, got {}", result);
2248
        });
2249
    }
2250
2251
    #[test]
2252
    fn test_avx512_min_non_aligned() {
2253
        avx512_test(|| {
2254
            let mut a: Vec<f32> = (0..18).map(|i| (i as f32) * 1.5 + 10.0).collect();
2255
            a[17] = -200.0; // Min is in remainder
2256
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2257
            let result = unsafe { Avx512Backend::min(&a) };
2258
            assert_eq!(result, -200.0, "Expected -200.0, got {}", result);
2259
        });
2260
    }
2261
2262
    #[test]
2263
    fn test_avx512_min_positive_values() {
2264
        avx512_test(|| {
2265
            let a = vec![5.0, 2.0, 10.0, 1.0, 8.0];
2266
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2267
            let result = unsafe { Avx512Backend::min(&a) };
2268
            assert_eq!(result, 1.0, "Expected 1.0, got {}", result);
2269
        });
2270
    }
2271
2272
    #[test]
2273
    fn test_avx512_min_equivalence_to_scalar() {
2274
        avx512_test(|| {
2275
            let sizes = vec![1, 7, 15, 16, 17, 32, 63, 100, 1000];
2276
            for size in sizes {
2277
                let a: Vec<f32> = (0..size).map(|i| ((i * 7) % 100) as f32 - 50.0).collect();
2278
                // SAFETY: CPU feature verified at runtime, slices bounds-checked
2279
                let result_avx512 = unsafe { Avx512Backend::min(&a) };
2280
                // SAFETY: CPU feature verified at runtime, slices bounds-checked
2281
                let result_scalar = unsafe { ScalarBackend::min(&a) };
2282
                assert_eq!(
2283
                    result_avx512, result_scalar,
2284
                    "Backend mismatch at size {}: AVX512={}, Scalar={}",
2285
                    size, result_avx512, result_scalar
2286
                );
2287
            }
2288
        });
2289
    }
2290
2291
    // ============================================================================
2292
    // argmax() tests
2293
    // ============================================================================
2294
2295
    #[test]
2296
    fn test_avx512_argmax_basic() {
2297
        avx512_test(|| {
2298
            let a = vec![1.0, 5.0, 3.0, 9.0, 2.0];
2299
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2300
            let result = unsafe { Avx512Backend::argmax(&a) };
2301
            assert_eq!(result, 3); // Index of 9.0
2302
        });
2303
    }
2304
2305
    #[test]
2306
    fn test_avx512_argmax_aligned_16() {
2307
        avx512_test(|| {
2308
            let a: Vec<f32> = (0..16).map(|i| i as f32).collect();
2309
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2310
            let result = unsafe { Avx512Backend::argmax(&a) };
2311
            assert_eq!(result, 15); // Maximum is at index 15
2312
        });
2313
    }
2314
2315
    #[test]
2316
    fn test_avx512_argmax_non_aligned_18() {
2317
        avx512_test(|| {
2318
            let a: Vec<f32> = (0..18).map(|i| i as f32).collect();
2319
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2320
            let result = unsafe { Avx512Backend::argmax(&a) };
2321
            assert_eq!(result, 17); // Maximum is at index 17
2322
        });
2323
    }
2324
2325
    #[test]
2326
    fn test_avx512_argmax_negative_values() {
2327
        avx512_test(|| {
2328
            let a = vec![-5.0, -2.0, -8.0, -1.0, -10.0];
2329
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2330
            let result = unsafe { Avx512Backend::argmax(&a) };
2331
            assert_eq!(result, 3); // Index of -1.0
2332
        });
2333
    }
2334
2335
    #[test]
2336
    fn test_avx512_argmax_max_at_start() {
2337
        avx512_test(|| {
2338
            let a = vec![100.0, 1.0, 2.0, 3.0, 4.0];
2339
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2340
            let result = unsafe { Avx512Backend::argmax(&a) };
2341
            assert_eq!(result, 0); // Maximum is at index 0
2342
        });
2343
    }
2344
2345
    #[test]
2346
    fn test_avx512_argmax_backend_equivalence() {
2347
        avx512_test(|| {
2348
            let sizes = [16, 17, 100, 1000, 10000, 16384, 16385, 100000, 1000000];
2349
            for &size in &sizes {
2350
                let a: Vec<f32> = (0..size).map(|i| ((i * 13) % 100) as f32 - 50.0).collect();
2351
                // SAFETY: CPU feature verified at runtime, slices bounds-checked
2352
                let result_avx512 = unsafe { Avx512Backend::argmax(&a) };
2353
                // SAFETY: CPU feature verified at runtime, slices bounds-checked
2354
                let result_scalar = unsafe { ScalarBackend::argmax(&a) };
2355
                assert_eq!(
2356
                    result_avx512, result_scalar,
2357
                    "Backend mismatch at size {}: AVX512={}, Scalar={}",
2358
                    size, result_avx512, result_scalar
2359
                );
2360
            }
2361
        });
2362
    }
2363
2364
    // ============================================================================
2365
    // argmin() tests
2366
    // ============================================================================
2367
2368
    #[test]
2369
    fn test_avx512_argmin_basic() {
2370
        avx512_test(|| {
2371
            let a = vec![5.0, 1.0, 9.0, 3.0, 2.0];
2372
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2373
            let result = unsafe { Avx512Backend::argmin(&a) };
2374
            assert_eq!(result, 1); // Index of 1.0
2375
        });
2376
    }
2377
2378
    #[test]
2379
    fn test_avx512_argmin_aligned_16() {
2380
        avx512_test(|| {
2381
            let a: Vec<f32> = (0..16).rev().map(|i| i as f32).collect();
2382
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2383
            let result = unsafe { Avx512Backend::argmin(&a) };
2384
            assert_eq!(result, 15); // Minimum is at index 15
2385
        });
2386
    }
2387
2388
    #[test]
2389
    fn test_avx512_argmin_non_aligned_18() {
2390
        avx512_test(|| {
2391
            let a: Vec<f32> = (0..18).rev().map(|i| i as f32).collect();
2392
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2393
            let result = unsafe { Avx512Backend::argmin(&a) };
2394
            assert_eq!(result, 17); // Minimum is at index 17
2395
        });
2396
    }
2397
2398
    #[test]
2399
    fn test_avx512_argmin_positive_values() {
2400
        avx512_test(|| {
2401
            let a = vec![10.0, 5.0, 8.0, 2.0, 15.0];
2402
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2403
            let result = unsafe { Avx512Backend::argmin(&a) };
2404
            assert_eq!(result, 3); // Index of 2.0
2405
        });
2406
    }
2407
2408
    #[test]
2409
    fn test_avx512_argmin_min_at_start() {
2410
        avx512_test(|| {
2411
            let a = vec![1.0, 100.0, 200.0, 300.0, 400.0];
2412
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2413
            let result = unsafe { Avx512Backend::argmin(&a) };
2414
            assert_eq!(result, 0); // Minimum is at index 0
2415
        });
2416
    }
2417
2418
    #[test]
2419
    fn test_avx512_argmin_backend_equivalence() {
2420
        avx512_test(|| {
2421
            let sizes = [16, 17, 100, 1000, 10000, 16384, 16385, 100000, 1000000];
2422
            for &size in &sizes {
2423
                let a: Vec<f32> = (0..size).map(|i| ((i * 13) % 100) as f32 - 50.0).collect();
2424
                // SAFETY: CPU feature verified at runtime, slices bounds-checked
2425
                let result_avx512 = unsafe { Avx512Backend::argmin(&a) };
2426
                // SAFETY: CPU feature verified at runtime, slices bounds-checked
2427
                let result_scalar = unsafe { ScalarBackend::argmin(&a) };
2428
                assert_eq!(
2429
                    result_avx512, result_scalar,
2430
                    "Backend mismatch at size {}: AVX512={}, Scalar={}",
2431
                    size, result_avx512, result_scalar
2432
                );
2433
            }
2434
        });
2435
    }
2436
2437
    // ============================================================
2438
    // norm_l2 Tests
2439
    // ============================================================
2440
2441
    #[test]
2442
    fn test_avx512_norm_l2_basic() {
2443
        avx512_test(|| {
2444
            let a = vec![3.0, 4.0];
2445
            // Expected: sqrt(3^2 + 4^2) = sqrt(9 + 16) = sqrt(25) = 5.0
2446
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2447
            let result = unsafe { Avx512Backend::norm_l2(&a) };
2448
            assert!((result - 5.0).abs() < 1e-5, "Expected 5.0, got {}", result);
2449
        });
2450
    }
2451
2452
    #[test]
2453
    fn test_avx512_norm_l2_empty() {
2454
        avx512_test(|| {
2455
            let a: Vec<f32> = vec![];
2456
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2457
            let result = unsafe { Avx512Backend::norm_l2(&a) };
2458
            assert_eq!(result, 0.0, "L2 norm of empty vector should be 0.0");
2459
        });
2460
    }
2461
2462
    #[test]
2463
    fn test_avx512_norm_l2_single() {
2464
        avx512_test(|| {
2465
            let a = vec![7.0];
2466
            // Expected: sqrt(7^2) = 7.0
2467
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2468
            let result = unsafe { Avx512Backend::norm_l2(&a) };
2469
            assert!((result - 7.0).abs() < 1e-5, "Expected 7.0, got {}", result);
2470
        });
2471
    }
2472
2473
    #[test]
2474
    fn test_avx512_norm_l2_aligned_16() {
2475
        avx512_test(|| {
2476
            // Test with exactly 16 elements (one AVX-512 register)
2477
            let a = vec![1.0; 16];
2478
            // Expected: sqrt(16 * 1^2) = sqrt(16) = 4.0
2479
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2480
            let result = unsafe { Avx512Backend::norm_l2(&a) };
2481
            assert!((result - 4.0).abs() < 1e-5, "Expected 4.0, got {}", result);
2482
        });
2483
    }
2484
2485
    #[test]
2486
    fn test_avx512_norm_l2_non_aligned() {
2487
        avx512_test(|| {
2488
            // Test with 18 elements (16 + 2 remainder)
2489
            let a: Vec<f32> = (0..18).map(|i| (i as f32) + 1.0).collect();
2490
            // Expected: sqrt(sum((i+1)^2 for i in 0..18))
2491
            let expected = (0..18)
2492
                .map(|i| ((i as f32) + 1.0) * ((i as f32) + 1.0))
2493
                .sum::<f32>()
2494
                .sqrt();
2495
2496
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2497
            let result = unsafe { Avx512Backend::norm_l2(&a) };
2498
            assert!(
2499
                (result - expected).abs() < 1e-3,
2500
                "Expected {}, got {}",
2501
                expected,
2502
                result
2503
            );
2504
        });
2505
    }
2506
2507
    #[test]
2508
    fn test_avx512_norm_l2_large() {
2509
        avx512_test(|| {
2510
            // Test with 1000 elements
2511
            let size = 1000;
2512
            let a: Vec<f32> = (0..size).map(|i| (i as f32) * 0.1).collect();
2513
            let expected = (0..size)
2514
                .map(|i| ((i as f32) * 0.1) * ((i as f32) * 0.1))
2515
                .sum::<f32>()
2516
                .sqrt();
2517
2518
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2519
            let result = unsafe { Avx512Backend::norm_l2(&a) };
2520
            let rel_error = if expected.abs() > 1.0 {
2521
                (result - expected).abs() / expected.abs()
2522
            } else {
2523
                (result - expected).abs()
2524
            };
2525
            assert!(
2526
                rel_error < 1e-4,
2527
                "Expected {}, got {}, relative error: {}",
2528
                expected,
2529
                result,
2530
                rel_error
2531
            );
2532
        });
2533
    }
2534
2535
    #[test]
2536
    fn test_avx512_norm_l2_equivalence_to_scalar() {
2537
        avx512_test(|| {
2538
            // Backend equivalence: AVX-512 should produce same results as Scalar
2539
            let sizes = vec![0, 1, 7, 15, 16, 17, 32, 63, 100, 1000];
2540
2541
            for size in sizes {
2542
                let a: Vec<f32> = (0..size).map(|i| (i as f32 * 0.7) - 10.0).collect();
2543
2544
                // SAFETY: CPU feature verified at runtime, slices bounds-checked
2545
                let result_avx512 = unsafe { Avx512Backend::norm_l2(&a) };
2546
                // SAFETY: CPU feature verified at runtime, slices bounds-checked
2547
                let result_scalar = unsafe { ScalarBackend::norm_l2(&a) };
2548
2549
                // Use relative tolerance for larger values
2550
                let tolerance = if result_scalar.abs() > 1.0 {
2551
                    result_scalar.abs() * 1e-5
2552
                } else {
2553
                    1e-5
2554
                };
2555
2556
                assert!(
2557
                    (result_avx512 - result_scalar).abs() < tolerance,
2558
                    "Backend mismatch at size {}: AVX512={}, Scalar={}, diff={}",
2559
                    size,
2560
                    result_avx512,
2561
                    result_scalar,
2562
                    (result_avx512 - result_scalar).abs()
2563
                );
2564
            }
2565
        });
2566
    }
2567
2568
    #[test]
2569
    fn test_avx512_norm_l2_negative_values() {
2570
        avx512_test(|| {
2571
            let a = vec![-3.0, -4.0];
2572
            // Expected: sqrt((-3)^2 + (-4)^2) = sqrt(9 + 16) = 5.0
2573
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2574
            let result = unsafe { Avx512Backend::norm_l2(&a) };
2575
            assert!((result - 5.0).abs() < 1e-5, "Expected 5.0, got {}", result);
2576
        });
2577
    }
2578
2579
    #[test]
2580
    fn test_avx512_norm_l1_scalar_fallback() {
2581
        avx512_test(|| {
2582
            // Test scalar fallback for norm_l1 (not yet SIMD optimized)
2583
            let test_cases = vec![
2584
                vec![],
2585
                vec![5.0],
2586
                vec![-3.0, 1.0, -4.0, 1.0, 5.0],
2587
                vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0],
2588
            ];
2589
2590
            for test_vec in test_cases {
2591
                // SAFETY: Test code calling backend trait methods
2592
                let avx512_result = unsafe { Avx512Backend::norm_l1(&test_vec) };
2593
                // SAFETY: CPU feature verified at runtime, slices bounds-checked
2594
                let scalar_result = unsafe { ScalarBackend::norm_l1(&test_vec) };
2595
2596
                assert!(
2597
                    (avx512_result - scalar_result).abs() < 1e-5,
2598
                    "norm_l1 mismatch for {:?}: avx512={}, scalar={}",
2599
                    test_vec,
2600
                    avx512_result,
2601
                    scalar_result
2602
                );
2603
            }
2604
        });
2605
    }
2606
2607
    #[test]
2608
    fn test_avx512_norm_linf_scalar_fallback() {
2609
        avx512_test(|| {
2610
            // Test scalar fallback for norm_linf (not yet SIMD optimized)
2611
            let test_cases = vec![
2612
                vec![],
2613
                vec![5.0],
2614
                vec![-3.0, 1.0, -4.0, 1.0, 5.0],
2615
                vec![-10.0, 5.0, 3.0, 7.0, -2.0, 8.0, 4.0],
2616
                vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0],
2617
            ];
2618
2619
            for test_vec in test_cases {
2620
                // SAFETY: Test code calling backend trait methods
2621
                let avx512_result = unsafe { Avx512Backend::norm_linf(&test_vec) };
2622
                // SAFETY: CPU feature verified at runtime, slices bounds-checked
2623
                let scalar_result = unsafe { ScalarBackend::norm_linf(&test_vec) };
2624
2625
                assert!(
2626
                    (avx512_result - scalar_result).abs() < 1e-5,
2627
                    "norm_linf mismatch for {:?}: avx512={}, scalar={}",
2628
                    test_vec,
2629
                    avx512_result,
2630
                    scalar_result
2631
                );
2632
            }
2633
        });
2634
    }
2635
2636
    #[test]
2637
    fn test_avx512_sum_kahan_scalar_fallback() {
2638
        avx512_test(|| {
2639
            // Test scalar fallback for sum_kahan (not yet SIMD optimized)
2640
            let test_vec = vec![1.0e10, 1.0, -1.0e10, 1.0]; // Tests numerical stability
2641
2642
            // SAFETY: Test code calling backend trait methods
2643
            let avx512_result = unsafe { Avx512Backend::sum_kahan(&test_vec) };
2644
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
2645
            let scalar_result = unsafe { ScalarBackend::sum_kahan(&test_vec) };
2646
2647
            assert!(
2648
                (avx512_result - scalar_result).abs() < 1e-5,
2649
                "sum_kahan mismatch: avx512={}, scalar={}",
2650
                avx512_result,
2651
                scalar_result
2652
            );
2653
        });
2654
    }
2655
2656
    #[test]
2657
    fn test_avx512_scale_scalar_fallback() {
2658
        avx512_test(|| {
2659
            // Test scalar fallback for scale (not yet SIMD optimized)
2660
            let a = vec![1.0, 2.0, 3.0, 4.0, 5.0];
2661
            let scalar = 2.5;
2662
            let mut avx512_result = vec![0.0; a.len()];
2663
            let mut scalar_result = vec![0.0; a.len()];
2664
2665
            // SAFETY: Test code calling backend trait methods
2666
            unsafe {
2667
                Avx512Backend::scale(&a, scalar, &mut avx512_result);
2668
                ScalarBackend::scale(&a, scalar, &mut scalar_result);
2669
            }
2670
2671
            for i in 0..a.len() {
2672
                assert!(
2673
                    (avx512_result[i] - scalar_result[i]).abs() < 1e-5,
2674
                    "scale mismatch at index {}: avx512={}, scalar={}",
2675
                    i,
2676
                    avx512_result[i],
2677
                    scalar_result[i]
2678
                );
2679
            }
2680
        });
2681
    }
2682
2683
    #[test]
2684
    fn test_avx512_abs_scalar_fallback() {
2685
        avx512_test(|| {
2686
            // Test scalar fallback for abs (not yet SIMD optimized)
2687
            let a = vec![-3.0, 1.0, -4.0, 0.0, 5.0, -2.0];
2688
            let mut avx512_result = vec![0.0; a.len()];
2689
            let mut scalar_result = vec![0.0; a.len()];
2690
2691
            // SAFETY: Test code calling backend trait methods
2692
            unsafe {
2693
                Avx512Backend::abs(&a, &mut avx512_result);
2694
                ScalarBackend::abs(&a, &mut scalar_result);
2695
            }
2696
2697
            for i in 0..a.len() {
2698
                assert!(
2699
                    (avx512_result[i] - scalar_result[i]).abs() < 1e-5,
2700
                    "abs mismatch at index {}: avx512={}, scalar={}",
2701
                    i,
2702
                    avx512_result[i],
2703
                    scalar_result[i]
2704
                );
2705
            }
2706
        });
2707
    }
2708
2709
    #[test]
2710
    fn test_avx512_clamp_scalar_fallback() {
2711
        avx512_test(|| {
2712
            // Test scalar fallback for clamp (not yet SIMD optimized)
2713
            let a = vec![-5.0, 0.0, 3.0, 7.0, 10.0];
2714
            let min_val = 0.0;
2715
            let max_val = 5.0;
2716
            let mut avx512_result = vec![0.0; a.len()];
2717
            let mut scalar_result = vec![0.0; a.len()];
2718
2719
            // SAFETY: Test code calling backend trait methods
2720
            unsafe {
2721
                Avx512Backend::clamp(&a, min_val, max_val, &mut avx512_result);
2722
                ScalarBackend::clamp(&a, min_val, max_val, &mut scalar_result);
2723
            }
2724
2725
            for i in 0..a.len() {
2726
                assert!(
2727
                    (avx512_result[i] - scalar_result[i]).abs() < 1e-5,
2728
                    "clamp mismatch at index {}: avx512={}, scalar={}",
2729
                    i,
2730
                    avx512_result[i],
2731
                    scalar_result[i]
2732
                );
2733
            }
2734
        });
2735
    }
2736
2737
    #[test]
2738
    fn test_avx512_lerp_scalar_fallback() {
2739
        avx512_test(|| {
2740
            // Test scalar fallback for lerp (not yet SIMD optimized)
2741
            let a = vec![0.0, 1.0, 2.0, 3.0];
2742
            let b = vec![10.0, 20.0, 30.0, 40.0];
2743
            let t = 0.5;
2744
            let mut avx512_result = vec![0.0; a.len()];
2745
            let mut scalar_result = vec![0.0; a.len()];
2746
2747
            // SAFETY: Test code calling backend trait methods
2748
            unsafe {
2749
                Avx512Backend::lerp(&a, &b, t, &mut avx512_result);
2750
                ScalarBackend::lerp(&a, &b, t, &mut scalar_result);
2751
            }
2752
2753
            for i in 0..a.len() {
2754
                assert!(
2755
                    (avx512_result[i] - scalar_result[i]).abs() < 1e-5,
2756
                    "lerp mismatch at index {}: avx512={}, scalar={}",
2757
                    i,
2758
                    avx512_result[i],
2759
                    scalar_result[i]
2760
                );
2761
            }
2762
        });
2763
    }
2764
2765
    #[test]
2766
    fn test_avx512_fma_scalar_fallback() {
2767
        avx512_test(|| {
2768
            // Test scalar fallback for fma (not yet SIMD optimized)
2769
            let a = vec![1.0, 2.0, 3.0, 4.0];
2770
            let b = vec![2.0, 3.0, 4.0, 5.0];
2771
            let c = vec![1.0, 1.0, 1.0, 1.0];
2772
            let mut avx512_result = vec![0.0; a.len()];
2773
            let mut scalar_result = vec![0.0; a.len()];
2774
2775
            // SAFETY: Test code calling backend trait methods
2776
            unsafe {
2777
                Avx512Backend::fma(&a, &b, &c, &mut avx512_result);
2778
                ScalarBackend::fma(&a, &b, &c, &mut scalar_result);
2779
            }
2780
2781
            for i in 0..a.len() {
2782
                assert!(
2783
                    (avx512_result[i] - scalar_result[i]).abs() < 1e-5,
2784
                    "fma mismatch at index {}: avx512={}, scalar={}",
2785
                    i,
2786
                    avx512_result[i],
2787
                    scalar_result[i]
2788
                );
2789
            }
2790
        });
2791
    }
2792
2793
    // ===== Mul Tests =====
2794
2795
    #[test]
2796
    fn test_avx512_mul_basic() {
2797
        if !is_x86_feature_detected!("avx512f") {
2798
            return;
2799
        }
2800
2801
        let a = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0];
2802
        let b = vec![2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0];
2803
        let mut result = vec![0.0; 8];
2804
2805
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
2806
        unsafe {
2807
            Avx512Backend::mul(&a, &b, &mut result);
2808
        }
2809
2810
        let expected = [2.0, 6.0, 12.0, 20.0, 30.0, 42.0, 56.0, 72.0];
2811
        for i in 0..8 {
2812
            assert!(
2813
                (result[i] - expected[i]).abs() < 1e-5,
2814
                "mul mismatch at {}: {} vs {}",
2815
                i,
2816
                result[i],
2817
                expected[i]
2818
            );
2819
        }
2820
    }
2821
2822
    #[test]
2823
    fn test_avx512_mul_equivalence_to_scalar() {
2824
        if !is_x86_feature_detected!("avx512f") {
2825
            return;
2826
        }
2827
2828
        let a: Vec<f32> = (0..100).map(|i| (i as f32) * 0.1).collect();
2829
        let b: Vec<f32> = (0..100).map(|i| (i as f32) * 0.2).collect();
2830
        let mut avx512_result = vec![0.0; 100];
2831
        let mut scalar_result = vec![0.0; 100];
2832
2833
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
2834
        unsafe {
2835
            Avx512Backend::mul(&a, &b, &mut avx512_result);
2836
            ScalarBackend::mul(&a, &b, &mut scalar_result);
2837
        }
2838
2839
        for i in 0..100 {
2840
            assert!(
2841
                (avx512_result[i] - scalar_result[i]).abs() < 1e-5,
2842
                "mul mismatch at {}: avx512={}, scalar={}",
2843
                i,
2844
                avx512_result[i],
2845
                scalar_result[i]
2846
            );
2847
        }
2848
    }
2849
2850
    // ===== Div Tests =====
2851
2852
    #[test]
2853
    fn test_avx512_div_basic() {
2854
        if !is_x86_feature_detected!("avx512f") {
2855
            return;
2856
        }
2857
2858
        let a = vec![10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0];
2859
        let b = vec![2.0, 4.0, 5.0, 8.0, 10.0, 12.0, 14.0, 16.0];
2860
        let mut result = vec![0.0; 8];
2861
2862
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
2863
        unsafe {
2864
            Avx512Backend::div(&a, &b, &mut result);
2865
        }
2866
2867
        let expected = [5.0, 5.0, 6.0, 5.0, 5.0, 5.0, 5.0, 5.0];
2868
        for i in 0..8 {
2869
            assert!(
2870
                (result[i] - expected[i]).abs() < 1e-5,
2871
                "div mismatch at {}: {} vs {}",
2872
                i,
2873
                result[i],
2874
                expected[i]
2875
            );
2876
        }
2877
    }
2878
2879
    #[test]
2880
    fn test_avx512_div_equivalence_to_scalar() {
2881
        if !is_x86_feature_detected!("avx512f") {
2882
            return;
2883
        }
2884
2885
        let a: Vec<f32> = (1..101).map(|i| (i as f32) * 10.0).collect();
2886
        let b: Vec<f32> = (1..101).map(|i| (i as f32) * 2.0).collect();
2887
        let mut avx512_result = vec![0.0; 100];
2888
        let mut scalar_result = vec![0.0; 100];
2889
2890
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
2891
        unsafe {
2892
            Avx512Backend::div(&a, &b, &mut avx512_result);
2893
            ScalarBackend::div(&a, &b, &mut scalar_result);
2894
        }
2895
2896
        for i in 0..100 {
2897
            assert!(
2898
                (avx512_result[i] - scalar_result[i]).abs() < 1e-5,
2899
                "div mismatch at {}: avx512={}, scalar={}",
2900
                i,
2901
                avx512_result[i],
2902
                scalar_result[i]
2903
            );
2904
        }
2905
    }
2906
2907
    // ===== Sub Tests =====
2908
2909
    #[test]
2910
    fn test_avx512_sub_basic() {
2911
        if !is_x86_feature_detected!("avx512f") {
2912
            return;
2913
        }
2914
2915
        let a = vec![10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0];
2916
        let b = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0];
2917
        let mut result = vec![0.0; 8];
2918
2919
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
2920
        unsafe {
2921
            Avx512Backend::sub(&a, &b, &mut result);
2922
        }
2923
2924
        let expected = [9.0, 18.0, 27.0, 36.0, 45.0, 54.0, 63.0, 72.0];
2925
        for i in 0..8 {
2926
            assert!(
2927
                (result[i] - expected[i]).abs() < 1e-5,
2928
                "sub mismatch at {}: {} vs {}",
2929
                i,
2930
                result[i],
2931
                expected[i]
2932
            );
2933
        }
2934
    }
2935
2936
    #[test]
2937
    fn test_avx512_sub_equivalence_to_scalar() {
2938
        if !is_x86_feature_detected!("avx512f") {
2939
            return;
2940
        }
2941
2942
        let a: Vec<f32> = (0..100).map(|i| (i as f32) * 2.0).collect();
2943
        let b: Vec<f32> = (0..100).map(|i| (i as f32) * 0.5).collect();
2944
        let mut avx512_result = vec![0.0; 100];
2945
        let mut scalar_result = vec![0.0; 100];
2946
2947
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
2948
        unsafe {
2949
            Avx512Backend::sub(&a, &b, &mut avx512_result);
2950
            ScalarBackend::sub(&a, &b, &mut scalar_result);
2951
        }
2952
2953
        for i in 0..100 {
2954
            assert!(
2955
                (avx512_result[i] - scalar_result[i]).abs() < 1e-5,
2956
                "sub mismatch at {}: avx512={}, scalar={}",
2957
                i,
2958
                avx512_result[i],
2959
                scalar_result[i]
2960
            );
2961
        }
2962
    }
2963
2964
    // ===== Sqrt Tests =====
2965
2966
    #[test]
2967
    fn test_avx512_sqrt_basic() {
2968
        if !is_x86_feature_detected!("avx512f") {
2969
            return;
2970
        }
2971
2972
        let a = vec![1.0, 4.0, 9.0, 16.0, 25.0, 36.0, 49.0, 64.0];
2973
        let mut result = vec![0.0; 8];
2974
2975
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
2976
        unsafe {
2977
            Avx512Backend::sqrt(&a, &mut result);
2978
        }
2979
2980
        let expected = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0];
2981
        for i in 0..8 {
2982
            assert!(
2983
                (result[i] - expected[i]).abs() < 1e-5,
2984
                "sqrt mismatch at {}: {} vs {}",
2985
                i,
2986
                result[i],
2987
                expected[i]
2988
            );
2989
        }
2990
    }
2991
2992
    #[test]
2993
    fn test_avx512_sqrt_equivalence_to_scalar() {
2994
        if !is_x86_feature_detected!("avx512f") {
2995
            return;
2996
        }
2997
2998
        let a: Vec<f32> = (1..101).map(|i| (i as f32) * (i as f32)).collect();
2999
        let mut avx512_result = vec![0.0; 100];
3000
        let mut scalar_result = vec![0.0; 100];
3001
3002
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3003
        unsafe {
3004
            Avx512Backend::sqrt(&a, &mut avx512_result);
3005
            ScalarBackend::sqrt(&a, &mut scalar_result);
3006
        }
3007
3008
        for i in 0..100 {
3009
            assert!(
3010
                (avx512_result[i] - scalar_result[i]).abs() < 1e-5,
3011
                "sqrt mismatch at {}: avx512={}, scalar={}",
3012
                i,
3013
                avx512_result[i],
3014
                scalar_result[i]
3015
            );
3016
        }
3017
    }
3018
3019
    // ===== Exp Tests =====
3020
3021
    #[test]
3022
    fn test_avx512_exp_basic() {
3023
        if !is_x86_feature_detected!("avx512f") {
3024
            return;
3025
        }
3026
3027
        let a = vec![0.0, 1.0, 2.0, -1.0, 0.5, -0.5, 3.0, -3.0];
3028
        let mut result = vec![0.0; 8];
3029
3030
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3031
        unsafe {
3032
            Avx512Backend::exp(&a, &mut result);
3033
        }
3034
3035
        for i in 0..8 {
3036
            let expected = a[i].exp();
3037
            assert!(
3038
                (result[i] - expected).abs() < 1e-5
3039
                    || (result[i] - expected).abs() / expected < 1e-5,
3040
                "exp mismatch at {}: {} vs {} (input: {})",
3041
                i,
3042
                result[i],
3043
                expected,
3044
                a[i]
3045
            );
3046
        }
3047
    }
3048
3049
    #[test]
3050
    fn test_avx512_exp_equivalence_to_scalar() {
3051
        if !is_x86_feature_detected!("avx512f") {
3052
            return;
3053
        }
3054
3055
        let a: Vec<f32> = (0..100).map(|i| ((i as f32) - 50.0) * 0.1).collect();
3056
        let mut avx512_result = vec![0.0; 100];
3057
        let mut scalar_result = vec![0.0; 100];
3058
3059
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3060
        unsafe {
3061
            Avx512Backend::exp(&a, &mut avx512_result);
3062
            ScalarBackend::exp(&a, &mut scalar_result);
3063
        }
3064
3065
        for i in 0..100 {
3066
            let rel_error = if scalar_result[i] != 0.0 {
3067
                (avx512_result[i] - scalar_result[i]).abs() / scalar_result[i]
3068
            } else {
3069
                (avx512_result[i] - scalar_result[i]).abs()
3070
            };
3071
            assert!(
3072
                rel_error < 1e-5,
3073
                "exp mismatch at {}: avx512={}, scalar={}, rel_error={}",
3074
                i,
3075
                avx512_result[i],
3076
                scalar_result[i],
3077
                rel_error
3078
            );
3079
        }
3080
    }
3081
3082
    // ===== Ln Tests =====
3083
3084
    #[test]
3085
    fn test_avx512_ln_basic() {
3086
        if !is_x86_feature_detected!("avx512f") {
3087
            return;
3088
        }
3089
3090
        let a = vec![1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 100.0, 1000.0];
3091
        let mut result = vec![0.0; 8];
3092
3093
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3094
        unsafe {
3095
            Avx512Backend::ln(&a, &mut result);
3096
        }
3097
3098
        for i in 0..8 {
3099
            let expected = a[i].ln();
3100
            assert!(
3101
                (result[i] - expected).abs() < 1e-5
3102
                    || (result[i] - expected).abs() / expected.abs() < 1e-4,
3103
                "ln mismatch at {}: {} vs {} (input: {})",
3104
                i,
3105
                result[i],
3106
                expected,
3107
                a[i]
3108
            );
3109
        }
3110
    }
3111
3112
    #[test]
3113
    fn test_avx512_ln_equivalence_to_scalar() {
3114
        if !is_x86_feature_detected!("avx512f") {
3115
            return;
3116
        }
3117
3118
        let a: Vec<f32> = (1..101).map(|i| (i as f32) * 0.5).collect();
3119
        let mut avx512_result = vec![0.0; 100];
3120
        let mut scalar_result = vec![0.0; 100];
3121
3122
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3123
        unsafe {
3124
            Avx512Backend::ln(&a, &mut avx512_result);
3125
            ScalarBackend::ln(&a, &mut scalar_result);
3126
        }
3127
3128
        for i in 0..100 {
3129
            let abs_error = (avx512_result[i] - scalar_result[i]).abs();
3130
            let rel_error = if scalar_result[i].abs() > 1e-10 {
3131
                abs_error / scalar_result[i].abs()
3132
            } else {
3133
                abs_error
3134
            };
3135
            assert!(
3136
                rel_error < 1e-4 || abs_error < 1e-5,
3137
                "ln mismatch at {}: avx512={}, scalar={}, abs_error={}, rel_error={}",
3138
                i,
3139
                avx512_result[i],
3140
                scalar_result[i],
3141
                abs_error,
3142
                rel_error
3143
            );
3144
        }
3145
    }
3146
3147
    // ===== Log2 Tests =====
3148
3149
    #[test]
3150
    fn test_avx512_log2_basic() {
3151
        if !is_x86_feature_detected!("avx512f") {
3152
            return;
3153
        }
3154
3155
        let a = vec![1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0];
3156
        let mut result = vec![0.0; 8];
3157
3158
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3159
        unsafe {
3160
            Avx512Backend::log2(&a, &mut result);
3161
        }
3162
3163
        let expected = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0];
3164
        for i in 0..8 {
3165
            assert!(
3166
                (result[i] - expected[i]).abs() < 1e-5,
3167
                "log2 mismatch at {}: {} vs {}",
3168
                i,
3169
                result[i],
3170
                expected[i]
3171
            );
3172
        }
3173
    }
3174
3175
    // ===== Log10 Tests =====
3176
3177
    #[test]
3178
    fn test_avx512_log10_basic() {
3179
        if !is_x86_feature_detected!("avx512f") {
3180
            return;
3181
        }
3182
3183
        let a = vec![1.0, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1e6, 1e7];
3184
        let mut result = vec![0.0; 8];
3185
3186
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3187
        unsafe {
3188
            Avx512Backend::log10(&a, &mut result);
3189
        }
3190
3191
        let expected = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0];
3192
        for i in 0..8 {
3193
            assert!(
3194
                (result[i] - expected[i]).abs() < 1e-4,
3195
                "log10 mismatch at {}: {} vs {}",
3196
                i,
3197
                result[i],
3198
                expected[i]
3199
            );
3200
        }
3201
    }
3202
3203
    // ===== Recip Tests =====
3204
3205
    #[test]
3206
    fn test_avx512_recip_basic() {
3207
        if !is_x86_feature_detected!("avx512f") {
3208
            return;
3209
        }
3210
3211
        let a = vec![1.0, 2.0, 4.0, 5.0, 10.0, 20.0, 50.0, 100.0];
3212
        let mut result = vec![0.0; 8];
3213
3214
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3215
        unsafe {
3216
            Avx512Backend::recip(&a, &mut result);
3217
        }
3218
3219
        let expected = [1.0, 0.5, 0.25, 0.2, 0.1, 0.05, 0.02, 0.01];
3220
        for i in 0..8 {
3221
            assert!(
3222
                (result[i] - expected[i]).abs() < 1e-5,
3223
                "recip mismatch at {}: {} vs {}",
3224
                i,
3225
                result[i],
3226
                expected[i]
3227
            );
3228
        }
3229
    }
3230
3231
    #[test]
3232
    fn test_avx512_recip_equivalence_to_scalar() {
3233
        if !is_x86_feature_detected!("avx512f") {
3234
            return;
3235
        }
3236
3237
        let a: Vec<f32> = (1..101).map(|i| (i as f32) * 2.0).collect();
3238
        let mut avx512_result = vec![0.0; 100];
3239
        let mut scalar_result = vec![0.0; 100];
3240
3241
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3242
        unsafe {
3243
            Avx512Backend::recip(&a, &mut avx512_result);
3244
            ScalarBackend::recip(&a, &mut scalar_result);
3245
        }
3246
3247
        for i in 0..100 {
3248
            assert!(
3249
                (avx512_result[i] - scalar_result[i]).abs() < 1e-5,
3250
                "recip mismatch at {}: avx512={}, scalar={}",
3251
                i,
3252
                avx512_result[i],
3253
                scalar_result[i]
3254
            );
3255
        }
3256
    }
3257
3258
    // ===== ReLU Tests =====
3259
3260
    #[test]
3261
    fn test_avx512_relu_basic() {
3262
        if !is_x86_feature_detected!("avx512f") {
3263
            return;
3264
        }
3265
3266
        let a = vec![-2.0, -1.0, 0.0, 1.0, 2.0, 3.0, -5.0, 10.0];
3267
        let mut result = vec![0.0; 8];
3268
3269
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3270
        unsafe {
3271
            Avx512Backend::relu(&a, &mut result);
3272
        }
3273
3274
        let expected = [0.0, 0.0, 0.0, 1.0, 2.0, 3.0, 0.0, 10.0];
3275
        for i in 0..8 {
3276
            assert!(
3277
                (result[i] - expected[i]).abs() < 1e-5,
3278
                "relu mismatch at {}: {} vs {}",
3279
                i,
3280
                result[i],
3281
                expected[i]
3282
            );
3283
        }
3284
    }
3285
3286
    // ===== Sigmoid Tests =====
3287
3288
    #[test]
3289
    fn test_avx512_sigmoid_basic() {
3290
        if !is_x86_feature_detected!("avx512f") {
3291
            return;
3292
        }
3293
3294
        let a = vec![0.0, 1.0, -1.0, 2.0, -2.0, 5.0, -5.0, 10.0];
3295
        let mut avx512_result = vec![0.0; 8];
3296
        let mut scalar_result = vec![0.0; 8];
3297
3298
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3299
        unsafe {
3300
            Avx512Backend::sigmoid(&a, &mut avx512_result);
3301
            ScalarBackend::sigmoid(&a, &mut scalar_result);
3302
        }
3303
3304
        for i in 0..8 {
3305
            assert!(
3306
                (avx512_result[i] - scalar_result[i]).abs() < 1e-4,
3307
                "sigmoid mismatch at {}: avx512={}, scalar={}",
3308
                i,
3309
                avx512_result[i],
3310
                scalar_result[i]
3311
            );
3312
        }
3313
    }
3314
3315
    // ===== GELU Tests =====
3316
3317
    #[test]
3318
    fn test_avx512_gelu_basic() {
3319
        if !is_x86_feature_detected!("avx512f") {
3320
            return;
3321
        }
3322
3323
        let a = vec![0.0, 1.0, -1.0, 2.0, -2.0, 0.5, -0.5, 3.0];
3324
        let mut avx512_result = vec![0.0; 8];
3325
        let mut scalar_result = vec![0.0; 8];
3326
3327
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3328
        unsafe {
3329
            Avx512Backend::gelu(&a, &mut avx512_result);
3330
            ScalarBackend::gelu(&a, &mut scalar_result);
3331
        }
3332
3333
        for i in 0..8 {
3334
            assert!(
3335
                (avx512_result[i] - scalar_result[i]).abs() < 1e-4,
3336
                "gelu mismatch at {}: avx512={}, scalar={}",
3337
                i,
3338
                avx512_result[i],
3339
                scalar_result[i]
3340
            );
3341
        }
3342
    }
3343
3344
    // ===== Swish Tests =====
3345
3346
    #[test]
3347
    fn test_avx512_swish_basic() {
3348
        if !is_x86_feature_detected!("avx512f") {
3349
            return;
3350
        }
3351
3352
        let a = vec![0.0, 1.0, -1.0, 2.0, -2.0, 0.5, -0.5, 3.0];
3353
        let mut avx512_result = vec![0.0; 8];
3354
        let mut scalar_result = vec![0.0; 8];
3355
3356
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3357
        unsafe {
3358
            Avx512Backend::swish(&a, &mut avx512_result);
3359
            ScalarBackend::swish(&a, &mut scalar_result);
3360
        }
3361
3362
        for i in 0..8 {
3363
            assert!(
3364
                (avx512_result[i] - scalar_result[i]).abs() < 1e-4,
3365
                "swish mismatch at {}: avx512={}, scalar={}",
3366
                i,
3367
                avx512_result[i],
3368
                scalar_result[i]
3369
            );
3370
        }
3371
    }
3372
3373
    // ===== Ceil Tests =====
3374
3375
    #[test]
3376
    fn test_avx512_ceil_basic() {
3377
        if !is_x86_feature_detected!("avx512f") {
3378
            return;
3379
        }
3380
3381
        let a = vec![1.1, 1.9, -1.1, -1.9, 0.0, 2.5, -2.5, 3.3];
3382
        let mut result = vec![0.0; 8];
3383
3384
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3385
        unsafe {
3386
            Avx512Backend::ceil(&a, &mut result);
3387
        }
3388
3389
        let expected = [2.0, 2.0, -1.0, -1.0, 0.0, 3.0, -2.0, 4.0];
3390
        for i in 0..8 {
3391
            assert!(
3392
                (result[i] - expected[i]).abs() < 1e-5,
3393
                "ceil mismatch at {}: {} vs {}",
3394
                i,
3395
                result[i],
3396
                expected[i]
3397
            );
3398
        }
3399
    }
3400
3401
    // ===== Floor Tests =====
3402
3403
    #[test]
3404
    fn test_avx512_floor_basic() {
3405
        if !is_x86_feature_detected!("avx512f") {
3406
            return;
3407
        }
3408
3409
        let a = vec![1.1, 1.9, -1.1, -1.9, 0.0, 2.5, -2.5, 3.3];
3410
        let mut result = vec![0.0; 8];
3411
3412
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3413
        unsafe {
3414
            Avx512Backend::floor(&a, &mut result);
3415
        }
3416
3417
        let expected = [1.0, 1.0, -2.0, -2.0, 0.0, 2.0, -3.0, 3.0];
3418
        for i in 0..8 {
3419
            assert!(
3420
                (result[i] - expected[i]).abs() < 1e-5,
3421
                "floor mismatch at {}: {} vs {}",
3422
                i,
3423
                result[i],
3424
                expected[i]
3425
            );
3426
        }
3427
    }
3428
3429
    // ===== Round Tests =====
3430
3431
    #[test]
3432
    fn test_avx512_round_basic() {
3433
        if !is_x86_feature_detected!("avx512f") {
3434
            return;
3435
        }
3436
3437
        let a = vec![1.1, 1.5, 1.9, -1.1, -1.5, -1.9, 2.5, -2.5];
3438
        let mut result = vec![0.0; 8];
3439
3440
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3441
        unsafe {
3442
            Avx512Backend::round(&a, &mut result);
3443
        }
3444
3445
        // Note: AVX-512 round uses "round away from zero" for .5 values
3446
        // 1.5 → 2.0, 2.5 → 3.0, -1.5 → -2.0, -2.5 → -3.0
3447
        let expected = [1.0, 2.0, 2.0, -1.0, -2.0, -2.0, 3.0, -3.0];
3448
        for i in 0..8 {
3449
            assert!(
3450
                (result[i] - expected[i]).abs() < 1e-5,
3451
                "round mismatch at {}: {} vs {}",
3452
                i,
3453
                result[i],
3454
                expected[i]
3455
            );
3456
        }
3457
    }
3458
3459
    // ===== Trigonometric Tests =====
3460
3461
    #[test]
3462
    fn test_avx512_sin_basic() {
3463
        if !is_x86_feature_detected!("avx512f") {
3464
            return;
3465
        }
3466
3467
        let a = vec![0.0, 1.0, -1.0, 2.0, -2.0, 0.5, -0.5, 3.0];
3468
        let mut avx512_result = vec![0.0; 8];
3469
        let mut scalar_result = vec![0.0; 8];
3470
3471
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3472
        unsafe {
3473
            Avx512Backend::sin(&a, &mut avx512_result);
3474
            ScalarBackend::sin(&a, &mut scalar_result);
3475
        }
3476
3477
        for i in 0..8 {
3478
            assert!(
3479
                (avx512_result[i] - scalar_result[i]).abs() < 1e-4,
3480
                "sin mismatch at {}: avx512={}, scalar={}",
3481
                i,
3482
                avx512_result[i],
3483
                scalar_result[i]
3484
            );
3485
        }
3486
    }
3487
3488
    #[test]
3489
    fn test_avx512_cos_basic() {
3490
        if !is_x86_feature_detected!("avx512f") {
3491
            return;
3492
        }
3493
3494
        let a = vec![0.0, 1.0, -1.0, 2.0, -2.0, 0.5, -0.5, 3.0];
3495
        let mut avx512_result = vec![0.0; 8];
3496
        let mut scalar_result = vec![0.0; 8];
3497
3498
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3499
        unsafe {
3500
            Avx512Backend::cos(&a, &mut avx512_result);
3501
            ScalarBackend::cos(&a, &mut scalar_result);
3502
        }
3503
3504
        for i in 0..8 {
3505
            assert!(
3506
                (avx512_result[i] - scalar_result[i]).abs() < 1e-4,
3507
                "cos mismatch at {}: avx512={}, scalar={}",
3508
                i,
3509
                avx512_result[i],
3510
                scalar_result[i]
3511
            );
3512
        }
3513
    }
3514
3515
    #[test]
3516
    fn test_avx512_tan_basic() {
3517
        if !is_x86_feature_detected!("avx512f") {
3518
            return;
3519
        }
3520
3521
        let a = vec![0.0, 0.5, -0.5, 1.0, -1.0, 0.25, -0.25, 0.75];
3522
        let mut avx512_result = vec![0.0; 8];
3523
        let mut scalar_result = vec![0.0; 8];
3524
3525
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3526
        unsafe {
3527
            Avx512Backend::tan(&a, &mut avx512_result);
3528
            ScalarBackend::tan(&a, &mut scalar_result);
3529
        }
3530
3531
        for i in 0..8 {
3532
            assert!(
3533
                (avx512_result[i] - scalar_result[i]).abs() < 1e-4,
3534
                "tan mismatch at {}: avx512={}, scalar={}",
3535
                i,
3536
                avx512_result[i],
3537
                scalar_result[i]
3538
            );
3539
        }
3540
    }
3541
3542
    #[test]
3543
    fn test_avx512_tanh_basic() {
3544
        if !is_x86_feature_detected!("avx512f") {
3545
            return;
3546
        }
3547
3548
        let a = vec![0.0, 1.0, -1.0, 2.0, -2.0, 0.5, -0.5, 3.0];
3549
        let mut avx512_result = vec![0.0; 8];
3550
        let mut scalar_result = vec![0.0; 8];
3551
3552
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3553
        unsafe {
3554
            Avx512Backend::tanh(&a, &mut avx512_result);
3555
            ScalarBackend::tanh(&a, &mut scalar_result);
3556
        }
3557
3558
        for i in 0..8 {
3559
            assert!(
3560
                (avx512_result[i] - scalar_result[i]).abs() < 1e-4,
3561
                "tanh mismatch at {}: avx512={}, scalar={}",
3562
                i,
3563
                avx512_result[i],
3564
                scalar_result[i]
3565
            );
3566
        }
3567
    }
3568
3569
    // ===== Edge Case and Remainder Tests =====
3570
3571
    #[test]
3572
    fn test_avx512_mul_large_with_remainder() {
3573
        if !is_x86_feature_detected!("avx512f") {
3574
            return;
3575
        }
3576
3577
        // Test with size that's not multiple of 16 (AVX-512 processes 16 f32s)
3578
        let size = 123; // Not divisible by 16
3579
        let a: Vec<f32> = (0..size).map(|i| (i as f32) * 0.5).collect();
3580
        let b: Vec<f32> = (0..size).map(|i| (i as f32) * 0.3).collect();
3581
        let mut avx512_result = vec![0.0; size];
3582
        let mut scalar_result = vec![0.0; size];
3583
3584
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3585
        unsafe {
3586
            Avx512Backend::mul(&a, &b, &mut avx512_result);
3587
            ScalarBackend::mul(&a, &b, &mut scalar_result);
3588
        }
3589
3590
        for i in 0..size {
3591
            assert!(
3592
                (avx512_result[i] - scalar_result[i]).abs() < 1e-5,
3593
                "mul remainder mismatch at {}: avx512={}, scalar={}",
3594
                i,
3595
                avx512_result[i],
3596
                scalar_result[i]
3597
            );
3598
        }
3599
    }
3600
3601
    #[test]
3602
    fn test_avx512_sqrt_large_with_remainder() {
3603
        if !is_x86_feature_detected!("avx512f") {
3604
            return;
3605
        }
3606
3607
        let size = 97;
3608
        let a: Vec<f32> = (1..=size).map(|i| (i as f32) * (i as f32)).collect();
3609
        let mut avx512_result = vec![0.0; size];
3610
        let mut scalar_result = vec![0.0; size];
3611
3612
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3613
        unsafe {
3614
            Avx512Backend::sqrt(&a, &mut avx512_result);
3615
            ScalarBackend::sqrt(&a, &mut scalar_result);
3616
        }
3617
3618
        for i in 0..size {
3619
            assert!(
3620
                (avx512_result[i] - scalar_result[i]).abs() < 1e-5,
3621
                "sqrt remainder mismatch at {}: avx512={}, scalar={}",
3622
                i,
3623
                avx512_result[i],
3624
                scalar_result[i]
3625
            );
3626
        }
3627
    }
3628
3629
    #[test]
3630
    fn test_avx512_exp_large_with_remainder() {
3631
        if !is_x86_feature_detected!("avx512f") {
3632
            return;
3633
        }
3634
3635
        let size = 85;
3636
        let a: Vec<f32> = (0..size).map(|i| ((i as f32) - 40.0) * 0.1).collect();
3637
        let mut avx512_result = vec![0.0; size];
3638
        let mut scalar_result = vec![0.0; size];
3639
3640
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3641
        unsafe {
3642
            Avx512Backend::exp(&a, &mut avx512_result);
3643
            ScalarBackend::exp(&a, &mut scalar_result);
3644
        }
3645
3646
        for i in 0..size {
3647
            let rel_error = if scalar_result[i] != 0.0 {
3648
                (avx512_result[i] - scalar_result[i]).abs() / scalar_result[i]
3649
            } else {
3650
                (avx512_result[i] - scalar_result[i]).abs()
3651
            };
3652
            assert!(
3653
                rel_error < 1e-5,
3654
                "exp remainder mismatch at {}: avx512={}, scalar={}, rel_error={}",
3655
                i,
3656
                avx512_result[i],
3657
                scalar_result[i],
3658
                rel_error
3659
            );
3660
        }
3661
    }
3662
3663
    #[test]
3664
    fn test_avx512_relu_large_with_remainder() {
3665
        if !is_x86_feature_detected!("avx512f") {
3666
            return;
3667
        }
3668
3669
        let size = 111;
3670
        let a: Vec<f32> = (0..size).map(|i| ((i as f32) - 50.0) * 0.5).collect();
3671
        let mut avx512_result = vec![0.0; size];
3672
        let mut scalar_result = vec![0.0; size];
3673
3674
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3675
        unsafe {
3676
            Avx512Backend::relu(&a, &mut avx512_result);
3677
            ScalarBackend::relu(&a, &mut scalar_result);
3678
        }
3679
3680
        for i in 0..size {
3681
            assert_eq!(
3682
                avx512_result[i], scalar_result[i],
3683
                "relu remainder mismatch at {}: avx512={}, scalar={}",
3684
                i, avx512_result[i], scalar_result[i]
3685
            );
3686
        }
3687
    }
3688
3689
    #[test]
3690
    fn test_avx512_sigmoid_large_with_remainder() {
3691
        if !is_x86_feature_detected!("avx512f") {
3692
            return;
3693
        }
3694
3695
        let size = 99;
3696
        let a: Vec<f32> = (0..size).map(|i| ((i as f32) - 50.0) * 0.2).collect();
3697
        let mut avx512_result = vec![0.0; size];
3698
        let mut scalar_result = vec![0.0; size];
3699
3700
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3701
        unsafe {
3702
            Avx512Backend::sigmoid(&a, &mut avx512_result);
3703
            ScalarBackend::sigmoid(&a, &mut scalar_result);
3704
        }
3705
3706
        for i in 0..size {
3707
            assert!(
3708
                (avx512_result[i] - scalar_result[i]).abs() < 1e-4,
3709
                "sigmoid remainder mismatch at {}: avx512={}, scalar={}",
3710
                i,
3711
                avx512_result[i],
3712
                scalar_result[i]
3713
            );
3714
        }
3715
    }
3716
3717
    #[test]
3718
    fn test_avx512_ceil_floor_round_remainder() {
3719
        if !is_x86_feature_detected!("avx512f") {
3720
            return;
3721
        }
3722
3723
        let size = 77;
3724
        let a: Vec<f32> = (0..size).map(|i| ((i as f32) / 10.0) - 3.0).collect();
3725
3726
        let mut ceil_result = vec![0.0; size];
3727
        let mut floor_result = vec![0.0; size];
3728
        let mut round_result = vec![0.0; size];
3729
3730
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3731
        unsafe {
3732
            Avx512Backend::ceil(&a, &mut ceil_result);
3733
            Avx512Backend::floor(&a, &mut floor_result);
3734
            Avx512Backend::round(&a, &mut round_result);
3735
        }
3736
3737
        for i in 0..size {
3738
            let expected_ceil = a[i].ceil();
3739
            let expected_floor = a[i].floor();
3740
3741
            assert!(
3742
                (ceil_result[i] - expected_ceil).abs() < 1e-5,
3743
                "ceil remainder mismatch at {}: {} vs {}",
3744
                i,
3745
                ceil_result[i],
3746
                expected_ceil
3747
            );
3748
            assert!(
3749
                (floor_result[i] - expected_floor).abs() < 1e-5,
3750
                "floor remainder mismatch at {}: {} vs {}",
3751
                i,
3752
                floor_result[i],
3753
                expected_floor
3754
            );
3755
        }
3756
    }
3757
3758
    #[test]
3759
    fn test_avx512_trig_large_with_remainder() {
3760
        if !is_x86_feature_detected!("avx512f") {
3761
            return;
3762
        }
3763
3764
        let size = 93;
3765
        let a: Vec<f32> = (0..size).map(|i| ((i as f32) / 30.0) - 1.5).collect();
3766
3767
        let mut sin_avx512 = vec![0.0; size];
3768
        let mut sin_scalar = vec![0.0; size];
3769
        let mut cos_avx512 = vec![0.0; size];
3770
        let mut cos_scalar = vec![0.0; size];
3771
3772
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3773
        unsafe {
3774
            Avx512Backend::sin(&a, &mut sin_avx512);
3775
            ScalarBackend::sin(&a, &mut sin_scalar);
3776
            Avx512Backend::cos(&a, &mut cos_avx512);
3777
            ScalarBackend::cos(&a, &mut cos_scalar);
3778
        }
3779
3780
        for i in 0..size {
3781
            assert!(
3782
                (sin_avx512[i] - sin_scalar[i]).abs() < 1e-4,
3783
                "sin remainder mismatch at {}: avx512={}, scalar={}",
3784
                i,
3785
                sin_avx512[i],
3786
                sin_scalar[i]
3787
            );
3788
            assert!(
3789
                (cos_avx512[i] - cos_scalar[i]).abs() < 1e-4,
3790
                "cos remainder mismatch at {}: avx512={}, scalar={}",
3791
                i,
3792
                cos_avx512[i],
3793
                cos_scalar[i]
3794
            );
3795
        }
3796
    }
3797
3798
    // ===== AVX512 SIMD Path Tests (32+ elements) =====
3799
    // These tests exercise the SIMD loops which require >= 16 elements
3800
3801
    #[test]
3802
    fn test_avx512_norm_l1_simd_path() {
3803
        if !is_x86_feature_detected!("avx512f") {
3804
            return;
3805
        }
3806
3807
        // 64 elements to ensure AVX512 loop runs 4 times (64 / 16 = 4)
3808
        let a: Vec<f32> = (0..64)
3809
            .map(|i| if i % 2 == 0 { i as f32 } else { -(i as f32) })
3810
            .collect();
3811
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3812
        let avx512_result = unsafe { Avx512Backend::norm_l1(&a) };
3813
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3814
        let scalar_result = unsafe { ScalarBackend::norm_l1(&a) };
3815
3816
        assert!(
3817
            (avx512_result - scalar_result).abs() < 1e-3,
3818
            "norm_l1 SIMD mismatch: avx512={}, scalar={}",
3819
            avx512_result,
3820
            scalar_result
3821
        );
3822
    }
3823
3824
    #[test]
3825
    fn test_avx512_norm_linf_simd_path() {
3826
        if !is_x86_feature_detected!("avx512f") {
3827
            return;
3828
        }
3829
3830
        // 64 elements with max absolute value at various positions
3831
        let mut a: Vec<f32> = (0..64).map(|i| i as f32).collect();
3832
        a[47] = -200.0; // Max absolute at position 47
3833
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3834
        let avx512_result = unsafe { Avx512Backend::norm_linf(&a) };
3835
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3836
        let scalar_result = unsafe { ScalarBackend::norm_linf(&a) };
3837
3838
        assert!(
3839
            (avx512_result - scalar_result).abs() < 1e-5,
3840
            "norm_linf SIMD mismatch: avx512={}, scalar={}",
3841
            avx512_result,
3842
            scalar_result
3843
        );
3844
    }
3845
3846
    #[test]
3847
    fn test_avx512_scale_simd_path() {
3848
        if !is_x86_feature_detected!("avx512f") {
3849
            return;
3850
        }
3851
3852
        // 64 elements to exercise SIMD loop
3853
        let a: Vec<f32> = (0..64).map(|i| i as f32).collect();
3854
        let scalar = 3.5;
3855
        let mut avx512_result = vec![0.0; 64];
3856
        let mut scalar_result = vec![0.0; 64];
3857
3858
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3859
        unsafe {
3860
            Avx512Backend::scale(&a, scalar, &mut avx512_result);
3861
            ScalarBackend::scale(&a, scalar, &mut scalar_result);
3862
        }
3863
3864
        for i in 0..64 {
3865
            assert!(
3866
                (avx512_result[i] - scalar_result[i]).abs() < 1e-5,
3867
                "scale SIMD mismatch at {}: avx512={}, scalar={}",
3868
                i,
3869
                avx512_result[i],
3870
                scalar_result[i]
3871
            );
3872
        }
3873
    }
3874
3875
    #[test]
3876
    fn test_avx512_abs_simd_path() {
3877
        if !is_x86_feature_detected!("avx512f") {
3878
            return;
3879
        }
3880
3881
        // 64 elements with mixed positive/negative
3882
        let a: Vec<f32> = (0..64)
3883
            .map(|i| if i % 2 == 0 { i as f32 } else { -(i as f32) })
3884
            .collect();
3885
        let mut avx512_result = vec![0.0; 64];
3886
        let mut scalar_result = vec![0.0; 64];
3887
3888
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3889
        unsafe {
3890
            Avx512Backend::abs(&a, &mut avx512_result);
3891
            ScalarBackend::abs(&a, &mut scalar_result);
3892
        }
3893
3894
        for i in 0..64 {
3895
            assert!(
3896
                (avx512_result[i] - scalar_result[i]).abs() < 1e-5,
3897
                "abs SIMD mismatch at {}: avx512={}, scalar={}",
3898
                i,
3899
                avx512_result[i],
3900
                scalar_result[i]
3901
            );
3902
        }
3903
    }
3904
3905
    #[test]
3906
    fn test_avx512_clamp_simd_path() {
3907
        if !is_x86_feature_detected!("avx512f") {
3908
            return;
3909
        }
3910
3911
        // 64 elements spanning the clamp range
3912
        let a: Vec<f32> = (0..64).map(|i| (i as f32) - 20.0).collect();
3913
        let min_val = 0.0;
3914
        let max_val = 30.0;
3915
        let mut avx512_result = vec![0.0; 64];
3916
        let mut scalar_result = vec![0.0; 64];
3917
3918
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3919
        unsafe {
3920
            Avx512Backend::clamp(&a, min_val, max_val, &mut avx512_result);
3921
            ScalarBackend::clamp(&a, min_val, max_val, &mut scalar_result);
3922
        }
3923
3924
        for i in 0..64 {
3925
            assert!(
3926
                (avx512_result[i] - scalar_result[i]).abs() < 1e-5,
3927
                "clamp SIMD mismatch at {}: avx512={}, scalar={}",
3928
                i,
3929
                avx512_result[i],
3930
                scalar_result[i]
3931
            );
3932
        }
3933
    }
3934
3935
    #[test]
3936
    fn test_avx512_lerp_simd_path() {
3937
        if !is_x86_feature_detected!("avx512f") {
3938
            return;
3939
        }
3940
3941
        // 64 elements
3942
        let a: Vec<f32> = (0..64).map(|i| i as f32).collect();
3943
        let b: Vec<f32> = (0..64).map(|i| (i as f32) * 2.0 + 10.0).collect();
3944
        let t = 0.3;
3945
        let mut avx512_result = vec![0.0; 64];
3946
        let mut scalar_result = vec![0.0; 64];
3947
3948
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3949
        unsafe {
3950
            Avx512Backend::lerp(&a, &b, t, &mut avx512_result);
3951
            ScalarBackend::lerp(&a, &b, t, &mut scalar_result);
3952
        }
3953
3954
        for i in 0..64 {
3955
            assert!(
3956
                (avx512_result[i] - scalar_result[i]).abs() < 1e-5,
3957
                "lerp SIMD mismatch at {}: avx512={}, scalar={}",
3958
                i,
3959
                avx512_result[i],
3960
                scalar_result[i]
3961
            );
3962
        }
3963
    }
3964
3965
    #[test]
3966
    fn test_avx512_fma_simd_path() {
3967
        if !is_x86_feature_detected!("avx512f") {
3968
            return;
3969
        }
3970
3971
        // 64 elements: a*b + c
3972
        let a: Vec<f32> = (0..64).map(|i| i as f32).collect();
3973
        let b: Vec<f32> = (0..64).map(|i| (i as f32) * 0.5 + 1.0).collect();
3974
        let c: Vec<f32> = (0..64).map(|i| (i as f32) * 0.25).collect();
3975
        let mut avx512_result = vec![0.0; 64];
3976
        let mut scalar_result = vec![0.0; 64];
3977
3978
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
3979
        unsafe {
3980
            Avx512Backend::fma(&a, &b, &c, &mut avx512_result);
3981
            ScalarBackend::fma(&a, &b, &c, &mut scalar_result);
3982
        }
3983
3984
        for i in 0..64 {
3985
            assert!(
3986
                (avx512_result[i] - scalar_result[i]).abs() < 1e-3,
3987
                "fma SIMD mismatch at {}: avx512={}, scalar={}",
3988
                i,
3989
                avx512_result[i],
3990
                scalar_result[i]
3991
            );
3992
        }
3993
    }
3994
3995
    #[test]
3996
    fn test_avx512_argmax_empty() {
3997
        if !is_x86_feature_detected!("avx512f") {
3998
            return;
3999
        }
4000
4001
        let a: Vec<f32> = vec![];
4002
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
4003
        let result = unsafe { Avx512Backend::argmax(&a) };
4004
        assert_eq!(result, 0);
4005
    }
4006
4007
    #[test]
4008
    fn test_avx512_argmin_empty() {
4009
        if !is_x86_feature_detected!("avx512f") {
4010
            return;
4011
        }
4012
4013
        let a: Vec<f32> = vec![];
4014
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
4015
        let result = unsafe { Avx512Backend::argmin(&a) };
4016
        assert_eq!(result, 0);
4017
    }
4018
4019
    #[test]
4020
    fn test_avx512_argmax_simd_path() {
4021
        if !is_x86_feature_detected!("avx512f") {
4022
            return;
4023
        }
4024
4025
        // 64 elements with max at position 35
4026
        let mut a: Vec<f32> = (0..64).map(|i| i as f32).collect();
4027
        a[35] = 1000.0;
4028
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
4029
        let avx512_result = unsafe { Avx512Backend::argmax(&a) };
4030
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
4031
        let scalar_result = unsafe { ScalarBackend::argmax(&a) };
4032
4033
        assert_eq!(
4034
            avx512_result, scalar_result,
4035
            "argmax SIMD mismatch: avx512={}, scalar={}",
4036
            avx512_result, scalar_result
4037
        );
4038
    }
4039
4040
    #[test]
4041
    fn test_avx512_argmin_simd_path() {
4042
        if !is_x86_feature_detected!("avx512f") {
4043
            return;
4044
        }
4045
4046
        // 64 elements with min at position 42
4047
        let mut a: Vec<f32> = (0..64).map(|i| i as f32).collect();
4048
        a[42] = -500.0;
4049
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
4050
        let avx512_result = unsafe { Avx512Backend::argmin(&a) };
4051
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
4052
        let scalar_result = unsafe { ScalarBackend::argmin(&a) };
4053
4054
        assert_eq!(
4055
            avx512_result, scalar_result,
4056
            "argmin SIMD mismatch: avx512={}, scalar={}",
4057
            avx512_result, scalar_result
4058
        );
4059
    }
4060
4061
    #[test]
4062
    fn test_avx512_scale_remainder() {
4063
        if !is_x86_feature_detected!("avx512f") {
4064
            return;
4065
        }
4066
4067
        // 47 elements (47 % 16 = 15 remainder)
4068
        let a: Vec<f32> = (0..47).map(|i| i as f32).collect();
4069
        let scalar = 2.0;
4070
        let mut avx512_result = vec![0.0; 47];
4071
        let mut scalar_result = vec![0.0; 47];
4072
4073
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
4074
        unsafe {
4075
            Avx512Backend::scale(&a, scalar, &mut avx512_result);
4076
            ScalarBackend::scale(&a, scalar, &mut scalar_result);
4077
        }
4078
4079
        for i in 0..47 {
4080
            assert!(
4081
                (avx512_result[i] - scalar_result[i]).abs() < 1e-5,
4082
                "scale remainder mismatch at {}: avx512={}, scalar={}",
4083
                i,
4084
                avx512_result[i],
4085
                scalar_result[i]
4086
            );
4087
        }
4088
    }
4089
4090
    #[test]
4091
    fn test_avx512_abs_remainder() {
4092
        if !is_x86_feature_detected!("avx512f") {
4093
            return;
4094
        }
4095
4096
        // 35 elements (35 % 16 = 3 remainder)
4097
        let a: Vec<f32> = (0..35)
4098
            .map(|i| if i % 2 == 0 { i as f32 } else { -(i as f32) })
4099
            .collect();
4100
        let mut avx512_result = vec![0.0; 35];
4101
        let mut scalar_result = vec![0.0; 35];
4102
4103
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
4104
        unsafe {
4105
            Avx512Backend::abs(&a, &mut avx512_result);
4106
            ScalarBackend::abs(&a, &mut scalar_result);
4107
        }
4108
4109
        for i in 0..35 {
4110
            assert!(
4111
                (avx512_result[i] - scalar_result[i]).abs() < 1e-5,
4112
                "abs remainder mismatch at {}: avx512={}, scalar={}",
4113
                i,
4114
                avx512_result[i],
4115
                scalar_result[i]
4116
            );
4117
        }
4118
    }
4119
4120
    // cuda-tile-behavior.md: Falsification test #81 - All backends produce equivalent results
4121
    #[test]
4122
    fn test_avx512_dot_2x_unroll_various_sizes() {
4123
        if !is_x86_feature_detected!("avx512f") {
4124
            eprintln!("Skipping AVX-512 test: CPU does not support AVX-512F");
4125
            return;
4126
        }
4127
4128
        use super::super::scalar::ScalarBackend;
4129
4130
        // Test various sizes to exercise all code paths:
4131
        // - Empty (edge case)
4132
        // - 1 element (scalar only)
4133
        // - 16 elements (one AVX-512 chunk)
4134
        // - 31 elements (one chunk + remainder)
4135
        // - 32 elements (exactly two chunks for unrolled path)
4136
        // - 33 elements (two chunks + remainder)
4137
        // - 64 elements (multiple unrolled iterations)
4138
        // - 100 elements (realistic workload)
4139
        // - 1000 elements (larger workload)
4140
        let sizes = [0, 1, 16, 31, 32, 33, 64, 100, 1000];
4141
4142
        for &size in &sizes {
4143
            if size == 0 {
4144
                continue; // Empty slice handled separately
4145
            }
4146
4147
            let a: Vec<f32> = (0..size).map(|i| (i as f32) * 0.1).collect();
4148
            let b: Vec<f32> = (0..size).map(|i| ((size - i) as f32) * 0.1).collect();
4149
4150
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
4151
            let avx512_result = unsafe { Avx512Backend::dot(&a, &b) };
4152
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
4153
            let scalar_result = unsafe { ScalarBackend::dot(&a, &b) };
4154
4155
            // Use relative tolerance for large results (2x unrolling changes operation order)
4156
            let tolerance = (1e-5 * scalar_result.abs()).max(1e-4);
4157
            assert!(
4158
                (avx512_result - scalar_result).abs() < tolerance,
4159
                "dot mismatch at size {}: avx512={}, scalar={}, diff={}, tolerance={}",
4160
                size,
4161
                avx512_result,
4162
                scalar_result,
4163
                (avx512_result - scalar_result).abs(),
4164
                tolerance
4165
            );
4166
        }
4167
    }
4168
4169
    // cuda-tile-behavior.md: Falsification test #83 - SIMD remainder handling
4170
    #[test]
4171
    fn test_avx512_dot_remainder_handling() {
4172
        if !is_x86_feature_detected!("avx512f") {
4173
            eprintln!("Skipping AVX-512 test: CPU does not support AVX-512F");
4174
            return;
4175
        }
4176
4177
        use super::super::scalar::ScalarBackend;
4178
4179
        // Test sizes that exercise remainder paths: n % 32 != 0 and n % 16 != 0
4180
        for size in 1..50 {
4181
            let a: Vec<f32> = (0..size).map(|i| (i + 1) as f32).collect();
4182
            let b: Vec<f32> = (0..size).map(|i| (i + 1) as f32).collect();
4183
4184
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
4185
            let avx512_result = unsafe { Avx512Backend::dot(&a, &b) };
4186
            // SAFETY: CPU feature verified at runtime, slices bounds-checked
4187
            let scalar_result = unsafe { ScalarBackend::dot(&a, &b) };
4188
4189
            assert!(
4190
                (avx512_result - scalar_result).abs() < 1e-4,
4191
                "remainder handling failed at size {}: avx512={}, scalar={}",
4192
                size,
4193
                avx512_result,
4194
                scalar_result
4195
            );
4196
        }
4197
    }
4198
4199
    // =========================================================================
4200
    // AVX-512 SIMD PATH COVERAGE TESTS
4201
    // These tests use 32+ elements to guarantee the SIMD loop (16 elements/iter)
4202
    // is executed, not just the scalar fallback.
4203
    // =========================================================================
4204
4205
    #[test]
4206
    fn test_avx512_gelu_simd_path() {
4207
        if !is_x86_feature_detected!("avx512f") {
4208
            return;
4209
        }
4210
        use super::super::scalar::ScalarBackend;
4211
4212
        // 32 elements = 2 full SIMD iterations
4213
        let a: Vec<f32> = (0..32).map(|i| (i as f32 - 16.0) * 0.25).collect();
4214
        let mut avx512_result = vec![0.0; 32];
4215
        let mut scalar_result = vec![0.0; 32];
4216
4217
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
4218
        unsafe {
4219
            Avx512Backend::gelu(&a, &mut avx512_result);
4220
            ScalarBackend::gelu(&a, &mut scalar_result);
4221
        }
4222
4223
        for i in 0..32 {
4224
            assert!(
4225
                (avx512_result[i] - scalar_result[i]).abs() < 1e-4,
4226
                "gelu SIMD mismatch at {}: avx512={}, scalar={}",
4227
                i, avx512_result[i], scalar_result[i]
4228
            );
4229
        }
4230
    }
4231
4232
    #[test]
4233
    fn test_avx512_swish_simd_path() {
4234
        if !is_x86_feature_detected!("avx512f") {
4235
            return;
4236
        }
4237
        use super::super::scalar::ScalarBackend;
4238
4239
        let a: Vec<f32> = (0..32).map(|i| (i as f32 - 16.0) * 0.5).collect();
4240
        let mut avx512_result = vec![0.0; 32];
4241
        let mut scalar_result = vec![0.0; 32];
4242
4243
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
4244
        unsafe {
4245
            Avx512Backend::swish(&a, &mut avx512_result);
4246
            ScalarBackend::swish(&a, &mut scalar_result);
4247
        }
4248
4249
        for i in 0..32 {
4250
            assert!(
4251
                (avx512_result[i] - scalar_result[i]).abs() < 1e-4,
4252
                "swish SIMD mismatch at {}: avx512={}, scalar={}",
4253
                i, avx512_result[i], scalar_result[i]
4254
            );
4255
        }
4256
    }
4257
4258
    #[test]
4259
    fn test_avx512_tanh_simd_path() {
4260
        if !is_x86_feature_detected!("avx512f") {
4261
            return;
4262
        }
4263
        use super::super::scalar::ScalarBackend;
4264
4265
        let a: Vec<f32> = (0..32).map(|i| (i as f32 - 16.0) * 0.2).collect();
4266
        let mut avx512_result = vec![0.0; 32];
4267
        let mut scalar_result = vec![0.0; 32];
4268
4269
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
4270
        unsafe {
4271
            Avx512Backend::tanh(&a, &mut avx512_result);
4272
            ScalarBackend::tanh(&a, &mut scalar_result);
4273
        }
4274
4275
        for i in 0..32 {
4276
            assert!(
4277
                (avx512_result[i] - scalar_result[i]).abs() < 1e-4,
4278
                "tanh SIMD mismatch at {}: avx512={}, scalar={}",
4279
                i, avx512_result[i], scalar_result[i]
4280
            );
4281
        }
4282
    }
4283
4284
    #[test]
4285
    fn test_avx512_log2_simd_path() {
4286
        if !is_x86_feature_detected!("avx512f") {
4287
            return;
4288
        }
4289
        use super::super::scalar::ScalarBackend;
4290
4291
        // Positive values only for log2
4292
        let a: Vec<f32> = (1..=32).map(|i| i as f32).collect();
4293
        let mut avx512_result = vec![0.0; 32];
4294
        let mut scalar_result = vec![0.0; 32];
4295
4296
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
4297
        unsafe {
4298
            Avx512Backend::log2(&a, &mut avx512_result);
4299
            ScalarBackend::log2(&a, &mut scalar_result);
4300
        }
4301
4302
        for i in 0..32 {
4303
            assert!(
4304
                (avx512_result[i] - scalar_result[i]).abs() < 1e-4,
4305
                "log2 SIMD mismatch at {}: avx512={}, scalar={}",
4306
                i, avx512_result[i], scalar_result[i]
4307
            );
4308
        }
4309
    }
4310
4311
    #[test]
4312
    fn test_avx512_log10_simd_path() {
4313
        if !is_x86_feature_detected!("avx512f") {
4314
            return;
4315
        }
4316
        use super::super::scalar::ScalarBackend;
4317
4318
        // Positive values only for log10
4319
        let a: Vec<f32> = (1..=32).map(|i| i as f32).collect();
4320
        let mut avx512_result = vec![0.0; 32];
4321
        let mut scalar_result = vec![0.0; 32];
4322
4323
        // SAFETY: CPU feature verified at runtime, slices bounds-checked
4324
        unsafe {
4325
            Avx512Backend::log10(&a, &mut avx512_result);
4326
            ScalarBackend::log10(&a, &mut scalar_result);
4327
        }
4328
4329
        for i in 0..32 {
4330
            assert!(
4331
                (avx512_result[i] - scalar_result[i]).abs() < 1e-4,
4332
                "log10 SIMD mismatch at {}: avx512={}, scalar={}",
4333
                i, avx512_result[i], scalar_result[i]
4334
            );
4335
        }
4336
    }
4337
}