/home/noah/src/trueno/src/backends/avx512/ops/reductions.rs
Line | Count | Source |
1 | | //! AVX-512 reduction operations (dot, sum, max, min, argmax, argmin). |
2 | | |
3 | | #[cfg(target_arch = "x86_64")] |
4 | | use std::arch::x86_64::*; |
5 | | |
6 | | /// AVX-512 dot product. |
7 | | #[inline] |
8 | | #[target_feature(enable = "avx512f")] |
9 | 0 | pub unsafe fn dot(a: &[f32], b: &[f32]) -> f32 { |
10 | 0 | let len = a.len(); |
11 | 0 | let mut i = 0; |
12 | 0 | let mut acc = _mm512_setzero_ps(); |
13 | | |
14 | 0 | while i + 16 <= len { |
15 | 0 | let va = _mm512_loadu_ps(a.as_ptr().add(i)); |
16 | 0 | let vb = _mm512_loadu_ps(b.as_ptr().add(i)); |
17 | 0 | acc = _mm512_fmadd_ps(va, vb, acc); |
18 | 0 | i += 16; |
19 | 0 | } |
20 | | |
21 | 0 | let mut result = _mm512_reduce_add_ps(acc); |
22 | 0 | result += a[i..].iter().zip(&b[i..]).map(|(x, y)| x * y).sum::<f32>(); |
23 | 0 | result |
24 | 0 | } |
25 | | |
26 | | /// AVX-512 vector sum. |
27 | | #[inline] |
28 | | #[target_feature(enable = "avx512f")] |
29 | 0 | pub unsafe fn sum(a: &[f32]) -> f32 { |
30 | 0 | let len = a.len(); |
31 | 0 | let mut i = 0; |
32 | 0 | let mut acc = _mm512_setzero_ps(); |
33 | | |
34 | 0 | while i + 16 <= len { |
35 | 0 | acc = _mm512_add_ps(acc, _mm512_loadu_ps(a.as_ptr().add(i))); |
36 | 0 | i += 16; |
37 | 0 | } |
38 | | |
39 | 0 | let mut result = _mm512_reduce_add_ps(acc); |
40 | 0 | result += a[i..].iter().sum::<f32>(); |
41 | 0 | result |
42 | 0 | } |
43 | | |
44 | | /// AVX-512 vector max. |
45 | | #[inline] |
46 | | #[target_feature(enable = "avx512f")] |
47 | 0 | pub unsafe fn max(a: &[f32]) -> f32 { |
48 | 0 | let len = a.len(); |
49 | 0 | let mut i = 0; |
50 | 0 | let mut vmax = _mm512_set1_ps(a[0]); |
51 | | |
52 | 0 | while i + 16 <= len { |
53 | 0 | vmax = _mm512_max_ps(vmax, _mm512_loadu_ps(a.as_ptr().add(i))); |
54 | 0 | i += 16; |
55 | 0 | } |
56 | | |
57 | 0 | let mut result = _mm512_reduce_max_ps(vmax); |
58 | 0 | for &val in &a[i..] { if val > result { result = val; } } |
59 | 0 | result |
60 | 0 | } |
61 | | |
62 | | /// AVX-512 vector min. |
63 | | #[inline] |
64 | | #[target_feature(enable = "avx512f")] |
65 | 0 | pub unsafe fn min(a: &[f32]) -> f32 { |
66 | 0 | let len = a.len(); |
67 | 0 | let mut i = 0; |
68 | 0 | let mut vmin = _mm512_set1_ps(a[0]); |
69 | | |
70 | 0 | while i + 16 <= len { |
71 | 0 | vmin = _mm512_min_ps(vmin, _mm512_loadu_ps(a.as_ptr().add(i))); |
72 | 0 | i += 16; |
73 | 0 | } |
74 | | |
75 | 0 | let mut result = _mm512_reduce_min_ps(vmin); |
76 | 0 | for &val in &a[i..] { if val < result { result = val; } } |
77 | 0 | result |
78 | 0 | } |
79 | | |
80 | | /// AVX-512 argmax. |
81 | | #[inline] |
82 | | #[target_feature(enable = "avx512f")] |
83 | 0 | pub unsafe fn argmax(a: &[f32]) -> usize { |
84 | 0 | let mut max_idx: usize = 0; |
85 | 0 | let mut max_val = a[0]; |
86 | 0 | for (i, &val) in a.iter().enumerate() { |
87 | 0 | if val > max_val { max_val = val; max_idx = i; } |
88 | | } |
89 | 0 | max_idx |
90 | 0 | } |
91 | | |
92 | | /// AVX-512 argmin. |
93 | | #[inline] |
94 | | #[target_feature(enable = "avx512f")] |
95 | 0 | pub unsafe fn argmin(a: &[f32]) -> usize { |
96 | 0 | let mut min_idx: usize = 0; |
97 | 0 | let mut min_val = a[0]; |
98 | 0 | for (i, &val) in a.iter().enumerate() { |
99 | 0 | if val < min_val { min_val = val; min_idx = i; } |
100 | | } |
101 | 0 | min_idx |
102 | 0 | } |
103 | | |
104 | | /// Kahan sum (scalar implementation). |
105 | | #[inline] |
106 | 0 | pub unsafe fn sum_kahan(a: &[f32]) -> f32 { |
107 | 0 | let mut sum = 0.0; |
108 | 0 | let mut c = 0.0; |
109 | 0 | for &x in a { |
110 | 0 | let y = x - c; |
111 | 0 | let t = sum + y; |
112 | 0 | c = (t - sum) - y; |
113 | 0 | sum = t; |
114 | 0 | } |
115 | 0 | sum |
116 | 0 | } |