/home/noah/src/trueno/src/backends/avx2/ops/reductions.rs
Line | Count | Source |
1 | | //! AVX2 reduction operations (dot, sum, max, min, argmax, argmin). |
2 | | |
3 | | #[cfg(target_arch = "x86_64")] |
4 | | use std::arch::x86_64::*; |
5 | | |
6 | | use crate::backends::VectorBackend; |
7 | | |
8 | | /// AVX2 dot product with 4-accumulator unrolling for ILP. |
9 | | #[inline] |
10 | | #[target_feature(enable = "avx2,fma")] |
11 | 229k | pub unsafe fn dot(a: &[f32], b: &[f32]) -> f32 { |
12 | 229k | let len = a.len(); |
13 | 229k | let mut i = 0; |
14 | | |
15 | 229k | let mut acc0 = _mm256_setzero_ps(); |
16 | 229k | let mut acc1 = _mm256_setzero_ps(); |
17 | 229k | let mut acc2 = _mm256_setzero_ps(); |
18 | 229k | let mut acc3 = _mm256_setzero_ps(); |
19 | | |
20 | 269k | while i + 32 <= len { |
21 | 39.9k | let va0 = _mm256_loadu_ps(a.as_ptr().add(i)); |
22 | 39.9k | let vb0 = _mm256_loadu_ps(b.as_ptr().add(i)); |
23 | 39.9k | let va1 = _mm256_loadu_ps(a.as_ptr().add(i + 8)); |
24 | 39.9k | let vb1 = _mm256_loadu_ps(b.as_ptr().add(i + 8)); |
25 | 39.9k | let va2 = _mm256_loadu_ps(a.as_ptr().add(i + 16)); |
26 | 39.9k | let vb2 = _mm256_loadu_ps(b.as_ptr().add(i + 16)); |
27 | 39.9k | let va3 = _mm256_loadu_ps(a.as_ptr().add(i + 24)); |
28 | 39.9k | let vb3 = _mm256_loadu_ps(b.as_ptr().add(i + 24)); |
29 | 39.9k | |
30 | 39.9k | acc0 = _mm256_fmadd_ps(va0, vb0, acc0); |
31 | 39.9k | acc1 = _mm256_fmadd_ps(va1, vb1, acc1); |
32 | 39.9k | acc2 = _mm256_fmadd_ps(va2, vb2, acc2); |
33 | 39.9k | acc3 = _mm256_fmadd_ps(va3, vb3, acc3); |
34 | 39.9k | |
35 | 39.9k | i += 32; |
36 | 39.9k | } |
37 | | |
38 | 680k | while i + 8 <= len { |
39 | 451k | let va = _mm256_loadu_ps(a.as_ptr().add(i)); |
40 | 451k | let vb = _mm256_loadu_ps(b.as_ptr().add(i)); |
41 | 451k | acc0 = _mm256_fmadd_ps(va, vb, acc0); |
42 | 451k | i += 8; |
43 | 451k | } |
44 | | |
45 | 229k | let acc01 = _mm256_add_ps(acc0, acc1); |
46 | 229k | let acc23 = _mm256_add_ps(acc2, acc3); |
47 | 229k | let acc = _mm256_add_ps(acc01, acc23); |
48 | | |
49 | 229k | let mut result = { |
50 | 229k | let sum_halves = _mm_add_ps(_mm256_castps256_ps128(acc), _mm256_extractf128_ps(acc, 1)); |
51 | 229k | let temp = _mm_add_ps(sum_halves, _mm_movehl_ps(sum_halves, sum_halves)); |
52 | 229k | let temp = _mm_add_ss(temp, _mm_shuffle_ps(temp, temp, 1)); |
53 | 229k | _mm_cvtss_f32(temp) |
54 | | }; |
55 | | |
56 | 229k | result += a[i..].iter().zip(&b[i..]).map(|(x, y)| x206 * y206 ).sum::<f32>(); |
57 | 229k | result |
58 | 229k | } |
59 | | |
60 | | /// AVX2 vector sum. |
61 | | #[inline] |
62 | | #[target_feature(enable = "avx2")] |
63 | 118 | pub unsafe fn sum(a: &[f32]) -> f32 { |
64 | 118 | let len = a.len(); |
65 | 118 | let mut i = 0; |
66 | 118 | let mut acc = _mm256_setzero_ps(); |
67 | | |
68 | 14.2k | while i + 8 <= len { |
69 | 14.0k | let va = _mm256_loadu_ps(a.as_ptr().add(i)); |
70 | 14.0k | acc = _mm256_add_ps(acc, va); |
71 | 14.0k | i += 8; |
72 | 14.0k | } |
73 | | |
74 | 118 | let mut result = { |
75 | 118 | let sum_halves = _mm_add_ps(_mm256_castps256_ps128(acc), _mm256_extractf128_ps(acc, 1)); |
76 | 118 | let temp = _mm_add_ps(sum_halves, _mm_movehl_ps(sum_halves, sum_halves)); |
77 | 118 | let temp = _mm_add_ss(temp, _mm_shuffle_ps(temp, temp, 1)); |
78 | 118 | _mm_cvtss_f32(temp) |
79 | | }; |
80 | | |
81 | 118 | result += a[i..].iter().sum::<f32>(); |
82 | 118 | result |
83 | 118 | } |
84 | | |
85 | | /// AVX2 vector max. |
86 | | #[inline] |
87 | | #[target_feature(enable = "avx2")] |
88 | 10.4k | pub unsafe fn max(a: &[f32]) -> f32 { |
89 | 10.4k | let len = a.len(); |
90 | 10.4k | let mut i = 0; |
91 | 10.4k | let mut vmax = _mm256_set1_ps(a[0]); |
92 | | |
93 | 34.1k | while i + 8 <= len { |
94 | 23.7k | let va = _mm256_loadu_ps(a.as_ptr().add(i)); |
95 | 23.7k | vmax = _mm256_max_ps(vmax, va); |
96 | 23.7k | i += 8; |
97 | 23.7k | } |
98 | | |
99 | 10.4k | let mut result = { |
100 | 10.4k | let max_halves = _mm_max_ps(_mm256_castps256_ps128(vmax), _mm256_extractf128_ps(vmax, 1)); |
101 | 10.4k | let temp = _mm_max_ps(max_halves, _mm_movehl_ps(max_halves, max_halves)); |
102 | 10.4k | let temp = _mm_max_ss(temp, _mm_shuffle_ps(temp, temp, 1)); |
103 | 10.4k | _mm_cvtss_f32(temp) |
104 | | }; |
105 | | |
106 | 37.9k | for &val in &a[i..]10.4k { |
107 | 37.9k | if val > result { |
108 | 19.3k | result = val; |
109 | 19.3k | }18.6k |
110 | | } |
111 | 10.4k | result |
112 | 10.4k | } |
113 | | |
114 | | /// AVX2 vector min. |
115 | | #[inline] |
116 | | #[target_feature(enable = "avx2")] |
117 | 0 | pub unsafe fn min(a: &[f32]) -> f32 { |
118 | 0 | let len = a.len(); |
119 | 0 | let mut i = 0; |
120 | 0 | let mut vmin = _mm256_set1_ps(a[0]); |
121 | | |
122 | 0 | while i + 8 <= len { |
123 | 0 | let va = _mm256_loadu_ps(a.as_ptr().add(i)); |
124 | 0 | vmin = _mm256_min_ps(vmin, va); |
125 | 0 | i += 8; |
126 | 0 | } |
127 | | |
128 | 0 | let mut result = { |
129 | 0 | let min_halves = _mm_min_ps(_mm256_castps256_ps128(vmin), _mm256_extractf128_ps(vmin, 1)); |
130 | 0 | let temp = _mm_min_ps(min_halves, _mm_movehl_ps(min_halves, min_halves)); |
131 | 0 | let temp = _mm_min_ss(temp, _mm_shuffle_ps(temp, temp, 1)); |
132 | 0 | _mm_cvtss_f32(temp) |
133 | | }; |
134 | | |
135 | 0 | for &val in &a[i..] { |
136 | 0 | if val < result { |
137 | 0 | result = val; |
138 | 0 | } |
139 | | } |
140 | 0 | result |
141 | 0 | } |
142 | | |
143 | | /// AVX2 argmax. |
144 | | #[inline] |
145 | | #[target_feature(enable = "avx2")] |
146 | 0 | pub unsafe fn argmax(a: &[f32]) -> usize { |
147 | 0 | let len = a.len(); |
148 | 0 | let mut max_idx: usize = 0; |
149 | 0 | let mut max_val = a[0]; |
150 | 0 | let mut i = 0; |
151 | | |
152 | 0 | let mut vmax = _mm256_set1_ps(a[0]); |
153 | 0 | let mut vidx_max = _mm256_setzero_ps(); |
154 | 0 | let vidx_inc = _mm256_set1_ps(8.0); |
155 | 0 | let mut vcurrent_idx = _mm256_set_ps(7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0); |
156 | | |
157 | 0 | while i + 8 <= len { |
158 | 0 | let va = _mm256_loadu_ps(a.as_ptr().add(i)); |
159 | 0 | let mask = _mm256_cmp_ps(va, vmax, _CMP_GT_OQ); |
160 | 0 | vmax = _mm256_blendv_ps(vmax, va, mask); |
161 | 0 | vidx_max = _mm256_blendv_ps(vidx_max, vcurrent_idx, mask); |
162 | 0 | vcurrent_idx = _mm256_add_ps(vcurrent_idx, vidx_inc); |
163 | 0 | i += 8; |
164 | 0 | } |
165 | | |
166 | | // Extract max from vector |
167 | 0 | let mut vals = [0.0f32; 8]; |
168 | 0 | let mut idxs = [0.0f32; 8]; |
169 | 0 | _mm256_storeu_ps(vals.as_mut_ptr(), vmax); |
170 | 0 | _mm256_storeu_ps(idxs.as_mut_ptr(), vidx_max); |
171 | | |
172 | 0 | for j in 0..8 { |
173 | 0 | if vals[j] > max_val { |
174 | 0 | max_val = vals[j]; |
175 | 0 | max_idx = idxs[j] as usize; |
176 | 0 | } |
177 | | } |
178 | | |
179 | | // Check remaining elements |
180 | 0 | for (j, &val) in a[i..].iter().enumerate() { |
181 | 0 | if val > max_val { |
182 | 0 | max_val = val; |
183 | 0 | max_idx = i + j; |
184 | 0 | } |
185 | | } |
186 | | |
187 | 0 | max_idx |
188 | 0 | } |
189 | | |
190 | | /// AVX2 argmin. |
191 | | #[inline] |
192 | | #[target_feature(enable = "avx2")] |
193 | 0 | pub unsafe fn argmin(a: &[f32]) -> usize { |
194 | 0 | let len = a.len(); |
195 | 0 | let mut min_idx: usize = 0; |
196 | 0 | let mut min_val = a[0]; |
197 | 0 | let mut i = 0; |
198 | | |
199 | 0 | let mut vmin = _mm256_set1_ps(a[0]); |
200 | 0 | let mut vidx_min = _mm256_setzero_ps(); |
201 | 0 | let vidx_inc = _mm256_set1_ps(8.0); |
202 | 0 | let mut vcurrent_idx = _mm256_set_ps(7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0); |
203 | | |
204 | 0 | while i + 8 <= len { |
205 | 0 | let va = _mm256_loadu_ps(a.as_ptr().add(i)); |
206 | 0 | let mask = _mm256_cmp_ps(va, vmin, _CMP_LT_OQ); |
207 | 0 | vmin = _mm256_blendv_ps(vmin, va, mask); |
208 | 0 | vidx_min = _mm256_blendv_ps(vidx_min, vcurrent_idx, mask); |
209 | 0 | vcurrent_idx = _mm256_add_ps(vcurrent_idx, vidx_inc); |
210 | 0 | i += 8; |
211 | 0 | } |
212 | | |
213 | 0 | let mut vals = [0.0f32; 8]; |
214 | 0 | let mut idxs = [0.0f32; 8]; |
215 | 0 | _mm256_storeu_ps(vals.as_mut_ptr(), vmin); |
216 | 0 | _mm256_storeu_ps(idxs.as_mut_ptr(), vidx_min); |
217 | | |
218 | 0 | for j in 0..8 { |
219 | 0 | if vals[j] < min_val { |
220 | 0 | min_val = vals[j]; |
221 | 0 | min_idx = idxs[j] as usize; |
222 | 0 | } |
223 | | } |
224 | | |
225 | 0 | for (j, &val) in a[i..].iter().enumerate() { |
226 | 0 | if val < min_val { |
227 | 0 | min_val = val; |
228 | 0 | min_idx = i + j; |
229 | 0 | } |
230 | | } |
231 | | |
232 | 0 | min_idx |
233 | 0 | } |
234 | | |
235 | | /// Kahan sum for numerical stability (delegates to scalar). |
236 | | #[inline] |
237 | 0 | pub unsafe fn sum_kahan(a: &[f32]) -> f32 { |
238 | 0 | crate::backends::scalar::ScalarBackend::sum_kahan(a) |
239 | 0 | } |