Coverage Report

Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/home/noah/src/trueno/src/backends/avx2/ops/reductions.rs
Line
Count
Source
1
//! AVX2 reduction operations (dot, sum, max, min, argmax, argmin).
2
3
#[cfg(target_arch = "x86_64")]
4
use std::arch::x86_64::*;
5
6
use crate::backends::VectorBackend;
7
8
/// AVX2 dot product with 4-accumulator unrolling for ILP.
9
#[inline]
10
#[target_feature(enable = "avx2,fma")]
11
229k
pub unsafe fn dot(a: &[f32], b: &[f32]) -> f32 {
12
229k
    let len = a.len();
13
229k
    let mut i = 0;
14
15
229k
    let mut acc0 = _mm256_setzero_ps();
16
229k
    let mut acc1 = _mm256_setzero_ps();
17
229k
    let mut acc2 = _mm256_setzero_ps();
18
229k
    let mut acc3 = _mm256_setzero_ps();
19
20
269k
    while i + 32 <= len {
21
39.9k
        let va0 = _mm256_loadu_ps(a.as_ptr().add(i));
22
39.9k
        let vb0 = _mm256_loadu_ps(b.as_ptr().add(i));
23
39.9k
        let va1 = _mm256_loadu_ps(a.as_ptr().add(i + 8));
24
39.9k
        let vb1 = _mm256_loadu_ps(b.as_ptr().add(i + 8));
25
39.9k
        let va2 = _mm256_loadu_ps(a.as_ptr().add(i + 16));
26
39.9k
        let vb2 = _mm256_loadu_ps(b.as_ptr().add(i + 16));
27
39.9k
        let va3 = _mm256_loadu_ps(a.as_ptr().add(i + 24));
28
39.9k
        let vb3 = _mm256_loadu_ps(b.as_ptr().add(i + 24));
29
39.9k
30
39.9k
        acc0 = _mm256_fmadd_ps(va0, vb0, acc0);
31
39.9k
        acc1 = _mm256_fmadd_ps(va1, vb1, acc1);
32
39.9k
        acc2 = _mm256_fmadd_ps(va2, vb2, acc2);
33
39.9k
        acc3 = _mm256_fmadd_ps(va3, vb3, acc3);
34
39.9k
35
39.9k
        i += 32;
36
39.9k
    }
37
38
680k
    while i + 8 <= len {
39
451k
        let va = _mm256_loadu_ps(a.as_ptr().add(i));
40
451k
        let vb = _mm256_loadu_ps(b.as_ptr().add(i));
41
451k
        acc0 = _mm256_fmadd_ps(va, vb, acc0);
42
451k
        i += 8;
43
451k
    }
44
45
229k
    let acc01 = _mm256_add_ps(acc0, acc1);
46
229k
    let acc23 = _mm256_add_ps(acc2, acc3);
47
229k
    let acc = _mm256_add_ps(acc01, acc23);
48
49
229k
    let mut result = {
50
229k
        let sum_halves = _mm_add_ps(_mm256_castps256_ps128(acc), _mm256_extractf128_ps(acc, 1));
51
229k
        let temp = _mm_add_ps(sum_halves, _mm_movehl_ps(sum_halves, sum_halves));
52
229k
        let temp = _mm_add_ss(temp, _mm_shuffle_ps(temp, temp, 1));
53
229k
        _mm_cvtss_f32(temp)
54
    };
55
56
229k
    result += a[i..].iter().zip(&b[i..]).map(|(x, y)| 
x206
*
y206
).sum::<f32>();
57
229k
    result
58
229k
}
59
60
/// AVX2 vector sum.
61
#[inline]
62
#[target_feature(enable = "avx2")]
63
118
pub unsafe fn sum(a: &[f32]) -> f32 {
64
118
    let len = a.len();
65
118
    let mut i = 0;
66
118
    let mut acc = _mm256_setzero_ps();
67
68
14.2k
    while i + 8 <= len {
69
14.0k
        let va = _mm256_loadu_ps(a.as_ptr().add(i));
70
14.0k
        acc = _mm256_add_ps(acc, va);
71
14.0k
        i += 8;
72
14.0k
    }
73
74
118
    let mut result = {
75
118
        let sum_halves = _mm_add_ps(_mm256_castps256_ps128(acc), _mm256_extractf128_ps(acc, 1));
76
118
        let temp = _mm_add_ps(sum_halves, _mm_movehl_ps(sum_halves, sum_halves));
77
118
        let temp = _mm_add_ss(temp, _mm_shuffle_ps(temp, temp, 1));
78
118
        _mm_cvtss_f32(temp)
79
    };
80
81
118
    result += a[i..].iter().sum::<f32>();
82
118
    result
83
118
}
84
85
/// AVX2 vector max.
86
#[inline]
87
#[target_feature(enable = "avx2")]
88
10.4k
pub unsafe fn max(a: &[f32]) -> f32 {
89
10.4k
    let len = a.len();
90
10.4k
    let mut i = 0;
91
10.4k
    let mut vmax = _mm256_set1_ps(a[0]);
92
93
34.1k
    while i + 8 <= len {
94
23.7k
        let va = _mm256_loadu_ps(a.as_ptr().add(i));
95
23.7k
        vmax = _mm256_max_ps(vmax, va);
96
23.7k
        i += 8;
97
23.7k
    }
98
99
10.4k
    let mut result = {
100
10.4k
        let max_halves = _mm_max_ps(_mm256_castps256_ps128(vmax), _mm256_extractf128_ps(vmax, 1));
101
10.4k
        let temp = _mm_max_ps(max_halves, _mm_movehl_ps(max_halves, max_halves));
102
10.4k
        let temp = _mm_max_ss(temp, _mm_shuffle_ps(temp, temp, 1));
103
10.4k
        _mm_cvtss_f32(temp)
104
    };
105
106
37.9k
    for &val in &
a[i..]10.4k
{
107
37.9k
        if val > result {
108
19.3k
            result = val;
109
19.3k
        
}18.6k
110
    }
111
10.4k
    result
112
10.4k
}
113
114
/// AVX2 vector min.
115
#[inline]
116
#[target_feature(enable = "avx2")]
117
0
pub unsafe fn min(a: &[f32]) -> f32 {
118
0
    let len = a.len();
119
0
    let mut i = 0;
120
0
    let mut vmin = _mm256_set1_ps(a[0]);
121
122
0
    while i + 8 <= len {
123
0
        let va = _mm256_loadu_ps(a.as_ptr().add(i));
124
0
        vmin = _mm256_min_ps(vmin, va);
125
0
        i += 8;
126
0
    }
127
128
0
    let mut result = {
129
0
        let min_halves = _mm_min_ps(_mm256_castps256_ps128(vmin), _mm256_extractf128_ps(vmin, 1));
130
0
        let temp = _mm_min_ps(min_halves, _mm_movehl_ps(min_halves, min_halves));
131
0
        let temp = _mm_min_ss(temp, _mm_shuffle_ps(temp, temp, 1));
132
0
        _mm_cvtss_f32(temp)
133
    };
134
135
0
    for &val in &a[i..] {
136
0
        if val < result {
137
0
            result = val;
138
0
        }
139
    }
140
0
    result
141
0
}
142
143
/// AVX2 argmax.
144
#[inline]
145
#[target_feature(enable = "avx2")]
146
0
pub unsafe fn argmax(a: &[f32]) -> usize {
147
0
    let len = a.len();
148
0
    let mut max_idx: usize = 0;
149
0
    let mut max_val = a[0];
150
0
    let mut i = 0;
151
152
0
    let mut vmax = _mm256_set1_ps(a[0]);
153
0
    let mut vidx_max = _mm256_setzero_ps();
154
0
    let vidx_inc = _mm256_set1_ps(8.0);
155
0
    let mut vcurrent_idx = _mm256_set_ps(7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0);
156
157
0
    while i + 8 <= len {
158
0
        let va = _mm256_loadu_ps(a.as_ptr().add(i));
159
0
        let mask = _mm256_cmp_ps(va, vmax, _CMP_GT_OQ);
160
0
        vmax = _mm256_blendv_ps(vmax, va, mask);
161
0
        vidx_max = _mm256_blendv_ps(vidx_max, vcurrent_idx, mask);
162
0
        vcurrent_idx = _mm256_add_ps(vcurrent_idx, vidx_inc);
163
0
        i += 8;
164
0
    }
165
166
    // Extract max from vector
167
0
    let mut vals = [0.0f32; 8];
168
0
    let mut idxs = [0.0f32; 8];
169
0
    _mm256_storeu_ps(vals.as_mut_ptr(), vmax);
170
0
    _mm256_storeu_ps(idxs.as_mut_ptr(), vidx_max);
171
172
0
    for j in 0..8 {
173
0
        if vals[j] > max_val {
174
0
            max_val = vals[j];
175
0
            max_idx = idxs[j] as usize;
176
0
        }
177
    }
178
179
    // Check remaining elements
180
0
    for (j, &val) in a[i..].iter().enumerate() {
181
0
        if val > max_val {
182
0
            max_val = val;
183
0
            max_idx = i + j;
184
0
        }
185
    }
186
187
0
    max_idx
188
0
}
189
190
/// AVX2 argmin.
191
#[inline]
192
#[target_feature(enable = "avx2")]
193
0
pub unsafe fn argmin(a: &[f32]) -> usize {
194
0
    let len = a.len();
195
0
    let mut min_idx: usize = 0;
196
0
    let mut min_val = a[0];
197
0
    let mut i = 0;
198
199
0
    let mut vmin = _mm256_set1_ps(a[0]);
200
0
    let mut vidx_min = _mm256_setzero_ps();
201
0
    let vidx_inc = _mm256_set1_ps(8.0);
202
0
    let mut vcurrent_idx = _mm256_set_ps(7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0);
203
204
0
    while i + 8 <= len {
205
0
        let va = _mm256_loadu_ps(a.as_ptr().add(i));
206
0
        let mask = _mm256_cmp_ps(va, vmin, _CMP_LT_OQ);
207
0
        vmin = _mm256_blendv_ps(vmin, va, mask);
208
0
        vidx_min = _mm256_blendv_ps(vidx_min, vcurrent_idx, mask);
209
0
        vcurrent_idx = _mm256_add_ps(vcurrent_idx, vidx_inc);
210
0
        i += 8;
211
0
    }
212
213
0
    let mut vals = [0.0f32; 8];
214
0
    let mut idxs = [0.0f32; 8];
215
0
    _mm256_storeu_ps(vals.as_mut_ptr(), vmin);
216
0
    _mm256_storeu_ps(idxs.as_mut_ptr(), vidx_min);
217
218
0
    for j in 0..8 {
219
0
        if vals[j] < min_val {
220
0
            min_val = vals[j];
221
0
            min_idx = idxs[j] as usize;
222
0
        }
223
    }
224
225
0
    for (j, &val) in a[i..].iter().enumerate() {
226
0
        if val < min_val {
227
0
            min_val = val;
228
0
            min_idx = i + j;
229
0
        }
230
    }
231
232
0
    min_idx
233
0
}
234
235
/// Kahan sum for numerical stability (delegates to scalar).
236
#[inline]
237
0
pub unsafe fn sum_kahan(a: &[f32]) -> f32 {
238
0
    crate::backends::scalar::ScalarBackend::sum_kahan(a)
239
0
}