/home/noah/src/trueno/src/backends/sse2/ops/reductions.rs
Line | Count | Source |
1 | | //! SSE2 reduction operations (dot, sum, max, min, argmax, argmin). |
2 | | |
3 | | #[cfg(target_arch = "x86_64")] |
4 | | use std::arch::x86_64::*; |
5 | | |
6 | | use crate::backends::VectorBackend; |
7 | | |
8 | | /// SSE2 dot product. |
9 | | #[inline] |
10 | | #[target_feature(enable = "sse2")] |
11 | 0 | pub unsafe fn dot(a: &[f32], b: &[f32]) -> f32 { |
12 | 0 | let len = a.len(); |
13 | 0 | let mut i = 0; |
14 | 0 | let mut acc = _mm_setzero_ps(); |
15 | | |
16 | 0 | while i + 4 <= len { |
17 | 0 | let va = _mm_loadu_ps(a.as_ptr().add(i)); |
18 | 0 | let vb = _mm_loadu_ps(b.as_ptr().add(i)); |
19 | 0 | acc = _mm_add_ps(acc, _mm_mul_ps(va, vb)); |
20 | 0 | i += 4; |
21 | 0 | } |
22 | | |
23 | 0 | let mut result = horizontal_sum_ps(acc); |
24 | 0 | result += a[i..].iter().zip(&b[i..]).map(|(x, y)| x * y).sum::<f32>(); |
25 | 0 | result |
26 | 0 | } |
27 | | |
28 | | /// SSE2 vector sum. |
29 | | #[inline] |
30 | | #[target_feature(enable = "sse2")] |
31 | 0 | pub unsafe fn sum(a: &[f32]) -> f32 { |
32 | 0 | let len = a.len(); |
33 | 0 | let mut i = 0; |
34 | 0 | let mut acc = _mm_setzero_ps(); |
35 | | |
36 | 0 | while i + 4 <= len { |
37 | 0 | acc = _mm_add_ps(acc, _mm_loadu_ps(a.as_ptr().add(i))); |
38 | 0 | i += 4; |
39 | 0 | } |
40 | | |
41 | 0 | let mut result = horizontal_sum_ps(acc); |
42 | 0 | result += a[i..].iter().sum::<f32>(); |
43 | 0 | result |
44 | 0 | } |
45 | | |
46 | | /// SSE2 vector max. |
47 | | #[inline] |
48 | | #[target_feature(enable = "sse2")] |
49 | 0 | pub unsafe fn max(a: &[f32]) -> f32 { |
50 | 0 | let len = a.len(); |
51 | 0 | let mut i = 0; |
52 | 0 | let mut vmax = _mm_set1_ps(a[0]); |
53 | | |
54 | 0 | while i + 4 <= len { |
55 | 0 | vmax = _mm_max_ps(vmax, _mm_loadu_ps(a.as_ptr().add(i))); |
56 | 0 | i += 4; |
57 | 0 | } |
58 | | |
59 | 0 | let mut result = horizontal_max_ps(vmax); |
60 | 0 | for &val in &a[i..] { if val > result { result = val; } } |
61 | 0 | result |
62 | 0 | } |
63 | | |
64 | | /// SSE2 vector min. |
65 | | #[inline] |
66 | | #[target_feature(enable = "sse2")] |
67 | 0 | pub unsafe fn min(a: &[f32]) -> f32 { |
68 | 0 | let len = a.len(); |
69 | 0 | let mut i = 0; |
70 | 0 | let mut vmin = _mm_set1_ps(a[0]); |
71 | | |
72 | 0 | while i + 4 <= len { |
73 | 0 | vmin = _mm_min_ps(vmin, _mm_loadu_ps(a.as_ptr().add(i))); |
74 | 0 | i += 4; |
75 | 0 | } |
76 | | |
77 | 0 | let mut result = horizontal_min_ps(vmin); |
78 | 0 | for &val in &a[i..] { if val < result { result = val; } } |
79 | 0 | result |
80 | 0 | } |
81 | | |
82 | | /// SSE2 argmax. |
83 | | #[inline] |
84 | | #[target_feature(enable = "sse2")] |
85 | 0 | pub unsafe fn argmax(a: &[f32]) -> usize { |
86 | 0 | let _len = a.len(); |
87 | 0 | let mut max_idx: usize = 0; |
88 | 0 | let mut max_val = a[0]; |
89 | | |
90 | 0 | for (i, &val) in a.iter().enumerate() { |
91 | 0 | if val > max_val { |
92 | 0 | max_val = val; |
93 | 0 | max_idx = i; |
94 | 0 | } |
95 | | } |
96 | 0 | max_idx |
97 | 0 | } |
98 | | |
99 | | /// SSE2 argmin. |
100 | | #[inline] |
101 | | #[target_feature(enable = "sse2")] |
102 | 0 | pub unsafe fn argmin(a: &[f32]) -> usize { |
103 | 0 | let _len = a.len(); |
104 | 0 | let mut min_idx: usize = 0; |
105 | 0 | let mut min_val = a[0]; |
106 | | |
107 | 0 | for (i, &val) in a.iter().enumerate() { |
108 | 0 | if val < min_val { |
109 | 0 | min_val = val; |
110 | 0 | min_idx = i; |
111 | 0 | } |
112 | | } |
113 | 0 | min_idx |
114 | 0 | } |
115 | | |
116 | | /// Kahan sum (delegates to scalar). |
117 | | #[inline] |
118 | 0 | pub unsafe fn sum_kahan(a: &[f32]) -> f32 { |
119 | 0 | crate::backends::scalar::ScalarBackend::sum_kahan(a) |
120 | 0 | } |
121 | | |
122 | | // Helper: horizontal sum of 4 floats |
123 | | #[inline] |
124 | | #[target_feature(enable = "sse2")] |
125 | 0 | unsafe fn horizontal_sum_ps(v: __m128) -> f32 { |
126 | 0 | let temp = _mm_add_ps(v, _mm_movehl_ps(v, v)); |
127 | 0 | let temp = _mm_add_ss(temp, _mm_shuffle_ps(temp, temp, 1)); |
128 | 0 | _mm_cvtss_f32(temp) |
129 | 0 | } |
130 | | |
131 | | // Helper: horizontal max of 4 floats |
132 | | #[inline] |
133 | | #[target_feature(enable = "sse2")] |
134 | 0 | unsafe fn horizontal_max_ps(v: __m128) -> f32 { |
135 | 0 | let temp = _mm_max_ps(v, _mm_movehl_ps(v, v)); |
136 | 0 | let temp = _mm_max_ss(temp, _mm_shuffle_ps(temp, temp, 1)); |
137 | 0 | _mm_cvtss_f32(temp) |
138 | 0 | } |
139 | | |
140 | | // Helper: horizontal min of 4 floats |
141 | | #[inline] |
142 | | #[target_feature(enable = "sse2")] |
143 | 0 | unsafe fn horizontal_min_ps(v: __m128) -> f32 { |
144 | 0 | let temp = _mm_min_ps(v, _mm_movehl_ps(v, v)); |
145 | 0 | let temp = _mm_min_ss(temp, _mm_shuffle_ps(temp, temp, 1)); |
146 | 0 | _mm_cvtss_f32(temp) |
147 | 0 | } |