/home/noah/src/trueno/src/matrix/ops/arithmetic.rs
Line | Count | Source |
1 | | //! Matrix arithmetic operations |
2 | | //! |
3 | | //! This module provides matrix multiplication and related operations: |
4 | | //! - `matmul()` - Standard matrix multiplication with SIMD optimization |
5 | | //! - `batched_matmul()` - Batched 3D tensor multiplication |
6 | | //! - `batched_matmul_4d()` - 4D tensor multiplication for attention |
7 | | //! |
8 | | //! ## Domain Separation (PMAT-018) |
9 | | //! |
10 | | //! Arithmetic operations (multiplication, addition) are separate from storage |
11 | | //! operations (allocation, indexing). This allows optimizing compute kernels |
12 | | //! independently of memory layout decisions. |
13 | | //! |
14 | | //! ## Performance Hierarchy |
15 | | //! |
16 | | //! 1. GPU for large matrices (≥500×500) - 2-10x speedup |
17 | | //! 2. BLIS/SIMD for medium-large matrices (>64×64) - 2-8x speedup |
18 | | //! 3. Naive for small matrices - lowest overhead |
19 | | |
20 | | use crate::TruenoError; |
21 | | |
22 | | #[cfg(feature = "tracing")] |
23 | | use tracing::instrument; |
24 | | |
25 | | use super::super::Matrix; |
26 | | |
27 | | impl Matrix<f32> { |
28 | | /// Matrix multiplication (matmul) |
29 | | /// |
30 | | /// Computes `C = A × B` where A is `m×n`, B is `n×p`, and C is `m×p`. |
31 | | /// |
32 | | /// # Arguments |
33 | | /// |
34 | | /// * `other` - The matrix to multiply with (right operand) |
35 | | /// |
36 | | /// # Returns |
37 | | /// |
38 | | /// A new matrix containing the result of matrix multiplication |
39 | | /// |
40 | | /// # Errors |
41 | | /// |
42 | | /// Returns `InvalidInput` if matrix dimensions are incompatible |
43 | | /// (i.e., `self.cols != other.rows`) |
44 | | /// |
45 | | /// # Example |
46 | | /// |
47 | | /// ``` |
48 | | /// use trueno::Matrix; |
49 | | /// |
50 | | /// let a = Matrix::from_vec(2, 2, vec![1.0, 2.0, 3.0, 4.0]).unwrap(); |
51 | | /// let b = Matrix::from_vec(2, 2, vec![5.0, 6.0, 7.0, 8.0]).unwrap(); |
52 | | /// let c = a.matmul(&b).unwrap(); |
53 | | /// |
54 | | /// // [[1, 2], [[5, 6], [[19, 22], |
55 | | /// // [3, 4]] × [7, 8]] = [43, 50]] |
56 | | /// assert_eq!(c.get(0, 0), Some(&19.0)); |
57 | | /// assert_eq!(c.get(0, 1), Some(&22.0)); |
58 | | /// assert_eq!(c.get(1, 0), Some(&43.0)); |
59 | | /// assert_eq!(c.get(1, 1), Some(&50.0)); |
60 | | /// ``` |
61 | | // ========================================================================= |
62 | | // HOT PATH - PERFORMANCE CRITICAL |
63 | | // ========================================================================= |
64 | | // Core matrix operation used in neural network forward passes. |
65 | | // Changes to inner loops REQUIRE benchmark verification: make bench-check |
66 | | // ========================================================================= |
67 | | #[cfg_attr(feature = "tracing", instrument(skip(self, other), fields(dims = %format!("{}x{} @ {}x{}", self.rows, self.cols, other.rows, other.cols))))] |
68 | 0 | pub fn matmul(&self, other: &Matrix<f32>) -> Result<Matrix<f32>, TruenoError> { |
69 | 0 | if self.cols != other.rows { |
70 | 0 | return Err(TruenoError::InvalidInput(format!( |
71 | 0 | "Matrix dimension mismatch for multiplication: {}×{} × {}×{} (inner dimensions {} and {} must match)", |
72 | 0 | self.rows, self.cols, other.rows, other.cols, self.cols, other.rows |
73 | 0 | ))); |
74 | 0 | } |
75 | | |
76 | | // Fast path for vector-matrix multiply (rows=1) |
77 | 0 | if self.rows == 1 { |
78 | 0 | return self.matmul_vector_matrix(other); |
79 | 0 | } |
80 | | |
81 | 0 | let mut result = Matrix::zeros_with_backend(self.rows, other.cols, self.backend); |
82 | | |
83 | | #[cfg(all(feature = "gpu", not(target_arch = "wasm32")))] |
84 | | const GPU_THRESHOLD: usize = 500; |
85 | | const SIMD_THRESHOLD: usize = 64; |
86 | | |
87 | | // Try GPU first for very large matrices |
88 | | #[cfg(all(feature = "gpu", not(target_arch = "wasm32")))] |
89 | | { |
90 | 0 | if self.rows >= GPU_THRESHOLD |
91 | 0 | && self.cols >= GPU_THRESHOLD |
92 | 0 | && other.cols >= GPU_THRESHOLD |
93 | | { |
94 | 0 | if let Ok(gpu_result) = self.matmul_gpu(other) { |
95 | 0 | return Ok(gpu_result); |
96 | 0 | } |
97 | 0 | } |
98 | | } |
99 | | |
100 | | // Use SIMD for medium-large matrices |
101 | 0 | if self.rows >= SIMD_THRESHOLD |
102 | 0 | || self.cols >= SIMD_THRESHOLD |
103 | 0 | || other.cols >= SIMD_THRESHOLD |
104 | | { |
105 | | #[cfg(target_arch = "wasm32")] |
106 | | { |
107 | | self.matmul_wasm_tiled(other, &mut result)?; |
108 | | } |
109 | | #[cfg(not(target_arch = "wasm32"))] |
110 | | { |
111 | 0 | crate::blis::gemm_blis( |
112 | 0 | self.rows, |
113 | 0 | other.cols, |
114 | 0 | self.cols, |
115 | 0 | &self.data, |
116 | 0 | &other.data, |
117 | 0 | &mut result.data, |
118 | 0 | None, |
119 | 0 | )?; |
120 | | } |
121 | | } else { |
122 | 0 | self.matmul_naive(other, &mut result)?; |
123 | | } |
124 | | |
125 | 0 | Ok(result) |
126 | 0 | } |
127 | | |
128 | | /// Batched matrix multiplication for 3D tensors. |
129 | | /// |
130 | | /// Computes `[batch, m, k] @ [batch, k, n] -> [batch, m, n]` using SIMD for each batch. |
131 | | #[cfg_attr( |
132 | | feature = "tracing", |
133 | | instrument(skip(a_data, b_data), fields(batch, m, k, n)) |
134 | | )] |
135 | 0 | pub fn batched_matmul( |
136 | 0 | a_data: &[f32], |
137 | 0 | b_data: &[f32], |
138 | 0 | batch: usize, |
139 | 0 | m: usize, |
140 | 0 | k: usize, |
141 | 0 | n: usize, |
142 | 0 | ) -> Result<Vec<f32>, TruenoError> { |
143 | 0 | let a_stride = m * k; |
144 | 0 | let b_stride = k * n; |
145 | 0 | let out_stride = m * n; |
146 | | |
147 | 0 | if a_data.len() != batch * a_stride { |
148 | 0 | return Err(TruenoError::InvalidInput(format!( |
149 | 0 | "A data size mismatch: expected {} ({}×{}×{}), got {}", |
150 | 0 | batch * a_stride, batch, m, k, a_data.len() |
151 | 0 | ))); |
152 | 0 | } |
153 | 0 | if b_data.len() != batch * b_stride { |
154 | 0 | return Err(TruenoError::InvalidInput(format!( |
155 | 0 | "B data size mismatch: expected {} ({}×{}×{}), got {}", |
156 | 0 | batch * b_stride, batch, k, n, b_data.len() |
157 | 0 | ))); |
158 | 0 | } |
159 | | |
160 | 0 | let mut output = vec![0.0f32; batch * out_stride]; |
161 | | |
162 | 0 | for ba in 0..batch { |
163 | 0 | let a_offset = ba * a_stride; |
164 | 0 | let b_offset = ba * b_stride; |
165 | 0 | let out_offset = ba * out_stride; |
166 | | |
167 | 0 | let a_slice = &a_data[a_offset..a_offset + a_stride]; |
168 | 0 | let b_slice = &b_data[b_offset..b_offset + b_stride]; |
169 | | |
170 | 0 | let a_mat = Matrix::from_slice(m, k, a_slice)?; |
171 | 0 | let b_mat = Matrix::from_slice(k, n, b_slice)?; |
172 | | |
173 | 0 | let result = a_mat.matmul(&b_mat)?; |
174 | 0 | output[out_offset..out_offset + out_stride].copy_from_slice(result.as_slice()); |
175 | | } |
176 | | |
177 | 0 | Ok(output) |
178 | 0 | } |
179 | | |
180 | | /// Batched matrix multiplication for 4D tensors (attention pattern). |
181 | | /// |
182 | | /// Computes `[batch, heads, m, k] @ [batch, heads, k, n] -> [batch, heads, m, n]` |
183 | | #[cfg_attr( |
184 | | feature = "tracing", |
185 | | instrument(skip(a_data, b_data), fields(batch, heads, m, k, n)) |
186 | | )] |
187 | 0 | pub fn batched_matmul_4d( |
188 | 0 | a_data: &[f32], |
189 | 0 | b_data: &[f32], |
190 | 0 | batch: usize, |
191 | 0 | heads: usize, |
192 | 0 | m: usize, |
193 | 0 | k: usize, |
194 | 0 | n: usize, |
195 | 0 | ) -> Result<Vec<f32>, TruenoError> { |
196 | 0 | let a_head_stride = m * k; |
197 | 0 | let b_head_stride = k * n; |
198 | 0 | let out_head_stride = m * n; |
199 | 0 | let total_heads = batch * heads; |
200 | | |
201 | 0 | let expected_a = total_heads * a_head_stride; |
202 | 0 | let expected_b = total_heads * b_head_stride; |
203 | 0 | if a_data.len() != expected_a { |
204 | 0 | return Err(TruenoError::InvalidInput(format!( |
205 | 0 | "A data size mismatch: expected {} ({}×{}×{}×{}), got {}", |
206 | 0 | expected_a, batch, heads, m, k, a_data.len() |
207 | 0 | ))); |
208 | 0 | } |
209 | 0 | if b_data.len() != expected_b { |
210 | 0 | return Err(TruenoError::InvalidInput(format!( |
211 | 0 | "B data size mismatch: expected {} ({}×{}×{}×{}), got {}", |
212 | 0 | expected_b, batch, heads, k, n, b_data.len() |
213 | 0 | ))); |
214 | 0 | } |
215 | | |
216 | 0 | let mut output = vec![0.0f32; total_heads * out_head_stride]; |
217 | | |
218 | 0 | for bh in 0..total_heads { |
219 | 0 | let a_offset = bh * a_head_stride; |
220 | 0 | let b_offset = bh * b_head_stride; |
221 | 0 | let out_offset = bh * out_head_stride; |
222 | | |
223 | 0 | let a_slice = &a_data[a_offset..a_offset + a_head_stride]; |
224 | 0 | let b_slice = &b_data[b_offset..b_offset + b_head_stride]; |
225 | | |
226 | 0 | let a_mat = Matrix::from_slice(m, k, a_slice)?; |
227 | 0 | let b_mat = Matrix::from_slice(k, n, b_slice)?; |
228 | | |
229 | 0 | let result = a_mat.matmul(&b_mat)?; |
230 | 0 | output[out_offset..out_offset + out_head_stride].copy_from_slice(result.as_slice()); |
231 | | } |
232 | | |
233 | 0 | Ok(output) |
234 | 0 | } |
235 | | |
236 | | /// Fast path for vector-matrix multiplication (1×K @ K×N → 1×N) |
237 | | #[cfg_attr(feature = "tracing", instrument(skip(self, other), fields(k = self.cols, n = other.cols)))] |
238 | 0 | fn matmul_vector_matrix(&self, other: &Matrix<f32>) -> Result<Matrix<f32>, TruenoError> { |
239 | 0 | debug_assert_eq!(self.rows, 1); |
240 | | |
241 | 0 | let k = self.cols; |
242 | 0 | let n = other.cols; |
243 | 0 | let mut result = Matrix::zeros_with_backend(1, n, self.backend); |
244 | | |
245 | 0 | for ki in 0..k { |
246 | 0 | let a_k = self.data[ki]; |
247 | 0 | if a_k == 0.0 { |
248 | 0 | continue; |
249 | 0 | } |
250 | | |
251 | 0 | let b_row_start = ki * n; |
252 | 0 | for j in 0..n { |
253 | 0 | result.data[j] += a_k * other.data[b_row_start + j]; |
254 | 0 | } |
255 | | } |
256 | | |
257 | 0 | Ok(result) |
258 | 0 | } |
259 | | |
260 | | /// Naive O(n³) matrix multiplication (baseline for small matrices) |
261 | 0 | fn matmul_naive( |
262 | 0 | &self, |
263 | 0 | other: &Matrix<f32>, |
264 | 0 | result: &mut Matrix<f32>, |
265 | 0 | ) -> Result<(), TruenoError> { |
266 | 0 | for i in 0..self.rows { |
267 | 0 | for j in 0..other.cols { |
268 | 0 | let mut sum = 0.0; |
269 | 0 | for k in 0..self.cols { |
270 | 0 | sum += self.get(i, k).expect("bounds validated") |
271 | 0 | * other.get(k, j).expect("bounds validated"); |
272 | 0 | } |
273 | 0 | *result.get_mut(i, j).expect("bounds validated") = sum; |
274 | | } |
275 | | } |
276 | 0 | Ok(()) |
277 | 0 | } |
278 | | |
279 | | /// WASM-optimized tiled matrix multiplication |
280 | | #[allow(dead_code)] |
281 | 0 | fn matmul_wasm_tiled( |
282 | 0 | &self, |
283 | 0 | other: &Matrix<f32>, |
284 | 0 | result: &mut Matrix<f32>, |
285 | 0 | ) -> Result<(), TruenoError> { |
286 | 0 | let m = self.rows; |
287 | 0 | let k = self.cols; |
288 | 0 | let n = other.cols; |
289 | | |
290 | 0 | for i in 0..m { |
291 | 0 | let a_row_start = i * k; |
292 | 0 | let result_row_start = i * n; |
293 | | |
294 | 0 | let simd_width = 8; |
295 | 0 | let n_simd = (n / simd_width) * simd_width; |
296 | | |
297 | | #[allow(clippy::needless_range_loop)] |
298 | 0 | for j0 in (0..n_simd).step_by(simd_width) { |
299 | 0 | let mut acc = [0.0f32; 8]; |
300 | | |
301 | 0 | for kk in 0..k { |
302 | 0 | let a_val = self.data[a_row_start + kk]; |
303 | 0 | let b_row_start = kk * n + j0; |
304 | | |
305 | 0 | for jj in 0..simd_width { |
306 | 0 | acc[jj] += a_val * other.data[b_row_start + jj]; |
307 | 0 | } |
308 | | } |
309 | | |
310 | 0 | for jj in 0..simd_width { |
311 | 0 | result.data[result_row_start + j0 + jj] = acc[jj]; |
312 | 0 | } |
313 | | } |
314 | | |
315 | 0 | for j in n_simd..n { |
316 | 0 | let mut sum = 0.0f32; |
317 | 0 | for kk in 0..k { |
318 | 0 | sum += self.data[a_row_start + kk] * other.data[kk * n + j]; |
319 | 0 | } |
320 | 0 | result.data[result_row_start + j] = sum; |
321 | | } |
322 | | } |
323 | | |
324 | 0 | Ok(()) |
325 | 0 | } |
326 | | |
327 | | /// GPU-accelerated matrix multiplication |
328 | | #[cfg(all(feature = "gpu", not(target_arch = "wasm32")))] |
329 | 0 | fn matmul_gpu(&self, other: &Matrix<f32>) -> Result<Matrix<f32>, TruenoError> { |
330 | | use crate::backends::gpu::GpuBackend; |
331 | | |
332 | 0 | if !GpuBackend::is_available() { |
333 | 0 | return Err(TruenoError::InvalidInput("GPU not available".to_string())); |
334 | 0 | } |
335 | | |
336 | 0 | let mut gpu = GpuBackend::new(); |
337 | 0 | let result_data = gpu |
338 | 0 | .matmul(&self.data, &other.data, self.rows, self.cols, other.cols) |
339 | 0 | .map_err(|e| TruenoError::InvalidInput(format!("GPU matmul failed: {}", e)))?; |
340 | | |
341 | 0 | let mut result = Matrix::zeros(self.rows, other.cols); |
342 | 0 | result.data = result_data; |
343 | | |
344 | 0 | Ok(result) |
345 | 0 | } |
346 | | } |
347 | | |
348 | | #[cfg(test)] |
349 | | mod tests { |
350 | | use super::*; |
351 | | |
352 | | #[test] |
353 | | fn test_matmul_basic() { |
354 | | let a = Matrix::from_vec(2, 2, vec![1.0, 2.0, 3.0, 4.0]).unwrap(); |
355 | | let b = Matrix::from_vec(2, 2, vec![5.0, 6.0, 7.0, 8.0]).unwrap(); |
356 | | let c = a.matmul(&b).unwrap(); |
357 | | |
358 | | assert_eq!(c.get(0, 0), Some(&19.0)); |
359 | | assert_eq!(c.get(0, 1), Some(&22.0)); |
360 | | assert_eq!(c.get(1, 0), Some(&43.0)); |
361 | | assert_eq!(c.get(1, 1), Some(&50.0)); |
362 | | } |
363 | | |
364 | | #[test] |
365 | | fn test_matmul_dimension_mismatch() { |
366 | | let a = Matrix::from_vec(2, 3, vec![1.0; 6]).unwrap(); |
367 | | let b = Matrix::from_vec(2, 2, vec![1.0; 4]).unwrap(); |
368 | | assert!(a.matmul(&b).is_err()); |
369 | | } |
370 | | |
371 | | #[test] |
372 | | fn test_matmul_identity() { |
373 | | let a = Matrix::from_vec(2, 2, vec![1.0, 2.0, 3.0, 4.0]).unwrap(); |
374 | | let i = Matrix::identity(2); |
375 | | let result = a.matmul(&i).unwrap(); |
376 | | assert_eq!(result.as_slice(), a.as_slice()); |
377 | | } |
378 | | |
379 | | #[test] |
380 | | fn test_batched_matmul() { |
381 | | let a = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]; // 2 batches of 2×2 |
382 | | let b = vec![1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0]; // 2 identity matrices |
383 | | let result = Matrix::batched_matmul(&a, &b, 2, 2, 2, 2).unwrap(); |
384 | | assert_eq!(result, a); // A × I = A |
385 | | } |
386 | | } |