## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/gpu/metrics.rs

    Line
    Count
    Source
    1
    //! Metrics & Health Monitoring (PMAT-802)
    2
    //!
    3
    //! M28: InferenceMetrics, HealthChecker, ShutdownCoordinator, GpuCompute, HybridScheduler.
    4
    5
    use crate::error::{RealizarError, Result};
    6
    use crate::tensor::Tensor;
    7
    use super::MatmulOp;
    8
    9
    // =============================================================================
    10
    // M28: Metrics & Health Monitoring (Phase 19)
    11
    // =============================================================================
    12
    13
    /// Inference metrics collector (M28 - IMP-067)
    14
    ///
    15
    /// Collects and aggregates inference performance metrics including
    16
    /// latency distribution and throughput.
    17
    #[derive(Debug)]
    18
    pub struct InferenceMetrics {
    19
        latencies: Vec<std::time::Duration>,
    20
        total_tokens: u64,
    21
        start_time: std::time::Instant,
    22
    }
    23
    24
    impl InferenceMetrics {
    25
        /// Create a new inference metrics collector
    26
        #[must_use]
    27
    12
        pub fn new() -> Self {
    28
    12
            Self {
    29
    12
                latencies: Vec::new(),
    30
    12
                total_tokens: 0,
    31
    12
                start_time: std::time::Instant::now(),
    32
    12
            }
    33
    12
        }
    34
    35
        /// Get total number of recorded inferences
    36
        #[must_use]
    37
    11
        pub fn total_inferences(&self) -> usize {
    38
    11
            self.latencies.len()
    39
    11
        }
    40
    41
        /// Get total number of tokens processed
    42
        #[must_use]
    43
    10
        pub fn total_tokens(&self) -> u64 {
    44
    10
            self.total_tokens
    45
    10
        }
    46
    47
        /// Record an inference with its latency and token count
    48
    24
        pub fn record_inference(&mut self, latency: std::time::Duration, tokens: usize) {
    49
    24
            self.latencies.push(latency);
    50
    24
            self.total_tokens += tokens as u64;
    51
    24
        }
    52
    53
        /// Get latency at given percentile (0-100)
    54
        ///
    55
        /// Returns None if no inferences recorded.
    56
        #[must_use]
    57
    9
        pub fn latency_percentile(&self, percentile: u8) -> Option<std::time::Duration> {
    58
    9
            if self.latencies.is_empty() {
    59
    3
                return None;
    60
    6
            }
    61
    62
    6
            let mut sorted = self.latencies.clone();
    63
    6
            sorted.sort();
    64
    65
    6
            let idx = ((percentile as usize) * sorted.len() / 100).min(sorted.len() - 1);
    66
    6
            Some(sorted[idx])
    67
    9
        }
    68
    69
        /// Calculate throughput in tokens per second
    70
        #[must_use]
    71
    2
        pub fn throughput(&self) -> f64 {
    72
    2
            let elapsed = self.start_time.elapsed().as_secs_f64();
    73
    2
            if elapsed > 0.0 {
    74
    2
                self.total_tokens as f64 / elapsed
    75
            } else {
    76
    0
                0.0
    77
            }
    78
    2
        }
    79
    80
        /// Reset all metrics
    81
    2
        pub fn reset(&mut self) {
    82
    2
            self.latencies.clear();
    83
    2
            self.total_tokens = 0;
    84
    2
            self.start_time = std::time::Instant::now();
    85
    2
        }
    86
    }
    87
    88
    impl Default for InferenceMetrics {
    89
    1
        fn default() -> Self {
    90
    1
            Self::new()
    91
    1
        }
    92
    }
    93
    94
    /// Type alias for health check function
    95
    pub type HealthCheckFn = Box<dyn Fn() -> bool + Send + Sync>;
    96
    97
    /// Health checker for system components (M28 - IMP-068)
    98
    ///
    99
    /// Monitors health status of system components via registered check functions.
    100
    pub struct HealthChecker {
    101
        checks: Vec<(String, HealthCheckFn)>,
    102
        last_results: std::collections::HashMap<String, bool>,
    103
    }
    104
    105
    impl std::fmt::Debug for HealthChecker {
    106
    2
        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
    107
    2
            f.debug_struct("HealthChecker")
    108
    2
                .field("check_count", &self.checks.len())
    109
    2
                .field("last_results", &self.last_results)
    110
    2
                .finish()
    111
    2
        }
    112
    }
    113
    114
    impl HealthChecker {
    115
        /// Create a new health checker
    116
        #[must_use]
    117
    13
        pub fn new() -> Self {
    118
    13
            Self {
    119
    13
                checks: Vec::new(),
    120
    13
                last_results: std::collections::HashMap::new(),
    121
    13
            }
    122
    13
        }
    123
    124
        /// Get number of registered checks
    125
        #[must_use]
    126
    6
        pub fn check_count(&self) -> usize {
    127
    6
            self.checks.len()
    128
    6
        }
    129
    130
        /// Register a health check function
    131
    14
        pub fn register_check(&mut self, name: &str, check: HealthCheckFn) {
    132
    14
            self.checks.push((name.to_string(), check));
    133
    14
        }
    134
    135
        /// Run all health checks and return results
    136
    8
        pub fn check_all(&mut self) -> std::collections::HashMap<String, bool> {
    137
    8
            let mut results = std::collections::HashMap::new();
    138
    22
            for (name14, check14) in &self.checks {
    139
    14
                let healthy = check();
    140
    14
                results.insert(name.clone(), healthy);
    141
    14
            }
    142
    8
            self.last_results.clone_from(&results);
    143
    8
            results
    144
    8
        }
    145
    146
        /// Check if system is overall healthy (all checks pass)
    147
        #[must_use]
    148
    12
        pub fn is_healthy(&self) -> bool {
    149
    12
            if self.checks.is_empty() {
    150
    6
                return true;
    151
    6
            }
    152
    6
            self.last_results.values().all(|&v| v)
    153
    12
        }
    154
    155
        /// Clear all registered checks
    156
    2
        pub fn clear(&mut self) {
    157
    2
            self.checks.clear();
    158
    2
            self.last_results.clear();
    159
    2
        }
    160
    }
    161
    162
    impl Default for HealthChecker {
    163
    1
        fn default() -> Self {
    164
    1
            Self::new()
    165
    1
        }
    166
    }
    167
    168
    /// Type alias for shutdown handler function
    169
    pub type ShutdownHandlerFn = Box<dyn Fn() + Send + Sync>;
    170
    171
    /// Graceful shutdown coordinator (M28 - IMP-069)
    172
    ///
    173
    /// Coordinates shutdown sequence with request draining and handler callbacks.
    174
    pub struct ShutdownCoordinator {
    175
        shutting_down: bool,
    176
        pending_requests: u32,
    177
        handlers: Vec<ShutdownHandlerFn>,
    178
    }
    179
    180
    impl std::fmt::Debug for ShutdownCoordinator {
    181
    2
        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
    182
    2
            f.debug_struct("ShutdownCoordinator")
    183
    2
                .field("shutting_down", &self.shutting_down)
    184
    2
                .field("pending_requests", &self.pending_requests)
    185
    2
                .field("handler_count", &self.handlers.len())
    186
    2
                .finish()
    187
    2
        }
    188
    }
    189
    190
    impl ShutdownCoordinator {
    191
        /// Create a new shutdown coordinator
    192
        #[must_use]
    193
    11
        pub fn new() -> Self {
    194
    11
            Self {
    195
    11
                shutting_down: false,
    196
    11
                pending_requests: 0,
    197
    11
                handlers: Vec::new(),
    198
    11
            }
    199
    11
        }
    200
    201
        /// Check if shutdown has been initiated
    202
        #[must_use]
    203
    6
        pub fn is_shutting_down(&self) -> bool {
    204
    6
            self.shutting_down
    205
    6
        }
    206
    207
        /// Get number of pending requests
    208
        #[must_use]
    209
    12
        pub fn pending_requests(&self) -> u32 {
    210
    12
            self.pending_requests
    211
    12
        }
    212
    213
        /// Get number of registered handlers
    214
        #[must_use]
    215
    4
        pub fn handler_count(&self) -> usize {
    216
    4
            self.handlers.len()
    217
    4
        }
    218
    219
        /// Register a shutdown handler
    220
    5
        pub fn register_handler(&mut self, handler: ShutdownHandlerFn) {
    221
    5
            self.handlers.push(handler);
    222
    5
        }
    223
    224
        /// Mark that a request has started
    225
    5
        pub fn request_started(&mut self) {
    226
    5
            self.pending_requests += 1;
    227
    5
        }
    228
    229
        /// Mark that a request has completed
    230
    7
        pub fn request_completed(&mut self) {
    231
    7
            self.pending_requests = self.pending_requests.saturating_sub(1);
    232
    7
        }
    233
    234
        /// Initiate shutdown sequence
    235
        ///
    236
        /// Calls all registered handlers.
    237
    6
        pub fn initiate_shutdown(&mut self) {
    238
    6
            if self.shutting_down {
    239
    2
                return;
    240
    4
            }
    241
    4
            self.shutting_down = true;
    242
    243
            // Call all handlers
    244
    7
            for handler3 in &self.handlers {
    245
    3
                handler();
    246
    3
            }
    247
    6
        }
    248
    249
        /// Check if shutdown is complete (initiated + no pending requests)
    250
        #[must_use]
    251
    4
        pub fn is_complete(&self) -> bool {
    252
    4
            self.shutting_down && self.pending_requests == 03
    253
    4
        }
    254
    }
    255
    256
    impl Default for ShutdownCoordinator {
    257
    1
        fn default() -> Self {
    258
    1
            Self::new()
    259
    1
        }
    260
    }
    261
    262
    /// Compute backend selection
    263
    #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
    264
    pub enum ComputeBackend {
    265
        /// GPU compute via trueno's wgpu backend
    266
        Gpu,
    267
        /// CPU compute (fallback)
    268
        Cpu,
    269
        /// Auto-select best available backend
    270
        #[default]
    271
        Auto,
    272
    }
    273
    274
    /// GPU compute context
    275
    ///
    276
    /// Provides GPU-accelerated operations with automatic fallback to CPU
    277
    /// when GPU is not available.
    278
    pub struct GpuCompute {
    279
        backend: ComputeBackend,
    280
        gpu: Option<trueno::backends::gpu::GpuBackend>,
    281
    }
    282
    283
    impl GpuCompute {
    284
        /// Create GPU compute context with auto-detected backend
    285
        ///
    286
        /// Attempts to initialize GPU backend, falls back to CPU if unavailable.
    287
        ///
    288
        /// # Errors
    289
        ///
    290
        /// Returns error if both GPU and CPU initialization fail (should not happen).
    291
    153
        pub fn auto() -> Result<Self> {
    292
    153
            Self::new(ComputeBackend::Auto)
    293
    153
        }
    294
    295
        /// Create GPU compute context with specified backend
    296
        ///
    297
        /// # Arguments
    298
        ///
    299
        /// * `backend` - Backend selection (Gpu, Cpu, or Auto)
    300
        ///
    301
        /// # Errors
    302
        ///
    303
        /// Returns error if:
    304
        /// - `Gpu` backend requested but GPU is not available
    305
        /// - Backend initialization fails
    306
    177
        pub fn new(backend: ComputeBackend) -> Result<Self> {
    307
    177
            match backend {
    308
                ComputeBackend::Gpu => {
    309
    2
                    if trueno::backends::gpu::GpuBackend::is_available() {
    310
    2
                        Ok(Self {
    311
    2
                            backend: ComputeBackend::Gpu,
    312
    2
                            gpu: Some(trueno::backends::gpu::GpuBackend::new()),
    313
    2
                        })
    314
                    } else {
    315
    0
                        Err(RealizarError::GpuError {
    316
    0
                            reason: "GPU not available".to_string(),
    317
    0
                        })
    318
                    }
    319
                },
    320
    22
                ComputeBackend::Cpu => Ok(Self {
    321
    22
                    backend: ComputeBackend::Cpu,
    322
    22
                    gpu: None,
    323
    22
                }),
    324
                ComputeBackend::Auto => {
    325
    153
                    if trueno::backends::gpu::GpuBackend::is_available() {
    326
    153
                        Ok(Self {
    327
    153
                            backend: ComputeBackend::Gpu,
    328
    153
                            gpu: Some(trueno::backends::gpu::GpuBackend::new()),
    329
    153
                        })
    330
                    } else {
    331
    0
                        Ok(Self {
    332
    0
                            backend: ComputeBackend::Cpu,
    333
    0
                            gpu: None,
    334
    0
                        })
    335
                    }
    336
                },
    337
            }
    338
    177
        }
    339
    340
        /// Check if GPU backend is active
    341
        #[must_use]
    342
    879
        pub fn is_gpu(&self) -> bool {
    343
    879
            self.backend == ComputeBackend::Gpu && self.gpu878.is_some878()
    344
    879
        }
    345
    346
        /// Get active backend type
    347
        #[must_use]
    348
    3
        pub fn backend(&self) -> ComputeBackend {
    349
    3
            self.backend
    350
    3
        }
    351
    352
        /// GPU-accelerated matrix multiplication
    353
        ///
    354
        /// Computes `C = A @ B` where:
    355
        /// - A is `[m, k]`
    356
        /// - B is `[k, n]`
    357
        /// - C is `[m, n]`
    358
        ///
    359
        /// # Arguments
    360
        ///
    361
        /// * `a` - Left matrix as flat f32 slice, row-major `[m, k]`
    362
        /// * `b` - Right matrix as flat f32 slice, row-major `[k, n]`
    363
        /// * `m` - Rows in A and C
    364
        /// * `k` - Cols in A, rows in B
    365
        /// * `n` - Cols in B and C
    366
        ///
    367
        /// # Errors
    368
        ///
    369
        /// Returns error if:
    370
        /// - Input dimensions don't match
    371
        /// - GPU compute fails
    372
        #[allow(clippy::many_single_char_names)]
    373
    458
        pub fn matmul(
    374
    458
            &mut self,
    375
    458
            a: &[f32],
    376
    458
            b: &[f32],
    377
    458
            m: usize,
    378
    458
            k: usize,
    379
    458
            n: usize,
    380
    458
        ) -> Result<Vec<f32>> {
    381
            // Validate dimensions
    382
    458
            if a.len() != m * k {
    383
    1
                return Err(RealizarError::InvalidShape {
    384
    1
                    reason: format!(
    385
    1
                        "Matrix A size {} doesn't match m*k={}*{}={}",
    386
    1
                        a.len(),
    387
    1
                        m,
    388
    1
                        k,
    389
    1
                        m * k
    390
    1
                    ),
    391
    1
                });
    392
    457
            }
    393
    457
            if b.len() != k * n {
    394
    0
                return Err(RealizarError::InvalidShape {
    395
    0
                    reason: format!(
    396
    0
                        "Matrix B size {} doesn't match k*n={}*{}={}",
    397
    0
                        b.len(),
    398
    0
                        k,
    399
    0
                        n,
    400
    0
                        k * n
    401
    0
                    ),
    402
    0
                });
    403
    457
            }
    404
    405
    457
            if let Some(gpu438) = &mut self.gpu {
    406
                // GPU path
    407
                #[allow(clippy::implicit_clone)]
    408
    438
                gpu.matmul(a, b, m, k, n)
    409
    438
                    .map_err(|e| RealizarError::GpuError {
    410
    0
                        reason: e.to_string(),
    411
    0
                    })
    412
            } else {
    413
                // CPU fallback: naive matmul
    414
    19
                Ok(cpu_matmul(a, b, m, k, n))
    415
            }
    416
    458
        }
    417
    418
        /// GPU-accelerated matrix multiplication with Tensor input/output
    419
        ///
    420
        /// # Arguments
    421
        ///
    422
        /// * `a` - Left tensor `[m, k]`
    423
        /// * `b` - Right tensor `[k, n]`
    424
        ///
    425
        /// # Errors
    426
        ///
    427
        /// Returns error if tensors are not 2D or dimensions don't match.
    428
        #[allow(clippy::many_single_char_names)]
    429
    2
        pub fn matmul_tensor(&mut self, a: &Tensor<f32>, b: &Tensor<f32>) -> Result<Tensor<f32>> {
    430
    2
            let a_shape = a.shape();
    431
    2
            let b_shape = b.shape();
    432
    433
    2
            if a_shape.len() != 2 || b_shape.len() != 2 {
    434
    0
                return Err(RealizarError::InvalidShape {
    435
    0
                    reason: "matmul_tensor requires 2D tensors".to_string(),
    436
    0
                });
    437
    2
            }
    438
    439
    2
            let m = a_shape[0];
    440
    2
            let k = a_shape[1];
    441
    2
            let k2 = b_shape[0];
    442
    2
            let n = b_shape[1];
    443
    444
    2
            if k != k2 {
    445
    1
                return Err(RealizarError::InvalidShape {
    446
    1
                    reason: format!("Inner dimensions don't match: A[{m},{k}] @ B[{k2},{n}]"),
    447
    1
                });
    448
    1
            }
    449
    450
    1
            let result = self.matmul(a.data(), b.data(), m, k, n)?0;
    451
    1
            Tensor::from_vec(vec![m, n], result)
    452
    2
        }
    453
    454
        /// GPU-accelerated vector dot product
    455
        ///
    456
        /// # Errors
    457
        ///
    458
        /// Returns error if vectors have different lengths or GPU compute fails.
    459
    3
        pub fn dot(&mut self, a: &[f32], b: &[f32]) -> Result<f32> {
    460
    3
            if a.len() != b.len() {
    461
    1
                return Err(RealizarError::InvalidShape {
    462
    1
                    reason: format!("Vector lengths don't match: {} vs {}", a.len(), b.len()),
    463
    1
                });
    464
    2
            }
    465
    466
    2
            if let Some(gpu0) = &mut self.gpu {
    467
                #[allow(clippy::implicit_clone)]
    468
    0
                gpu.dot(a, b).map_err(|e| RealizarError::GpuError {
    469
    0
                    reason: e.to_string(),
    470
    0
                })
    471
            } else {
    472
                // CPU fallback
    473
    3
                Ok(a2.iter2().zip2(b2.iter2()).map2(|(x, y)| x * y).sum2())
    474
            }
    475
    3
        }
    476
    477
        /// GPU-accelerated ReLU activation
    478
        ///
    479
        /// # Errors
    480
        ///
    481
        /// Returns error if GPU compute fails.
    482
    3
        pub fn relu(&mut self, input: &[f32]) -> Result<Vec<f32>> {
    483
    3
            if let Some(gpu0) = &mut self.gpu {
    484
                #[allow(clippy::implicit_clone)]
    485
    0
                gpu.relu(input).map_err(|e| RealizarError::GpuError {
    486
    0
                    reason: e.to_string(),
    487
    0
                })
    488
            } else {
    489
    5
                Ok(input3.iter3().map3(|&x| x.max(0.0)).collect3())
    490
            }
    491
    3
        }
    492
    493
        /// GPU-accelerated sigmoid activation
    494
        ///
    495
        /// # Errors
    496
        ///
    497
        /// Returns error if GPU compute fails.
    498
    2
        pub fn sigmoid(&mut self, input: &[f32]) -> Result<Vec<f32>> {
    499
    2
            if let Some(gpu0) = &mut self.gpu {
    500
                #[allow(clippy::implicit_clone)]
    501
    0
                gpu.sigmoid(input).map_err(|e| RealizarError::GpuError {
    502
    0
                    reason: e.to_string(),
    503
    0
                })
    504
            } else {
    505
    4
                Ok(input2.iter2().map2(|&x| 1.0 / (1.0 + (-x).exp())).collect2())
    506
            }
    507
    2
        }
    508
    }
    509
    510
    /// CPU fallback matmul implementation
    511
    #[allow(clippy::many_single_char_names)]
    512
    7.99k
    pub(crate) fn cpu_matmul(a: &[f32], b: &[f32], m: usize, k: usize, n: usize) -> Vec<f32> {
    513
        // For m=1 (vector-matrix multiply), use optimized path
    514
    7.99k
        if m == 1 {
    515
    7.58k
            return cpu_vector_matmul(a, b, k, n);
    516
    412
        }
    517
    518
    412
        let mut c = vec![0.0; m * n];
    519
    4.80k
        for i in 0..m412 {
    520
    894k
            for j in 0..n4.80k {
    521
    894k
                let mut sum = 0.0;
    522
    199M
                for p in 0..k894k {
    523
    199M
                    sum += a[i * k + p] * b[p * n + j];
    524
    199M
                }
    525
    894k
                c[i * n + j] = sum;
    526
            }
    527
        }
    528
    412
        c
    529
    7.99k
    }
    530
    531
    /// IMP-098: Parallelized vector-matrix multiply: a[1,k] @ b[k,n] -> c[1,n]
    532
    ///
    533
    /// Uses parallel output chunks for multi-core utilization.
    534
    /// Each thread accumulates its chunk of outputs independently.
    535
    #[allow(clippy::many_single_char_names)]
    536
    7.58k
    fn cpu_vector_matmul(a: &[f32], b: &[f32], k: usize, n: usize) -> Vec<f32> {
    537
        use rayon::prelude::*;
    538
    539
        // For small n, use sequential (avoids rayon overhead)
    540
    7.58k
        if n < 2048 {
    541
    7.58k
            return cpu_vector_matmul_seq(a, b, k, n);
    542
    0
        }
    543
    544
        // Parallel over output chunks
    545
        const CHUNK_SIZE: usize = 1024;
    546
    0
        let num_chunks = n.div_ceil(CHUNK_SIZE);
    547
    548
    0
        let chunks: Vec<Vec<f32>> = (0..num_chunks)
    549
    0
            .into_par_iter()
    550
    0
            .map(|chunk_idx| {
    551
    0
                let start = chunk_idx * CHUNK_SIZE;
    552
    0
                let end = (start + CHUNK_SIZE).min(n);
    553
    0
                let chunk_len = end - start;
    554
    0
                let mut chunk_c = vec![0.0f32; chunk_len];
    555
    556
                // Accumulate this chunk of outputs
    557
    0
                for (p, &a_val) in a.iter().enumerate() {
    558
    0
                    let row_start = p * n + start;
    559
    0
                    let row = &b[row_start..row_start + chunk_len];
    560
    0
                    for (j, &b_val) in row.iter().enumerate() {
    561
    0
                        chunk_c[j] += a_val * b_val;
    562
    0
                    }
    563
                }
    564
    0
                chunk_c
    565
    0
            })
    566
    0
            .collect();
    567
    568
        // Flatten chunks into result
    569
    0
        chunks.into_iter().flatten().collect()
    570
    7.58k
    }
    571
    572
    /// Sequential fallback for small outputs
    573
    #[allow(clippy::many_single_char_names)]
    574
    7.58k
    fn cpu_vector_matmul_seq(a: &[f32], b: &[f32], _k: usize, n: usize) -> Vec<f32> {
    575
    7.58k
        let mut c = vec![0.0f32; n];
    576
    577
        // Row-major accumulation: for each row of B, scale by corresponding a[p]
    578
    960k
        for (p, &a_val) in a7.58k.iter7.58k().enumerate7.58k() {
    579
    960k
            let row = &b[p * n..(p + 1) * n];
    580
    181M
            for (j, &b_val) in row960k.iter960k().enumerate960k() {
    581
    181M
                c[j] += a_val * b_val;
    582
    181M
            }
    583
        }
    584
    585
    7.58k
        c
    586
    7.58k
    }
    587
    588
    /// CPU matmul with B transposed: A @ B^T
    589
    /// a[m,k] @ b[n,k]^T -> c[m,n]
    590
    #[allow(clippy::many_single_char_names)]
    591
    41
    pub(crate) fn cpu_matmul_transpose_b(a: &[f32], b: &[f32], m: usize, k: usize, n: usize) -> Vec<f32> {
    592
    41
        let mut c = vec![0.0; m * n];
    593
    130
        for i in 0..m41 {
    594
    484
            for j in 0..n130 {
    595
    484
                let mut sum = 0.0;
    596
    14.9k
                for p in 0..k484 {
    597
    14.9k
                    // a[i,p] * b[j,p] (b is stored row-major as [n,k])
    598
    14.9k
                    sum += a[i * k + p] * b[j * k + p];
    599
    14.9k
                }
    600
    484
                c[i * n + j] = sum;
    601
            }
    602
        }
    603
    41
        c
    604
    41
    }
    605
    606
    /// Transpose a matrix from [rows, cols] to [cols, rows]
    607
    0
    pub(crate) fn transpose(data: &[f32], rows: usize, cols: usize) -> Vec<f32> {
    608
    0
        let mut result = vec![0.0; data.len()];
    609
    0
        for i in 0..rows {
    610
    0
            for j in 0..cols {
    611
    0
                result[j * rows + i] = data[i * cols + j];
    612
    0
            }
    613
        }
    614
    0
        result
    615
    0
    }
    616
    617
    /// IMP-096: Parallel SIMD vector-matrix multiply using transposed weights
    618
    ///
    619
    /// Computes a[1,k] @ weight_t[n,k]^T + bias[n] -> c[n]
    620
    /// Each output c[j] = dot(a, weight_t[j,:]) + bias[j]
    621
    ///
    622
    /// Uses transposed weights for row-major access pattern (contiguous dot products).
    623
    /// Parallelized with rayon. Compiler auto-vectorizes the inner dot product.
    624
    #[allow(clippy::many_single_char_names)]
    625
    0
    pub(crate) fn cpu_matmul_transposed_simd(
    626
    0
        a: &[f32],        // Input vector: [k]
    627
    0
        weight_t: &[f32], // Transposed weights: [n, k] (row-major)
    628
    0
        bias: &[f32],     // Bias: [n]
    629
    0
        k: usize,
    630
    0
        n: usize,
    631
    0
    ) -> Vec<f32> {
    632
        use rayon::prelude::*;
    633
    634
        // Process in chunks for better parallelism and cache locality
    635
        const CHUNK_SIZE: usize = 4096;
    636
    637
    0
        (0..n)
    638
    0
            .into_par_iter()
    639
    0
            .step_by(CHUNK_SIZE)
    640
    0
            .flat_map(|chunk_start| {
    641
    0
                let chunk_end = (chunk_start + CHUNK_SIZE).min(n);
    642
    0
                (chunk_start..chunk_end)
    643
    0
                    .map(|j| {
    644
                        // Row-major access: weight_t[j, :] is contiguous in memory
    645
    0
                        let row = &weight_t[j * k..(j + 1) * k];
    646
    647
                        // Compiler auto-vectorizes this dot product pattern
    648
    0
                        let dot: f32 = row.iter().zip(a.iter()).map(|(&w, &h)| w * h).sum();
    649
    0
                        dot + bias[j]
    650
    0
                    })
    651
    0
                    .collect::<Vec<_>>()
    652
    0
            })
    653
    0
            .collect()
    654
    0
    }
    655
    656
    /// GPU buffer pool for memory reuse and reduced allocation overhead
    657
    pub struct GpuBufferPool {
    658
        /// Available buffers indexed by size bucket
    659
        available_buffers: std::collections::HashMap<usize, Vec<Vec<f32>>>,
    660
        /// Size buckets for efficient pooling (powers of 2)
    661
        bucket_sizes: Vec<usize>,
    662
        /// Maximum cached buffers per bucket
    663
        max_per_bucket: usize,
    664
    }
    665
    666
    impl GpuBufferPool {
    667
        /// Create new buffer pool with default configuration
    668
        #[must_use]
    669
    167
        pub fn new() -> Self {
    670
            Self {
    671
    167
                available_buffers: std::collections::HashMap::new(),
    672
    2.50k
                bucket_sizes: (10..=24167).map167(|i| 1 << i).collect167(), // 1KB to 16MB
    673
                max_per_bucket: 4,
    674
            }
    675
    167
        }
    676
    677
        /// Get bucket size for requested allocation
    678
    42
        fn get_bucket(&self, size: usize) -> usize {
    679
    42
            *self
    680
    42
                .bucket_sizes
    681
    42
                .iter()
    682
    68
                .find42(|&&b| b >= size)
    683
    42
                .unwrap_or(&size)
    684
    42
        }
    685
    686
        /// Acquire buffer of at least `size` elements
    687
    24
        pub fn acquire(&mut self, size: usize) -> Vec<f32> {
    688
    24
            let bucket = self.get_bucket(size);
    689
    24
            if let Some(buffers4) = self.available_buffers.get_mut(&bucket) {
    690
    4
                if let Some(mut buf) = buffers.pop() {
    691
    4
                    buf.resize(size, 0.0);
    692
    4
                    return buf;
    693
    0
                }
    694
    20
            }
    695
    20
            vec![0.0; size]
    696
    24
        }
    697
    698
        /// Release buffer back to pool for reuse
    699
    18
        pub fn release(&mut self, mut buffer: Vec<f32>) {
    700
    18
            let bucket = self.get_bucket(buffer.capacity());
    701
    18
            let buffers = self.available_buffers.entry(bucket).or_default();
    702
    18
            if buffers.len() < self.max_per_bucket {
    703
    18
                buffer.clear();
    704
    18
                buffers.push(buffer);
    705
    18
            }0
    706
            // Otherwise just drop it
    707
    18
        }
    708
    709
        /// Clear all cached buffers
    710
    2
        pub fn clear(&mut self) {
    711
    2
            self.available_buffers.clear();
    712
    2
        }
    713
    714
        /// Get configured bucket sizes
    715
        #[must_use]
    716
    2
        pub fn bucket_sizes(&self) -> &[usize] {
    717
    2
            &self.bucket_sizes
    718
    2
        }
    719
    720
        /// Get pool statistics
    721
        #[must_use]
    722
    14
        pub fn stats(&self) -> GpuPoolStats {
    723
    14
            let total_buffers: usize = self.available_buffers.values().map(Vec::len).sum();
    724
    14
            let total_bytes: usize = self
    725
    14
                .available_buffers
    726
    14
                .iter()
    727
    14
                .map(|(bucket, buffers)| bucket11 * buffers.len() * 4)
    728
    14
                .sum();
    729
    14
            GpuPoolStats {
    730
    14
                cached_buffers: total_buffers,
    731
    14
                cached_bytes: total_bytes,
    732
    14
            }
    733
    14
        }
    734
    }
    735
    736
    impl Default for GpuBufferPool {
    737
    1
        fn default() -> Self {
    738
    1
            Self::new()
    739
    1
        }
    740
    }
    741
    742
    /// GPU buffer pool statistics
    743
    #[derive(Debug, Clone, Copy)]
    744
    pub struct GpuPoolStats {
    745
        /// Number of cached buffers
    746
        pub cached_buffers: usize,
    747
        /// Total cached bytes
    748
        pub cached_bytes: usize,
    749
    }
    750
    751
    /// Async GPU compute handle for non-blocking operations
    752
    ///
    753
    /// Per spec: "Async transfer - No host blocking"
    754
    pub struct AsyncGpuResult {
    755
        /// Result data when ready
    756
        result: Option<Vec<f32>>,
    757
        /// Whether computation is complete
    758
        ready: bool,
    759
    }
    760
    761
    impl AsyncGpuResult {
    762
        /// Create result that's immediately ready (CPU fallback)
    763
    6
        pub fn ready(data: Vec<f32>) -> Self {
    764
    6
            Self {
    765
    6
                result: Some(data),
    766
    6
                ready: true,
    767
    6
            }
    768
    6
        }
    769
    770
        /// Create pending result (GPU async)
    771
    5
        pub fn pending() -> Self {
    772
    5
            Self {
    773
    5
                result: None,
    774
    5
                ready: false,
    775
    5
            }
    776
    5
        }
    777
    778
        /// Check if result is ready
    779
        #[must_use]
    780
    9
        pub fn is_ready(&self) -> bool {
    781
    9
            self.ready
    782
    9
        }
    783
    784
        /// Mark as ready with result
    785
    4
        pub fn set_result(&mut self, data: Vec<f32>) {
    786
    4
            self.result = Some(data);
    787
    4
            self.ready = true;
    788
    4
        }
    789
    790
        /// Block until result is ready (for synchronization points)
    791
    6
        pub fn wait(self) -> Vec<f32> {
    792
    6
            self.result.expect("Result not ready")
    793
    6
        }
    794
    795
        /// Try to get result without blocking
    796
    5
        pub fn try_get(&self) -> Option<&Vec<f32>> {
    797
    5
            if self.ready {
    798
    3
                self.result.as_ref()
    799
            } else {
    800
    2
                None
    801
            }
    802
    5
        }
    803
    }
    804
    805
    /// Hybrid CPU/GPU scheduler
    806
    ///
    807
    /// Automatically selects optimal backend based on workload size.
    808
    pub struct HybridScheduler {
    809
        gpu_compute: GpuCompute,
    810
        /// Minimum matrix size (m*k*n) to use GPU
    811
        gpu_threshold: usize,
    812
        /// Buffer pool for memory reuse
    813
        buffer_pool: GpuBufferPool,
    814
    }
    815
    816
    impl HybridScheduler {
    817
        /// Create hybrid scheduler with auto-detected GPU
    818
        ///
    819
        /// # Errors
    820
        ///
    821
        /// Returns error if compute initialization fails.
    822
    36
        pub fn new() -> Result<Self> {
    823
            Ok(Self {
    824
    36
                gpu_compute: GpuCompute::auto()?0,
    825
    36
                gpu_threshold: 64 * 64 * 64, // 262K elements
    826
    36
                buffer_pool: GpuBufferPool::new(),
    827
            })
    828
    36
        }
    829
    830
        /// Create scheduler with custom threshold
    831
        ///
    832
        /// # Arguments
    833
        ///
    834
        /// * `gpu_threshold` - Minimum m*k*n to trigger GPU acceleration
    835
        ///
    836
        /// # Errors
    837
        ///
    838
        /// Returns error if compute initialization fails.
    839
    115
        pub fn with_threshold(gpu_threshold: usize) -> Result<Self> {
    840
            Ok(Self {
    841
    115
                gpu_compute: GpuCompute::auto()?0,
    842
    115
                gpu_threshold,
    843
    115
                buffer_pool: GpuBufferPool::new(),
    844
            })
    845
    115
        }
    846
    847
        /// Check if GPU is available
    848
        #[must_use]
    849
    11
        pub fn has_gpu(&self) -> bool {
    850
    11
            self.gpu_compute.is_gpu()
    851
    11
        }
    852
    853
        /// Get GPU threshold
    854
        #[must_use]
    855
    4
        pub fn gpu_threshold(&self) -> usize {
    856
    4
            self.gpu_threshold
    857
    4
        }
    858
    859
        /// Decide whether to use GPU for given workload
    860
        ///
    861
        /// IMP-097: For m=1 (single-token inference), CPU is faster due to:
    862
        /// - No GPU data transfer overhead
    863
        /// - No kernel launch latency
    864
        /// - CPU SIMD is sufficient for vector-matrix multiply
    865
        #[must_use]
    866
        #[allow(clippy::many_single_char_names)]
    867
    8.45k
        pub fn should_use_gpu(&self, m: usize, k: usize, n: usize) -> bool {
    868
            // IMP-097: Force CPU for single-token operations (m=1)
    869
            // GPU kernel launch overhead exceeds compute benefit for small batch sizes
    870
    8.45k
            if m <= 1 {
    871
    7.59k
                return false;
    872
    866
            }
    873
    866
            self.gpu_compute.is_gpu() && (m * k * n) >= self.gpu_threshold
    874
    8.45k
        }
    875
    876
        /// Execute matmul with automatic backend selection
    877
        ///
    878
        /// Uses GPU for large matrices, CPU for small ones.
    879
        ///
    880
        /// # Errors
    881
        ///
    882
        /// Returns error if compute fails.
    883
        #[allow(clippy::many_single_char_names)]
    884
    8.39k
        pub fn matmul(
    885
    8.39k
            &mut self,
    886
    8.39k
            a: &[f32],
    887
    8.39k
            b: &[f32],
    888
    8.39k
            m: usize,
    889
    8.39k
            k: usize,
    890
    8.39k
            n: usize,
    891
    8.39k
        ) -> Result<Vec<f32>> {
    892
    8.39k
            if self.should_use_gpu(m, k, n) {
    893
    426
                self.gpu_compute.matmul(a, b, m, k, n)
    894
            } else {
    895
    7.96k
                Ok(cpu_matmul(a, b, m, k, n))
    896
            }
    897
    8.39k
        }
    898
    899
        /// Execute matmul with pooled output buffer
    900
        ///
    901
        /// Reduces allocation overhead by reusing buffers.
    902
        ///
    903
        /// # Errors
    904
        ///
    905
        /// Returns error if compute fails.
    906
        #[allow(clippy::many_single_char_names)]
    907
    4
        pub fn matmul_pooled(
    908
    4
            &mut self,
    909
    4
            a: &[f32],
    910
    4
            b: &[f32],
    911
    4
            m: usize,
    912
    4
            k: usize,
    913
    4
            n: usize,
    914
    4
        ) -> Result<Vec<f32>> {
    915
            // Acquire buffer from pool
    916
    4
            let mut output = self.buffer_pool.acquire(m * n);
    917
    918
            // Compute result
    919
    4
            let result = if self.should_use_gpu(m, k, n) {
    920
    0
                self.gpu_compute.matmul(a, b, m, k, n)?
    921
            } else {
    922
    4
                cpu_matmul(a, b, m, k, n)
    923
            };
    924
    925
            // Copy to pooled buffer
    926
    4
            output.copy_from_slice(&result);
    927
    4
            Ok(output)
    928
    4
        }
    929
    930
        /// Release buffer back to pool
    931
        ///
    932
        /// Call this when done with a buffer returned by `matmul_pooled`.
    933
    4
        pub fn release_buffer(&mut self, buffer: Vec<f32>) {
    934
    4
            self.buffer_pool.release(buffer);
    935
    4
        }
    936
    937
        /// Get buffer pool statistics
    938
        #[must_use]
    939
    3
        pub fn pool_stats(&self) -> GpuPoolStats {
    940
    3
            self.buffer_pool.stats()
    941
    3
        }
    942
    943
        /// Execute matmul asynchronously (non-blocking on CPU fallback)
    944
        ///
    945
        /// Per spec: "Async transfer - No host blocking"
    946
        ///
    947
        /// # Errors
    948
        ///
    949
        /// Returns error if compute setup fails.
    950
        #[allow(clippy::many_single_char_names)]
    951
    1
        pub fn matmul_async(
    952
    1
            &mut self,
    953
    1
            a: &[f32],
    954
    1
            b: &[f32],
    955
    1
            m: usize,
    956
    1
            k: usize,
    957
    1
            n: usize,
    958
    1
        ) -> Result<AsyncGpuResult> {
    959
            // For CPU fallback, compute immediately
    960
            // For GPU, this would submit to command queue without blocking
    961
    1
            let result = if self.should_use_gpu(m, k, n) {
    962
    0
                self.gpu_compute.matmul(a, b, m, k, n)?
    963
            } else {
    964
    1
                cpu_matmul(a, b, m, k, n)
    965
            };
    966
    967
    1
            Ok(AsyncGpuResult::ready(result))
    968
    1
        }
    969
    970
        /// Process batch of matmuls with optimal scheduling
    971
        ///
    972
        /// Batches small operations for CPU, pipelines large ones for GPU.
    973
        ///
    974
        /// # Errors
    975
        ///
    976
        /// Returns error if any compute fails.
    977
    4
        pub fn matmul_batch(&mut self, operations: &[MatmulOp]) -> Result<Vec<Vec<f32>>> {
    978
    4
            let mut results = Vec::with_capacity(operations.len());
    979
    980
    9
            for (a5, b5, m5, k5, n5) in operations {
    981
    5
                let result = self.matmul(a, b, *m, *k, *n)?0;
    982
    5
                results.push(result);
    983
            }
    984
    985
    4
            Ok(results)
    986
    4
        }
    987
    988
        /// Execute matmul with B transposed: A @ B^T
    989
        ///
    990
        /// Computes C[m,n] = A[m,k] @ B[n,k]^T
    991
        /// where B is stored row-major as [n, k].
    992
        ///
    993
        /// # Errors
    994
        ///
    995
        /// Returns error if compute fails.
    996
        #[allow(clippy::many_single_char_names)]
    997
    41
        pub fn matmul_transpose_b(
    998
    41
            &mut self,
    999
    41
            a: &[f32],
    1000
    41
            b: &[f32],
    1001
    41
            m: usize,
    1002
    41
            k: usize,
    1003
    41
            n: usize,
    1004
    41
        ) -> Result<Vec<f32>> {
    1005
            // For attention: Q[seq, head_dim] @ K[seq, head_dim]^T = scores[seq, seq]
    1006
            // B is stored as [n, k], we need B^T which is [k, n]
    1007
    41
            if self.should_use_gpu(m, k, n) {
    1008
                // Transpose B and use GPU matmul
    1009
    0
                let b_t = transpose(b, n, k);
    1010
    0
                self.gpu_compute.matmul(a, &b_t, m, k, n)
    1011
            } else {
    1012
                // CPU: compute A @ B^T directly
    1013
    41
                Ok(cpu_matmul_transpose_b(a, b, m, k, n))
    1014
            }
    1015
    41
        }
    1016
    }
    1017