## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/gpu/mod.rs

    Line
    Count
    Source
    1
    //! GPU Acceleration Module (Phase 4)
    2
    //!
    3
    //! Provides GPU-accelerated compute primitives via trueno's wgpu backend.
    4
    //!
    5
    //! ## Architecture
    6
    //!
    7
    //! ```text
    8
    //! +-----------------------+
    9
    //! |    GpuCompute API     |  <- Safe public API
    10
    //! +-----------------------+
    11
    //! |   trueno::GpuBackend  |  <- wgpu-based GPU compute
    12
    //! +-----------------------+
    13
    //! |   wgpu Device/Queue   |  <- WebGPU abstraction
    14
    //! +-----------------------+
    15
    //! ```
    16
    //!
    17
    //! ## Usage
    18
    //!
    19
    //! ```rust,ignore
    20
    //! use realizar::gpu::{GpuCompute, ComputeBackend};
    21
    //!
    22
    //! // Auto-select best backend
    23
    //! let compute = GpuCompute::auto()?;
    24
    //!
    25
    //! // GPU matmul
    26
    //! let c = compute.matmul(&a, &b, m, k, n)?;
    27
    //! ```
    28
    //!
    29
    //! ## Performance Targets (Refs REALIZAR-PERF-SPEC-001)
    30
    //!
    31
    //! | Operation | GPU Target | CPU Baseline |
    32
    //! |-----------|------------|--------------|
    33
    //! | matmul    | 20x faster | 1x           |
    34
    //! | tok/s     | ≥100       | ≥25          |
    35
    36
    use crate::error::Result;
    37
    use std::collections::HashMap;
    38
    use std::time::Duration;
    39
    40
    // PMAT-802: Extracted modules
    41
    pub mod scheduler;
    42
    pub mod adapters;
    43
    pub mod backend;
    44
    pub mod mock_backend;
    45
    mod allocator;
    46
    mod diagnostics;
    47
    mod resilience;
    48
    mod simd_ops;
    49
    mod streaming_kv;
    50
    mod metrics;
    51
    mod batch_scheduling;
    52
    53
    // Types available without cuda feature
    54
    pub use scheduler::{GpuModel, GpuModelConfig, GpuGenerateConfig, WeightType, AttentionBuffers, BlockWeights};
    55
    #[cfg(feature = "cuda")]
    56
    pub use scheduler::CudaScheduler;
    57
    58
    // Allocator exports (M21, M22)
    59
    pub use allocator::{
    60
        CacheAlignedBuffer, TensorPool, ForwardArena, ScratchBuffer,
    61
        prefetch_read, sequential_sum, sum_with_prefetch, naive_matmul, blocked_matmul,
    62
    };
    63
    64
    // Diagnostics exports (M32)
    65
    pub use diagnostics::{
    66
        LogLevel, LogEntry, LogConfig, Logger, PhaseTimer,
    67
        MemoryReport, MemoryTracker, DiagnosticsSummary, DiagnosticsCollector,
    68
        DebugMode, RequestCapture, StateDump,
    69
    };
    70
    71
    // Resilience exports (M31)
    72
    pub use resilience::{
    73
        ErrorCategory, RetryDecision, RetryConfig, RetryPolicy,
    74
        CircuitState, CircuitConfig, CircuitBreaker,
    75
        RequestType, BulkheadPermit, BulkheadConfig, BulkheadStats, BulkheadManager,
    76
    };
    77
    78
    // SIMD ops exports (M18)
    79
    pub use simd_ops::{scalar_softmax, simd_softmax, scalar_rope, simd_rope};
    80
    81
    // Streaming KV cache exports (M6)
    82
    pub use streaming_kv::{StreamingKVCache, StreamingKVCacheFp16};
    83
    84
    // Metrics exports (M28)
    85
    pub use metrics::{
    86
        InferenceMetrics, HealthChecker, ShutdownCoordinator, ComputeBackend,
    87
        GpuCompute, GpuBufferPool, GpuPoolStats, AsyncGpuResult, HybridScheduler,
    88
    };
    89
    // Internal-only matmul functions (used by scheduler module)
    90
    pub(crate) use metrics::{cpu_matmul, cpu_matmul_transposed_simd};
    91
    92
    // Batch scheduling exports (M25, M26, M27)
    93
    pub use batch_scheduling::{
    94
        TokenBatch, SpeculativeBuffer, InferenceBatchScheduler, BatchId,
    95
        AsyncRequestQueue, InferenceEventNotifier, InferenceCompletionHandler,
    96
        TimeoutManager, RequestId, PriorityRequest, PriorityRequestQueue, Priority,
    97
        TokenRateLimiter, ResourceTracker, AllocationId,
    98
    };
    99
    100
    // ============================================================================
    101
    // GPU Buffer Limits (IMP-090, IMP-091)
    102
    // ============================================================================
    103
    104
    /// Maximum GPU buffer size in bytes (wgpu limit: 256MB)
    105
    const MAX_GPU_BUFFER_BYTES: usize = 256 * 1024 * 1024;
    106
    107
    /// Vocab size threshold above which we use CPU for embedding/lm_head
    108
    /// This is calculated as: MAX_GPU_BUFFER_BYTES / (hidden_dim * sizeof(f32))
    109
    /// For hidden_dim=1536: 256MB / (1536 * 4) = 43,690 tokens
    110
    /// We use 65536 as a round threshold that works for most models
    111
    pub const LARGE_VOCAB_THRESHOLD: usize = 65536;
    112
    113
    /// Check if a matrix operation would exceed GPU buffer limits (IMP-090)
    114
    ///
    115
    /// Returns true if the operation should use CPU fallback
    116
    #[inline]
    117
    #[must_use]
    118
    615
    pub fn exceeds_gpu_buffer_limit(elements: usize) -> bool {
    119
    615
        elements * std::mem::size_of::<f32>() > MAX_GPU_BUFFER_BYTES
    120
    615
    }
    121
    122
    /// Matmul batch operation: (A matrix, B matrix, m rows, k cols, n cols)
    123
    pub type MatmulOp = (Vec<f32>, Vec<f32>, usize, usize, usize);
    124
    125
    // ============================================================================
    126
    // Contiguous Attention Buffer (M19 - IMP-040)
    127
    // ============================================================================
    128
    129
    /// Contiguous memory buffer for attention tensors (M19 - IMP-040)
    130
    ///
    131
    /// Pre-allocates a single contiguous block for Q, K, V, O tensors
    132
    /// to reduce memory fragmentation during attention computation.
    133
    #[derive(Debug)]
    134
    pub struct ContiguousAttentionBuffer {
    135
        /// Single contiguous allocation for all tensors
    136
        data: Vec<f32>,
    137
        /// Maximum sequence length
    138
        max_seq_len: usize,
    139
        /// Number of attention heads (stored for future use)
    140
        #[allow(dead_code)]
    141
        num_heads: usize,
    142
        /// Dimension per head (stored for future use)
    143
        #[allow(dead_code)]
    144
        head_dim: usize,
    145
        /// Size of each tensor (Q, K, V, O have same size)
    146
        tensor_size: usize,
    147
    }
    148
    149
    impl ContiguousAttentionBuffer {
    150
        /// Create a new contiguous attention buffer
    151
        #[must_use]
    152
    8
        pub fn new(max_seq_len: usize, num_heads: usize, head_dim: usize) -> Self {
    153
    8
            let tensor_size = max_seq_len * num_heads * head_dim;
    154
            // Allocate 4x for Q, K, V, O in single contiguous block
    155
    8
            let data = vec![0.0f32; tensor_size * 4];
    156
    157
    8
            Self {
    158
    8
                data,
    159
    8
                max_seq_len,
    160
    8
                num_heads,
    161
    8
                head_dim,
    162
    8
                tensor_size,
    163
    8
            }
    164
    8
        }
    165
    166
        /// Check if buffer is contiguous (always true for this implementation)
    167
        #[must_use]
    168
    5
        pub fn is_contiguous(&self) -> bool {
    169
            // Buffer is contiguous by construction
    170
    5
            true
    171
    5
        }
    172
    173
        /// Get views into Q, K, V, O tensors
    174
        #[must_use]
    175
    6
        pub fn get_views(&self) -> (&[f32], &[f32], &[f32], &[f32]) {
    176
    6
            let q_start = 0;
    177
    6
            let k_start = self.tensor_size;
    178
    6
            let v_start = self.tensor_size * 2;
    179
    6
            let o_start = self.tensor_size * 3;
    180
    181
    6
            (
    182
    6
                &self.data[q_start..k_start],
    183
    6
                &self.data[k_start..v_start],
    184
    6
                &self.data[v_start..o_start],
    185
    6
                &self.data[o_start..],
    186
    6
            )
    187
    6
        }
    188
    189
        /// Get mutable views into Q, K, V, O tensors
    190
    3
        pub fn get_views_mut(&mut self) -> (&mut [f32], &mut [f32], &mut [f32], &mut [f32]) {
    191
    3
            let tensor_size = self.tensor_size;
    192
    193
            // Split the data into 4 mutable slices
    194
    3
            let (q, rest) = self.data.split_at_mut(tensor_size);
    195
    3
            let (k, rest) = rest.split_at_mut(tensor_size);
    196
    3
            let (v, o) = rest.split_at_mut(tensor_size);
    197
    198
    3
            (q, k, v, o)
    199
    3
        }
    200
    201
        /// Reset buffer to zeros for reuse
    202
    3
        pub fn reset(&mut self) {
    203
    3
            self.data.fill(0.0);
    204
    3
        }
    205
    206
        /// Get maximum sequence length
    207
        #[must_use]
    208
    2
        pub fn max_seq_len(&self) -> usize {
    209
    2
            self.max_seq_len
    210
    2
        }
    211
    }
    212
    213
    // ============================================================================
    214
    // Batch Processing & Parallel Execution (M20 - IMP-043/044/045)
    215
    // ============================================================================
    216
    217
    /// Batch token embedding lookup (M20 - IMP-043)
    218
    ///
    219
    /// Performs vectorized embedding lookup for multiple tokens at once.
    220
    /// More efficient than individual lookups due to better memory access patterns.
    221
    #[must_use]
    222
    1.01k
    pub fn batch_embed(embedding_table: &[f32], tokens: &[usize], hidden_dim: usize) -> Vec<f32> {
    223
    1.01k
        if tokens.is_empty() || embedding_table1.01k.is_empty1.01k() {
    224
    3
            return Vec::new();
    225
    1.01k
        }
    226
    227
    1.01k
        let mut result = Vec::with_capacity(tokens.len() * hidden_dim);
    228
    229
        // Batch copy embeddings
    230
    9.07k
        for &token8.06k in tokens {
    231
    8.06k
            let start_idx = token * hidden_dim;
    232
    8.06k
            let end_idx = start_idx + hidden_dim;
    233
    8.06k
            if end_idx <= embedding_table.len() {
    234
    8.06k
                result.extend_from_slice(&embedding_table[start_idx..end_idx]);
    235
    8.06k
            } else {
    236
    1
                // Pad with zeros for out-of-bounds tokens
    237
    1
                result.extend(std::iter::repeat_n(0.0, hidden_dim));
    238
    1
            }
    239
        }
    240
    241
    1.01k
        result
    242
    1.01k
    }
    243
    244
    /// Sequential FFN computation (baseline for comparison)
    245
    ///
    246
    /// Standard two-layer feed-forward network: up projection -> activation -> down projection
    247
    #[must_use]
    248
    260
    pub fn sequential_ffn(
    249
    260
        input: &[f32],
    250
    260
        w_up: &[f32],
    251
    260
        w_down: &[f32],
    252
    260
        hidden_dim: usize,
    253
    260
        intermediate_dim: usize,
    254
    260
    ) -> Vec<f32> {
    255
    260
        if input.is_empty() {
    256
    1
            return Vec::new();
    257
    259
        }
    258
    259
        // Up projection: (hidden_dim,) @ (hidden_dim, intermediate_dim) -> (intermediate_dim,)
    260
    259
        let mut intermediate = vec![0.0f32; intermediate_dim];
    261
    130k
        for i in 0..intermediate_dim259 {
    262
    130k
            let mut sum = 0.0f32;
    263
    33.3M
            for j in 0..hidden_dim130k {
    264
    33.3M
                sum += input[j] * w_up[j * intermediate_dim + i];
    265
    33.3M
            }
    266
            // GELU activation
    267
    130k
            intermediate[i] =
    268
    130k
                sum * 0.5 * (1.0 + (sum * 0.797_884_5 * (1.0 + 0.044_715 * sum * sum)).tanh());
    269
        }
    270
    271
        // Down projection: (intermediate_dim,) @ (intermediate_dim, hidden_dim) -> (hidden_dim,)
    272
    259
        let mut output = vec![0.0f32; hidden_dim];
    273
    65.1k
        for i in 0..hidden_dim259 {
    274
    65.1k
            let mut sum = 0.0f32;
    275
    33.3M
            for j in 0..intermediate_dim65.1k {
    276
    33.3M
                sum += intermediate[j] * w_down[j * hidden_dim + i];
    277
    33.3M
            }
    278
    65.1k
            output[i] = sum;
    279
        }
    280
    281
    259
        output
    282
    260
    }
    283
    284
    /// Parallel FFN computation (M20 - IMP-044)
    285
    ///
    286
    /// Uses rayon parallelism for the down projection matmul.
    287
    #[must_use]
    288
    259
    pub fn parallel_ffn(
    289
    259
        input: &[f32],
    290
    259
        w_up: &[f32],
    291
    259
        w_down: &[f32],
    292
    259
        hidden_dim: usize,
    293
    259
        intermediate_dim: usize,
    294
    259
    ) -> Vec<f32> {
    295
        use rayon::prelude::*;
    296
    297
    259
        if input.is_empty() {
    298
    1
            return Vec::new();
    299
    258
        }
    300
    301
        // Up projection with GELU (sequential - typically smaller)
    302
    258
        let intermediate: Vec<f32> = (0..intermediate_dim)
    303
    130k
            .map258(|i| {
    304
    130k
                let sum: f32 = (0..hidden_dim)
    305
    33.3M
                    .map130k(|j| input[j] * w_up[j * intermediate_dim + i])
    306
    130k
                    .sum();
    307
                // GELU activation
    308
    130k
                sum * 0.5 * (1.0 + (sum * 0.797_884_5 * (1.0 + 0.044_715 * sum * sum)).tanh())
    309
    130k
            })
    310
    258
            .collect();
    311
    312
        // Down projection with rayon parallelism
    313
    258
        let output: Vec<f32> = (0..hidden_dim)
    314
    258
            .into_par_iter()
    315
    65.1k
            .map258(|i| {
    316
    65.1k
                (0..intermediate_dim)
    317
    33.3M
                    .map65.1k(|j| intermediate[j] * w_down[j * hidden_dim + i])
    318
    65.1k
                    .sum()
    319
    65.1k
            })
    320
    258
            .collect();
    321
    322
    258
        output
    323
    259
    }
    324
    325
    /// Standard two-pass layer normalization (baseline for comparison)
    326
    ///
    327
    /// First pass computes mean, second pass computes variance.
    328
    #[must_use]
    329
    1.01k
    pub fn standard_layernorm(input: &[f32], gamma: &[f32], beta: &[f32], eps: f32) -> Vec<f32> {
    330
    1.01k
        if input.is_empty() {
    331
    1
            return Vec::new();
    332
    1.01k
        }
    333
    334
    1.01k
        let n = input.len() as f32;
    335
    336
        // First pass: compute mean
    337
    1.01k
        let mean: f32 = input.iter().sum::<f32>() / n;
    338
    339
        // Second pass: compute variance
    340
    257k
        let variance1.01k: f321.01k = input1.01k.iter1.01k().map1.01k(|&x| (x - mean).powi(2)).sum1.01k::<f32>() / n1.01k;
    341
    342
    1.01k
        let std_dev = (variance + eps).sqrt();
    343
    344
        // Normalize and apply gamma/beta
    345
    1.01k
        input
    346
    1.01k
            .iter()
    347
    1.01k
            .enumerate()
    348
    257k
            .map1.01k(|(i, &x)| {
    349
    257k
                let normalized = (x - mean) / std_dev;
    350
    257k
                normalized * gamma.get(i).copied().unwrap_or(1.0) + beta.get(i).copied().unwrap_or(0.0)
    351
    257k
            })
    352
    1.01k
            .collect()
    353
    1.01k
    }
    354
    355
    /// Fused single-pass layer normalization using Welford's algorithm (M20 - IMP-045)
    356
    ///
    357
    /// Computes mean and variance in a single pass, reducing memory bandwidth.
    358
    #[must_use]
    359
    1.01k
    pub fn fused_layernorm(input: &[f32], gamma: &[f32], beta: &[f32], eps: f32) -> Vec<f32> {
    360
    1.01k
        if input.is_empty() {
    361
    1
            return Vec::new();
    362
    1.01k
        }
    363
    364
    1.01k
        let n = input.len();
    365
    366
        // Welford's online algorithm for mean and variance
    367
    1.01k
        let mut mean = 0.0f32;
    368
    1.01k
        let mut m2 = 0.0f32;
    369
    370
    257k
        for (i, &x) in input1.01k.iter1.01k().enumerate1.01k() {
    371
    257k
            let delta = x - mean;
    372
    257k
            mean += delta / (i + 1) as f32;
    373
    257k
            let delta2 = x - mean;
    374
    257k
            m2 += delta * delta2;
    375
    257k
        }
    376
    377
    1.01k
        let variance = m2 / n as f32;
    378
    1.01k
        let std_dev = (variance + eps).sqrt();
    379
    1.01k
        let inv_std = 1.0 / std_dev;
    380
    381
        // Normalize and apply gamma/beta in single pass
    382
    1.01k
        input
    383
    1.01k
            .iter()
    384
    1.01k
            .enumerate()
    385
    257k
            .map1.01k(|(i, &x)| {
    386
    257k
                let normalized = (x - mean) * inv_std;
    387
    257k
                normalized * gamma.get(i).copied().unwrap_or(1.0) + beta.get(i).copied().unwrap_or(0.0)
    388
    257k
            })
    389
    1.01k
            .collect()
    390
    1.01k
    }
    391
    392
    // ============================================================================
    393
    // Phase 14: Quantized Compute Kernels (M23)
    394
    // ============================================================================
    395
    396
    /// Quantized dot product for Q4_0 blocks (M23 - IMP-052)
    397
    ///
    398
    /// Computes dot product directly on Q4_0 quantized data without full dequantization.
    399
    /// Q4_0 format: 2 bytes (f16 scale) + 16 bytes (32 x 4-bit values)
    400
    #[must_use]
    401
    6
    pub fn quantized_dot_q4(block_a: &[u8], block_b: &[u8]) -> f32 {
    402
    6
        if block_a.len() < 18 || block_b.len() < 185 {
    403
    1
            return 0.0;
    404
    5
        }
    405
    406
        // Extract scales (f16 little-endian)
    407
    5
        let scale_a = half::f16::from_le_bytes([block_a[0], block_a[1]]).to_f32();
    408
    5
        let scale_b = half::f16::from_le_bytes([block_b[0], block_b[1]]).to_f32();
    409
    410
        // Accumulate dot product over packed 4-bit values
    411
    5
        let mut acc = 0i32;
    412
    413
    85
        for i80 in 0..16 {
    414
    80
            let byte_a = block_a[2 + i];
    415
    80
            let byte_b = block_b[2 + i];
    416
    80
    417
    80
            // Extract low and high nibbles, center at 8
    418
    80
            let a_lo = (byte_a & 0x0F) as i32 - 8;
    419
    80
            let a_hi = ((byte_a >> 4) & 0x0F) as i32 - 8;
    420
    80
            let b_lo = (byte_b & 0x0F) as i32 - 8;
    421
    80
            let b_hi = ((byte_b >> 4) & 0x0F) as i32 - 8;
    422
    80
    423
    80
            acc += a_lo * b_lo + a_hi * b_hi;
    424
    80
        }
    425
    426
        // Apply combined scale
    427
    5
        (acc as f32) * scale_a * scale_b
    428
    6
    }
    429
    430
    /// Quantized dot product for Q8_0 blocks (M23 - IMP-052)
    431
    ///
    432
    /// Computes dot product directly on Q8_0 quantized data without full dequantization.
    433
    /// Q8_0 format: 2 bytes (f16 scale) + 32 bytes (32 x i8 values)
    434
    #[must_use]
    435
    5
    pub fn quantized_dot_q8(block_a: &[u8], block_b: &[u8]) -> f32 {
    436
    5
        if block_a.len() < 34 || block_b.len() < 344 {
    437
    1
            return 0.0;
    438
    4
        }
    439
    440
        // Extract scales (f16 little-endian)
    441
    4
        let scale_a = half::f16::from_le_bytes([block_a[0], block_a[1]]).to_f32();
    442
    4
        let scale_b = half::f16::from_le_bytes([block_b[0], block_b[1]]).to_f32();
    443
    444
        // Accumulate dot product over i8 values
    445
    4
        let mut acc = 0i32;
    446
    447
    132
        for i128 in 0..32 {
    448
    128
            let a_val = block_a[2 + i] as i8 as i32;
    449
    128
            let b_val = block_b[2 + i] as i8 as i32;
    450
    128
            acc += a_val * b_val;
    451
    128
        }
    452
    453
        // Apply combined scale
    454
    4
        (acc as f32) * scale_a * scale_b
    455
    5
    }
    456
    457
    /// Quantized matrix-vector multiply for Q4_0 weights (M23 - IMP-053)
    458
    ///
    459
    /// Computes y = W @ x where W is Q4_0 quantized without full dequantization.
    460
    /// Each row of W consists of ceil(cols/32) Q4_0 blocks.
    461
    #[must_use]
    462
    4
    pub fn quantized_matvec_q4(weights: &[u8], input: &[f32], rows: usize, cols: usize) -> Vec<f32> {
    463
        const Q4_BLOCK_SIZE: usize = 18; // 2 bytes scale + 16 bytes data
    464
        const Q4_BLOCK_VALUES: usize = 32;
    465
    466
    4
        let blocks_per_row = cols.div_ceil(Q4_BLOCK_VALUES);
    467
    4
        let row_bytes = blocks_per_row * Q4_BLOCK_SIZE;
    468
    469
    4
        let mut output = vec![0.0f32; rows];
    470
    471
    8
        for (row, out_val) in output.iter_mut()4.enumerate4().take4(rows4) {
    472
    8
            let row_offset = row * row_bytes;
    473
    8
            let mut acc = 0.0f32;
    474
    475
    8
            for block_idx in 0..blocks_per_row {
    476
    8
                let block_offset = row_offset + block_idx * Q4_BLOCK_SIZE;
    477
    478
    8
                if block_offset + Q4_BLOCK_SIZE > weights.len() {
    479
    0
                    break;
    480
    8
                }
    481
    482
                // Extract scale
    483
    8
                let scale =
    484
    8
                    half::f16::from_le_bytes([weights[block_offset], weights[block_offset + 1]])
    485
    8
                        .to_f32();
    486
    487
                // Process 32 values in this block
    488
    8
                let input_offset = block_idx * Q4_BLOCK_VALUES;
    489
    490
    136
                for i128 in 0..16 {
    491
    128
                    let byte = weights[block_offset + 2 + i];
    492
    128
                    let val_lo = (byte & 0x0F) as i32 - 8;
    493
    128
                    let val_hi = ((byte >> 4) & 0x0F) as i32 - 8;
    494
    495
    128
                    let in_idx_lo = input_offset + i * 2;
    496
    128
                    let in_idx_hi = input_offset + i * 2 + 1;
    497
    498
    128
                    if in_idx_lo < cols {
    499
    128
                        acc += (val_lo as f32) * scale * input[in_idx_lo];
    500
    128
                    }0
    501
    128
                    if in_idx_hi < cols {
    502
    128
                        acc += (val_hi as f32) * scale * input[in_idx_hi];
    503
    128
                    }0
    504
                }
    505
            }
    506
    507
    8
            *out_val = acc;
    508
        }
    509
    510
    4
        output
    511
    4
    }
    512
    513
    /// Quantized matrix-vector multiply for Q8_0 weights (M23 - IMP-053)
    514
    ///
    515
    /// Computes y = W @ x where W is Q8_0 quantized without full dequantization.
    516
    /// Each row of W consists of ceil(cols/32) Q8_0 blocks.
    517
    #[must_use]
    518
    3
    pub fn quantized_matvec_q8(weights: &[u8], input: &[f32], rows: usize, cols: usize) -> Vec<f32> {
    519
        const Q8_BLOCK_SIZE: usize = 34; // 2 bytes scale + 32 bytes data
    520
        const Q8_BLOCK_VALUES: usize = 32;
    521
    522
    3
        let blocks_per_row = cols.div_ceil(Q8_BLOCK_VALUES);
    523
    3
        let row_bytes = blocks_per_row * Q8_BLOCK_SIZE;
    524
    525
    3
        let mut output = vec![0.0f32; rows];
    526
    527
    8
        for (row, out_val) in output.iter_mut()3.enumerate3().take3(rows3) {
    528
    8
            let row_offset = row * row_bytes;
    529
    8
            let mut acc = 0.0f32;
    530
    531
    8
            for block_idx in 0..blocks_per_row {
    532
    8
                let block_offset = row_offset + block_idx * Q8_BLOCK_SIZE;
    533
    534
    8
                if block_offset + Q8_BLOCK_SIZE > weights.len() {
    535
    0
                    break;
    536
    8
                }
    537
    538
                // Extract scale
    539
    8
                let scale =
    540
    8
                    half::f16::from_le_bytes([weights[block_offset], weights[block_offset + 1]])
    541
    8
                        .to_f32();
    542
    543
                // Process 32 values in this block
    544
    8
                let input_offset = block_idx * Q8_BLOCK_VALUES;
    545
    546
    264
                for i256 in 0..32 {
    547
    256
                    let val = weights[block_offset + 2 + i] as i8 as i32;
    548
    256
                    let in_idx = input_offset + i;
    549
    550
    256
                    if in_idx < cols {
    551
    256
                        acc += (val as f32) * scale * input[in_idx];
    552
    256
                    }0
    553
                }
    554
            }
    555
    556
    8
            *out_val = acc;
    557
        }
    558
    559
    3
        output
    560
    3
    }
    561
    562
    /// Mixed precision accumulator for quantized computations (M23 - IMP-054)
    563
    ///
    564
    /// Accumulates values in f32 precision while processing quantized data,
    565
    /// ensuring numerical accuracy during block-wise operations.
    566
    #[derive(Debug, Clone, Default)]
    567
    pub struct QuantizedAccumulator {
    568
        /// Running sum in f32 precision
    569
        sum: f32,
    570
    }
    571
    572
    impl QuantizedAccumulator {
    573
        /// Create a new zeroed accumulator
    574
        #[must_use]
    575
    7
        pub fn new() -> Self {
    576
    7
            Self { sum: 0.0 }
    577
    7
        }
    578
    579
        /// Get the current accumulated sum
    580
        #[must_use]
    581
    13
        pub fn sum(&self) -> f32 {
    582
    13
            self.sum
    583
    13
        }
    584
    585
        /// Reset the accumulator to zero
    586
    4
        pub fn reset(&mut self) {
    587
    4
            self.sum = 0.0;
    588
    4
        }
    589
    590
        /// Add a scaled value to the accumulator
    591
        #[inline]
    592
    7
        pub fn add_scaled(&mut self, value: f32, scale: f32) {
    593
    7
            self.sum += value * scale;
    594
    7
        }
    595
    596
        /// Add a block contribution (block_sum * block_scale)
    597
        #[inline]
    598
    14
        pub fn add_block(&mut self, block_sum: f32, block_scale: f32) {
    599
    14
            self.sum += block_sum * block_scale;
    600
    14
        }
    601
    }
    602
    603
    // =============================================================================
    604
    // M24: Streaming & Pipelining (Phase 15)
    605
    // =============================================================================
    606
    607
    /// Double buffer for overlapping compute with memory operations (M24 - IMP-055)
    608
    ///
    609
    /// Enables loading next layer weights while computing current layer.
    610
    /// Front buffer is read-only for compute, back buffer is writable for loading.
    611
    #[derive(Debug)]
    612
    pub struct DoubleBuffer<T> {
    613
        front: Vec<T>,
    614
        back: Vec<T>,
    615
    }
    616
    617
    impl<T: Default + Clone> DoubleBuffer<T> {
    618
        /// Create a new double buffer with given capacity
    619
        #[must_use]
    620
    7
        pub fn new(capacity: usize) -> Self {
    621
    7
            Self {
    622
    7
                front: vec![T::default(); capacity],
    623
    7
                back: vec![T::default(); capacity],
    624
    7
            }
    625
    7
        }
    626
    627
        /// Get the capacity of each buffer
    628
        #[must_use]
    629
    3
        pub fn capacity(&self) -> usize {
    630
    3
            self.front.len()
    631
    3
        }
    632
    633
        /// Get immutable reference to front buffer (for reading/compute)
    634
        #[must_use]
    635
    9
        pub fn front(&self) -> &[T] {
    636
    9
            &self.front
    637
    9
        }
    638
    639
        /// Get mutable reference to back buffer (for writing/loading)
    640
    6
        pub fn back_mut(&mut self) -> &mut [T] {
    641
    6
            &mut self.back
    642
    6
        }
    643
    644
        /// Swap front and back buffers
    645
    6
        pub fn swap(&mut self) {
    646
    6
            std::mem::swap(&mut self.front, &mut self.back);
    647
    6
        }
    648
    }
    649
    650
    /// Chunked token processor for improved cache utilization (M24 - IMP-056)
    651
    ///
    652
    /// Processes tokens in configurable chunks to improve memory locality
    653
    /// and cache efficiency during batch processing.
    654
    #[derive(Debug, Clone)]
    655
    pub struct ChunkedProcessor {
    656
        chunk_size: usize,
    657
    }
    658
    659
    impl ChunkedProcessor {
    660
        /// Create a new chunked processor with given chunk size
    661
        #[must_use]
    662
    9
        pub fn new(chunk_size: usize) -> Self {
    663
    9
            Self { chunk_size }
    664
    9
        }
    665
    666
        /// Get the chunk size
    667
        #[must_use]
    668
    2
        pub fn chunk_size(&self) -> usize {
    669
    2
            self.chunk_size
    670
    2
        }
    671
    672
        /// Calculate number of chunks needed for given input length
    673
        #[must_use]
    674
    15
        pub fn num_chunks(&self, total_len: usize) -> usize {
    675
    15
            if total_len == 0 {
    676
    4
                return 0;
    677
    11
            }
    678
    11
            total_len.div_ceil(self.chunk_size)
    679
    15
        }
    680
    681
        /// Get bounds (start, end) for a specific chunk index
    682
        #[must_use]
    683
    14
        pub fn chunk_bounds(&self, chunk_idx: usize, total_len: usize) -> (usize, usize) {
    684
    14
            let start = chunk_idx * self.chunk_size;
    685
    14
            let end = (start + self.chunk_size).min(total_len);
    686
    14
            (start, end)
    687
    14
        }
    688
    689
        /// Process data in chunks, accumulating results
    690
    4
        pub fn process_chunks<T, F>(&self, data: &[T], mut process: F) -> f32
    691
    4
        where
    692
    4
            F: FnMut(&[T]) -> f32,
    693
        {
    694
    4
            let mut total = 0.0f32;
    695
    4
            let num_chunks = self.num_chunks(data.len());
    696
    697
    6
            for chunk_idx in 0..num_chunks4 {
    698
    6
                let (start, end) = self.chunk_bounds(chunk_idx, data.len());
    699
    6
                total += process(&data[start..end]);
    700
    6
            }
    701
    702
    4
            total
    703
    4
        }
    704
    }
    705
    706
    /// Inference pipeline stages (M24 - IMP-057)
    707
    ///
    708
    /// Represents the different stages of transformer inference.
    709
    #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
    710
    #[repr(u8)]
    711
    pub enum GpuPipelineStage {
    712
        /// Token embedding lookup
    713
        Embed = 0,
    714
        /// Self-attention computation
    715
        Attention = 1,
    716
        /// Feed-forward network
    717
        FFN = 2,
    718
        /// Output projection and sampling
    719
        Output = 3,
    720
    }
    721
    722
    /// Inference pipeline coordinator (M24 - IMP-057)
    723
    ///
    724
    /// Manages multi-stage inference pipeline with timing tracking.
    725
    #[derive(Debug)]
    726
    pub struct InferencePipeline {
    727
        num_stages: usize,
    728
        stage_times: std::collections::HashMap<GpuPipelineStage, f32>,
    729
    }
    730
    731
    impl InferencePipeline {
    732
        /// Create a new pipeline with given number of stages
    733
        #[must_use]
    734
    6
        pub fn new(num_stages: usize) -> Self {
    735
    6
            Self {
    736
    6
                num_stages,
    737
    6
                stage_times: std::collections::HashMap::new(),
    738
    6
            }
    739
    6
        }
    740
    741
        /// Get number of stages in the pipeline
    742
        #[must_use]
    743
    3
        pub fn num_stages(&self) -> usize {
    744
    3
            self.num_stages
    745
    3
        }
    746
    747
        /// Record timing for a pipeline stage (in milliseconds)
    748
    13
        pub fn record_stage_time(&mut self, stage: GpuPipelineStage, time_ms: f32) {
    749
    13
            self.stage_times.insert(stage, time_ms);
    750
    13
        }
    751
    752
        /// Get total pipeline latency (sum of all stage times)
    753
        #[must_use]
    754
    8
        pub fn total_latency(&self) -> f32 {
    755
    8
            self.stage_times.values().sum()
    756
    8
        }
    757
    758
        /// Get breakdown of stage timings
    759
        #[must_use]
    760
    3
        pub fn stage_breakdown(&self) -> &std::collections::HashMap<GpuPipelineStage, f32> {
    761
    3
            &self.stage_times
    762
    3
        }
    763
    764
        /// Reset pipeline for new forward pass
    765
    3
        pub fn reset(&mut self) {
    766
    3
            self.stage_times.clear();
    767
    3
        }
    768
    }
    769
    // M29: Error Recovery & Graceful Degradation (Phase 20)
    770
    // ============================================================================
    771
    772
    /// Error classification for recovery decisions
    773
    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
    774
    pub enum ErrorClassification {
    775
        /// Transient error - may succeed on retry
    776
        Transient,
    777
        /// Fatal error - should not retry
    778
        Fatal,
    779
        /// GPU-specific error - may fallback to CPU
    780
        GpuFailure,
    781
    }
    782
    783
    /// Recovery action to take
    784
    #[derive(Debug, Clone)]
    785
    pub enum RecoveryAction {
    786
        /// Retry the operation with a delay
    787
        Retry {
    788
            /// Delay before retry
    789
            delay: Duration,
    790
        },
    791
        /// Fallback to CPU inference
    792
        FallbackToCpu,
    793
        /// Give up and propagate error
    794
        Fail,
    795
    }
    796
    797
    /// Error recovery strategy with exponential backoff
    798
    pub struct ErrorRecoveryStrategy {
    799
        max_retries: u32,
    800
        base_delay: Duration,
    801
        max_delay: Duration,
    802
        jitter: f64,
    803
    }
    804
    805
    impl ErrorRecoveryStrategy {
    806
        /// Create new error recovery strategy
    807
        #[must_use]
    808
    1
        pub fn new() -> Self {
    809
    1
            Self {
    810
    1
                max_retries: 3,
    811
    1
                base_delay: Duration::from_millis(100),
    812
    1
                max_delay: Duration::from_secs(10),
    813
    1
                jitter: 0.1,
    814
    1
            }
    815
    1
        }
    816
    817
        /// Set maximum retries
    818
        #[must_use]
    819
    1
        pub fn with_max_retries(mut self, max_retries: u32) -> Self {
    820
    1
            self.max_retries = max_retries;
    821
    1
            self
    822
    1
        }
    823
    824
        /// Set base delay
    825
        #[must_use]
    826
    1
        pub fn with_base_delay(mut self, base_delay: Duration) -> Self {
    827
    1
            self.base_delay = base_delay;
    828
    1
            self
    829
    1
        }
    830
    831
        /// Set maximum delay
    832
        #[must_use]
    833
    1
        pub fn with_max_delay(mut self, max_delay: Duration) -> Self {
    834
    1
            self.max_delay = max_delay;
    835
    1
            self
    836
    1
        }
    837
    838
        /// Set jitter factor (0.0 - 1.0)
    839
        #[must_use]
    840
    1
        pub fn with_jitter(mut self, jitter: f64) -> Self {
    841
    1
            self.jitter = jitter.clamp(0.0, 1.0);
    842
    1
            self
    843
    1
        }
    844
    845
        /// Get max retries
    846
        #[must_use]
    847
    1
        pub fn max_retries(&self) -> u32 {
    848
    1
            self.max_retries
    849
    1
        }
    850
    851
        /// Classify an error
    852
        #[must_use]
    853
    3
        pub fn classify_error(&self, error: &std::io::Error) -> ErrorClassification {
    854
    3
            match error.kind() {
    855
                std::io::ErrorKind::TimedOut
    856
                | std::io::ErrorKind::ConnectionReset
    857
                | std::io::ErrorKind::ConnectionAborted
    858
                | std::io::ErrorKind::Interrupted
    859
    2
                | std::io::ErrorKind::WouldBlock => ErrorClassification::Transient,
    860
    861
                std::io::ErrorKind::Other => {
    862
    0
                    let msg = error.to_string().to_lowercase();
    863
    0
                    if msg.contains("gpu") || msg.contains("cuda") || msg.contains("wgpu") {
    864
    0
                        ErrorClassification::GpuFailure
    865
                    } else {
    866
    0
                        ErrorClassification::Transient
    867
                    }
    868
                },
    869
    870
    1
                _ => ErrorClassification::Fatal,
    871
            }
    872
    3
        }
    873
    874
        /// Determine recovery action
    875
        #[must_use]
    876
    2
        pub fn determine_action(&self, error: &std::io::Error, attempt: u32) -> RecoveryAction {
    877
    2
            if attempt >= self.max_retries {
    878
    1
                return RecoveryAction::Fail;
    879
    1
            }
    880
    881
    1
            match self.classify_error(error) {
    882
    1
                ErrorClassification::Transient => RecoveryAction::Retry {
    883
    1
                    delay: self.calculate_delay(attempt),
    884
    1
                },
    885
    0
                ErrorClassification::GpuFailure => RecoveryAction::FallbackToCpu,
    886
    0
                ErrorClassification::Fatal => RecoveryAction::Fail,
    887
            }
    888
    2
        }
    889
    890
        /// Determine action with explicit GPU fallback
    891
        #[must_use]
    892
    1
        pub fn determine_action_with_fallback(
    893
    1
            &self,
    894
    1
            error: &std::io::Error,
    895
    1
            attempt: u32,
    896
    1
        ) -> RecoveryAction {
    897
    1
            let msg = error.to_string().to_lowercase();
    898
    1
            if msg.contains("gpu") || msg.contains("unavailable")0 {
    899
    1
                RecoveryAction::FallbackToCpu
    900
            } else {
    901
    0
                self.determine_action(error, attempt)
    902
            }
    903
    1
        }
    904
    905
        /// Calculate delay for retry attempt with exponential backoff
    906
        #[must_use]
    907
    4
        pub fn calculate_delay(&self, attempt: u32) -> Duration {
    908
    4
            let base_ms = self.base_delay.as_millis() as f64;
    909
    4
            let exp_delay = base_ms * 2.0_f64.powi(attempt as i32);
    910
    4
            let capped_delay = exp_delay.min(self.max_delay.as_millis() as f64);
    911
    912
            // Add jitter
    913
    4
            let jitter_range = capped_delay * self.jitter;
    914
    4
            let jittered = capped_delay + (jitter_range * 0.5); // Simplified jitter
    915
    916
    4
            Duration::from_millis(jittered as u64)
    917
    4
        }
    918
    }
    919
    920
    impl Default for ErrorRecoveryStrategy {
    921
    0
        fn default() -> Self {
    922
    0
            Self::new()
    923
    0
        }
    924
    }
    925
    926
    /// Degradation mode for system state
    927
    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
    928
    pub enum DegradationMode {
    929
        /// Normal operation
    930
        Normal,
    931
        /// Running on CPU fallback
    932
        CpuFallback,
    933
        /// Memory pressure - reduced capacity
    934
        MemoryPressure,
    935
        /// Low latency priority mode
    936
        LowLatency,
    937
        /// High throughput priority mode
    938
        HighThroughput,
    939
    }
    940
    941
    /// System load metrics
    942
    #[derive(Debug, Clone, Copy)]
    943
    pub struct SystemLoad {
    944
        /// CPU utilization percentage
    945
        pub cpu_percent: f64,
    946
        /// Memory utilization percentage
    947
        pub memory_percent: f64,
    948
        /// Current queue depth
    949
        pub queue_depth: u32,
    950
    }
    951
    952
    /// Graceful degradation manager
    953
    pub struct DegradationManager {
    954
        gpu_available: bool,
    955
        memory_pressure: f64,
    956
        system_load: Option<SystemLoad>,
    957
        latency_priority: bool,
    958
        mode: DegradationMode,
    959
    }
    960
    961
    impl DegradationManager {
    962
        /// Create new degradation manager
    963
        #[must_use]
    964
    1
        pub fn new() -> Self {
    965
    1
            Self {
    966
    1
                gpu_available: true,
    967
    1
                memory_pressure: 0.0,
    968
    1
                system_load: None,
    969
    1
                latency_priority: false,
    970
    1
                mode: DegradationMode::Normal,
    971
    1
            }
    972
    1
        }
    973
    974
        /// Get current degradation mode
    975
        #[must_use]
    976
    4
        pub fn current_mode(&self) -> DegradationMode {
    977
    4
            self.mode
    978
    4
        }
    979
    980
        /// Set GPU availability
    981
    3
        pub fn set_gpu_available(&mut self, available: bool) {
    982
    3
            self.gpu_available = available;
    983
    3
            self.update_mode();
    984
    3
        }
    985
    986
        /// Update memory pressure (0.0 - 1.0)
    987
    2
        pub fn update_memory_pressure(&mut self, pressure: f64) {
    988
    2
            self.memory_pressure = pressure.clamp(0.0, 1.0);
    989
    2
            self.update_mode();
    990
    2
        }
    991
    992
        /// Update system load
    993
    2
        pub fn update_system_load(&mut self, load: SystemLoad) {
    994
    2
            self.system_load = Some(load);
    995
    2
            self.update_mode();
    996
    2
        }
    997
    998
        /// Set latency priority mode
    999
    2
        pub fn set_latency_priority(&mut self, priority: bool) {
    1000
    2
            self.latency_priority = priority;
    1001
    2
            self.update_mode();
    1002
    2
        }
    1003
    1004
        /// Get recommended batch size based on system state
    1005
        #[must_use]
    1006
    1
        pub fn recommended_batch_size(&self, requested: usize) -> usize {
    1007
    1
            if self.memory_pressure > 0.8 {
    1008
                // Reduce batch size under memory pressure
    1009
    1
                (requested as f64 * (1.0 - self.memory_pressure)).max(1.0) as usize
    1010
            } else {
    1011
    0
                requested
    1012
            }
    1013
    1
        }
    1014
    1015
        /// Get recommended max context length based on system state
    1016
        #[must_use]
    1017
    1
        pub fn recommended_max_context(&self, requested: usize) -> usize {
    1018
    1
            if let Some(load) = &self.system_load {
    1019
    1
                if load.cpu_percent > 90.0 || load.memory_percent > 80.00 || load.queue_depth > 500 {
    1020
                    // Reduce context length under high load
    1021
    1
                    (requested as f64 * 0.75).max(256.0) as usize
    1022
                } else {
    1023
    0
                    requested
    1024
                }
    1025
            } else {
    1026
    0
                requested
    1027
            }
    1028
    1
        }
    1029
    1030
    9
        fn update_mode(&mut self) {
    1031
    9
            self.mode = if !self.gpu_available {
    1032
    1
                DegradationMode::CpuFallback
    1033
    8
            } else if self.latency_priority {
    1034
    3
                DegradationMode::LowLatency
    1035
    5
            } else if self.memory_pressure > 0.8 {
    1036
    2
                DegradationMode::MemoryPressure
    1037
    3
            } else if let Some(load2) = &self.system_load {
    1038
    2
                if load.cpu_percent > 90.0 || load.memory_percent > 80.01 {
    1039
    1
                    DegradationMode::MemoryPressure
    1040
                } else {
    1041
    1
                    DegradationMode::Normal
    1042
                }
    1043
            } else {
    1044
    1
                DegradationMode::Normal
    1045
            };
    1046
    9
        }
    1047
    }
    1048
    1049
    impl Default for DegradationManager {
    1050
    0
        fn default() -> Self {
    1051
    0
            Self::new()
    1052
    0
        }
    1053
    }
    1054
    1055
    /// Request outcome for failure tracking
    1056
    #[derive(Debug, Clone)]
    1057
    pub enum RequestOutcome {
    1058
        /// Request completed successfully
    1059
        Success,
    1060
        /// Request failed with error message
    1061
        Failed(String),
    1062
    }
    1063
    1064
    /// Failure isolator with circuit breaker
    1065
    pub struct FailureIsolator {
    1066
        active_requests: std::sync::atomic::AtomicU64,
    1067
        success_count: std::sync::atomic::AtomicU64,
    1068
        failure_count: std::sync::atomic::AtomicU64,
    1069
        consecutive_failures: std::sync::atomic::AtomicU32,
    1070
        circuit_open: std::sync::atomic::AtomicBool,
    1071
        next_request_id: std::sync::atomic::AtomicU64,
    1072
        failure_threshold: u32,
    1073
        cleanups: std::sync::Mutex<HashMap<u64, Box<dyn FnOnce() + Send>>>,
    1074
    }
    1075
    1076
    impl FailureIsolator {
    1077
        /// Create new failure isolator
    1078
        #[must_use]
    1079
    1
        pub fn new() -> Self {
    1080
    1
            Self {
    1081
    1
                active_requests: std::sync::atomic::AtomicU64::new(0),
    1082
    1
                success_count: std::sync::atomic::AtomicU64::new(0),
    1083
    1
                failure_count: std::sync::atomic::AtomicU64::new(0),
    1084
    1
                consecutive_failures: std::sync::atomic::AtomicU32::new(0),
    1085
    1
                circuit_open: std::sync::atomic::AtomicBool::new(false),
    1086
    1
                next_request_id: std::sync::atomic::AtomicU64::new(0),
    1087
    1
                failure_threshold: 5,
    1088
    1
                cleanups: std::sync::Mutex::new(HashMap::new()),
    1089
    1
            }
    1090
    1
        }
    1091
    1092
        /// Get number of active requests
    1093
        #[must_use]
    1094
    3
        pub fn active_requests(&self) -> u64 {
    1095
    3
            self.active_requests
    1096
    3
                .load(std::sync::atomic::Ordering::SeqCst)
    1097
    3
        }
    1098
    1099
        /// Get success count
    1100
        #[must_use]
    1101
    1
        pub fn success_count(&self) -> u64 {
    1102
    1
            self.success_count.load(std::sync::atomic::Ordering::SeqCst)
    1103
    1
        }
    1104
    1105
        /// Get failure count
    1106
        #[must_use]
    1107
    1
        pub fn failure_count(&self) -> u64 {
    1108
    1
            self.failure_count.load(std::sync::atomic::Ordering::SeqCst)
    1109
    1
        }
    1110
    1111
        /// Check if circuit is open
    1112
        #[must_use]
    1113
    4
        pub fn is_circuit_open(&self) -> bool {
    1114
    4
            self.circuit_open.load(std::sync::atomic::Ordering::SeqCst)
    1115
    4
        }
    1116
    1117
        /// Start a new isolated request
    1118
        #[must_use]
    1119
    8
        pub fn start_request(&self) -> u64 {
    1120
    8
            self.active_requests
    1121
    8
                .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
    1122
    8
            self.next_request_id
    1123
    8
                .fetch_add(1, std::sync::atomic::Ordering::SeqCst)
    1124
    8
        }
    1125
    1126
        /// Try to start a request (fails if circuit is open)
    1127
        ///
    1128
        /// # Errors
    1129
        /// Returns error if circuit breaker is open.
    1130
    2
        pub fn try_start_request(&self) -> std::result::Result<u64, &'static str> {
    1131
    2
            if self.is_circuit_open() {
    1132
    1
                Err("Circuit breaker is open")
    1133
            } else {
    1134
    1
                Ok(self.start_request())
    1135
            }
    1136
    2
        }
    1137
    1138
        /// Register cleanup handler for a request
    1139
    1
        pub fn register_cleanup<F: FnOnce() + Send + 'static>(&self, request_id: u64, cleanup: F) {
    1140
    1
            if let Ok(mut cleanups) = self.cleanups.lock() {
    1141
    1
                cleanups.insert(request_id, Box::new(cleanup));
    1142
    1
            }0
    1143
    1
        }
    1144
    1145
        /// Complete a request with outcome
    1146
    7
        pub fn complete_request(&self, request_id: u64, outcome: &RequestOutcome) {
    1147
    7
            self.active_requests
    1148
    7
                .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
    1149
    1150
    7
            match outcome {
    1151
    1
                RequestOutcome::Success => {
    1152
    1
                    self.success_count
    1153
    1
                        .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
    1154
    1
                    self.consecutive_failures
    1155
    1
                        .store(0, std::sync::atomic::Ordering::SeqCst);
    1156
    1
                },
    1157
                RequestOutcome::Failed(_) => {
    1158
    6
                    self.failure_count
    1159
    6
                        .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
    1160
    6
                    let failures = self
    1161
    6
                        .consecutive_failures
    1162
    6
                        .fetch_add(1, std::sync::atomic::Ordering::SeqCst)
    1163
    6
                        + 1;
    1164
    1165
                    // Open circuit if threshold exceeded
    1166
    6
                    if failures >= self.failure_threshold {
    1167
    2
                        self.circuit_open
    1168
    2
                            .store(true, std::sync::atomic::Ordering::SeqCst);
    1169
    4
                    }
    1170
    1171
                    // Run cleanup handler
    1172
    6
                    if let Ok(mut cleanups) = self.cleanups.lock() {
    1173
    6
                        if let Some(cleanup1) = cleanups.remove(&request_id) {
    1174
    1
                            cleanup();
    1175
    5
                        }
    1176
    0
                    }
    1177
                },
    1178
            }
    1179
    7
        }
    1180
    1181
        /// Reset circuit breaker
    1182
    1
        pub fn reset_circuit(&self) {
    1183
    1
            self.circuit_open
    1184
    1
                .store(false, std::sync::atomic::Ordering::SeqCst);
    1185
    1
            self.consecutive_failures
    1186
    1
                .store(0, std::sync::atomic::Ordering::SeqCst);
    1187
    1
        }
    1188
    }
    1189
    1190
    impl Default for FailureIsolator {
    1191
    0
        fn default() -> Self {
    1192
    0
            Self::new()
    1193
    0
        }
    1194
    }
    1195
    1196
    /// Isolated request handle (unused but kept for API completeness)
    1197
    #[allow(dead_code)]
    1198
    pub struct IsolatedRequest {
    1199
        id: u64,
    1200
    }
    1201
    1202
    // ============================================================================
    1203
    // M30: Connection Pooling & Resource Limits (IMP-073, IMP-074, IMP-075)
    1204
    // ============================================================================
    1205
    1206
    /// Connection pool configuration (IMP-073)
    1207
    #[derive(Debug, Clone)]
    1208
    pub struct ConnectionConfig {
    1209
        max_connections: usize,
    1210
        min_connections: usize,
    1211
        idle_timeout: Duration,
    1212
    }
    1213
    1214
    impl ConnectionConfig {
    1215
        /// Create new connection config with defaults
    1216
        #[must_use]
    1217
    2
        pub fn new() -> Self {
    1218
    2
            Self {
    1219
    2
                max_connections: 10,
    1220
    2
                min_connections: 1,
    1221
    2
                idle_timeout: Duration::from_secs(300),
    1222
    2
            }
    1223
    2
        }
    1224
    1225
        /// Set maximum connections
    1226
        #[must_use]
    1227
    1
        pub fn with_max_connections(mut self, max: usize) -> Self {
    1228
    1
            self.max_connections = max;
    1229
    1
            self
    1230
    1
        }
    1231
    1232
        /// Set minimum connections
    1233
        #[must_use]
    1234
    2
        pub fn with_min_connections(mut self, min: usize) -> Self {
    1235
    2
            self.min_connections = min;
    1236
    2
            self
    1237
    2
        }
    1238
    1239
        /// Set idle timeout
    1240
        #[must_use]
    1241
    1
        pub fn with_idle_timeout(mut self, timeout: Duration) -> Self {
    1242
    1
            self.idle_timeout = timeout;
    1243
    1
            self
    1244
    1
        }
    1245
    }
    1246
    1247
    impl Default for ConnectionConfig {
    1248
    0
        fn default() -> Self {
    1249
    0
            Self::new()
    1250
    0
        }
    1251
    }
    1252
    1253
    /// Connection state for health checking
    1254
    #[derive(Debug, Clone, PartialEq, Eq)]
    1255
    pub enum ConnectionState {
    1256
        /// Connection is healthy
    1257
        Healthy,
    1258
        /// Connection is stale and needs recycling
    1259
        Stale,
    1260
        /// Connection is broken
    1261
        Broken,
    1262
    }
    1263
    1264
    /// Connection handle
    1265
    #[derive(Debug)]
    1266
    pub struct Connection {
    1267
        #[allow(dead_code)]
    1268
        id: u64,
    1269
        created_at: std::time::Instant,
    1270
    }
    1271
    1272
    /// Connection pool with bounded capacity (IMP-073)
    1273
    pub struct ConnectionPool {
    1274
        config: ConnectionConfig,
    1275
        active: std::sync::atomic::AtomicUsize,
    1276
        idle: std::sync::Mutex<Vec<Connection>>,
    1277
        next_id: std::sync::atomic::AtomicU64,
    1278
    }
    1279
    1280
    impl ConnectionPool {
    1281
        /// Create new connection pool
    1282
        #[must_use]
    1283
    2
        pub fn new(config: ConnectionConfig) -> Self {
    1284
    2
            Self {
    1285
    2
                config,
    1286
    2
                active: std::sync::atomic::AtomicUsize::new(0),
    1287
    2
                idle: std::sync::Mutex::new(Vec::new()),
    1288
    2
                next_id: std::sync::atomic::AtomicU64::new(0),
    1289
    2
            }
    1290
    2
        }
    1291
    1292
        /// Get max connections
    1293
        #[must_use]
    1294
    1
        pub fn max_connections(&self) -> usize {
    1295
    1
            self.config.max_connections
    1296
    1
        }
    1297
    1298
        /// Get min connections
    1299
        #[must_use]
    1300
    1
        pub fn min_connections(&self) -> usize {
    1301
    1
            self.config.min_connections
    1302
    1
        }
    1303
    1304
        /// Get active connection count
    1305
        #[must_use]
    1306
    2
        pub fn active_connections(&self) -> usize {
    1307
    2
            self.active.load(std::sync::atomic::Ordering::SeqCst)
    1308
    2
        }
    1309
    1310
        /// Get idle connection count
    1311
        #[must_use]
    1312
    2
        pub fn idle_connections(&self) -> usize {
    1313
    2
            self.idle.lock().expect("mutex poisoned").len()
    1314
    2
        }
    1315
    1316
        /// Acquire a connection (blocking)
    1317
        ///
    1318
        /// # Errors
    1319
        /// Returns error if pool is exhausted.
    1320
    13
        pub fn acquire(&self) -> std::result::Result<Connection, &'static str> {
    1321
            // Try to get from idle pool first
    1322
            {
    1323
    13
                let mut idle = self.idle.lock().expect("mutex poisoned");
    1324
    13
                if let Some(conn2) = idle.pop() {
    1325
    2
                    self.active
    1326
    2
                        .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
    1327
    2
                    return Ok(conn);
    1328
    11
                }
    1329
            }
    1330
    1331
            // Create new if under limit
    1332
    11
            let current = self.active.load(std::sync::atomic::Ordering::SeqCst);
    1333
    11
            if current < self.config.max_connections {
    1334
    10
                self.active
    1335
    10
                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
    1336
    10
                let id = self
    1337
    10
                    .next_id
    1338
    10
                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
    1339
    10
                return Ok(Connection {
    1340
    10
                    id,
    1341
    10
                    created_at: std::time::Instant::now(),
    1342
    10
                });
    1343
    1
            }
    1344
    1345
    1
            Err("Pool exhausted")
    1346
    13
        }
    1347
    1348
        /// Try to acquire a connection (non-blocking)
    1349
        ///
    1350
        /// # Errors
    1351
        /// Returns error if pool is exhausted.
    1352
    1
        pub fn try_acquire(&self) -> std::result::Result<Connection, &'static str> {
    1353
    1
            self.acquire()
    1354
    1
        }
    1355
    1356
        /// Release a connection back to pool
    1357
    12
        pub fn release(&self, conn: Connection) {
    1358
    12
            self.active
    1359
    12
                .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
    1360
    12
            let mut idle = self.idle.lock().expect("mutex poisoned");
    1361
    12
            idle.push(conn);
    1362
    12
        }
    1363
    1364
        /// Check connection health
    1365
        #[must_use]
    1366
    1
        pub fn check_health(&self, conn: &Connection) -> ConnectionState {
    1367
    1
            let age = conn.created_at.elapsed();
    1368
    1
            if age > self.config.idle_timeout {
    1369
    0
                ConnectionState::Stale
    1370
            } else {
    1371
    1
                ConnectionState::Healthy
    1372
            }
    1373
    1
        }
    1374
    1375
        /// Warm the pool to min_connections
    1376
    1
        pub fn warm(&self) {
    1377
    1
            let current_idle = self.idle_connections();
    1378
    1
            let need = self.config.min_connections.saturating_sub(current_idle);
    1379
    1380
    1
            let mut idle = self.idle.lock().expect("mutex poisoned");
    1381
    3
            for _ in 0..need1 {
    1382
    3
                let id = self
    1383
    3
                    .next_id
    1384
    3
                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
    1385
    3
                idle.push(Connection {
    1386
    3
                    id,
    1387
    3
                    created_at: std::time::Instant::now(),
    1388
    3
                });
    1389
    3
            }
    1390
    1
        }
    1391
    }
    1392
    1393
    /// Resource configuration (IMP-074)
    1394
    #[derive(Debug, Clone)]
    1395
    #[allow(clippy::struct_field_names)]
    1396
    pub struct ResourceConfig {
    1397
        max_memory_per_request: u64,
    1398
        max_total_memory: u64,
    1399
        max_compute_time: Duration,
    1400
        max_queue_depth: usize,
    1401
    }
    1402
    1403
    impl ResourceConfig {
    1404
        /// Create new resource config with defaults
    1405
        #[must_use]
    1406
    1
        pub fn new() -> Self {
    1407
    1
            Self {
    1408
    1
                max_memory_per_request: 512 * 1024 * 1024, // 512MB
    1409
    1
                max_total_memory: 4 * 1024 * 1024 * 1024,  // 4GB
    1410
    1
                max_compute_time: Duration::from_secs(30),
    1411
    1
                max_queue_depth: 100,
    1412
    1
            }
    1413
    1
        }
    1414
    1415
        /// Set max memory per request
    1416
        #[must_use]
    1417
    1
        pub fn with_max_memory_per_request(mut self, bytes: u64) -> Self {
    1418
    1
            self.max_memory_per_request = bytes;
    1419
    1
            self
    1420
    1
        }
    1421
    1422
        /// Set max total memory
    1423
        #[must_use]
    1424
    1
        pub fn with_max_total_memory(mut self, bytes: u64) -> Self {
    1425
    1
            self.max_total_memory = bytes;
    1426
    1
            self
    1427
    1
        }
    1428
    1429
        /// Set max compute time
    1430
        #[must_use]
    1431
    1
        pub fn with_max_compute_time(mut self, time: Duration) -> Self {
    1432
    1
            self.max_compute_time = time;
    1433
    1
            self
    1434
    1
        }
    1435
    1436
        /// Set max queue depth
    1437
        #[must_use]
    1438
    1
        pub fn with_max_queue_depth(mut self, depth: usize) -> Self {
    1439
    1
            self.max_queue_depth = depth;
    1440
    1
            self
    1441
    1
        }
    1442
    }
    1443
    1444
    impl Default for ResourceConfig {
    1445
    0
        fn default() -> Self {
    1446
    0
            Self::new()
    1447
    0
        }
    1448
    }
    1449
    1450
    /// Result of limit check
    1451
    #[derive(Debug, Clone)]
    1452
    pub enum LimitResult {
    1453
        /// Request is allowed
    1454
        Allowed,
    1455
        /// Request is denied with reason
    1456
        Denied {
    1457
            /// Reason for denial
    1458
            reason: String,
    1459
        },
    1460
        /// Backpressure should be applied
    1461
        Backpressure,
    1462
    }
    1463
    1464
    /// Resource limiter (IMP-074)
    1465
    pub struct ResourceLimiter {
    1466
        config: ResourceConfig,
    1467
        current_memory: std::sync::atomic::AtomicU64,
    1468
        queue_depth: std::sync::atomic::AtomicUsize,
    1469
    }
    1470
    1471
    impl ResourceLimiter {
    1472
        /// Create new resource limiter
    1473
        #[must_use]
    1474
    1
        pub fn new(config: ResourceConfig) -> Self {
    1475
    1
            Self {
    1476
    1
                config,
    1477
    1
                current_memory: std::sync::atomic::AtomicU64::new(0),
    1478
    1
                queue_depth: std::sync::atomic::AtomicUsize::new(0),
    1479
    1
            }
    1480
    1
        }
    1481
    1482
        /// Check if memory request is within limits
    1483
        #[must_use]
    1484
    3
        pub fn check_memory(&self, bytes: u64) -> LimitResult {
    1485
    3
            if bytes > self.config.max_memory_per_request {
    1486
    1
                return LimitResult::Denied {
    1487
    1
                    reason: format!(
    1488
    1
                        "Request {} bytes exceeds per-request limit {} bytes",
    1489
    1
                        bytes, self.config.max_memory_per_request
    1490
    1
                    ),
    1491
    1
                };
    1492
    2
            }
    1493
    1494
    2
            let current = self
    1495
    2
                .current_memory
    1496
    2
                .load(std::sync::atomic::Ordering::SeqCst);
    1497
    2
            if current + bytes > self.config.max_total_memory {
    1498
    0
                return LimitResult::Denied {
    1499
    0
                    reason: format!(
    1500
    0
                        "Total memory {} + {} would exceed limit {}",
    1501
    0
                        current, bytes, self.config.max_total_memory
    1502
    0
                    ),
    1503
    0
                };
    1504
    2
            }
    1505
    1506
    2
            LimitResult::Allowed
    1507
    3
        }
    1508
    1509
        /// Allocate memory
    1510
        ///
    1511
        /// # Errors
    1512
        /// Returns error if memory limit exceeded.
    1513
    1
        pub fn allocate(&self, bytes: u64) -> std::result::Result<(), &'static str> {
    1514
    1
            if let LimitResult::Denied { .. } = self.check_memory(bytes) {
    1515
    0
                return Err("Memory limit exceeded");
    1516
    1
            }
    1517
    1
            self.current_memory
    1518
    1
                .fetch_add(bytes, std::sync::atomic::Ordering::SeqCst);
    1519
    1
            Ok(())
    1520
    1
        }
    1521
    1522
        /// Deallocate memory
    1523
    1
        pub fn deallocate(&self, bytes: u64) {
    1524
    1
            self.current_memory
    1525
    1
                .fetch_sub(bytes, std::sync::atomic::Ordering::SeqCst);
    1526
    1
        }
    1527
    1528
        /// Get current memory usage
    1529
        #[must_use]
    1530
    2
        pub fn current_memory(&self) -> u64 {
    1531
    2
            self.current_memory
    1532
    2
                .load(std::sync::atomic::Ordering::SeqCst)
    1533
    2
        }
    1534
    1535
        /// Enqueue a request
    1536
    100
        pub fn enqueue(&self) -> LimitResult {
    1537
    100
            let current = self
    1538
    100
                .queue_depth
    1539
    100
                .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
    1540
    100
            if current >= self.config.max_queue_depth {
    1541
    0
                self.queue_depth
    1542
    0
                    .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
    1543
    0
                LimitResult::Backpressure
    1544
            } else {
    1545
    100
                LimitResult::Allowed
    1546
            }
    1547
    100
        }
    1548
    1549
        /// Try to enqueue (returns backpressure if full)
    1550
        #[must_use]
    1551
    1
        pub fn try_enqueue(&self) -> LimitResult {
    1552
    1
            let current = self.queue_depth.load(std::sync::atomic::Ordering::SeqCst);
    1553
    1
            if current >= self.config.max_queue_depth {
    1554
    1
                LimitResult::Backpressure
    1555
            } else {
    1556
    0
                self.enqueue()
    1557
            }
    1558
    1
        }
    1559
    1560
        /// Dequeue a request
    1561
    100
        pub fn dequeue(&self) {
    1562
    100
            let current = self.queue_depth.load(std::sync::atomic::Ordering::SeqCst);
    1563
    100
            if current > 0 {
    1564
    100
                self.queue_depth
    1565
    100
                    .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
    1566
    100
            }0
    1567
    100
        }
    1568
    1569
        /// Start compute timer
    1570
        #[must_use]
    1571
    1
        pub fn start_compute(&self) -> std::time::Instant {
    1572
    1
            std::time::Instant::now()
    1573
    1
        }
    1574
    }
    1575
    1576
    /// Resource metrics snapshot (IMP-075)
    1577
    #[derive(Debug, Clone)]
    1578
    pub struct ResourceMetrics {
    1579
        /// Current memory usage in bytes
    1580
        pub memory_bytes: u64,
    1581
        /// GPU utilization percentage (0-100)
    1582
        pub gpu_utilization: f64,
    1583
        /// Current queue depth
    1584
        pub queue_depth: usize,
    1585
        /// Last recorded latency in milliseconds
    1586
        pub last_latency_ms: u64,
    1587
    }
    1588
    1589
    /// Latency statistics
    1590
    #[derive(Debug, Clone)]
    1591
    pub struct LatencyStats {
    1592
        /// Minimum latency in ms
    1593
        pub min_ms: u64,
    1594
        /// Maximum latency in ms
    1595
        pub max_ms: u64,
    1596
        /// Average latency in ms
    1597
        pub avg_ms: u64,
    1598
    }
    1599
    1600
    /// Resource monitor snapshot
    1601
    #[derive(Debug, Clone)]
    1602
    pub struct ResourceSnapshot {
    1603
        /// Unix timestamp
    1604
        pub timestamp: u64,
    1605
        /// Memory in bytes
    1606
        pub memory_bytes: u64,
    1607
        /// GPU utilization
    1608
        pub gpu_utilization: f64,
    1609
        /// Queue depth
    1610
        pub queue_depth: usize,
    1611
    }
    1612
    1613
    /// Resource monitor (IMP-075)
    1614
    pub struct ResourceMonitor {
    1615
        memory_bytes: std::sync::atomic::AtomicU64,
    1616
        gpu_utilization: std::sync::Mutex<f64>,
    1617
        queue_depth: std::sync::atomic::AtomicUsize,
    1618
        latencies: std::sync::Mutex<Vec<u64>>,
    1619
        last_latency_ms: std::sync::atomic::AtomicU64,
    1620
    }
    1621
    1622
    impl ResourceMonitor {
    1623
        /// Create new resource monitor
    1624
        #[must_use]
    1625
    1
        pub fn new() -> Self {
    1626
    1
            Self {
    1627
    1
                memory_bytes: std::sync::atomic::AtomicU64::new(0),
    1628
    1
                gpu_utilization: std::sync::Mutex::new(0.0),
    1629
    1
                queue_depth: std::sync::atomic::AtomicUsize::new(0),
    1630
    1
                latencies: std::sync::Mutex::new(Vec::new()),
    1631
    1
                last_latency_ms: std::sync::atomic::AtomicU64::new(0),
    1632
    1
            }
    1633
    1
        }
    1634
    1635
        /// Record memory usage
    1636
    1
        pub fn record_memory_usage(&self, bytes: u64) {
    1637
    1
            self.memory_bytes
    1638
    1
                .store(bytes, std::sync::atomic::Ordering::SeqCst);
    1639
    1
        }
    1640
    1641
        /// Record GPU utilization
    1642
    1
        pub fn record_gpu_utilization(&self, utilization: f64) {
    1643
    1
            *self.gpu_utilization.lock().expect("mutex poisoned") = utilization;
    1644
    1
        }
    1645
    1646
        /// Record queue depth
    1647
    1
        pub fn record_queue_depth(&self, depth: usize) {
    1648
    1
            self.queue_depth
    1649
    1
                .store(depth, std::sync::atomic::Ordering::SeqCst);
    1650
    1
        }
    1651
    1652
        /// Record latency
    1653
    6
        pub fn record_latency(&self, duration: Duration) {
    1654
    6
            let ms = duration.as_millis() as u64;
    1655
    6
            self.last_latency_ms
    1656
    6
                .store(ms, std::sync::atomic::Ordering::SeqCst);
    1657
    6
            self.latencies.lock().expect("mutex poisoned").push(ms);
    1658
    6
        }
    1659
    1660
        /// Get current metrics
    1661
        #[must_use]
    1662
    4
        pub fn current_metrics(&self) -> ResourceMetrics {
    1663
    4
            ResourceMetrics {
    1664
    4
                memory_bytes: self.memory_bytes.load(std::sync::atomic::Ordering::SeqCst),
    1665
    4
                gpu_utilization: *self.gpu_utilization.lock().expect("mutex poisoned"),
    1666
    4
                queue_depth: self.queue_depth.load(std::sync::atomic::Ordering::SeqCst),
    1667
    4
                last_latency_ms: self
    1668
    4
                    .last_latency_ms
    1669
    4
                    .load(std::sync::atomic::Ordering::SeqCst),
    1670
    4
            }
    1671
    4
        }
    1672
    1673
        /// Get latency statistics
    1674
        #[must_use]
    1675
    1
        pub fn latency_stats(&self) -> LatencyStats {
    1676
    1
            let latencies = self.latencies.lock().expect("mutex poisoned");
    1677
    1
            if latencies.is_empty() {
    1678
    0
                return LatencyStats {
    1679
    0
                    min_ms: 0,
    1680
    0
                    max_ms: 0,
    1681
    0
                    avg_ms: 0,
    1682
    0
                };
    1683
    1
            }
    1684
    1685
    1
            let min_ms = *latencies.iter().min().unwrap_or(&0);
    1686
    1
            let max_ms = *latencies.iter().max().unwrap_or(&0);
    1687
    1
            let sum: u64 = latencies.iter().sum();
    1688
    1
            let avg_ms = sum / latencies.len() as u64;
    1689
    1690
    1
            LatencyStats {
    1691
    1
                min_ms,
    1692
    1
                max_ms,
    1693
    1
                avg_ms,
    1694
    1
            }
    1695
    1
        }
    1696
    1697
        /// Get snapshot for reporting
    1698
        #[must_use]
    1699
    1
        pub fn snapshot(&self) -> ResourceSnapshot {
    1700
            use std::time::{SystemTime, UNIX_EPOCH};
    1701
    1702
    1
            let timestamp = SystemTime::now()
    1703
    1
                .duration_since(UNIX_EPOCH)
    1704
    1
                .unwrap_or_default()
    1705
    1
                .as_secs();
    1706
    1707
    1
            ResourceSnapshot {
    1708
    1
                timestamp,
    1709
    1
                memory_bytes: self.memory_bytes.load(std::sync::atomic::Ordering::SeqCst),
    1710
    1
                gpu_utilization: *self.gpu_utilization.lock().expect("mutex poisoned"),
    1711
    1
                queue_depth: self.queue_depth.load(std::sync::atomic::Ordering::SeqCst),
    1712
    1
            }
    1713
    1
        }
    1714
    }
    1715
    1716
    impl Default for ResourceMonitor {
    1717
    0
        fn default() -> Self {
    1718
    0
            Self::new()
    1719
    0
        }
    1720
    }
    1721
    1722
    // ============================================================================
    1723
    // M33: GGUF HTTP Serving Integration (IMP-082, IMP-083)
    1724
    // Per spec v2.15.0: Wire GpuModel to HTTP server
    1725
    // ============================================================================
    1726
    1727
    /// State for holding a loaded GGUF model in HTTP server context (IMP-082)
    1728
    ///
    1729
    /// This struct wraps a GpuModel and provides thread-safe access for
    1730
    /// the HTTP server to perform inference requests.
    1731
    ///
    1732
    /// # Example
    1733
    ///
    1734
    /// ```rust,ignore
    1735
    /// use realizar::gpu::GgufModelState;
    1736
    ///
    1737
    /// let state = GgufModelState::new();
    1738
    /// assert!(!state.is_loaded());
    1739
    ///
    1740
    /// // Load model
    1741
    /// let state = load_gguf_to_gpu(vocab_size, hidden_dim, num_layers)?;
    1742
    /// assert!(state.is_loaded());
    1743
    /// ```
    1744
    pub struct GgufModelState {
    1745
        /// Loaded GPU model (None if not loaded)
    1746
        model: Option<GpuModel>,
    1747
        /// Model name/path
    1748
        model_name: Option<String>,
    1749
        /// Whether model is ready for inference
    1750
        ready: bool,
    1751
    }
    1752
    1753
    impl std::fmt::Debug for GgufModelState {
    1754
    1
        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
    1755
    1
            f.debug_struct("GgufModelState")
    1756
    1
                .field("model_name", &self.model_name)
    1757
    1
                .field("ready", &self.ready)
    1758
    1
                .field("is_loaded", &self.model.is_some())
    1759
    1
                .finish()
    1760
    1
        }
    1761
    }
    1762
    1763
    impl GgufModelState {
    1764
        /// Create empty state (no model loaded)
    1765
        #[must_use]
    1766
    2
        pub fn new() -> Self {
    1767
    2
            Self {
    1768
    2
                model: None,
    1769
    2
                model_name: None,
    1770
    2
                ready: false,
    1771
    2
            }
    1772
    2
        }
    1773
    1774
        /// Create state with a loaded model
    1775
        #[must_use]
    1776
    1
        pub fn with_model(model: GpuModel, name: String) -> Self {
    1777
    1
            Self {
    1778
    1
                model: Some(model),
    1779
    1
                model_name: Some(name),
    1780
    1
                ready: true,
    1781
    1
            }
    1782
    1
        }
    1783
    1784
        /// Check if a model is loaded
    1785
        #[must_use]
    1786
    2
        pub fn is_loaded(&self) -> bool {
    1787
    2
            self.model.is_some()
    1788
    2
        }
    1789
    1790
        /// Check if model is ready for inference
    1791
        #[must_use]
    1792
    2
        pub fn is_ready(&self) -> bool {
    1793
    2
            self.ready && self.model1.is_some1()
    1794
    2
        }
    1795
    1796
        /// Get model name
    1797
        #[must_use]
    1798
    1
        pub fn model_name(&self) -> Option<&str> {
    1799
    1
            self.model_name.as_deref()
    1800
    1
        }
    1801
    1802
        /// Get vocab size (0 if no model loaded)
    1803
        #[must_use]
    1804
    2
        pub fn vocab_size(&self) -> usize {
    1805
    2
            self.model.as_ref().map_or(0, |m| m1.config1().vocab_size)
    1806
    2
        }
    1807
    1808
        /// Get reference to the model (for inference)
    1809
        #[must_use]
    1810
    0
        pub fn model(&self) -> Option<&GpuModel> {
    1811
    0
            self.model.as_ref()
    1812
    0
        }
    1813
    1814
        /// Get mutable reference to the model
    1815
    0
        pub fn model_mut(&mut self) -> Option<&mut GpuModel> {
    1816
    0
            self.model.as_mut()
    1817
    0
        }
    1818
    }
    1819
    1820
    impl Default for GgufModelState {
    1821
    0
        fn default() -> Self {
    1822
    0
            Self::new()
    1823
    0
        }
    1824
    }
    1825
    1826
    /// Load GGUF model to GPU (IMP-083)
    1827
    ///
    1828
    /// Creates a minimal GPU model from configuration parameters.
    1829
    /// This is the pipeline entry point for serving GGUF models via HTTP.
    1830
    ///
    1831
    /// # Arguments
    1832
    ///
    1833
    /// * `vocab_size` - Vocabulary size
    1834
    /// * `hidden_dim` - Hidden dimension
    1835
    /// * `num_layers` - Number of transformer layers
    1836
    ///
    1837
    /// # Returns
    1838
    ///
    1839
    /// * `Ok(GgufModelState)` - State with loaded model ready for inference
    1840
    /// * `Err(RealizarError)` - If model creation fails
    1841
    ///
    1842
    /// # Errors
    1843
    ///
    1844
    /// Returns error if GPU initialization fails or model creation fails.
    1845
    ///
    1846
    /// # Example
    1847
    ///
    1848
    /// ```rust,ignore
    1849
    /// use realizar::gpu::load_gguf_to_gpu;
    1850
    ///
    1851
    /// let state = load_gguf_to_gpu(32000, 4096, 32)?;
    1852
    /// assert!(state.is_ready());
    1853
    /// ```
    1854
    1
    pub fn load_gguf_to_gpu(
    1855
    1
        vocab_size: usize,
    1856
    1
        hidden_dim: usize,
    1857
    1
        num_layers: usize,
    1858
    1
    ) -> Result<GgufModelState> {
    1859
        // Create GPU model config
    1860
    1
        let num_heads = hidden_dim / 64; // Standard head dim of 64
    1861
    1
        let config = GpuModelConfig {
    1862
    1
            vocab_size,
    1863
    1
            hidden_dim,
    1864
    1
            num_heads,
    1865
    1
            num_kv_heads: num_heads, // Standard MHA (no GQA)
    1866
    1
            num_layers,
    1867
    1
            intermediate_dim: hidden_dim * 4, // Standard FFN expansion
    1868
    1
            eps: 1e-5,
    1869
    1
            rope_theta: 10000.0, // Standard RoPE base frequency
    1870
    1
        };
    1871
    1872
        // Create GPU model
    1873
    1
        let model = GpuModel::new(config)?0;
    1874
    1875
        // Wrap in state
    1876
    1
        let model_name = format!("test_{}x{}x{}", vocab_size, hidden_dim, num_layers);
    1877
    1
        Ok(GgufModelState::with_model(model, model_name))
    1878
    1
    }
    1879
    1880
    #[cfg(test)]
    1881
    1882
    #[cfg(test)]
    1883
    mod tests;