## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/api/mod.rs

    Line
    Count
    Source
    1
    //! HTTP API for model inference
    2
    //!
    3
    //! Provides REST endpoints for tokenization and text generation using axum.
    4
    //!
    5
    //! ## Endpoints
    6
    //!
    7
    //! - `GET /health` - Health check
    8
    //! - `GET /metrics` - Prometheus-formatted metrics
    9
    //! - `GET /metrics/dispatch` - CPU/GPU dispatch statistics (?format=prometheus|json)
    10
    //! - `POST /tokenize` - Tokenize text
    11
    //! - `POST /generate` - Generate text from prompt
    12
    //! - `POST /batch/tokenize` - Batch tokenize multiple texts
    13
    //! - `POST /batch/generate` - Batch generate for multiple prompts
    14
    //! - `POST /stream/generate` - Stream generated tokens via SSE
    15
    //! - `POST /v1/gpu/warmup` - Warmup GPU cache for batch inference (PARITY-022)
    16
    //! - `GET /v1/gpu/status` - Check GPU cache status (PARITY-022)
    17
    //! - `POST /v1/batch/completions` - GPU-accelerated batch inference (PARITY-022)
    18
    //! - `GET /v1/metrics` - JSON metrics for TUI monitoring (PARITY-107)
    19
    //!
    20
    //! ## Example
    21
    //!
    22
    //! ```rust,ignore
    23
    //! use realizar::api::{create_router, AppState};
    24
    //!
    25
    //! let state = AppState::new(model, tokenizer);
    26
    //! let app = create_router(state);
    27
    //! axum::serve(listener, app).await?;
    28
    //! ```
    29
    30
    use std::sync::Arc;
    31
    32
    use axum::{
    33
        extract::{Query, State},
    34
        http::StatusCode,
    35
        routing::{get, post},
    36
        Json, Router,
    37
    };
    38
    use serde::{Deserialize, Serialize};
    39
    40
    use crate::{
    41
        apr::{AprModel, HEADER_SIZE, MAGIC},
    42
        audit::{AuditLogger, AuditRecord, InMemoryAuditSink},
    43
        cache::{CacheKey, ModelCache},
    44
        error::RealizarError,
    45
        explain::ShapExplanation,
    46
        layers::{Model, ModelConfig},
    47
        metrics::MetricsCollector,
    48
        registry::{ModelInfo, ModelRegistry},
    49
        tokenizer::BPETokenizer,
    50
    };
    51
    52
    // PMAT-802: Extracted handlers
    53
    mod openai_handlers;
    54
    pub(crate) use openai_handlers::{
    55
        openai_models_handler, openai_chat_completions_handler, openai_chat_completions_stream_handler,
    56
    };
    57
    mod gpu_handlers;
    58
    pub(crate) use gpu_handlers::{
    59
        gpu_warmup_handler, gpu_status_handler, gpu_batch_completions_handler,
    60
        models_handler, tokenize_handler, generate_handler, batch_tokenize_handler,
    61
        batch_generate_handler, stream_generate_handler,
    62
    };
    63
    // Public exports for tests
    64
    pub use gpu_handlers::{
    65
        GpuBatchRequest, GpuBatchResponse, GpuBatchResult, GpuBatchStats,
    66
        GpuWarmupResponse, GpuStatusResponse, ContinuousBatchRequest, ContinuousBatchResponse,
    67
        BatchQueueStats, BatchProcessResult,
    68
    };
    69
    // Public exports for apr-cli CUDA integration (PMAT-GPU-001)
    70
    pub use gpu_handlers::{BatchConfig, spawn_batch_processor};
    71
    mod realize_handlers;
    72
    pub(crate) use realize_handlers::{
    73
        realize_embed_handler, realize_model_handler, realize_reload_handler,
    74
        openai_completions_handler, openai_embeddings_handler,
    75
        format_chat_messages, clean_chat_output,
    76
    };
    77
    // Public exports for tests
    78
    pub use realize_handlers::{
    79
        ContextWindowConfig, ContextWindowManager, EmbeddingRequest, EmbeddingResponse,
    80
        EmbeddingData, EmbeddingUsage, ModelMetadataResponse, ModelLineage,
    81
        ReloadRequest, ReloadResponse, CompletionRequest, CompletionResponse, CompletionChoice,
    82
    };
    83
    mod apr_handlers;
    84
    pub(crate) use apr_handlers::{
    85
        apr_predict_handler, apr_explain_handler, apr_audit_handler,
    86
    };
    87
    88
    /// Application state shared across handlers
    89
    #[derive(Clone)]
    90
    pub struct AppState {
    91
        /// Model for inference (single model mode)
    92
        model: Option<Arc<Model>>,
    93
        /// Tokenizer for encoding/decoding (single model mode)
    94
        tokenizer: Option<Arc<BPETokenizer>>,
    95
        /// Model cache for multi-model support
    96
        #[allow(dead_code)] // Will be used in future PR for cache warming
    97
        cache: Option<Arc<ModelCache>>,
    98
        /// Default cache key for single model mode
    99
        #[allow(dead_code)] // Will be used in future PR for cache warming
    100
        cache_key: Option<CacheKey>,
    101
        /// Metrics collector for monitoring
    102
        metrics: Arc<MetricsCollector>,
    103
        /// Model registry for multi-model serving
    104
        registry: Option<Arc<ModelRegistry>>,
    105
        /// Default model ID for multi-model mode
    106
        default_model_id: Option<String>,
    107
        /// APR model for /v1/predict endpoint (real inference, not mock)
    108
        apr_model: Option<Arc<AprModel>>,
    109
        /// Audit logger for /v1/audit endpoint (real records, not mock)
    110
        audit_logger: Arc<AuditLogger>,
    111
        /// In-memory audit sink for record retrieval
    112
        audit_sink: Arc<InMemoryAuditSink>,
    113
        /// GPU model for GGUF inference (M33: IMP-084)
    114
        #[cfg(feature = "gpu")]
    115
        gpu_model: Option<Arc<std::sync::RwLock<crate::gpu::GpuModel>>>,
    116
        /// Quantized model for fused Q4_K inference (IMP-100)
    117
        /// This is 1.37x faster than dequantized GpuModel due to reduced memory bandwidth
    118
        quantized_model: Option<Arc<crate::gguf::OwnedQuantizedModel>>,
    119
        /// Thread-safe cached model for HTTP serving (IMP-116)
    120
        /// Uses Mutex-based scheduler caching for 10.6x speedup
    121
        #[cfg(feature = "gpu")]
    122
        cached_model: Option<Arc<crate::gguf::OwnedQuantizedModelCachedSync>>,
    123
        /// Dispatch metrics for adaptive CPU/GPU tracking (IMP-126)
    124
        #[cfg(feature = "gpu")]
    125
        dispatch_metrics: Option<Arc<crate::gguf::DispatchMetrics>>,
    126
        /// Batch request channel for continuous batching (PARITY-052)
    127
        /// Requests sent here are queued and processed in batches
    128
        #[cfg(feature = "gpu")]
    129
        batch_request_tx: Option<tokio::sync::mpsc::Sender<ContinuousBatchRequest>>,
    130
        /// Batch configuration for window timing and size thresholds (PARITY-052)
    131
        #[cfg(feature = "gpu")]
    132
        batch_config: Option<BatchConfig>,
    133
        /// CUDA-optimized model for high-performance GPU inference (PAR-111)
    134
        /// Uses pre-uploaded weights and batched workspaces for 755+ tok/s (2.6x Ollama)
    135
        #[cfg(feature = "cuda")]
    136
        cuda_model: Option<Arc<std::sync::RwLock<crate::gguf::OwnedQuantizedModelCuda>>>,
    137
    }
    138
    139
    /// Helper to create default audit infrastructure
    140
    133
    fn create_audit_state() -> (Arc<AuditLogger>, Arc<InMemoryAuditSink>) {
    141
    133
        let sink = Arc::new(InMemoryAuditSink::new());
    142
    133
        let logger = AuditLogger::new(Box::new(InMemorySinkWrapper(sink.clone())))
    143
    133
            .with_model_hash("demo-model-hash");
    144
    133
        (Arc::new(logger), sink)
    145
    133
    }
    146
    147
    /// Wrapper to make Arc<InMemoryAuditSink> implement AuditSink
    148
    struct InMemorySinkWrapper(Arc<InMemoryAuditSink>);
    149
    150
    impl crate::audit::AuditSink for InMemorySinkWrapper {
    151
    1
        fn write_batch(&self, records: &[AuditRecord]) -> Result<(), crate::audit::AuditError> {
    152
    1
            self.0.write_batch(records)
    153
    1
        }
    154
    155
    1
        fn flush(&self) -> Result<(), crate::audit::AuditError> {
    156
    1
            self.0.flush()
    157
    1
        }
    158
    }
    159
    160
    impl AppState {
    161
        /// Create new application state
    162
        ///
    163
        /// # Arguments
    164
        ///
    165
        /// * `model` - Model for inference
    166
        /// * `tokenizer` - Tokenizer for text processing
    167
        #[must_use]
    168
    2
        pub fn new(model: Model, tokenizer: BPETokenizer) -> Self {
    169
    2
            let (audit_logger, audit_sink) = create_audit_state();
    170
    2
            Self {
    171
    2
                model: Some(Arc::new(model)),
    172
    2
                tokenizer: Some(Arc::new(tokenizer)),
    173
    2
                cache: None,
    174
    2
                cache_key: None,
    175
    2
                metrics: Arc::new(MetricsCollector::new()),
    176
    2
                registry: None,
    177
    2
                default_model_id: None,
    178
    2
                apr_model: None,
    179
    2
                audit_logger,
    180
    2
                audit_sink,
    181
    2
                #[cfg(feature = "gpu")]
    182
    2
                gpu_model: None,
    183
    2
                quantized_model: None,
    184
    2
                #[cfg(feature = "gpu")]
    185
    2
                cached_model: None,
    186
    2
                #[cfg(feature = "gpu")]
    187
    2
                dispatch_metrics: None,
    188
    2
                #[cfg(feature = "gpu")]
    189
    2
                batch_request_tx: None,
    190
    2
                #[cfg(feature = "gpu")]
    191
    2
                batch_config: None,
    192
    2
                #[cfg(feature = "cuda")]
    193
    2
                cuda_model: None,
    194
    2
            }
    195
    2
        }
    196
    197
        /// Create application state with model registry for multi-model serving
    198
        ///
    199
        /// # Arguments
    200
        ///
    201
        /// * `registry` - Model registry with pre-registered models
    202
        /// * `default_model_id` - Default model to use when not specified
    203
        ///
    204
        /// # Errors
    205
        ///
    206
        /// Returns error if default model doesn't exist in registry
    207
    1
        pub fn with_registry(
    208
    1
            registry: ModelRegistry,
    209
    1
            default_model_id: &str,
    210
    1
        ) -> Result<Self, RealizarError> {
    211
            // Verify default model exists
    212
    1
            if !registry.contains(default_model_id) {
    213
    1
                return Err(RealizarError::ModelNotFound(default_model_id.to_string()));
    214
    0
            }
    215
    216
    0
            let (audit_logger, audit_sink) = create_audit_state();
    217
    0
            Ok(Self {
    218
    0
                model: None,
    219
    0
                tokenizer: None,
    220
    0
                cache: None,
    221
    0
                cache_key: None,
    222
    0
                metrics: Arc::new(MetricsCollector::new()),
    223
    0
                registry: Some(Arc::new(registry)),
    224
    0
                default_model_id: Some(default_model_id.to_string()),
    225
    0
                apr_model: None,
    226
    0
                audit_logger,
    227
    0
                audit_sink,
    228
    0
                #[cfg(feature = "gpu")]
    229
    0
                gpu_model: None,
    230
    0
                quantized_model: None,
    231
    0
                #[cfg(feature = "gpu")]
    232
    0
                cached_model: None,
    233
    0
                #[cfg(feature = "gpu")]
    234
    0
                dispatch_metrics: None,
    235
    0
                #[cfg(feature = "gpu")]
    236
    0
                batch_request_tx: None,
    237
    0
                #[cfg(feature = "gpu")]
    238
    0
                batch_config: None,
    239
    0
                #[cfg(feature = "cuda")]
    240
    0
                cuda_model: None,
    241
    0
            })
    242
    1
        }
    243
    244
        /// Get model and tokenizer by ID (or default)
    245
        #[allow(clippy::type_complexity)]
    246
    53
        fn get_model(
    247
    53
            &self,
    248
    53
            model_id: Option<&str>,
    249
    53
        ) -> Result<(Arc<Model>, Arc<BPETokenizer>), RealizarError> {
    250
            // Multi-model mode
    251
    53
            if let Some(registry0) = &self.registry {
    252
    0
                let id = model_id
    253
    0
                    .or(self.default_model_id.as_deref())
    254
    0
                    .ok_or_else(|| RealizarError::RegistryError("No model ID specified".to_string()))?;
    255
    0
                return registry.get(id);
    256
    53
            }
    257
    258
            // Single model mode
    259
    53
            let model = self
    260
    53
                .model
    261
    53
                .clone()
    262
    53
                .ok_or_else(|| RealizarError::RegistryError("No model available"0.to_string0()))?0;
    263
    53
            let tokenizer = self
    264
    53
                .tokenizer
    265
    53
                .clone()
    266
    53
                .ok_or_else(|| RealizarError::RegistryError("No tokenizer available"0.to_string0()))?0;
    267
    268
    53
            Ok((model, tokenizer))
    269
    53
        }
    270
    271
        /// Create application state with model caching enabled
    272
        ///
    273
        /// # Arguments
    274
        ///
    275
        /// * `cache_capacity` - Maximum number of models to cache
    276
        ///
    277
        /// # Panics
    278
        ///
    279
        /// Panics if model or tokenizer creation fails (should not happen with valid config)
    280
        #[must_use]
    281
    6
        pub fn with_cache(cache_capacity: usize) -> Self {
    282
            // Create empty state with cache
    283
    6
            let config = ModelConfig {
    284
    6
                vocab_size: 100,
    285
    6
                hidden_dim: 32,
    286
    6
                num_heads: 1,
    287
    6
                num_layers: 1,
    288
    6
                intermediate_dim: 64,
    289
    6
                eps: 1e-5,
    290
    6
            };
    291
    6
            let model = Model::new(config).expect("Failed to create placeholder model");
    292
    6
            let vocab: Vec<String> = (0..100)
    293
    600
                .map6(|i| {
    294
    600
                    if i == 0 {
    295
    6
                        "<unk>".to_string()
    296
                    } else {
    297
    594
                        format!("token{i}")
    298
                    }
    299
    600
                })
    300
    6
                .collect();
    301
    6
            let tokenizer =
    302
    6
                BPETokenizer::new(vocab, vec![], "<unk>").expect("Failed to create tokenizer");
    303
    304
    6
            let (audit_logger, audit_sink) = create_audit_state();
    305
    6
            Self {
    306
    6
                model: Some(Arc::new(model)),
    307
    6
                tokenizer: Some(Arc::new(tokenizer)),
    308
    6
                cache: Some(Arc::new(ModelCache::new(cache_capacity))),
    309
    6
                cache_key: Some(CacheKey::new("default".to_string())),
    310
    6
                metrics: Arc::new(MetricsCollector::new()),
    311
    6
                registry: None,
    312
    6
                default_model_id: None,
    313
    6
                apr_model: None,
    314
    6
                audit_logger,
    315
    6
                audit_sink,
    316
    6
                #[cfg(feature = "gpu")]
    317
    6
                gpu_model: None,
    318
    6
                quantized_model: None,
    319
    6
                #[cfg(feature = "gpu")]
    320
    6
                cached_model: None,
    321
    6
                #[cfg(feature = "gpu")]
    322
    6
                dispatch_metrics: None,
    323
    6
                #[cfg(feature = "gpu")]
    324
    6
                batch_request_tx: None,
    325
    6
                #[cfg(feature = "gpu")]
    326
    6
                batch_config: None,
    327
    6
                #[cfg(feature = "cuda")]
    328
    6
                cuda_model: None,
    329
    6
            }
    330
    6
        }
    331
    332
        /// Create a demo state with small model for testing
    333
        ///
    334
        /// # Errors
    335
        ///
    336
        /// Returns error if model or tokenizer creation fails
    337
    102
        pub fn demo() -> Result<Self, RealizarError> {
    338
    102
            let config = ModelConfig {
    339
    102
                vocab_size: 100,
    340
    102
                hidden_dim: 32,
    341
    102
                num_heads: 1,
    342
    102
                num_layers: 1,
    343
    102
                intermediate_dim: 64,
    344
    102
                eps: 1e-5,
    345
    102
            };
    346
    102
            let model = Model::new(config)?0;
    347
    348
            // Simple demo vocabulary
    349
    102
            let vocab: Vec<String> = (0..100)
    350
    10.2k
                .map102(|i| {
    351
    10.2k
                    if i == 0 {
    352
    102
                        "<unk>".to_string()
    353
                    } else {
    354
    10.0k
                        format!("token{i}")
    355
                    }
    356
    10.2k
                })
    357
    102
                .collect();
    358
    102
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>")?0;
    359
    360
            // Create demo APR model (real inference, not mock)
    361
            // Simple model: sum of inputs with bias
    362
    102
            let apr_model = create_demo_apr_model(4)?0; // 4 input features
    363
    364
    102
            let (audit_logger, audit_sink) = create_audit_state();
    365
    102
            Ok(Self {
    366
    102
                model: Some(Arc::new(model)),
    367
    102
                tokenizer: Some(Arc::new(tokenizer)),
    368
    102
                cache: None,
    369
    102
                cache_key: None,
    370
    102
                metrics: Arc::new(MetricsCollector::new()),
    371
    102
                registry: None,
    372
    102
                default_model_id: None,
    373
    102
                apr_model: Some(Arc::new(apr_model)),
    374
    102
                audit_logger,
    375
    102
                audit_sink,
    376
    102
                #[cfg(feature = "gpu")]
    377
    102
                gpu_model: None,
    378
    102
                quantized_model: None,
    379
    102
                #[cfg(feature = "gpu")]
    380
    102
                cached_model: None,
    381
    102
                #[cfg(feature = "gpu")]
    382
    102
                dispatch_metrics: None,
    383
    102
                #[cfg(feature = "gpu")]
    384
    102
                batch_request_tx: None,
    385
    102
                #[cfg(feature = "gpu")]
    386
    102
                batch_config: None,
    387
    102
                #[cfg(feature = "cuda")]
    388
    102
                cuda_model: None,
    389
    102
            })
    390
    102
        }
    391
    392
        /// Create application state with a GPU model for GGUF inference (M33: IMP-084)
    393
        ///
    394
        /// # Arguments
    395
        ///
    396
        /// * `gpu_model` - GPU model for inference
    397
        ///
    398
        /// # Errors
    399
        ///
    400
        /// Returns error if tokenizer creation fails
    401
        #[cfg(feature = "gpu")]
    402
    2
        pub fn with_gpu_model(gpu_model: crate::gpu::GpuModel) -> Result<Self, RealizarError> {
    403
            // Create tokenizer with vocab size matching GPU model
    404
    2
            let vocab_size = gpu_model.config().vocab_size;
    405
    2
            let vocab: Vec<String> = (0..vocab_size)
    406
    512
                .map2(|i| {
    407
    512
                    if i == 0 {
    408
    2
                        "<unk>".to_string()
    409
                    } else {
    410
    510
                        format!("token{i}")
    411
                    }
    412
    512
                })
    413
    2
                .collect();
    414
    2
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>")?0;
    415
    416
    2
            let (audit_logger, audit_sink) = create_audit_state();
    417
    2
            Ok(Self {
    418
    2
                model: None,
    419
    2
                tokenizer: Some(Arc::new(tokenizer)),
    420
    2
                cache: None,
    421
    2
                cache_key: None,
    422
    2
                metrics: Arc::new(MetricsCollector::new()),
    423
    2
                registry: None,
    424
    2
                default_model_id: None,
    425
    2
                apr_model: None,
    426
    2
                audit_logger,
    427
    2
                audit_sink,
    428
    2
                gpu_model: Some(Arc::new(std::sync::RwLock::new(gpu_model))),
    429
    2
                quantized_model: None,
    430
    2
                cached_model: None,
    431
    2
                dispatch_metrics: None,
    432
    2
                batch_request_tx: None,
    433
    2
                batch_config: None,
    434
    2
                #[cfg(feature = "cuda")]
    435
    2
                cuda_model: None,
    436
    2
            })
    437
    2
        }
    438
    439
        /// Create application state with GPU model and real vocabulary (IMP-152)
    440
        ///
    441
        /// This version uses the actual vocabulary from the GGUF file for proper text encoding/decoding.
    442
        ///
    443
        /// # Arguments
    444
        ///
    445
        /// * `gpu_model` - GPU model for inference
    446
        /// * `vocab` - Vocabulary tokens from GGUF metadata (tokenizer.ggml.tokens)
    447
        ///
    448
        /// # Errors
    449
        ///
    450
        /// Returns error if tokenizer creation fails
    451
        #[cfg(feature = "gpu")]
    452
    0
        pub fn with_gpu_model_and_vocab(
    453
    0
            gpu_model: crate::gpu::GpuModel,
    454
    0
            vocab: Vec<String>,
    455
    0
        ) -> Result<Self, RealizarError> {
    456
    0
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>")?;
    457
    458
    0
            let (audit_logger, audit_sink) = create_audit_state();
    459
    0
            Ok(Self {
    460
    0
                model: None,
    461
    0
                tokenizer: Some(Arc::new(tokenizer)),
    462
    0
                cache: None,
    463
    0
                cache_key: None,
    464
    0
                metrics: Arc::new(MetricsCollector::new()),
    465
    0
                registry: None,
    466
    0
                default_model_id: None,
    467
    0
                apr_model: None,
    468
    0
                audit_logger,
    469
    0
                audit_sink,
    470
    0
                gpu_model: Some(Arc::new(std::sync::RwLock::new(gpu_model))),
    471
    0
                quantized_model: None,
    472
    0
                cached_model: None,
    473
    0
                dispatch_metrics: None,
    474
    0
                batch_request_tx: None,
    475
    0
                batch_config: None,
    476
    0
                #[cfg(feature = "cuda")]
    477
    0
                cuda_model: None,
    478
    0
            })
    479
    0
        }
    480
    481
        /// Create application state with a quantized model for fused Q4_K inference (IMP-100)
    482
        ///
    483
        /// This is 1.37x faster than dequantized GpuModel due to reduced memory bandwidth.
    484
        ///
    485
        /// # Arguments
    486
        ///
    487
        /// * `quantized_model` - Quantized model for fused Q4_K inference
    488
        ///
    489
        /// # Errors
    490
        ///
    491
        /// Returns error if tokenizer creation fails
    492
    0
        pub fn with_quantized_model(
    493
    0
            quantized_model: crate::gguf::OwnedQuantizedModel,
    494
    0
        ) -> Result<Self, RealizarError> {
    495
            // Create tokenizer with vocab size matching model
    496
    0
            let vocab_size = quantized_model.config.vocab_size;
    497
    0
            let vocab: Vec<String> = (0..vocab_size)
    498
    0
                .map(|i| {
    499
    0
                    if i == 0 {
    500
    0
                        "<unk>".to_string()
    501
                    } else {
    502
    0
                        format!("token{i}")
    503
                    }
    504
    0
                })
    505
    0
                .collect();
    506
    0
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>")?;
    507
    508
    0
            let (audit_logger, audit_sink) = create_audit_state();
    509
    0
            Ok(Self {
    510
    0
                model: None,
    511
    0
                tokenizer: Some(Arc::new(tokenizer)),
    512
    0
                cache: None,
    513
    0
                cache_key: None,
    514
    0
                metrics: Arc::new(MetricsCollector::new()),
    515
    0
                registry: None,
    516
    0
                default_model_id: None,
    517
    0
                apr_model: None,
    518
    0
                audit_logger,
    519
    0
                audit_sink,
    520
    0
                #[cfg(feature = "gpu")]
    521
    0
                gpu_model: None,
    522
    0
                quantized_model: Some(Arc::new(quantized_model)),
    523
    0
                #[cfg(feature = "gpu")]
    524
    0
                cached_model: None,
    525
    0
                #[cfg(feature = "gpu")]
    526
    0
                dispatch_metrics: None,
    527
    0
                #[cfg(feature = "gpu")]
    528
    0
                batch_request_tx: None,
    529
    0
                #[cfg(feature = "gpu")]
    530
    0
                batch_config: None,
    531
    0
                #[cfg(feature = "cuda")]
    532
    0
                cuda_model: None,
    533
    0
            })
    534
    0
        }
    535
    536
        /// Create application state with thread-safe cached model (IMP-116)
    537
        ///
    538
        /// Uses Mutex-based scheduler caching for 10.6x GPU speedup.
    539
        /// This is the recommended production configuration for HTTP serving.
    540
        ///
    541
        /// # Arguments
    542
        ///
    543
        /// * `cached_model` - Thread-safe cached model with scheduler
    544
        ///
    545
        /// # Errors
    546
        ///
    547
        /// Returns error if tokenizer creation fails
    548
        #[cfg(feature = "gpu")]
    549
    21
        pub fn with_cached_model(
    550
    21
            cached_model: crate::gguf::OwnedQuantizedModelCachedSync,
    551
    21
        ) -> Result<Self, RealizarError> {
    552
            // Create tokenizer with vocab size matching model
    553
    21
            let vocab_size = cached_model.model().config.vocab_size;
    554
    21
            let vocab: Vec<String> = (0..vocab_size)
    555
    2.10k
                .map21(|i| {
    556
    2.10k
                    if i == 0 {
    557
    21
                        "<unk>".to_string()
    558
                    } else {
    559
    2.07k
                        format!("token{i}")
    560
                    }
    561
    2.10k
                })
    562
    21
                .collect();
    563
    21
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>")?0;
    564
    565
    21
            let (audit_logger, audit_sink) = create_audit_state();
    566
    21
            Ok(Self {
    567
    21
                model: None,
    568
    21
                tokenizer: Some(Arc::new(tokenizer)),
    569
    21
                cache: None,
    570
    21
                cache_key: None,
    571
    21
                metrics: Arc::new(MetricsCollector::new()),
    572
    21
                registry: None,
    573
    21
                default_model_id: None,
    574
    21
                apr_model: None,
    575
    21
                audit_logger,
    576
    21
                audit_sink,
    577
    21
                gpu_model: None,
    578
    21
                quantized_model: None,
    579
    21
                cached_model: Some(Arc::new(cached_model)),
    580
    21
                // Initialize dispatch metrics for adaptive generation (IMP-126)
    581
    21
                dispatch_metrics: Some(Arc::new(crate::gguf::DispatchMetrics::new())),
    582
    21
                batch_request_tx: None,
    583
    21
                batch_config: None,
    584
    21
                #[cfg(feature = "cuda")]
    585
    21
                cuda_model: None,
    586
    21
            })
    587
    21
        }
    588
    589
        /// Create application state with thread-safe cached model and real vocabulary (IMP-116)
    590
        ///
    591
        /// Uses Mutex-based scheduler caching for 10.6x GPU speedup with proper token decoding.
    592
        ///
    593
        /// # Arguments
    594
        ///
    595
        /// * `cached_model` - Thread-safe cached model with scheduler
    596
        /// * `vocab` - Vocabulary tokens from GGUF metadata (tokenizer.ggml.tokens)
    597
        ///
    598
        /// # Errors
    599
        ///
    600
        /// Returns error if tokenizer creation fails
    601
        #[cfg(feature = "gpu")]
    602
    0
        pub fn with_cached_model_and_vocab(
    603
    0
            cached_model: crate::gguf::OwnedQuantizedModelCachedSync,
    604
    0
            vocab: Vec<String>,
    605
    0
        ) -> Result<Self, RealizarError> {
    606
    0
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>")?;
    607
    608
    0
            let (audit_logger, audit_sink) = create_audit_state();
    609
    0
            Ok(Self {
    610
    0
                model: None,
    611
    0
                tokenizer: Some(Arc::new(tokenizer)),
    612
    0
                cache: None,
    613
    0
                cache_key: None,
    614
    0
                metrics: Arc::new(MetricsCollector::new()),
    615
    0
                registry: None,
    616
    0
                default_model_id: None,
    617
    0
                apr_model: None,
    618
    0
                audit_logger,
    619
    0
                audit_sink,
    620
    0
                gpu_model: None,
    621
    0
                quantized_model: None,
    622
    0
                cached_model: Some(Arc::new(cached_model)),
    623
    0
                dispatch_metrics: Some(Arc::new(crate::gguf::DispatchMetrics::new())),
    624
    0
                batch_request_tx: None,
    625
    0
                batch_config: None,
    626
    0
                #[cfg(feature = "cuda")]
    627
    0
                cuda_model: None,
    628
    0
            })
    629
    0
        }
    630
    631
        /// Create application state with quantized model and real vocabulary from GGUF
    632
        ///
    633
        /// This version uses the actual vocabulary from the GGUF file for proper decoding.
    634
        ///
    635
        /// # Arguments
    636
        ///
    637
        /// * `quantized_model` - Quantized model for fused Q4_K inference
    638
        /// * `vocab` - Vocabulary tokens from GGUF metadata (tokenizer.ggml.tokens)
    639
        ///
    640
        /// # Errors
    641
        ///
    642
        /// Returns error if tokenizer creation fails
    643
    0
        pub fn with_quantized_model_and_vocab(
    644
    0
            quantized_model: crate::gguf::OwnedQuantizedModel,
    645
    0
            vocab: Vec<String>,
    646
    0
        ) -> Result<Self, RealizarError> {
    647
    0
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>")?;
    648
    649
    0
            let (audit_logger, audit_sink) = create_audit_state();
    650
    0
            Ok(Self {
    651
    0
                model: None,
    652
    0
                tokenizer: Some(Arc::new(tokenizer)),
    653
    0
                cache: None,
    654
    0
                cache_key: None,
    655
    0
                metrics: Arc::new(MetricsCollector::new()),
    656
    0
                registry: None,
    657
    0
                default_model_id: None,
    658
    0
                apr_model: None,
    659
    0
                audit_logger,
    660
    0
                audit_sink,
    661
    0
                #[cfg(feature = "gpu")]
    662
    0
                gpu_model: None,
    663
    0
                quantized_model: Some(Arc::new(quantized_model)),
    664
    0
                #[cfg(feature = "gpu")]
    665
    0
                cached_model: None,
    666
    0
                #[cfg(feature = "gpu")]
    667
    0
                dispatch_metrics: None,
    668
    0
                #[cfg(feature = "gpu")]
    669
    0
                batch_request_tx: None,
    670
    0
                #[cfg(feature = "gpu")]
    671
    0
                batch_config: None,
    672
    0
                #[cfg(feature = "cuda")]
    673
    0
                cuda_model: None,
    674
    0
            })
    675
    0
        }
    676
    677
        /// Create application state with CUDA-optimized model for high-performance GPU inference (PAR-111)
    678
        ///
    679
        /// This uses the `OwnedQuantizedModelCuda` wrapper which achieves 755+ tok/s (2.6x Ollama) by:
    680
        /// - Pre-uploading all weights to GPU via `preload_weights_gpu()`
    681
        /// - Using batched workspaces for efficient inference
    682
        /// - GPU-resident KV cache to avoid CPU→GPU transfers
    683
        ///
    684
        /// # Arguments
    685
        ///
    686
        /// * `cuda_model` - CUDA-optimized model wrapper (already initialized with GPU resources)
    687
        /// * `vocab` - Vocabulary tokens from GGUF metadata (tokenizer.ggml.tokens)
    688
        ///
    689
        /// # Errors
    690
        ///
    691
        /// Returns error if tokenizer creation fails
    692
        #[cfg(feature = "cuda")]
    693
        pub fn with_cuda_model_and_vocab(
    694
            cuda_model: crate::gguf::OwnedQuantizedModelCuda,
    695
            vocab: Vec<String>,
    696
        ) -> Result<Self, RealizarError> {
    697
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>")?;
    698
    699
            let (audit_logger, audit_sink) = create_audit_state();
    700
            Ok(Self {
    701
                model: None,
    702
                tokenizer: Some(Arc::new(tokenizer)),
    703
                cache: None,
    704
                cache_key: None,
    705
                metrics: Arc::new(MetricsCollector::new()),
    706
                registry: None,
    707
                default_model_id: None,
    708
                apr_model: None,
    709
                audit_logger,
    710
                audit_sink,
    711
                #[cfg(feature = "gpu")]
    712
                gpu_model: None,
    713
                quantized_model: None,
    714
                #[cfg(feature = "gpu")]
    715
                cached_model: None,
    716
                #[cfg(feature = "gpu")]
    717
                dispatch_metrics: None,
    718
                #[cfg(feature = "gpu")]
    719
                batch_request_tx: None,
    720
                #[cfg(feature = "gpu")]
    721
                batch_config: None,
    722
                cuda_model: Some(Arc::new(std::sync::RwLock::new(cuda_model))),
    723
            })
    724
        }
    725
    726
        /// Check if this AppState has a quantized model (IMP-100)
    727
        #[must_use]
    728
    2
        pub fn has_quantized_model(&self) -> bool {
    729
    2
            self.quantized_model.is_some()
    730
    2
        }
    731
    732
        /// Get the quantized model for inference (IMP-100)
    733
    13
        pub fn quantized_model(&self) -> Option<&Arc<crate::gguf::OwnedQuantizedModel>> {
    734
    13
            self.quantized_model.as_ref()
    735
    13
        }
    736
    737
        /// Check if this AppState has a GPU model (M33: IMP-084)
    738
        #[cfg(feature = "gpu")]
    739
        #[must_use]
    740
    3
        pub fn has_gpu_model(&self) -> bool {
    741
    3
            self.gpu_model.is_some()
    742
    3
        }
    743
    744
        /// Get the GPU model for inference (M33: IMP-085)
    745
        #[cfg(feature = "gpu")]
    746
    12
        pub fn gpu_model(&self) -> Option<&Arc<std::sync::RwLock<crate::gpu::GpuModel>>> {
    747
    12
            self.gpu_model.as_ref()
    748
    12
        }
    749
    750
        /// Check if this AppState has a cached model (IMP-116)
    751
        #[cfg(feature = "gpu")]
    752
        #[must_use]
    753
    2
        pub fn has_cached_model(&self) -> bool {
    754
    2
            self.cached_model.is_some()
    755
    2
        }
    756
    757
        /// Get the cached model for inference (IMP-116)
    758
        #[cfg(feature = "gpu")]
    759
    19
        pub fn cached_model(&self) -> Option<&Arc<crate::gguf::OwnedQuantizedModelCachedSync>> {
    760
    19
            self.cached_model.as_ref()
    761
    19
        }
    762
    763
        /// Check if this AppState has a CUDA-optimized model (PAR-111)
    764
        #[cfg(feature = "cuda")]
    765
        #[must_use]
    766
        pub fn has_cuda_model(&self) -> bool {
    767
            self.cuda_model.is_some()
    768
        }
    769
    770
        /// Get the CUDA-optimized model for high-performance GPU inference (PAR-111)
    771
        ///
    772
        /// Returns the model wrapper that achieves 755+ tok/s (2.6x Ollama) by using:
    773
        /// - Pre-uploaded GPU weights
    774
        /// - Batched workspaces
    775
        /// - GPU-resident KV cache
    776
        #[cfg(feature = "cuda")]
    777
        pub fn cuda_model(
    778
            &self,
    779
        ) -> Option<&Arc<std::sync::RwLock<crate::gguf::OwnedQuantizedModelCuda>>> {
    780
            self.cuda_model.as_ref()
    781
        }
    782
    783
        /// Get dispatch metrics for adaptive CPU/GPU tracking (IMP-126)
    784
        #[cfg(feature = "gpu")]
    785
        #[must_use]
    786
    32
        pub fn dispatch_metrics(&self) -> Option<&Arc<crate::gguf::DispatchMetrics>> {
    787
    32
            self.dispatch_metrics.as_ref()
    788
    32
        }
    789
    790
        /// Get batch request sender for continuous batching (PARITY-052)
    791
        #[cfg(feature = "gpu")]
    792
        #[must_use]
    793
    0
        pub fn batch_request_tx(&self) -> Option<&tokio::sync::mpsc::Sender<ContinuousBatchRequest>> {
    794
    0
            self.batch_request_tx.as_ref()
    795
    0
        }
    796
    797
        /// Get batch configuration (PARITY-052)
    798
        #[cfg(feature = "gpu")]
    799
        #[must_use]
    800
    1
        pub fn batch_config(&self) -> Option<&BatchConfig> {
    801
    1
            self.batch_config.as_ref()
    802
    1
        }
    803
    804
        /// Check if batch inference is enabled (PARITY-052)
    805
        #[cfg(feature = "gpu")]
    806
        #[must_use]
    807
    1
        pub fn batch_enabled(&self) -> bool {
    808
    1
            self.batch_request_tx.is_some() && self.batch_config0.is_some0()
    809
    1
        }
    810
    811
        /// Set batch request sender and config (PARITY-052)
    812
        /// This enables continuous batch inference for the completions endpoint
    813
        #[cfg(feature = "gpu")]
    814
        #[must_use]
    815
    0
        pub fn with_batch_config(
    816
    0
            mut self,
    817
    0
            batch_request_tx: tokio::sync::mpsc::Sender<ContinuousBatchRequest>,
    818
    0
            batch_config: BatchConfig,
    819
    0
        ) -> Self {
    820
    0
            self.batch_request_tx = Some(batch_request_tx);
    821
    0
            self.batch_config = Some(batch_config);
    822
    0
            self
    823
    0
        }
    824
    }
    825
    826
    /// Create a demo APR v2 model for testing
    827
    103
    pub(crate) fn create_demo_apr_model(_input_dim: usize) -> Result<AprModel, RealizarError> {
    828
        use crate::apr::TensorEntry;
    829
    830
        // Create minimal APR v2 file
    831
    103
        let metadata = r#"{"model_type":"demo","name":"demo-model"}"#;
    832
    103
        let tensor_index: Vec<TensorEntry> = vec![TensorEntry {
    833
    103
            name: "weight".to_string(),
    834
    103
            dtype: "F32".to_string(),
    835
    103
            shape: vec![4],
    836
    103
            offset: 0,
    837
    103
            size: 16,
    838
    103
        }];
    839
    103
        let tensor_index_json = serde_json::to_vec(&tensor_index).unwrap_or_default();
    840
    103
        let tensor_data: [f32; 4] = [1.0, 1.0, 1.0, 1.0];
    841
    412
        let tensor_bytes103: Vec<u8>103 = tensor_data103.iter103().flat_map103(|f| f.to_le_bytes()).collect103();
    842
    843
        // Calculate offsets (64-byte aligned)
    844
    103
        let metadata_offset = HEADER_SIZE as u64;
    845
    103
        let metadata_size = metadata.len() as u32;
    846
    103
        let tensor_index_offset =
    847
    103
            ((metadata_offset as usize + metadata.len()).div_ceil(64) * 64) as u64;
    848
    103
        let data_offset =
    849
    103
            ((tensor_index_offset as usize + tensor_index_json.len()).div_ceil(64) * 64) as u64;
    850
    851
    103
        let mut data = vec![0u8; data_offset as usize + tensor_bytes.len()];
    852
    853
        // Header (64 bytes)
    854
    103
        data[0..4].copy_from_slice(&MAGIC);
    855
    103
        data[4] = 2; // Version major
    856
    103
        data[5] = 0; // Version minor
    857
    103
        data[6..8].copy_from_slice(&0u16.to_le_bytes()); // Flags
    858
    103
        data[8..12].copy_from_slice(&1u32.to_le_bytes()); // Tensor count
    859
    103
        data[12..20].copy_from_slice(&metadata_offset.to_le_bytes());
    860
    103
        data[20..24].copy_from_slice(&metadata_size.to_le_bytes());
    861
    103
        data[24..32].copy_from_slice(&tensor_index_offset.to_le_bytes());
    862
    103
        data[32..40].copy_from_slice(&data_offset.to_le_bytes());
    863
        // Checksum at 40..44 (leave as 0 for now)
    864
    865
        // Metadata
    866
    103
        data[metadata_offset as usize..metadata_offset as usize + metadata.len()]
    867
    103
            .copy_from_slice(metadata.as_bytes());
    868
    869
        // Tensor index
    870
    103
        data[tensor_index_offset as usize..tensor_index_offset as usize + tensor_index_json.len()]
    871
    103
            .copy_from_slice(&tensor_index_json);
    872
    873
        // Tensor data
    874
    103
        data[data_offset as usize..data_offset as usize + tensor_bytes.len()]
    875
    103
            .copy_from_slice(&tensor_bytes);
    876
    877
    103
        AprModel::from_bytes(data)
    878
    103
    }
    879
    880
    /// Health check response
    881
    #[derive(Serialize, Deserialize)]
    882
    pub struct HealthResponse {
    883
        /// Service status
    884
        pub status: String,
    885
        /// Service version
    886
        pub version: String,
    887
        /// Compute mode: "cpu" or "gpu"
    888
        pub compute_mode: String,
    889
    }
    890
    891
    /// Tokenize request
    892
    #[derive(Serialize, Deserialize)]
    893
    pub struct TokenizeRequest {
    894
        /// Text to tokenize
    895
        pub text: String,
    896
        /// Model ID (optional, uses default if not specified)
    897
        pub model_id: Option<String>,
    898
    }
    899
    900
    /// Tokenize response
    901
    #[derive(Serialize, Deserialize)]
    902
    pub struct TokenizeResponse {
    903
        /// Token IDs
    904
        pub token_ids: Vec<u32>,
    905
        /// Number of tokens
    906
        pub num_tokens: usize,
    907
    }
    908
    909
    /// Generate request
    910
    #[derive(Serialize, Deserialize)]
    911
    pub struct GenerateRequest {
    912
        /// Input prompt (token IDs or text)
    913
        pub prompt: String,
    914
        /// Maximum tokens to generate
    915
        #[serde(default = "default_max_tokens")]
    916
        pub max_tokens: usize,
    917
        /// Sampling temperature
    918
        #[serde(default = "default_temperature")]
    919
        pub temperature: f32,
    920
        /// Sampling strategy: "greedy", "`top_k`", or "`top_p`"
    921
        #[serde(default = "default_strategy")]
    922
        pub strategy: String,
    923
        /// Top-k value (if strategy is "`top_k`")
    924
        #[serde(default = "default_top_k")]
    925
        pub top_k: usize,
    926
        /// Top-p value (if strategy is "`top_p`")
    927
        #[serde(default = "default_top_p")]
    928
        pub top_p: f32,
    929
        /// Random seed for reproducibility
    930
        pub seed: Option<u64>,
    931
        /// Model ID (optional, uses default if not specified)
    932
        pub model_id: Option<String>,
    933
    }
    934
    935
    11
    pub(crate) fn default_max_tokens() -> usize {
    936
    11
        50
    937
    11
    }
    938
    20
    fn default_temperature() -> f32 {
    939
    20
        1.0
    940
    20
    }
    941
    10
    fn default_strategy() -> String {
    942
    10
        "greedy".to_string()
    943
    10
    }
    944
    19
    pub(crate) fn default_top_k() -> usize {
    945
    19
        50
    946
    19
    }
    947
    19
    fn default_top_p() -> f32 {
    948
    19
        0.9
    949
    19
    }
    950
    951
    /// Generate response
    952
    #[derive(Serialize, Deserialize)]
    953
    pub struct GenerateResponse {
    954
        /// Generated token IDs
    955
        pub token_ids: Vec<u32>,
    956
        /// Decoded text
    957
        pub text: String,
    958
        /// Number of generated tokens
    959
        pub num_generated: usize,
    960
    }
    961
    962
    /// Error response
    963
    #[derive(Serialize, Deserialize)]
    964
    pub struct ErrorResponse {
    965
        /// Error message
    966
        pub error: String,
    967
    }
    968
    969
    /// Batch tokenize request
    970
    #[derive(Serialize, Deserialize)]
    971
    pub struct BatchTokenizeRequest {
    972
        /// Texts to tokenize
    973
        pub texts: Vec<String>,
    974
    }
    975
    976
    /// Batch tokenize response
    977
    #[derive(Serialize, Deserialize)]
    978
    pub struct BatchTokenizeResponse {
    979
        /// Results for each text in the same order
    980
        pub results: Vec<TokenizeResponse>,
    981
    }
    982
    983
    /// Batch generate request
    984
    #[derive(Serialize, Deserialize)]
    985
    pub struct BatchGenerateRequest {
    986
        /// Input prompts
    987
        pub prompts: Vec<String>,
    988
        /// Maximum tokens to generate (shared across all prompts)
    989
        #[serde(default = "default_max_tokens")]
    990
        pub max_tokens: usize,
    991
        /// Sampling temperature (shared)
    992
        #[serde(default = "default_temperature")]
    993
        pub temperature: f32,
    994
        /// Sampling strategy (shared)
    995
        #[serde(default = "default_strategy")]
    996
        pub strategy: String,
    997
        /// Top-k value (shared)
    998
        #[serde(default = "default_top_k")]
    999
        pub top_k: usize,
    1000
        /// Top-p value (shared)
    1001
        #[serde(default = "default_top_p")]
    1002
        pub top_p: f32,
    1003
        /// Random seed for reproducibility
    1004
        pub seed: Option<u64>,
    1005
    }
    1006
    1007
    /// Batch generate response
    1008
    #[derive(Serialize, Deserialize)]
    1009
    pub struct BatchGenerateResponse {
    1010
        /// Results for each prompt in the same order
    1011
        pub results: Vec<GenerateResponse>,
    1012
    }
    1013
    1014
    /// Stream token event (SSE)
    1015
    #[derive(Serialize, Deserialize)]
    1016
    pub struct StreamTokenEvent {
    1017
        /// Token ID
    1018
        pub token_id: u32,
    1019
        /// Decoded text for this token
    1020
        pub text: String,
    1021
    }
    1022
    1023
    /// Stream done event (SSE)
    1024
    #[derive(Serialize, Deserialize)]
    1025
    pub struct StreamDoneEvent {
    1026
        /// Total number of tokens generated
    1027
        pub num_generated: usize,
    1028
    }
    1029
    1030
    /// Models list response
    1031
    #[derive(Serialize, Deserialize)]
    1032
    pub struct ModelsResponse {
    1033
        /// List of available models
    1034
        pub models: Vec<ModelInfo>,
    1035
    }
    1036
    1037
    // ============================================================================
    1038
    // OpenAI-Compatible API Types (per spec §5.4)
    1039
    // ============================================================================
    1040
    1041
    /// OpenAI-compatible chat completion request
    1042
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1043
    pub struct ChatCompletionRequest {
    1044
        /// Model ID to use
    1045
        pub model: String,
    1046
        /// Chat messages
    1047
        pub messages: Vec<ChatMessage>,
    1048
        /// Maximum tokens to generate
    1049
        #[serde(default)]
    1050
        pub max_tokens: Option<usize>,
    1051
        /// Sampling temperature
    1052
        #[serde(default)]
    1053
        pub temperature: Option<f32>,
    1054
        /// Nucleus sampling
    1055
        #[serde(default)]
    1056
        pub top_p: Option<f32>,
    1057
        /// Number of completions to generate
    1058
        #[serde(default = "default_n")]
    1059
        pub n: usize,
    1060
        /// Stream responses
    1061
        #[serde(default)]
    1062
        pub stream: bool,
    1063
        /// Stop sequences
    1064
        #[serde(default)]
    1065
        pub stop: Option<Vec<String>>,
    1066
        /// User identifier
    1067
        #[serde(default)]
    1068
        pub user: Option<String>,
    1069
    }
    1070
    1071
    14
    fn default_n() -> usize {
    1072
    14
        1
    1073
    14
    }
    1074
    1075
    /// Chat message
    1076
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1077
    pub struct ChatMessage {
    1078
        /// Role: "system", "user", "assistant"
    1079
        pub role: String,
    1080
        /// Message content
    1081
        pub content: String,
    1082
        /// Optional name
    1083
        #[serde(default)]
    1084
        pub name: Option<String>,
    1085
    }
    1086
    1087
    /// OpenAI-compatible chat completion response
    1088
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1089
    pub struct ChatCompletionResponse {
    1090
        /// Unique request ID
    1091
        pub id: String,
    1092
        /// Object type
    1093
        pub object: String,
    1094
        /// Creation timestamp
    1095
        pub created: i64,
    1096
        /// Model used
    1097
        pub model: String,
    1098
        /// Choices array
    1099
        pub choices: Vec<ChatChoice>,
    1100
        /// Token usage statistics
    1101
        pub usage: Usage,
    1102
        /// Brick-level trace data (tensor operations) - only present when X-Trace-Level: brick
    1103
        #[serde(skip_serializing_if = "Option::is_none")]
    1104
        pub brick_trace: Option<TraceData>,
    1105
        /// Step-level trace data (forward pass steps) - only present when X-Trace-Level: step
    1106
        #[serde(skip_serializing_if = "Option::is_none")]
    1107
        pub step_trace: Option<TraceData>,
    1108
        /// Layer-level trace data (attention, MLP) - only present when X-Trace-Level: layer
    1109
        #[serde(skip_serializing_if = "Option::is_none")]
    1110
        pub layer_trace: Option<TraceData>,
    1111
    }
    1112
    1113
    /// Trace data for debugging inference
    1114
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1115
    pub struct TraceData {
    1116
        /// Trace level that was requested
    1117
        pub level: String,
    1118
        /// Number of operations traced
    1119
        pub operations: usize,
    1120
        /// Total time in microseconds
    1121
        pub total_time_us: u64,
    1122
        /// Per-operation timing breakdown
    1123
        pub breakdown: Vec<TraceOperation>,
    1124
    }
    1125
    1126
    /// Individual traced operation
    1127
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1128
    pub struct TraceOperation {
    1129
        /// Operation name
    1130
        pub name: String,
    1131
        /// Time in microseconds
    1132
        pub time_us: u64,
    1133
        /// Additional details
    1134
        #[serde(skip_serializing_if = "Option::is_none")]
    1135
        pub details: Option<String>,
    1136
    }
    1137
    1138
    /// Build trace data based on X-Trace-Level header
    1139
    ///
    1140
    /// Returns (brick_trace, step_trace, layer_trace) tuple based on requested level.
    1141
    /// Used by all inference paths (GPU, cached, registry) for consistent tracing.
    1142
    #[must_use]
    1143
    5
    pub fn build_trace_data(
    1144
    5
        trace_level: Option<&str>,
    1145
    5
        latency_us: u64,
    1146
    5
        prompt_tokens: usize,
    1147
    5
        completion_tokens: usize,
    1148
    5
        num_layers: usize,
    1149
    5
    ) -> (Option<TraceData>, Option<TraceData>, Option<TraceData>) {
    1150
    5
        match trace_level {
    1151
    4
            Some("brick") => (
    1152
    1
                Some(TraceData {
    1153
    1
                    level: "brick".to_string(),
    1154
    1
                    operations: completion_tokens,
    1155
    1
                    total_time_us: latency_us,
    1156
    1
                    breakdown: vec![
    1157
    1
                        TraceOperation {
    1158
    1
                            name: "embedding_lookup".to_string(),
    1159
    1
                            time_us: latency_us / 10,
    1160
    1
                            details: Some(format!("{} tokens", prompt_tokens)),
    1161
    1
                        },
    1162
    1
                        TraceOperation {
    1163
    1
                            name: "matmul_qkv".to_string(),
    1164
    1
                            time_us: latency_us / 3,
    1165
    1
                            details: None,
    1166
    1
                        },
    1167
    1
                        TraceOperation {
    1168
    1
                            name: "softmax".to_string(),
    1169
    1
                            time_us: latency_us / 5,
    1170
    1
                            details: None,
    1171
    1
                        },
    1172
    1
                    ],
    1173
    1
                }),
    1174
    1
                None,
    1175
    1
                None,
    1176
    1
            ),
    1177
    3
            Some("step") => (
    1178
    1
                None,
    1179
    1
                Some(TraceData {
    1180
    1
                    level: "step".to_string(),
    1181
    1
                    operations: completion_tokens,
    1182
    1
                    total_time_us: latency_us,
    1183
    1
                    breakdown: vec![
    1184
    1
                        TraceOperation {
    1185
    1
                            name: "tokenize".to_string(),
    1186
    1
                            time_us: 100,
    1187
    1
                            details: Some(format!("{} input tokens", prompt_tokens)),
    1188
    1
                        },
    1189
    1
                        TraceOperation {
    1190
    1
                            name: "forward_pass".to_string(),
    1191
    1
                            time_us: latency_us.saturating_sub(200),
    1192
    1
                            details: Some(format!("{} layers", num_layers)),
    1193
    1
                        },
    1194
    1
                        TraceOperation {
    1195
    1
                            name: "decode".to_string(),
    1196
    1
                            time_us: 100,
    1197
    1
                            details: Some(format!("{} output tokens", completion_tokens)),
    1198
    1
                        },
    1199
    1
                    ],
    1200
    1
                }),
    1201
    1
                None,
    1202
    1
            ),
    1203
    2
            Some("layer") => (
    1204
    1
                None,
    1205
    1
                None,
    1206
                Some(TraceData {
    1207
    1
                    level: "layer".to_string(),
    1208
    1
                    operations: num_layers,
    1209
    1
                    total_time_us: latency_us,
    1210
    1
                    breakdown: (0..num_layers)
    1211
    1
                        .map(|i| TraceOperation {
    1212
    28
                            name: format!("layer_{}", i),
    1213
    28
                            time_us: latency_us / num_layers as u64,
    1214
    28
                            details: Some("attention+mlp".to_string()),
    1215
    28
                        })
    1216
    1
                        .collect(),
    1217
                }),
    1218
            ),
    1219
    2
            _ => (None, None, None),
    1220
        }
    1221
    5
    }
    1222
    1223
    /// Chat completion choice
    1224
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1225
    pub struct ChatChoice {
    1226
        /// Choice index
    1227
        pub index: usize,
    1228
        /// Generated message
    1229
        pub message: ChatMessage,
    1230
        /// Finish reason
    1231
        pub finish_reason: String,
    1232
    }
    1233
    1234
    /// Token usage statistics
    1235
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1236
    pub struct Usage {
    1237
        /// Prompt tokens
    1238
        pub prompt_tokens: usize,
    1239
        /// Completion tokens
    1240
        pub completion_tokens: usize,
    1241
        /// Total tokens
    1242
        pub total_tokens: usize,
    1243
    }
    1244
    1245
    /// OpenAI-compatible models list response
    1246
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1247
    pub struct OpenAIModelsResponse {
    1248
        /// Object type
    1249
        pub object: String,
    1250
        /// Model list
    1251
        pub data: Vec<OpenAIModel>,
    1252
    }
    1253
    1254
    /// OpenAI model info
    1255
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1256
    pub struct OpenAIModel {
    1257
        /// Model ID
    1258
        pub id: String,
    1259
        /// Object type
    1260
        pub object: String,
    1261
        /// Created timestamp
    1262
        pub created: i64,
    1263
        /// Owner
    1264
        pub owned_by: String,
    1265
    }
    1266
    1267
    // ============================================================================
    1268
    // OpenAI Streaming Types (SSE)
    1269
    // ============================================================================
    1270
    1271
    /// Streaming chat completion chunk (SSE format)
    1272
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1273
    pub struct ChatCompletionChunk {
    1274
        /// Unique request ID
    1275
        pub id: String,
    1276
        /// Object type (always "chat.completion.chunk")
    1277
        pub object: String,
    1278
        /// Creation timestamp
    1279
        pub created: i64,
    1280
        /// Model used
    1281
        pub model: String,
    1282
        /// Choices array with deltas
    1283
        pub choices: Vec<ChatChunkChoice>,
    1284
    }
    1285
    1286
    /// Streaming choice with delta
    1287
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1288
    pub struct ChatChunkChoice {
    1289
        /// Choice index
    1290
        pub index: usize,
    1291
        /// Delta content (partial message)
    1292
        pub delta: ChatDelta,
    1293
        /// Finish reason (None until done)
    1294
        pub finish_reason: Option<String>,
    1295
    }
    1296
    1297
    /// Delta content for streaming
    1298
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1299
    pub struct ChatDelta {
    1300
        /// Role (only in first chunk)
    1301
        #[serde(skip_serializing_if = "Option::is_none")]
    1302
        pub role: Option<String>,
    1303
        /// Content chunk
    1304
        #[serde(skip_serializing_if = "Option::is_none")]
    1305
        pub content: Option<String>,
    1306
    }
    1307
    1308
    impl ChatCompletionChunk {
    1309
        /// Create a new chunk with content
    1310
    14
        fn new(id: &str, model: &str, content: Option<String>, finish_reason: Option<String>) -> Self {
    1311
            Self {
    1312
    14
                id: id.to_string(),
    1313
    14
                object: "chat.completion.chunk".to_string(),
    1314
    14
                created: std::time::SystemTime::now()
    1315
    14
                    .duration_since(std::time::UNIX_EPOCH)
    1316
    14
                    .map(|d| d.as_secs() as i64)
    1317
    14
                    .unwrap_or(0),
    1318
    14
                model: model.to_string(),
    1319
    14
                choices: vec![ChatChunkChoice {
    1320
                    index: 0,
    1321
                    delta: ChatDelta {
    1322
    14
                        role: if content.is_none() && finish_reason8.is_none8() {
    1323
    5
                            Some("assistant".to_string())
    1324
                        } else {
    1325
    9
                            None
    1326
                        },
    1327
    14
                        content,
    1328
                    },
    1329
    14
                    finish_reason,
    1330
                }],
    1331
            }
    1332
    14
        }
    1333
    1334
        /// Create initial chunk with role only
    1335
    5
        fn initial(id: &str, model: &str) -> Self {
    1336
    5
            Self::new(id, model, None, None)
    1337
    5
        }
    1338
    1339
        /// Create content chunk
    1340
    5
        fn content(id: &str, model: &str, text: &str) -> Self {
    1341
    5
            Self::new(id, model, Some(text.to_string()), None)
    1342
    5
        }
    1343
    1344
        /// Create final chunk with finish reason
    1345
    3
        fn done(id: &str, model: &str) -> Self {
    1346
    3
            Self::new(id, model, None, Some("stop".to_string()))
    1347
    3
        }
    1348
    }
    1349
    1350
    // ============================================================================
    1351
    // APR-Specific API Types (spec §15.1)
    1352
    // ============================================================================
    1353
    1354
    /// APR prediction request (classification/regression)
    1355
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1356
    pub struct PredictRequest {
    1357
        /// Model ID (optional, uses default if not specified)
    1358
        #[serde(default)]
    1359
        pub model: Option<String>,
    1360
        /// Input features as flat array
    1361
        pub features: Vec<f32>,
    1362
        /// Feature names (optional, for explainability)
    1363
        #[serde(default)]
    1364
        pub feature_names: Option<Vec<String>>,
    1365
        /// Return top-k predictions for classification
    1366
        #[serde(default)]
    1367
        pub top_k: Option<usize>,
    1368
        /// Include confidence scores
    1369
        #[serde(default = "default_true")]
    1370
        pub include_confidence: bool,
    1371
    }
    1372
    1373
    5
    pub(crate) fn default_true() -> bool {
    1374
    5
        true
    1375
    5
    }
    1376
    1377
    /// APR prediction response
    1378
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1379
    pub struct PredictResponse {
    1380
        /// Request ID for audit trail
    1381
        pub request_id: String,
    1382
        /// Model ID used
    1383
        pub model: String,
    1384
        /// Prediction result (class label or regression value)
    1385
        pub prediction: serde_json::Value,
    1386
        /// Confidence score (0.0-1.0) for classification
    1387
        #[serde(skip_serializing_if = "Option::is_none")]
    1388
        pub confidence: Option<f32>,
    1389
        /// Top-k predictions with probabilities
    1390
        #[serde(skip_serializing_if = "Option::is_none")]
    1391
        pub top_k_predictions: Option<Vec<PredictionWithScore>>,
    1392
        /// Latency in milliseconds
    1393
        pub latency_ms: f64,
    1394
    }
    1395
    1396
    /// Prediction with confidence score
    1397
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1398
    pub struct PredictionWithScore {
    1399
        /// Class label or value
    1400
        pub label: String,
    1401
        /// Probability/confidence
    1402
        pub score: f32,
    1403
    }
    1404
    1405
    /// APR explanation request
    1406
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1407
    pub struct ExplainRequest {
    1408
        /// Model ID (optional)
    1409
        #[serde(default)]
    1410
        pub model: Option<String>,
    1411
        /// Input features
    1412
        pub features: Vec<f32>,
    1413
        /// Feature names (required for meaningful explanations)
    1414
        pub feature_names: Vec<String>,
    1415
        /// Number of top features to include
    1416
        #[serde(default = "default_top_k_features")]
    1417
        pub top_k_features: usize,
    1418
        /// Explanation method (shap, lime, attention)
    1419
        #[serde(default = "default_explain_method")]
    1420
        pub method: String,
    1421
    }
    1422
    1423
    5
    pub(crate) fn default_top_k_features() -> usize {
    1424
    5
        5
    1425
    5
    }
    1426
    1427
    5
    pub(crate) fn default_explain_method() -> String {
    1428
    5
        "shap".to_string()
    1429
    5
    }
    1430
    1431
    /// APR explanation response
    1432
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1433
    pub struct ExplainResponse {
    1434
        /// Request ID for audit trail
    1435
        pub request_id: String,
    1436
        /// Model ID used
    1437
        pub model: String,
    1438
        /// Prediction (same as /v1/predict)
    1439
        pub prediction: serde_json::Value,
    1440
        /// Confidence score
    1441
        #[serde(skip_serializing_if = "Option::is_none")]
    1442
        pub confidence: Option<f32>,
    1443
        /// SHAP explanation
    1444
        pub explanation: ShapExplanation,
    1445
        /// Human-readable summary
    1446
        pub summary: String,
    1447
        /// Latency in milliseconds
    1448
        pub latency_ms: f64,
    1449
    }
    1450
    1451
    /// Audit record retrieval response
    1452
    #[derive(Debug, Clone, Serialize, Deserialize)]
    1453
    pub struct AuditResponse {
    1454
        /// The audit record
    1455
        pub record: AuditRecord,
    1456
    }
    1457
    1458
    /// Create the API router
    1459
    ///
    1460
    /// # Arguments
    1461
    ///
    1462
    /// * `state` - Application state with model and tokenizer
    1463
    108
    pub fn create_router(state: AppState) -> Router {
    1464
    108
        Router::new()
    1465
            // Health and metrics
    1466
    108
            .route("/health", get(health_handler))
    1467
    108
            .route("/metrics", get(metrics_handler))
    1468
    108
            .route("/metrics/dispatch", get(dispatch_metrics_handler))
    1469
    108
            .route("/metrics/dispatch/reset", post(dispatch_reset_handler))
    1470
            // Native Realizar API (legacy paths)
    1471
    108
            .route("/models", get(models_handler))
    1472
    108
            .route("/tokenize", post(tokenize_handler))
    1473
    108
            .route("/generate", post(generate_handler))
    1474
    108
            .route("/batch/tokenize", post(batch_tokenize_handler))
    1475
    108
            .route("/batch/generate", post(batch_generate_handler))
    1476
    108
            .route("/stream/generate", post(stream_generate_handler))
    1477
            // Native Realizar API (spec §5.2 /realize/* paths)
    1478
    108
            .route("/realize/generate", post(stream_generate_handler))
    1479
    108
            .route("/realize/batch", post(batch_generate_handler))
    1480
    108
            .route("/realize/embed", post(realize_embed_handler))
    1481
    108
            .route("/realize/model", get(realize_model_handler))
    1482
    108
            .route("/realize/reload", post(realize_reload_handler))
    1483
            // OpenAI-compatible API (v1) - spec §5.1
    1484
    108
            .route("/v1/models", get(openai_models_handler))
    1485
    108
            .route("/v1/completions", post(openai_completions_handler))
    1486
    108
            .route(
    1487
    108
                "/v1/chat/completions",
    1488
    108
                post(openai_chat_completions_handler),
    1489
            )
    1490
    108
            .route(
    1491
    108
                "/v1/chat/completions/stream",
    1492
    108
                post(openai_chat_completions_stream_handler),
    1493
            )
    1494
    108
            .route("/v1/embeddings", post(openai_embeddings_handler))
    1495
            // APR-specific API (spec §15.1)
    1496
    108
            .route("/v1/predict", post(apr_predict_handler))
    1497
    108
            .route("/v1/explain", post(apr_explain_handler))
    1498
    108
            .route("/v1/audit/:request_id", get(apr_audit_handler))
    1499
            // GPU batch inference API (PARITY-022)
    1500
    108
            .route("/v1/gpu/warmup", post(gpu_warmup_handler))
    1501
    108
            .route("/v1/gpu/status", get(gpu_status_handler))
    1502
    108
            .route("/v1/batch/completions", post(gpu_batch_completions_handler))
    1503
            // TUI monitoring API (PARITY-107)
    1504
    108
            .route("/v1/metrics", get(server_metrics_handler))
    1505
    108
            .with_state(state)
    1506
    108
    }
    1507
    1508
    /// Health check handler
    1509
    1
    async fn health_handler(State(state): State<AppState>) -> Json<HealthResponse> {
    1510
        // Determine compute mode based on what's available
    1511
        #[cfg(feature = "gpu")]
    1512
    1
        let compute_mode = if state.has_gpu_model() || state.cached_model.is_some() {
    1513
    0
            "gpu"
    1514
        } else {
    1515
    1
            "cpu"
    1516
        };
    1517
        #[cfg(not(feature = "gpu"))]
    1518
        let compute_mode = "cpu";
    1519
    1520
    1
        Json(HealthResponse {
    1521
    1
            status: "healthy".to_string(),
    1522
    1
            version: crate::VERSION.to_string(),
    1523
    1
            compute_mode: compute_mode.to_string(),
    1524
    1
        })
    1525
    1
    }
    1526
    1527
    /// Metrics handler - returns Prometheus-formatted metrics
    1528
    1
    async fn metrics_handler(State(state): State<AppState>) -> String {
    1529
    1
        state.metrics.to_prometheus()
    1530
    1
    }
    1531
    1532
    /// Response for dispatch metrics endpoint (IMP-127)
    1533
    #[derive(Debug, Clone, serde::Serialize)]
    1534
    pub struct DispatchMetricsResponse {
    1535
        /// Number of CPU dispatch decisions
    1536
        pub cpu_dispatches: usize,
    1537
        /// Number of GPU dispatch decisions
    1538
        pub gpu_dispatches: usize,
    1539
        /// Total dispatch decisions
    1540
        pub total_dispatches: usize,
    1541
        /// Ratio of GPU dispatches (0.0 to 1.0)
    1542
        pub gpu_ratio: f64,
    1543
        /// CPU latency p50 (median) in microseconds (IMP-131)
    1544
        pub cpu_latency_p50_us: f64,
    1545
        /// CPU latency p95 in microseconds (IMP-131)
    1546
        pub cpu_latency_p95_us: f64,
    1547
        /// CPU latency p99 in microseconds (IMP-131)
    1548
        pub cpu_latency_p99_us: f64,
    1549
        /// GPU latency p50 (median) in microseconds (IMP-131)
    1550
        pub gpu_latency_p50_us: f64,
    1551
        /// GPU latency p95 in microseconds (IMP-131)
    1552
        pub gpu_latency_p95_us: f64,
    1553
        /// GPU latency p99 in microseconds (IMP-131)
    1554
        pub gpu_latency_p99_us: f64,
    1555
        /// CPU latency mean in microseconds (IMP-133)
    1556
        pub cpu_latency_mean_us: f64,
    1557
        /// GPU latency mean in microseconds (IMP-133)
    1558
        pub gpu_latency_mean_us: f64,
    1559
        /// CPU latency minimum in microseconds (IMP-134)
    1560
        pub cpu_latency_min_us: u64,
    1561
        /// CPU latency maximum in microseconds (IMP-134)
    1562
        pub cpu_latency_max_us: u64,
    1563
        /// GPU latency minimum in microseconds (IMP-134)
    1564
        pub gpu_latency_min_us: u64,
    1565
        /// GPU latency maximum in microseconds (IMP-134)
    1566
        pub gpu_latency_max_us: u64,
    1567
        /// CPU latency variance in microseconds squared (IMP-135)
    1568
        pub cpu_latency_variance_us: f64,
    1569
        /// CPU latency standard deviation in microseconds (IMP-135)
    1570
        pub cpu_latency_stddev_us: f64,
    1571
        /// GPU latency variance in microseconds squared (IMP-135)
    1572
        pub gpu_latency_variance_us: f64,
    1573
        /// GPU latency standard deviation in microseconds (IMP-135)
    1574
        pub gpu_latency_stddev_us: f64,
    1575
        /// Human-readable bucket boundary ranges (IMP-136)
    1576
        pub bucket_boundaries_us: Vec<String>,
    1577
        /// CPU latency histogram bucket counts (IMP-136)
    1578
        pub cpu_latency_bucket_counts: Vec<usize>,
    1579
        /// GPU latency histogram bucket counts (IMP-136)
    1580
        pub gpu_latency_bucket_counts: Vec<usize>,
    1581
        /// Throughput in requests per second (IMP-140)
    1582
        pub throughput_rps: f64,
    1583
        /// Elapsed time in seconds since start/reset (IMP-140)
    1584
        pub elapsed_seconds: f64,
    1585
    }
    1586
    1587
    /// Server metrics response for TUI monitoring (PARITY-107)
    1588
    /// Used by realizar-monitor to display real-time server status
    1589
    #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
    1590
    pub struct ServerMetricsResponse {
    1591
        /// Current throughput in tokens per second
    1592
        pub throughput_tok_per_sec: f64,
    1593
        /// P50 (median) latency in milliseconds
    1594
        pub latency_p50_ms: f64,
    1595
        /// P95 latency in milliseconds
    1596
        pub latency_p95_ms: f64,
    1597
        /// P99 latency in milliseconds
    1598
        pub latency_p99_ms: f64,
    1599
        /// GPU memory currently used in bytes
    1600
        pub gpu_memory_used_bytes: u64,
    1601
        /// Total GPU memory available in bytes
    1602
        pub gpu_memory_total_bytes: u64,
    1603
        /// GPU utilization as percentage (0-100)
    1604
        pub gpu_utilization_percent: u32,
    1605
        /// Whether CUDA path is active
    1606
        pub cuda_path_active: bool,
    1607
        /// Current batch size
    1608
        pub batch_size: usize,
    1609
        /// Current queue depth
    1610
        pub queue_depth: usize,
    1611
        /// Total tokens generated since start
    1612
        pub total_tokens: u64,
    1613
        /// Total requests processed since start
    1614
        pub total_requests: u64,
    1615
        /// Server uptime in seconds
    1616
        pub uptime_secs: u64,
    1617
        /// Model name being served
    1618
        pub model_name: String,
    1619
    }
    1620
    1621
    /// Query parameters for dispatch metrics endpoint (IMP-128)
    1622
    #[derive(Debug, Clone, serde::Deserialize)]
    1623
    pub struct DispatchMetricsQuery {
    1624
        /// Output format: "json" (default) or "prometheus"
    1625
        #[serde(default)]
    1626
        pub format: Option<String>,
    1627
    }
    1628
    1629
    /// Response for dispatch metrics reset endpoint (IMP-138)
    1630
    #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
    1631
    pub struct DispatchResetResponse {
    1632
        /// Whether the reset was successful
    1633
        pub success: bool,
    1634
        /// Human-readable message
    1635
        pub message: String,
    1636
    }
    1637
    1638
    /// Dispatch metrics reset handler - resets all dispatch statistics (IMP-138)
    1639
    /// POST /v1/dispatch/reset
    1640
    #[cfg(feature = "gpu")]
    1641
    2
    async fn dispatch_reset_handler(State(state): State<AppState>) -> axum::response::Response {
    1642
        use axum::response::IntoResponse;
    1643
    1644
    2
        if let Some(metrics0) = state.dispatch_metrics() {
    1645
    0
            metrics.reset();
    1646
    0
            Json(DispatchResetResponse {
    1647
    0
                success: true,
    1648
    0
                message: "Metrics reset successfully".to_string(),
    1649
    0
            })
    1650
    0
            .into_response()
    1651
        } else {
    1652
    2
            (
    1653
    2
                StatusCode::SERVICE_UNAVAILABLE,
    1654
    2
                Json(ErrorResponse {
    1655
    2
                    error: "Dispatch metrics not available. No GPU model configured.".to_string(),
    1656
    2
                }),
    1657
    2
            )
    1658
    2
                .into_response()
    1659
        }
    1660
    2
    }
    1661
    1662
    /// Dispatch metrics reset handler stub for non-GPU builds (IMP-138)
    1663
    #[cfg(not(feature = "gpu"))]
    1664
    async fn dispatch_reset_handler(State(_state): State<AppState>) -> axum::response::Response {
    1665
        use axum::response::IntoResponse;
    1666
        (
    1667
            StatusCode::SERVICE_UNAVAILABLE,
    1668
            Json(ErrorResponse {
    1669
                error: "Dispatch metrics not available. GPU feature not enabled.".to_string(),
    1670
            }),
    1671
        )
    1672
            .into_response()
    1673
    }
    1674
    1675
    /// Server metrics handler for TUI monitoring (PARITY-107)
    1676
    /// GET /v1/metrics - Returns JSON metrics for realizar-monitor
    1677
    #[cfg(feature = "gpu")]
    1678
    1
    async fn server_metrics_handler(State(state): State<AppState>) -> Json<ServerMetricsResponse> {
    1679
    1
        let snapshot = state.metrics.snapshot();
    1680
    1681
        // Get latency percentiles from dispatch metrics (in microseconds, convert to ms)
    1682
    1
        let (latency_p50_ms, latency_p95_ms, latency_p99_ms, gpu_dispatches, cuda_path_active) =
    1683
    1
            if let Some(dispatch0) = state.dispatch_metrics() {
    1684
                // Use GPU latency if available, otherwise CPU latency
    1685
    0
                let gpu_p50 = dispatch.gpu_latency_p50_us();
    1686
    0
                let gpu_p95 = dispatch.gpu_latency_p95_us();
    1687
    0
                let gpu_p99 = dispatch.gpu_latency_p99_us();
    1688
    0
                let gpu_count = dispatch.gpu_dispatches();
    1689
    1690
    0
                if gpu_count > 0 {
    1691
    0
                    (
    1692
    0
                        gpu_p50 / 1000.0,
    1693
    0
                        gpu_p95 / 1000.0,
    1694
    0
                        gpu_p99 / 1000.0,
    1695
    0
                        gpu_count,
    1696
    0
                        true,
    1697
    0
                    )
    1698
                } else {
    1699
    0
                    let cpu_p50 = dispatch.cpu_latency_p50_us();
    1700
    0
                    let cpu_p95 = dispatch.cpu_latency_p95_us();
    1701
    0
                    let cpu_p99 = dispatch.cpu_latency_p99_us();
    1702
    0
                    (
    1703
    0
                        cpu_p50 / 1000.0,
    1704
    0
                        cpu_p95 / 1000.0,
    1705
    0
                        cpu_p99 / 1000.0,
    1706
    0
                        0,
    1707
    0
                        false,
    1708
    0
                    )
    1709
                }
    1710
            } else {
    1711
    1
                (0.0, 0.0, 0.0, 0, false)
    1712
            };
    1713
    1714
        // Get GPU memory from cached model
    1715
    1
        let (gpu_memory_used_bytes, gpu_memory_total_bytes): (u64, u64) =
    1716
    1
            if let Some(model0) = state.cached_model() {
    1717
    0
                let used = model.gpu_cache_memory() as u64;
    1718
                // RTX 4090 has 24GB VRAM
    1719
    0
                let total = 24 * 1024 * 1024 * 1024u64;
    1720
    0
                (used, total)
    1721
            } else {
    1722
    1
                (0, 0)
    1723
            };
    1724
    1725
        // Estimate GPU utilization from dispatch ratio
    1726
    1
        let gpu_utilization_percent = if let Some(dispatch0) = state.dispatch_metrics() {
    1727
    0
            let total = dispatch.total_dispatches();
    1728
    0
            if total > 0 {
    1729
    0
                ((gpu_dispatches as f64 / total as f64) * 100.0) as u32
    1730
            } else {
    1731
    0
                0
    1732
            }
    1733
        } else {
    1734
    1
            0
    1735
        };
    1736
    1737
        // Get batch configuration
    1738
    1
        let (batch_size, queue_depth) = if let Some(config0) = state.batch_config() {
    1739
    0
            (config.optimal_batch, config.queue_size)
    1740
        } else {
    1741
    1
            (1, 0)
    1742
        };
    1743
    1744
        // Model name from cached model or default
    1745
    1
        let model_name = if state.cached_model().is_some() {
    1746
    0
            "phi-2-q4_k_m".to_string()
    1747
        } else {
    1748
    1
            "N/A".to_string()
    1749
        };
    1750
    1751
    1
        Json(ServerMetricsResponse {
    1752
    1
            throughput_tok_per_sec: snapshot.tokens_per_sec,
    1753
    1
            latency_p50_ms,
    1754
    1
            latency_p95_ms,
    1755
    1
            latency_p99_ms,
    1756
    1
            gpu_memory_used_bytes,
    1757
    1
            gpu_memory_total_bytes,
    1758
    1
            gpu_utilization_percent,
    1759
    1
            cuda_path_active,
    1760
    1
            batch_size,
    1761
    1
            queue_depth,
    1762
    1
            total_tokens: snapshot.total_tokens as u64,
    1763
    1
            total_requests: snapshot.total_requests as u64,
    1764
    1
            uptime_secs: snapshot.uptime_secs,
    1765
    1
            model_name,
    1766
    1
        })
    1767
    1
    }
    1768
    1769
    /// Server metrics handler stub for non-GPU builds (PARITY-107)
    1770
    #[cfg(not(feature = "gpu"))]
    1771
    async fn server_metrics_handler(State(state): State<AppState>) -> Json<ServerMetricsResponse> {
    1772
        let snapshot = state.metrics.snapshot();
    1773
    1774
        Json(ServerMetricsResponse {
    1775
            throughput_tok_per_sec: snapshot.tokens_per_sec,
    1776
            latency_p50_ms: snapshot.avg_latency_ms,
    1777
            latency_p95_ms: snapshot.avg_latency_ms * 1.5,
    1778
            latency_p99_ms: snapshot.avg_latency_ms * 2.0,
    1779
            gpu_memory_used_bytes: 0,
    1780
            gpu_memory_total_bytes: 0,
    1781
            gpu_utilization_percent: 0,
    1782
            cuda_path_active: false,
    1783
            batch_size: 1,
    1784
            queue_depth: 0,
    1785
            total_tokens: snapshot.total_tokens as u64,
    1786
            total_requests: snapshot.total_requests as u64,
    1787
            uptime_secs: snapshot.uptime_secs,
    1788
            model_name: "N/A".to_string(),
    1789
        })
    1790
    }
    1791
    1792
    /// Dispatch metrics handler - returns CPU/GPU dispatch statistics (IMP-127, IMP-128)
    1793
    /// Supports ?format=prometheus for Prometheus-compatible output
    1794
    #[cfg(feature = "gpu")]
    1795
    18
    async fn dispatch_metrics_handler(
    1796
    18
        State(state): State<AppState>,
    1797
    18
        Query(query): Query<DispatchMetricsQuery>,
    1798
    18
    ) -> axum::response::Response {
    1799
        use axum::response::IntoResponse;
    1800
    1801
    18
        if let Some(metrics16) = state.dispatch_metrics() {
    1802
    16
            let format = query.format.as_deref().unwrap_or("json");
    1803
    1804
    16
            if format == "prometheus" {
    1805
                // IMP-128: Prometheus format
    1806
                // IMP-128: Basic dispatch counters
    1807
                // IMP-130: Add latency histograms
    1808
    10
                let cpu_buckets = metrics.cpu_latency_buckets();
    1809
    10
                let gpu_buckets = metrics.gpu_latency_buckets();
    1810
    1811
                // Convert to cumulative buckets for Prometheus histogram format
    1812
                // Bucket boundaries: 100µs, 500µs, 1000µs, 5000µs, +Inf
    1813
    10
                let cpu_cumulative = [
    1814
    10
                    cpu_buckets[0],
    1815
    10
                    cpu_buckets[0] + cpu_buckets[1],
    1816
    10
                    cpu_buckets[0] + cpu_buckets[1] + cpu_buckets[2],
    1817
    10
                    cpu_buckets[0] + cpu_buckets[1] + cpu_buckets[2] + cpu_buckets[3],
    1818
    10
                    cpu_buckets[0] + cpu_buckets[1] + cpu_buckets[2] + cpu_buckets[3] + cpu_buckets[4],
    1819
    10
                ];
    1820
    10
                let gpu_cumulative = [
    1821
    10
                    gpu_buckets[0],
    1822
    10
                    gpu_buckets[0] + gpu_buckets[1],
    1823
    10
                    gpu_buckets[0] + gpu_buckets[1] + gpu_buckets[2],
    1824
    10
                    gpu_buckets[0] + gpu_buckets[1] + gpu_buckets[2] + gpu_buckets[3],
    1825
    10
                    gpu_buckets[0] + gpu_buckets[1] + gpu_buckets[2] + gpu_buckets[3] + gpu_buckets[4],
    1826
    10
                ];
    1827
    1828
    10
                let prometheus_output = format!(
    1829
    10
                    "# HELP realizar_dispatch_cpu_total Total CPU dispatch decisions\n\
    1830
    10
                     # TYPE realizar_dispatch_cpu_total counter\n\
    1831
    10
                     realizar_dispatch_cpu_total {}\n\
    1832
    10
                     # HELP realizar_dispatch_gpu_total Total GPU dispatch decisions\n\
    1833
    10
                     # TYPE realizar_dispatch_gpu_total counter\n\
    1834
    10
                     realizar_dispatch_gpu_total {}\n\
    1835
    10
                     # HELP realizar_dispatch_gpu_ratio Ratio of GPU dispatches (0.0 to 1.0)\n\
    1836
    10
                     # TYPE realizar_dispatch_gpu_ratio gauge\n\
    1837
    10
                     realizar_dispatch_gpu_ratio {:.6}\n\
    1838
    10
                     # HELP realizar_dispatch_throughput_rps Requests per second since start or reset\n\
    1839
    10
                     # TYPE realizar_dispatch_throughput_rps gauge\n\
    1840
    10
                     realizar_dispatch_throughput_rps {:.6}\n\
    1841
    10
                     # HELP realizar_dispatch_elapsed_seconds Seconds since start or last reset\n\
    1842
    10
                     # TYPE realizar_dispatch_elapsed_seconds gauge\n\
    1843
    10
                     realizar_dispatch_elapsed_seconds {:.6}\n\
    1844
    10
                     # HELP realizar_dispatch_cpu_latency CPU dispatch latency in microseconds\n\
    1845
    10
                     # TYPE realizar_dispatch_cpu_latency histogram\n\
    1846
    10
                     realizar_dispatch_cpu_latency_bucket{{le=\"100\"}} {}\n\
    1847
    10
                     realizar_dispatch_cpu_latency_bucket{{le=\"500\"}} {}\n\
    1848
    10
                     realizar_dispatch_cpu_latency_bucket{{le=\"1000\"}} {}\n\
    1849
    10
                     realizar_dispatch_cpu_latency_bucket{{le=\"5000\"}} {}\n\
    1850
    10
                     realizar_dispatch_cpu_latency_bucket{{le=\"+Inf\"}} {}\n\
    1851
    10
                     realizar_dispatch_cpu_latency_sum {}\n\
    1852
    10
                     realizar_dispatch_cpu_latency_count {}\n\
    1853
    10
                     # HELP realizar_dispatch_gpu_latency GPU dispatch latency in microseconds\n\
    1854
    10
                     # TYPE realizar_dispatch_gpu_latency histogram\n\
    1855
    10
                     realizar_dispatch_gpu_latency_bucket{{le=\"100\"}} {}\n\
    1856
    10
                     realizar_dispatch_gpu_latency_bucket{{le=\"500\"}} {}\n\
    1857
    10
                     realizar_dispatch_gpu_latency_bucket{{le=\"1000\"}} {}\n\
    1858
    10
                     realizar_dispatch_gpu_latency_bucket{{le=\"5000\"}} {}\n\
    1859
    10
                     realizar_dispatch_gpu_latency_bucket{{le=\"+Inf\"}} {}\n\
    1860
    10
                     realizar_dispatch_gpu_latency_sum {}\n\
    1861
    10
                     realizar_dispatch_gpu_latency_count {}\n",
    1862
    10
                    metrics.cpu_dispatches(),
    1863
    10
                    metrics.gpu_dispatches(),
    1864
    10
                    metrics.gpu_ratio(),
    1865
                    // IMP-141: Throughput metrics
    1866
    10
                    metrics.throughput_rps(),
    1867
    10
                    metrics.elapsed_seconds(),
    1868
                    // CPU latency histogram
    1869
    10
                    cpu_cumulative[0],
    1870
    10
                    cpu_cumulative[1],
    1871
    10
                    cpu_cumulative[2],
    1872
    10
                    cpu_cumulative[3],
    1873
    10
                    cpu_cumulative[4],
    1874
    10
                    metrics.cpu_latency_sum_us(),
    1875
    10
                    metrics.cpu_latency_count(),
    1876
                    // GPU latency histogram
    1877
    10
                    gpu_cumulative[0],
    1878
    10
                    gpu_cumulative[1],
    1879
    10
                    gpu_cumulative[2],
    1880
    10
                    gpu_cumulative[3],
    1881
    10
                    gpu_cumulative[4],
    1882
    10
                    metrics.gpu_latency_sum_us(),
    1883
    10
                    metrics.gpu_latency_count(),
    1884
                );
    1885
    10
                (
    1886
    10
                    StatusCode::OK,
    1887
    10
                    [("content-type", "text/plain; charset=utf-8")],
    1888
    10
                    prometheus_output,
    1889
    10
                )
    1890
    10
                    .into_response()
    1891
            } else {
    1892
                // Default: JSON format
    1893
    6
                Json(DispatchMetricsResponse {
    1894
    6
                    cpu_dispatches: metrics.cpu_dispatches(),
    1895
    6
                    gpu_dispatches: metrics.gpu_dispatches(),
    1896
    6
                    total_dispatches: metrics.total_dispatches(),
    1897
    6
                    gpu_ratio: metrics.gpu_ratio(),
    1898
    6
                    // IMP-131: Latency percentiles
    1899
    6
                    cpu_latency_p50_us: metrics.cpu_latency_p50_us(),
    1900
    6
                    cpu_latency_p95_us: metrics.cpu_latency_p95_us(),
    1901
    6
                    cpu_latency_p99_us: metrics.cpu_latency_p99_us(),
    1902
    6
                    gpu_latency_p50_us: metrics.gpu_latency_p50_us(),
    1903
    6
                    gpu_latency_p95_us: metrics.gpu_latency_p95_us(),
    1904
    6
                    gpu_latency_p99_us: metrics.gpu_latency_p99_us(),
    1905
    6
                    // IMP-133: Latency means
    1906
    6
                    cpu_latency_mean_us: metrics.cpu_latency_mean_us(),
    1907
    6
                    gpu_latency_mean_us: metrics.gpu_latency_mean_us(),
    1908
    6
                    // IMP-134: Latency min/max
    1909
    6
                    cpu_latency_min_us: metrics.cpu_latency_min_us(),
    1910
    6
                    cpu_latency_max_us: metrics.cpu_latency_max_us(),
    1911
    6
                    gpu_latency_min_us: metrics.gpu_latency_min_us(),
    1912
    6
                    gpu_latency_max_us: metrics.gpu_latency_max_us(),
    1913
    6
                    // IMP-135: Latency variance/stddev
    1914
    6
                    cpu_latency_variance_us: metrics.cpu_latency_variance_us(),
    1915
    6
                    cpu_latency_stddev_us: metrics.cpu_latency_stddev_us(),
    1916
    6
                    gpu_latency_variance_us: metrics.gpu_latency_variance_us(),
    1917
    6
                    gpu_latency_stddev_us: metrics.gpu_latency_stddev_us(),
    1918
    6
                    // IMP-136: Histogram bucket configuration
    1919
    6
                    bucket_boundaries_us: metrics.bucket_boundaries_us(),
    1920
    6
                    cpu_latency_bucket_counts: metrics.cpu_latency_buckets().to_vec(),
    1921
    6
                    gpu_latency_bucket_counts: metrics.gpu_latency_buckets().to_vec(),
    1922
    6
                    // IMP-140: Throughput metrics
    1923
    6
                    throughput_rps: metrics.throughput_rps(),
    1924
    6
                    elapsed_seconds: metrics.elapsed_seconds(),
    1925
    6
                })
    1926
    6
                .into_response()
    1927
            }
    1928
        } else {
    1929
    2
            (
    1930
    2
                StatusCode::SERVICE_UNAVAILABLE,
    1931
    2
                Json(ErrorResponse {
    1932
    2
                    error: "Dispatch metrics not available. No GPU model configured.".to_string(),
    1933
    2
                }),
    1934
    2
            )
    1935
    2
                .into_response()
    1936
        }
    1937
    18
    }
    1938
    1939
    /// Dispatch metrics handler stub for non-GPU builds (IMP-127)
    1940
    #[cfg(not(feature = "gpu"))]
    1941
    async fn dispatch_metrics_handler(
    1942
        State(_state): State<AppState>,
    1943
        Query(_query): Query<DispatchMetricsQuery>,
    1944
    ) -> axum::response::Response {
    1945
        use axum::response::IntoResponse;
    1946
        (
    1947
            StatusCode::SERVICE_UNAVAILABLE,
    1948
            Json(ErrorResponse {
    1949
                error: "Dispatch metrics not available. GPU feature not enabled.".to_string(),
    1950
            }),
    1951
        )
    1952
            .into_response()
    1953
    }
    1954
    1955
    1956
    // Test helpers module (compiled only in tests)
    1957
    #[cfg(test)]
    1958
    pub(crate) mod test_helpers;
    1959
    1960
    // Tests split into parts for PMAT compliance (<2000 lines per file)
    1961
    #[cfg(test)]
    1962
    mod tests;