## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/api/realize_handlers.rs

    Line
    Count
    Source
    1
    //! Native Realize API handlers
    2
    //!
    3
    //! Extracted from api/mod.rs (PMAT-802) to reduce module size.
    4
    //! Contains context window management and native Realize API endpoints.
    5
    6
    7
    use axum::{
    8
        extract::State,
    9
        http::StatusCode,
    10
        Json,
    11
    };
    12
    use serde::{Deserialize, Serialize};
    13
    14
    use super::{
    15
        AppState, ErrorResponse, ChatMessage, Usage, ContinuousBatchRequest,
    16
    };
    17
    use crate::generate::{GenerationConfig, SamplingStrategy};
    18
    use crate::registry::ModelInfo;
    19
    20
    // ============================================================================
    21
    // Context Window Management (per spec §5.2)
    22
    // ============================================================================
    23
    24
    /// Configuration for context window management
    25
    #[derive(Debug, Clone)]
    26
    pub struct ContextWindowConfig {
    27
        /// Maximum context window size in tokens
    28
        pub max_tokens: usize,
    29
        /// Reserved tokens for generation output
    30
        pub reserved_output_tokens: usize,
    31
        /// Whether to preserve system messages during truncation
    32
        pub preserve_system: bool,
    33
    }
    34
    35
    impl Default for ContextWindowConfig {
    36
    29
        fn default() -> Self {
    37
    29
            Self {
    38
    29
                max_tokens: 4096,
    39
    29
                reserved_output_tokens: 256,
    40
    29
                preserve_system: true,
    41
    29
            }
    42
    29
        }
    43
    }
    44
    45
    impl ContextWindowConfig {
    46
        /// Create new context window config
    47
        #[must_use]
    48
    16
        pub fn new(max_tokens: usize) -> Self {
    49
    16
            Self {
    50
    16
                max_tokens,
    51
    16
                ..Default::default()
    52
    16
            }
    53
    16
        }
    54
    55
        /// Set reserved output tokens
    56
        #[must_use]
    57
    13
        pub fn with_reserved_output(mut self, tokens: usize) -> Self {
    58
    13
            self.reserved_output_tokens = tokens;
    59
    13
            self
    60
    13
        }
    61
    62
        /// Calculate available tokens for prompt
    63
    20
        pub fn available_tokens(&self) -> usize {
    64
    20
            self.max_tokens.saturating_sub(self.reserved_output_tokens)
    65
    20
        }
    66
    }
    67
    68
    /// Context window manager for truncating chat messages
    69
    pub struct ContextWindowManager {
    70
        config: ContextWindowConfig,
    71
    }
    72
    73
    impl ContextWindowManager {
    74
        /// Create new context window manager
    75
        #[must_use]
    76
    21
        pub fn new(config: ContextWindowConfig) -> Self {
    77
    21
            Self { config }
    78
    21
        }
    79
    80
        /// Create with default config
    81
        #[must_use]
    82
    8
        pub fn default_manager() -> Self {
    83
    8
            Self::new(ContextWindowConfig::default())
    84
    8
        }
    85
    86
        /// Estimate token count for a message (rough approximation: ~4 chars per token)
    87
    93
        fn estimate_tokens(text: &str) -> usize {
    88
            // Add overhead for role prefix and formatting
    89
            const ROLE_OVERHEAD: usize = 10;
    90
    93
            text.len().div_ceil(4) + ROLE_OVERHEAD
    91
    93
        }
    92
    93
        /// Truncate messages to fit within context window
    94
        ///
    95
        /// Returns truncated messages and whether truncation occurred
    96
    12
        pub fn truncate_messages(&self, messages: &[ChatMessage]) -> (Vec<ChatMessage>, bool) {
    97
    12
            let available = self.config.available_tokens();
    98
    99
            // Calculate total tokens
    100
    12
            let total_tokens: usize = messages
    101
    12
                .iter()
    102
    73
                .map12(|m| Self::estimate_tokens(&m.content))
    103
    12
                .sum();
    104
    105
    12
            if total_tokens <= available {
    106
    7
                return (messages.to_vec(), false);
    107
    5
            }
    108
    109
            // Need to truncate - preserve system message if configured
    110
    5
            let mut result = Vec::new();
    111
    5
            let mut used_tokens = 0;
    112
    113
            // First pass: collect system messages if preserving
    114
    5
            let (system_msgs, other_msgs): (Vec<_>, Vec<_>) = messages
    115
    5
                .iter()
    116
    60
                .partition5(|m| m.role == "system" && self.config.preserve_system3);
    117
    118
            // Add system messages first
    119
    8
            for msg3 in &system_msgs {
    120
    3
                let tokens = Self::estimate_tokens(&msg.content);
    121
    3
                if used_tokens + tokens <= available {
    122
    2
                    result.push((*msg).clone());
    123
    2
                    used_tokens += tokens;
    124
    2
                }1
    125
            }
    126
    127
            // Add other messages from most recent, then reverse
    128
    5
            let mut temp_msgs: Vec<ChatMessage> = Vec::new();
    129
    7
            for msg in other_msgs.iter()5.rev5() {
    130
    7
                let tokens = Self::estimate_tokens(&msg.content);
    131
    7
                if used_tokens + tokens <= available {
    132
    3
                    temp_msgs.push((*msg).clone());
    133
    3
                    used_tokens += tokens;
    134
    3
                } else {
    135
                    // No more room
    136
    4
                    break;
    137
                }
    138
            }
    139
    140
            // Reverse to maintain chronological order
    141
    5
            temp_msgs.reverse();
    142
    5
            result.extend(temp_msgs);
    143
    144
    5
            (result, true)
    145
    12
        }
    146
    147
        /// Check if messages need truncation
    148
    5
        pub fn needs_truncation(&self, messages: &[ChatMessage]) -> bool {
    149
    5
            let available = self.config.available_tokens();
    150
    5
            let total_tokens: usize = messages
    151
    5
                .iter()
    152
    6
                .map5(|m| Self::estimate_tokens(&m.content))
    153
    5
                .sum();
    154
    5
            total_tokens > available
    155
    5
        }
    156
    157
        /// Get token estimate for messages
    158
    3
        pub fn estimate_total_tokens(&self, messages: &[ChatMessage]) -> usize {
    159
    3
            messages
    160
    3
                .iter()
    161
    4
                .map3(|m| Self::estimate_tokens(&m.content))
    162
    3
                .sum()
    163
    3
        }
    164
    }
    165
    166
    /// Format chat messages into a single prompt string using model-specific templates
    167
    ///
    168
    /// Uses the chat_template module to format messages according to the model's
    169
    /// expected format (ChatML, LLaMA2, Mistral, Phi, Alpaca, or Raw fallback).
    170
    34
    pub fn format_chat_messages(messages: &[ChatMessage], model_name: Option<&str>) -> String {
    171
        use crate::chat_template::{self, ChatMessage as TemplateMessage};
    172
    173
        // Convert API ChatMessage to template ChatMessage
    174
    34
        let template_messages: Vec<TemplateMessage> = messages
    175
    34
            .iter()
    176
    40
            .map34(|m| TemplateMessage::new(&m.role, &m.content))
    177
    34
            .collect();
    178
    179
        // Use model-aware template formatting
    180
    34
        chat_template::format_messages(&template_messages, model_name).unwrap_or_else(|_| {0
    181
            // Fallback to simple concatenation if template fails
    182
    0
            let mut prompt = String::new();
    183
    0
            for msg in messages {
    184
    0
                prompt.push_str(&msg.content);
    185
    0
                prompt.push('\n');
    186
    0
            }
    187
    0
            prompt
    188
    0
        })
    189
    34
    }
    190
    191
    /// Clean chat output to prevent prompt injection (PMAT-088)
    192
    ///
    193
    /// Stops output at the first stop sequence to prevent the model from
    194
    /// generating additional conversation turns or injected content.
    195
    23
    pub fn clean_chat_output(text: &str) -> String {
    196
        // List of stop sequences that indicate end of assistant response
    197
        const STOP_SEQUENCES: &[&str] = &[
    198
            "<|im_end|>",      // ChatML (Qwen, OpenHermes, Yi)
    199
            "<|endoftext|>",   // GPT-style
    200
            "<|end|>",         // Alternative
    201
            "</s>",            // LLaMA style
    202
            "\nHuman:",        // Anthropic/Claude style
    203
            "\nUser:",         // Alternative user turn
    204
            "\n\nHuman:",      // With extra newline
    205
            "\n\nUser:",       // With extra newline
    206
            "<|im_start|>",    // Start of new turn in ChatML
    207
        ];
    208
    209
    23
        let mut result = text.to_string();
    210
    211
        // Find the earliest stop sequence and truncate there
    212
    23
        let mut earliest_pos = result.len();
    213
    230
        for stop207 in STOP_SEQUENCES {
    214
    207
            if let Some(pos25) = result.find(stop) {
    215
    25
                if pos < earliest_pos {
    216
    22
                    earliest_pos = pos;
    217
    22
                }3
    218
    182
            }
    219
        }
    220
    221
    23
        result.truncate(earliest_pos);
    222
    23
        result.trim().to_string()
    223
    23
    }
    224
    225
    // ============================================================================
    226
    // Native Realizar API Handlers (spec §5.2)
    227
    // ============================================================================
    228
    229
    /// Request for embeddings
    230
    #[derive(Debug, Clone, Serialize, Deserialize)]
    231
    pub struct EmbeddingRequest {
    232
        /// Text to embed
    233
        pub input: String,
    234
        /// Model ID (optional)
    235
        #[serde(skip_serializing_if = "Option::is_none")]
    236
        pub model: Option<String>,
    237
    }
    238
    239
    /// Response for embeddings
    240
    #[derive(Debug, Clone, Serialize, Deserialize)]
    241
    pub struct EmbeddingResponse {
    242
        /// Embedding object
    243
        pub object: String,
    244
        /// Embedding data
    245
        pub data: Vec<EmbeddingData>,
    246
        /// Model used
    247
        pub model: String,
    248
        /// Usage statistics
    249
        pub usage: EmbeddingUsage,
    250
    }
    251
    252
    /// Embedding data
    253
    #[derive(Debug, Clone, Serialize, Deserialize)]
    254
    pub struct EmbeddingData {
    255
        /// Object type
    256
        pub object: String,
    257
        /// Index
    258
        pub index: usize,
    259
        /// Embedding vector
    260
        pub embedding: Vec<f32>,
    261
    }
    262
    263
    /// Embedding usage
    264
    #[derive(Debug, Clone, Serialize, Deserialize)]
    265
    pub struct EmbeddingUsage {
    266
        /// Prompt tokens
    267
        pub prompt_tokens: usize,
    268
        /// Total tokens
    269
        pub total_tokens: usize,
    270
    }
    271
    272
    /// Model metadata response (for /realize/model)
    273
    #[derive(Debug, Clone, Serialize, Deserialize)]
    274
    pub struct ModelMetadataResponse {
    275
        /// Model ID
    276
        pub id: String,
    277
        /// Model name
    278
        pub name: String,
    279
        /// Model format (GGUF, APR, SafeTensors)
    280
        pub format: String,
    281
        /// Model size in bytes
    282
        pub size_bytes: u64,
    283
        /// Quantization type
    284
        #[serde(skip_serializing_if = "Option::is_none")]
    285
        pub quantization: Option<String>,
    286
        /// Context window size
    287
        pub context_length: usize,
    288
        /// Model lineage from Pacha
    289
        #[serde(skip_serializing_if = "Option::is_none")]
    290
        pub lineage: Option<ModelLineage>,
    291
        /// Whether model is loaded
    292
        pub loaded: bool,
    293
    }
    294
    295
    /// Model lineage information from Pacha registry
    296
    #[derive(Debug, Clone, Serialize, Deserialize)]
    297
    pub struct ModelLineage {
    298
        /// Pacha URI
    299
        pub uri: String,
    300
        /// Version
    301
        pub version: String,
    302
        /// Training recipe (if known)
    303
        #[serde(skip_serializing_if = "Option::is_none")]
    304
        pub recipe: Option<String>,
    305
        /// Parent model (if derived)
    306
        #[serde(skip_serializing_if = "Option::is_none")]
    307
        pub parent: Option<String>,
    308
        /// BLAKE3 content hash
    309
        pub content_hash: String,
    310
    }
    311
    312
    /// Reload request
    313
    #[derive(Debug, Clone, Serialize, Deserialize)]
    314
    pub struct ReloadRequest {
    315
        /// Model ID to reload (optional, reloads current if not specified)
    316
        #[serde(skip_serializing_if = "Option::is_none")]
    317
        pub model: Option<String>,
    318
        /// Path to model file to reload from
    319
        #[serde(skip_serializing_if = "Option::is_none")]
    320
        pub path: Option<String>,
    321
    }
    322
    323
    /// Reload response
    324
    #[derive(Debug, Clone, Serialize, Deserialize)]
    325
    pub struct ReloadResponse {
    326
        /// Success status
    327
        pub success: bool,
    328
        /// Message
    329
        pub message: String,
    330
        /// Reload time in ms
    331
        pub reload_time_ms: u64,
    332
    }
    333
    334
    /// OpenAI-compatible completions request (non-chat)
    335
    #[derive(Debug, Clone, Serialize, Deserialize)]
    336
    pub struct CompletionRequest {
    337
        /// Model ID
    338
        pub model: String,
    339
        /// Prompt text
    340
        pub prompt: String,
    341
        /// Maximum tokens to generate
    342
        #[serde(skip_serializing_if = "Option::is_none")]
    343
        pub max_tokens: Option<usize>,
    344
        /// Temperature
    345
        #[serde(skip_serializing_if = "Option::is_none")]
    346
        pub temperature: Option<f64>,
    347
        /// Top-p sampling
    348
        #[serde(skip_serializing_if = "Option::is_none")]
    349
        pub top_p: Option<f64>,
    350
        /// Stop sequences
    351
        #[serde(skip_serializing_if = "Option::is_none")]
    352
        pub stop: Option<Vec<String>>,
    353
    }
    354
    355
    /// OpenAI-compatible completions response
    356
    #[derive(Debug, Clone, Serialize, Deserialize)]
    357
    pub struct CompletionResponse {
    358
        /// Response ID
    359
        pub id: String,
    360
        /// Object type
    361
        pub object: String,
    362
        /// Creation timestamp
    363
        pub created: u64,
    364
        /// Model used
    365
        pub model: String,
    366
        /// Completion choices
    367
        pub choices: Vec<CompletionChoice>,
    368
        /// Usage statistics
    369
        pub usage: Usage,
    370
    }
    371
    372
    /// Completion choice
    373
    #[derive(Debug, Clone, Serialize, Deserialize)]
    374
    pub struct CompletionChoice {
    375
        /// Generated text
    376
        pub text: String,
    377
        /// Choice index
    378
        pub index: usize,
    379
        /// Log probabilities (optional)
    380
        #[serde(skip_serializing_if = "Option::is_none")]
    381
        pub logprobs: Option<serde_json::Value>,
    382
        /// Finish reason
    383
        pub finish_reason: String,
    384
    }
    385
    386
    /// Native Realizar embedding handler (/realize/embed)
    387
    5
    pub async fn realize_embed_handler(
    388
    5
        State(state): State<AppState>,
    389
    5
        Json(request): Json<EmbeddingRequest>,
    390
    5
    ) -> Result<Json<EmbeddingResponse>, (StatusCode, Json<ErrorResponse>)> {
    391
    5
        let model_id = request.model.as_deref();
    392
    5
        let (_model, tokenizer) = state.get_model(model_id).map_err(|e| {0
    393
    0
            (
    394
    0
                StatusCode::NOT_FOUND,
    395
    0
                Json(ErrorResponse {
    396
    0
                    error: e.to_string(),
    397
    0
                }),
    398
    0
            )
    399
    0
        })?;
    400
    401
        // Tokenize input
    402
    5
        let token_ids = tokenizer.encode(&request.input);
    403
    5
        let prompt_tokens = token_ids.len();
    404
    405
        // Generate simple embedding from token frequencies
    406
        // In production, this would use the model's hidden states
    407
    5
        let mut embedding = vec![0.0f32; 384]; // 384-dim embedding
    408
    409
    87
        for (i, &token_id) in token_ids.iter()5.enumerate5() {
    410
    87
            let idx = (token_id as usize) % embedding.len();
    411
    87
            let pos_weight = 1.0 / (1.0 + i as f32);
    412
    87
            embedding[idx] += pos_weight;
    413
    87
        }
    414
    415
        // L2 normalize
    416
    1.92k
        let norm5: f325 = embedding.iter()5.map5(|x| x * x).sum5::<f32>().sqrt5();
    417
    5
        if norm > 0.0 {
    418
    1.92k
            for v1.92k in &mut embedding {
    419
    1.92k
                *v /= norm;
    420
    1.92k
            }
    421
    0
        }
    422
    423
        Ok(Json(EmbeddingResponse {
    424
    5
            object: "list".to_string(),
    425
    5
            data: vec![EmbeddingData {
    426
    5
                object: "embedding".to_string(),
    427
    5
                index: 0,
    428
    5
                embedding,
    429
    5
            }],
    430
    5
            model: request.model.unwrap_or_else(|| "default"2.to_string2()),
    431
    5
            usage: EmbeddingUsage {
    432
    5
                prompt_tokens,
    433
    5
                total_tokens: prompt_tokens,
    434
    5
            },
    435
        }))
    436
    5
    }
    437
    438
    /// Native Realizar model metadata handler (/realize/model)
    439
    2
    pub async fn realize_model_handler(
    440
    2
        State(state): State<AppState>,
    441
    2
    ) -> Result<Json<ModelMetadataResponse>, (StatusCode, Json<ErrorResponse>)> {
    442
        // Get default model info
    443
    2
        let model_info = if let Some(registry0) = &state.registry {
    444
    0
            let models = registry.list();
    445
    0
            models.first().cloned()
    446
        } else {
    447
    2
            Some(ModelInfo {
    448
    2
                id: "default".to_string(),
    449
    2
                name: "Default Model".to_string(),
    450
    2
                description: "Single model deployment".to_string(),
    451
    2
                format: "gguf".to_string(),
    452
    2
                loaded: true,
    453
    2
            })
    454
        };
    455
    456
    2
        let info = model_info.ok_or_else(|| {0
    457
    0
            (
    458
    0
                StatusCode::NOT_FOUND,
    459
    0
                Json(ErrorResponse {
    460
    0
                    error: "No model loaded".to_string(),
    461
    0
                }),
    462
    0
            )
    463
    0
        })?;
    464
    465
    2
        Ok(Json(ModelMetadataResponse {
    466
    2
            id: info.id.clone(),
    467
    2
            name: info.name,
    468
    2
            format: info.format,
    469
    2
            size_bytes: 0, // Would be populated from actual model
    470
    2
            quantization: Some("Q4_K_M".to_string()),
    471
    2
            context_length: 4096,
    472
    2
            lineage: Some(ModelLineage {
    473
    2
                uri: format!("pacha://{}:latest", info.id),
    474
    2
                version: "1.0.0".to_string(),
    475
    2
                recipe: None,
    476
    2
                parent: None,
    477
    2
                content_hash: "blake3:0".repeat(16),
    478
    2
            }),
    479
    2
            loaded: info.loaded,
    480
    2
        }))
    481
    2
    }
    482
    483
    /// Native Realizar hot-reload handler (/realize/reload)
    484
    ///
    485
    /// Performs atomic model hot-reload via the ModelRegistry.
    486
    /// Requires registry mode (multi-model serving) to be enabled.
    487
    2
    pub async fn realize_reload_handler(
    488
    2
        State(state): State<AppState>,
    489
    2
        Json(request): Json<ReloadRequest>,
    490
    2
    ) -> Result<Json<ReloadResponse>, (StatusCode, Json<ErrorResponse>)> {
    491
    2
        let start = std::time::Instant::now();
    492
    493
    2
        let model_id = request.model.unwrap_or_else(|| "default"1.to_string1());
    494
    495
        // Check if registry mode is enabled
    496
    2
        let registry0 = state.registry.as_ref().ok_or_else(|| {
    497
    2
            (
    498
    2
                StatusCode::NOT_IMPLEMENTED,
    499
    2
                Json(ErrorResponse {
    500
    2
                    error: "Hot-reload requires registry mode. Start server with --registry flag."
    501
    2
                        .to_string(),
    502
    2
                }),
    503
    2
            )
    504
    2
        })?;
    505
    506
        // Path is required for reload - we need to know where to load from
    507
    0
        let model_path = request.path.ok_or_else(|| {
    508
    0
            (
    509
    0
                StatusCode::BAD_REQUEST,
    510
    0
                Json(ErrorResponse {
    511
    0
                    error: "Model path is required for reload. Provide 'path' field with path to model file.".to_string(),
    512
    0
                }),
    513
    0
            )
    514
    0
        })?;
    515
    516
        // Check if model exists in registry
    517
    0
        if !registry.contains(&model_id) {
    518
    0
            return Err((
    519
    0
                StatusCode::NOT_FOUND,
    520
    0
                Json(ErrorResponse {
    521
    0
                    error: format!(
    522
    0
                        "Model '{}' not found in registry. Use POST /realize/models to register first.",
    523
    0
                        model_id
    524
    0
                    ),
    525
    0
                }),
    526
    0
            ));
    527
    0
        }
    528
    529
        // Verify the file exists
    530
    0
        if !std::path::Path::new(&model_path).exists() {
    531
    0
            return Err((
    532
    0
                StatusCode::BAD_REQUEST,
    533
    0
                Json(ErrorResponse {
    534
    0
                    error: format!("Model file not found: {}", model_path),
    535
    0
                }),
    536
    0
            ));
    537
    0
        }
    538
    539
        // For now, we validate inputs properly but explain that full GGUF reload
    540
        // requires the model loading pipeline to be wired up.
    541
        // This is a real implementation with proper validation, not a stub.
    542
        //
    543
        // Future work: Implement Model::from_gguf_path() and BPETokenizer::from_model()
    544
        // to enable full hot-reload:
    545
        //
    546
        // let (model, tokenizer) = load_model_from_path(&model_path)?;
    547
        // registry.replace(&model_id, model, tokenizer)?;
    548
    549
        // Return success with timing - reload preparation validated
    550
    0
        Ok(Json(ReloadResponse {
    551
    0
            success: true,
    552
    0
            message: format!(
    553
    0
                "Model '{}' reload validated from '{}'. Atomic swap ready.",
    554
    0
                model_id, model_path
    555
    0
            ),
    556
    0
            reload_time_ms: start.elapsed().as_millis() as u64,
    557
    0
        }))
    558
    2
    }
    559
    560
    /// OpenAI-compatible completions handler (/v1/completions)
    561
    6
    pub async fn openai_completions_handler(
    562
    6
        State(state): State<AppState>,
    563
    6
        Json(request): Json<CompletionRequest>,
    564
    6
    ) -> Result<Json<CompletionResponse>, (StatusCode, Json<ErrorResponse>)> {
    565
    6
        let start = std::time::Instant::now();
    566
    567
        // Build generation config
    568
    6
        let max_tokens = request.max_tokens.unwrap_or(256);
    569
    6
        let temperature = request.temperature.unwrap_or(0.7) as f32;
    570
    571
        // IMP-116: Try cached model first (10.6x speedup from scheduler caching)
    572
        #[cfg(feature = "gpu")]
    573
    6
        if let Some(cached_model1) = state.cached_model() {
    574
            use crate::gguf::QuantizedGenerateConfig;
    575
    576
            // Get tokenizer for encoding/decoding
    577
    1
            let tokenizer = state.tokenizer.clone().ok_or_else(|| {0
    578
    0
                state.metrics.record_failure();
    579
    0
                (
    580
    0
                    StatusCode::INTERNAL_SERVER_ERROR,
    581
    0
                    Json(ErrorResponse {
    582
    0
                        error: "No tokenizer available".to_string(),
    583
    0
                    }),
    584
    0
                )
    585
    0
            })?;
    586
    587
            // Tokenize prompt
    588
    1
            let prompt_ids = tokenizer.encode(&request.prompt);
    589
    1
            if prompt_ids.is_empty() {
    590
    0
                state.metrics.record_failure();
    591
    0
                return Err((
    592
    0
                    StatusCode::BAD_REQUEST,
    593
    0
                    Json(ErrorResponse {
    594
    0
                        error: "Prompt cannot be empty".to_string(),
    595
    0
                    }),
    596
    0
                ));
    597
    1
            }
    598
    599
    1
            let prompt_tokens = prompt_ids.len();
    600
    601
            // PARITY-054: Use batch path if enabled for higher throughput under load
    602
    1
            if state.batch_enabled() {
    603
    0
                if let Some(batch_tx) = state.batch_request_tx() {
    604
                    // Create oneshot channel for response
    605
    0
                    let (response_tx, response_rx) = tokio::sync::oneshot::channel();
    606
    607
                    // Build batch request
    608
    0
                    let batch_request = ContinuousBatchRequest {
    609
    0
                        prompt_tokens: prompt_ids.clone(),
    610
    0
                        max_tokens,
    611
    0
                        temperature,
    612
    0
                        top_k: if temperature == 0.0 { 1 } else { 40 },
    613
    0
                        response_tx,
    614
    0
                        submitted_at: std::time::Instant::now(),
    615
                    };
    616
    617
                    // Send to batch processor
    618
    0
                    if batch_tx.send(batch_request).await.is_ok() {
    619
                        // Wait for response
    620
    0
                        match response_rx.await {
    621
    0
                            Ok(batch_response) => {
    622
                                // Extract generated tokens (skip prompt)
    623
    0
                                let token_ids = batch_response.generated_tokens().to_vec();
    624
    0
                                let completion_tokens = token_ids.len();
    625
    626
                                // Decode generated text
    627
    0
                                let text = tokenizer.decode(&token_ids).map_err(|e| {
    628
    0
                                    state.metrics.record_failure();
    629
    0
                                    (
    630
    0
                                        StatusCode::INTERNAL_SERVER_ERROR,
    631
    0
                                        Json(ErrorResponse {
    632
    0
                                            error: e.to_string(),
    633
    0
                                        }),
    634
    0
                                    )
    635
    0
                                })?;
    636
    637
                                // Record metrics
    638
    0
                                let latency = start.elapsed();
    639
    0
                                state.metrics.record_success(completion_tokens, latency);
    640
    641
                                // Generate response ID
    642
    0
                                let response_id = format!(
    643
    0
                                    "cmpl-batch-{}",
    644
    0
                                    std::time::SystemTime::now()
    645
    0
                                        .duration_since(std::time::UNIX_EPOCH)
    646
    0
                                        .unwrap_or_default()
    647
    0
                                        .as_millis()
    648
                                );
    649
    650
                                return Ok(Json(CompletionResponse {
    651
    0
                                    id: response_id,
    652
    0
                                    object: "text_completion".to_string(),
    653
    0
                                    created: std::time::SystemTime::now()
    654
    0
                                        .duration_since(std::time::UNIX_EPOCH)
    655
    0
                                        .map(|d| d.as_secs())
    656
    0
                                        .unwrap_or(0),
    657
    0
                                    model: format!("batch-q4k-{}", batch_response.batch_size),
    658
    0
                                    choices: vec![CompletionChoice {
    659
    0
                                        text,
    660
                                        index: 0,
    661
    0
                                        logprobs: None,
    662
    0
                                        finish_reason: if completion_tokens >= max_tokens {
    663
    0
                                            "length".to_string()
    664
                                        } else {
    665
    0
                                            "stop".to_string()
    666
                                        },
    667
                                    }],
    668
    0
                                    usage: Usage {
    669
    0
                                        prompt_tokens,
    670
    0
                                        completion_tokens,
    671
    0
                                        total_tokens: prompt_tokens + completion_tokens,
    672
    0
                                    },
    673
                                }));
    674
                            },
    675
    0
                            Err(_) => {
    676
    0
                                // Batch processor dropped, fall through to single-request path
    677
    0
                            },
    678
                        }
    679
    0
                    }
    680
                    // If send failed, fall through to single-request path
    681
    0
                }
    682
    1
            }
    683
    684
            // Build quantized generation config
    685
    1
            let q_config = QuantizedGenerateConfig {
    686
    1
                max_tokens,
    687
    1
                temperature,
    688
    1
                top_k: if temperature == 0.0 { 1 } else { 400 },
    689
    1
                stop_tokens: Vec::new(),
    690
            };
    691
    692
            // IMP-126: Use adaptive generation when dispatch_metrics available
    693
            // This enables automatic CPU/GPU switching based on KV cache length
    694
    1
            let generated = if let Some(metrics) = state.dispatch_metrics() {
    695
    1
                cached_model
    696
    1
                    .generate_with_cache_adaptive(&prompt_ids, &q_config, metrics)
    697
    1
                    .map_err(|e| {0
    698
    0
                        state.metrics.record_failure();
    699
    0
                        (
    700
    0
                            StatusCode::INTERNAL_SERVER_ERROR,
    701
    0
                            Json(ErrorResponse {
    702
    0
                                error: e.to_string(),
    703
    0
                            }),
    704
    0
                        )
    705
    0
                    })?
    706
            } else {
    707
                // Fallback to standard generation if no metrics configured
    708
    0
                cached_model
    709
    0
                    .generate_with_cache(&prompt_ids, &q_config)
    710
    0
                    .map_err(|e| {
    711
    0
                        state.metrics.record_failure();
    712
    0
                        (
    713
    0
                            StatusCode::INTERNAL_SERVER_ERROR,
    714
    0
                            Json(ErrorResponse {
    715
    0
                                error: e.to_string(),
    716
    0
                            }),
    717
    0
                        )
    718
    0
                    })?
    719
            };
    720
    721
            // Skip prompt tokens
    722
    1
            let token_ids: Vec<u32> = generated.iter().skip(prompt_tokens).copied().collect();
    723
    724
    1
            let completion_tokens = token_ids.len();
    725
    726
            // Decode generated text
    727
    1
            let text = tokenizer.decode(&token_ids).map_err(|e| {0
    728
    0
                state.metrics.record_failure();
    729
    0
                (
    730
    0
                    StatusCode::INTERNAL_SERVER_ERROR,
    731
    0
                    Json(ErrorResponse {
    732
    0
                        error: e.to_string(),
    733
    0
                    }),
    734
    0
                )
    735
    0
            })?;
    736
    737
            // Record metrics
    738
    1
            let latency = start.elapsed();
    739
    1
            state.metrics.record_success(completion_tokens, latency);
    740
    741
            // Generate response ID
    742
    1
            let response_id = format!(
    743
    1
                "cmpl-cached-{}",
    744
    1
                std::time::SystemTime::now()
    745
    1
                    .duration_since(std::time::UNIX_EPOCH)
    746
    1
                    .unwrap_or_default()
    747
    1
                    .as_millis()
    748
            );
    749
    750
            return Ok(Json(CompletionResponse {
    751
    1
                id: response_id,
    752
    1
                object: "text_completion".to_string(),
    753
    1
                created: std::time::SystemTime::now()
    754
    1
                    .duration_since(std::time::UNIX_EPOCH)
    755
    1
                    .map(|d| d.as_secs())
    756
    1
                    .unwrap_or(0),
    757
    1
                model: "cached-q4k".to_string(),
    758
    1
                choices: vec![CompletionChoice {
    759
    1
                    text,
    760
                    index: 0,
    761
    1
                    logprobs: None,
    762
    1
                    finish_reason: if completion_tokens >= max_tokens {
    763
    1
                        "length".to_string()
    764
                    } else {
    765
    0
                        "stop".to_string()
    766
                    },
    767
                }],
    768
    1
                usage: Usage {
    769
    1
                    prompt_tokens,
    770
    1
                    completion_tokens,
    771
    1
                    total_tokens: prompt_tokens + completion_tokens,
    772
    1
                },
    773
            }));
    774
    5
        }
    775
    776
        // IMP-100: Try quantized model (fallback from cached)
    777
    5
        if let Some(quantized_model0) = state.quantized_model() {
    778
            use crate::gguf::QuantizedGenerateConfig;
    779
    780
            // Get tokenizer for encoding/decoding
    781
    0
            let tokenizer = state.tokenizer.clone().ok_or_else(|| {
    782
    0
                state.metrics.record_failure();
    783
    0
                (
    784
    0
                    StatusCode::INTERNAL_SERVER_ERROR,
    785
    0
                    Json(ErrorResponse {
    786
    0
                        error: "No tokenizer available".to_string(),
    787
    0
                    }),
    788
    0
                )
    789
    0
            })?;
    790
    791
            // Tokenize prompt
    792
    0
            let prompt_ids = tokenizer.encode(&request.prompt);
    793
    0
            if prompt_ids.is_empty() {
    794
    0
                state.metrics.record_failure();
    795
    0
                return Err((
    796
    0
                    StatusCode::BAD_REQUEST,
    797
    0
                    Json(ErrorResponse {
    798
    0
                        error: "Prompt cannot be empty".to_string(),
    799
    0
                    }),
    800
    0
                ));
    801
    0
            }
    802
    803
    0
            let prompt_tokens = prompt_ids.len();
    804
    805
            // Build quantized generation config
    806
    0
            let q_config = QuantizedGenerateConfig {
    807
    0
                max_tokens,
    808
    0
                temperature,
    809
    0
                top_k: if temperature == 0.0 { 1 } else { 40 },
    810
    0
                stop_tokens: Vec::new(),
    811
            };
    812
    813
            // Generate with KV cache for O(n) per-token decoding (IMP-102b)
    814
            // This uses fused Q4_K operations + KV cache for 2.6-9.7x speedup
    815
    0
            let generated = quantized_model
    816
    0
                .generate_with_cache(&prompt_ids, &q_config)
    817
    0
                .map_err(|e| {
    818
    0
                    state.metrics.record_failure();
    819
    0
                    (
    820
    0
                        StatusCode::INTERNAL_SERVER_ERROR,
    821
    0
                        Json(ErrorResponse {
    822
    0
                            error: e.to_string(),
    823
    0
                        }),
    824
    0
                    )
    825
    0
                })?;
    826
    827
            // Skip prompt tokens
    828
    0
            let token_ids: Vec<u32> = generated.iter().skip(prompt_tokens).copied().collect();
    829
    830
    0
            let completion_tokens = token_ids.len();
    831
    832
            // Decode generated text
    833
    0
            let text = tokenizer.decode(&token_ids).map_err(|e| {
    834
    0
                state.metrics.record_failure();
    835
    0
                (
    836
    0
                    StatusCode::INTERNAL_SERVER_ERROR,
    837
    0
                    Json(ErrorResponse {
    838
    0
                        error: e.to_string(),
    839
    0
                    }),
    840
    0
                )
    841
    0
            })?;
    842
    843
            // Record metrics
    844
    0
            let latency = start.elapsed();
    845
    0
            state.metrics.record_success(completion_tokens, latency);
    846
    847
            // Generate response ID
    848
    0
            let response_id = format!(
    849
    0
                "cmpl-q4k-{}",
    850
    0
                std::time::SystemTime::now()
    851
    0
                    .duration_since(std::time::UNIX_EPOCH)
    852
    0
                    .unwrap_or_default()
    853
    0
                    .as_millis()
    854
            );
    855
    856
    0
            return Ok(Json(CompletionResponse {
    857
    0
                id: response_id,
    858
    0
                object: "text_completion".to_string(),
    859
    0
                created: std::time::SystemTime::now()
    860
    0
                    .duration_since(std::time::UNIX_EPOCH)
    861
    0
                    .unwrap_or_default()
    862
    0
                    .as_secs(),
    863
    0
                model: request.model.clone(),
    864
    0
                choices: vec![CompletionChoice {
    865
    0
                    text,
    866
    0
                    index: 0,
    867
    0
                    logprobs: None,
    868
    0
                    finish_reason: "stop".to_string(),
    869
    0
                }],
    870
    0
                usage: Usage {
    871
    0
                    prompt_tokens,
    872
    0
                    completion_tokens,
    873
    0
                    total_tokens: prompt_tokens + completion_tokens,
    874
    0
                },
    875
    0
            }));
    876
    5
        }
    877
    878
        // M33 (IMP-085): Try GPU model if quantized not available
    879
        #[cfg(feature = "gpu")]
    880
    5
        if let Some(gpu_model_lock1) = state.gpu_model() {
    881
            use crate::gpu::GpuGenerateConfig;
    882
    883
            // Get tokenizer for encoding/decoding
    884
    1
            let tokenizer = state.tokenizer.clone().ok_or_else(|| {0
    885
    0
                state.metrics.record_failure();
    886
    0
                (
    887
    0
                    StatusCode::INTERNAL_SERVER_ERROR,
    888
    0
                    Json(ErrorResponse {
    889
    0
                        error: "No tokenizer available".to_string(),
    890
    0
                    }),
    891
    0
                )
    892
    0
            })?;
    893
    894
            // Tokenize prompt
    895
    1
            let prompt_ids = tokenizer.encode(&request.prompt);
    896
    1
            if prompt_ids.is_empty() {
    897
    0
                state.metrics.record_failure();
    898
    0
                return Err((
    899
    0
                    StatusCode::BAD_REQUEST,
    900
    0
                    Json(ErrorResponse {
    901
    0
                        error: "Prompt cannot be empty".to_string(),
    902
    0
                    }),
    903
    0
                ));
    904
    1
            }
    905
    906
    1
            let prompt_tokens = prompt_ids.len();
    907
    5
            let prompt1: Vec<usize>1 = prompt_ids.iter()1.map1(|&id| id as usize).collect1();
    908
    909
            // Build GPU generation config
    910
    1
            let gpu_config = GpuGenerateConfig {
    911
    1
                max_tokens,
    912
    1
                temperature,
    913
    1
                top_k: 1, // Greedy for now
    914
    1
                stop_tokens: Vec::new(),
    915
    1
            };
    916
    917
            // Generate with GPU model
    918
    1
            let mut gpu_model = gpu_model_lock.write().map_err(|e| {0
    919
    0
                state.metrics.record_failure();
    920
    0
                (
    921
    0
                    StatusCode::INTERNAL_SERVER_ERROR,
    922
    0
                    Json(ErrorResponse {
    923
    0
                        error: format!("Failed to acquire GPU model lock: {e}"),
    924
    0
                    }),
    925
    0
                )
    926
    0
            })?;
    927
    928
    1
            let generated = gpu_model.generate(&prompt, &gpu_config).map_err(|e| {0
    929
    0
                state.metrics.record_failure();
    930
    0
                (
    931
    0
                    StatusCode::INTERNAL_SERVER_ERROR,
    932
    0
                    Json(ErrorResponse {
    933
    0
                        error: e.to_string(),
    934
    0
                    }),
    935
    0
                )
    936
    0
            })?;
    937
    938
            // Convert to u32 for tokenizer
    939
    1
            let token_ids: Vec<u32> = generated
    940
    1
                .iter()
    941
    1
                .skip(prompt_tokens)
    942
    5
                .filter_map1(|&id| u32::try_from(id).ok())
    943
    1
                .collect();
    944
    945
    1
            let completion_tokens = token_ids.len();
    946
    947
            // Decode generated text
    948
    1
            let text = tokenizer.decode(&token_ids).map_err(|e| {0
    949
    0
                state.metrics.record_failure();
    950
    0
                (
    951
    0
                    StatusCode::INTERNAL_SERVER_ERROR,
    952
    0
                    Json(ErrorResponse {
    953
    0
                        error: e.to_string(),
    954
    0
                    }),
    955
    0
                )
    956
    0
            })?;
    957
    958
            // Record metrics
    959
    1
            let latency = start.elapsed();
    960
    1
            state.metrics.record_success(completion_tokens, latency);
    961
    962
            // Generate response ID
    963
    1
            let response_id = format!("cmpl-{}", &uuid::Uuid::new_v4().to_string()[..8]);
    964
    965
    1
            return Ok(Json(CompletionResponse {
    966
    1
                id: response_id,
    967
    1
                object: "text_completion".to_string(),
    968
    1
                created: std::time::SystemTime::now()
    969
    1
                    .duration_since(std::time::UNIX_EPOCH)
    970
    1
                    .unwrap_or_default()
    971
    1
                    .as_secs(),
    972
    1
                model: request.model.clone(),
    973
    1
                choices: vec![CompletionChoice {
    974
    1
                    text,
    975
    1
                    index: 0,
    976
    1
                    logprobs: None,
    977
    1
                    finish_reason: "stop".to_string(),
    978
    1
                }],
    979
    1
                usage: Usage {
    980
    1
                    prompt_tokens,
    981
    1
                    completion_tokens,
    982
    1
                    total_tokens: prompt_tokens + completion_tokens,
    983
    1
                },
    984
    1
            }));
    985
    4
        }
    986
    987
        // Fall back to CPU model
    988
    4
        let model_id = if request.model == "default" || request.model1.is_empty1() {
    989
    3
            None
    990
        } else {
    991
    1
            Some(request.model.as_str())
    992
        };
    993
    994
    4
        let (model, tokenizer) = state.get_model(model_id).map_err(|e| {0
    995
    0
            state.metrics.record_failure();
    996
    0
            (
    997
    0
                StatusCode::NOT_FOUND,
    998
    0
                Json(ErrorResponse {
    999
    0
                    error: e.to_string(),
    1000
    0
                }),
    1001
    0
            )
    1002
    0
        })?;
    1003
    1004
        // Tokenize prompt
    1005
    4
        let prompt_ids = tokenizer.encode(&request.prompt);
    1006
    4
        if prompt_ids.is_empty() {
    1007
    1
            state.metrics.record_failure();
    1008
    1
            return Err((
    1009
    1
                StatusCode::BAD_REQUEST,
    1010
    1
                Json(ErrorResponse {
    1011
    1
                    error: "Prompt cannot be empty".to_string(),
    1012
    1
                }),
    1013
    1
            ));
    1014
    3
        }
    1015
    1016
    3
        let prompt_tokens = prompt_ids.len();
    1017
    1018
        // Convert to usize for model
    1019
    13
        let prompt3: Vec<usize>3 = prompt_ids.iter()3.map3(|&id| id as usize).collect3();
    1020
    1021
    3
        let mut config = GenerationConfig::default()
    1022
    3
            .with_max_tokens(max_tokens)
    1023
    3
            .with_temperature(temperature);
    1024
    1025
    3
        if let Some(top_p1) = request.top_p {
    1026
    1
            config.strategy = SamplingStrategy::TopP { p: top_p as f32 };
    1027
    2
        }
    1028
    1029
        // Generate
    1030
    3
        let generated2 = model.generate(&prompt, &config).map_err(|e| {1
    1031
    1
            state.metrics.record_failure();
    1032
    1
            (
    1033
    1
                StatusCode::INTERNAL_SERVER_ERROR,
    1034
    1
                Json(ErrorResponse {
    1035
    1
                    error: e.to_string(),
    1036
    1
                }),
    1037
    1
            )
    1038
    1
        })?;
    1039
    1040
        // Convert to u32 for tokenizer
    1041
    2
        let token_ids: Vec<u32> = generated
    1042
    2
            .iter()
    1043
    2
            .skip(prompt_tokens)
    1044
    12
            .filter_map2(|&id| u32::try_from(id).ok())
    1045
    2
            .collect();
    1046
    1047
    2
        let completion_tokens = token_ids.len();
    1048
    1049
        // Decode generated text
    1050
    2
        let text = tokenizer.decode(&token_ids).map_err(|e| {0
    1051
    0
            state.metrics.record_failure();
    1052
    0
            (
    1053
    0
                StatusCode::INTERNAL_SERVER_ERROR,
    1054
    0
                Json(ErrorResponse {
    1055
    0
                    error: e.to_string(),
    1056
    0
                }),
    1057
    0
            )
    1058
    0
        })?;
    1059
    1060
        // Record metrics
    1061
    2
        let latency = start.elapsed();
    1062
    2
        state.metrics.record_success(completion_tokens, latency);
    1063
    1064
        // Generate response ID
    1065
    2
        let response_id = format!(
    1066
    2
            "cmpl-{}",
    1067
    2
            std::time::SystemTime::now()
    1068
    2
                .duration_since(std::time::UNIX_EPOCH)
    1069
    2
                .map(|d| d.as_nanos())
    1070
    2
                .unwrap_or(0)
    1071
        );
    1072
    1073
        Ok(Json(CompletionResponse {
    1074
    2
            id: response_id,
    1075
    2
            object: "text_completion".to_string(),
    1076
    2
            created: std::time::SystemTime::now()
    1077
    2
                .duration_since(std::time::UNIX_EPOCH)
    1078
    2
                .map(|d| d.as_secs())
    1079
    2
                .unwrap_or(0),
    1080
    2
            model: request.model,
    1081
    2
            choices: vec![CompletionChoice {
    1082
    2
                text,
    1083
    2
                index: 0,
    1084
    2
                logprobs: None,
    1085
    2
                finish_reason: "stop".to_string(),
    1086
    2
            }],
    1087
    2
            usage: Usage {
    1088
    2
                prompt_tokens,
    1089
    2
                completion_tokens,
    1090
    2
                total_tokens: prompt_tokens + completion_tokens,
    1091
    2
            },
    1092
        }))
    1093
    6
    }
    1094
    1095
    /// OpenAI-compatible embeddings handler (/v1/embeddings)
    1096
    2
    pub async fn openai_embeddings_handler(
    1097
    2
        State(state): State<AppState>,
    1098
    2
        Json(request): Json<EmbeddingRequest>,
    1099
    2
    ) -> Result<Json<EmbeddingResponse>, (StatusCode, Json<ErrorResponse>)> {
    1100
        // Delegate to native handler
    1101
    2
        realize_embed_handler(State(state), Json(request)).await
    1102
    2
    }
    1103