## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/infer/mod.rs

    Line
    Count
    Source
    1
    //! High-level inference API for CLI tools
    2
    //!
    3
    //! This module provides a simple, high-level API for running inference
    4
    //! that can be used by CLI tools like `apr run` and `apr chat`.
    5
    //!
    6
    //! # Architecture (APR-CLI-DELEGATE-001)
    7
    //!
    8
    //! ```text
    9
    //! ┌─────────────┐     ┌─────────────┐     ┌─────────────┐
    10
    //! │  apr-cli    │ --> │  realizar   │ --> │   trueno    │
    11
    //! │  (100 LOC)  │     │   infer.rs  │     │   SIMD/GPU  │
    12
    //! └─────────────┘     └─────────────┘     └─────────────┘
    13
    //! ```
    14
    //!
    15
    //! The `apr run` command delegates ALL inference to this module.
    16
    //! This eliminates ~1800 lines of duplicated code in apr-cli.
    17
    //!
    18
    //! # Example
    19
    //!
    20
    //! ```rust,ignore
    21
    //! use realizar::infer::{InferenceConfig, run_inference};
    22
    //!
    23
    //! let config = InferenceConfig::new("model.gguf")
    24
    //!     .with_prompt("Hello, world!")
    25
    //!     .with_max_tokens(32);
    26
    //!
    27
    //! let result = run_inference(config)?;
    28
    //! println!("{}", result.text);
    29
    //! ```
    30
    31
    use crate::error::{RealizarError, Result};
    32
    use crate::format::{detect_format, ModelFormat};
    33
    use std::path::PathBuf;
    34
    use std::time::Instant;
    35
    36
    /// Configuration for inference
    37
    #[derive(Debug, Clone)]
    38
    pub struct InferenceConfig {
    39
        /// Path to model file (GGUF, APR, or SafeTensors)
    40
        pub model_path: PathBuf,
    41
        /// Text prompt for generation
    42
        pub prompt: Option<String>,
    43
        /// Token IDs for generation (alternative to prompt)
    44
        pub input_tokens: Option<Vec<u32>>,
    45
        /// Maximum tokens to generate
    46
        pub max_tokens: usize,
    47
        /// Temperature for sampling (0.0 = greedy)
    48
        pub temperature: f32,
    49
        /// Top-k sampling (0 = disabled)
    50
        pub top_k: usize,
    51
        /// Disable GPU acceleration
    52
        pub no_gpu: bool,
    53
        /// Enable inference tracing (APR-TRACE-001)
    54
        pub trace: bool,
    55
        /// Verbose tracing output
    56
        pub trace_verbose: bool,
    57
        /// Trace output file path
    58
        pub trace_output: Option<PathBuf>,
    59
        /// Specific trace steps to capture
    60
        pub trace_steps: Option<Vec<String>>,
    61
        /// Show verbose loading/progress output
    62
        pub verbose: bool,
    63
    }
    64
    65
    impl InferenceConfig {
    66
        /// Create a new inference config for a model file
    67
        #[must_use]
    68
    61
        pub fn new(model_path: impl Into<PathBuf>) -> Self {
    69
    61
            Self {
    70
    61
                model_path: model_path.into(),
    71
    61
                prompt: None,
    72
    61
                input_tokens: None,
    73
    61
                max_tokens: 32,
    74
    61
                temperature: 0.0, // Greedy by default
    75
    61
                top_k: 1,
    76
    61
                no_gpu: false,
    77
    61
                trace: false,
    78
    61
                trace_verbose: false,
    79
    61
                trace_output: None,
    80
    61
                trace_steps: None,
    81
    61
                verbose: false,
    82
    61
            }
    83
    61
        }
    84
    85
        /// Set the text prompt
    86
        #[must_use]
    87
    14
        pub fn with_prompt(mut self, prompt: impl Into<String>) -> Self {
    88
    14
            self.prompt = Some(prompt.into());
    89
    14
            self
    90
    14
        }
    91
    92
        /// Set input tokens directly
    93
        #[must_use]
    94
    7
        pub fn with_input_tokens(mut self, tokens: Vec<u32>) -> Self {
    95
    7
            self.input_tokens = Some(tokens);
    96
    7
            self
    97
    7
        }
    98
    99
        /// Set maximum tokens to generate
    100
        #[must_use]
    101
    14
        pub fn with_max_tokens(mut self, max_tokens: usize) -> Self {
    102
    14
            self.max_tokens = max_tokens;
    103
    14
            self
    104
    14
        }
    105
    106
        /// Set temperature (0.0 = greedy)
    107
        #[must_use]
    108
    9
        pub fn with_temperature(mut self, temperature: f32) -> Self {
    109
    9
            self.temperature = temperature;
    110
    9
            self
    111
    9
        }
    112
    113
        /// Set top-k sampling
    114
        #[must_use]
    115
    9
        pub fn with_top_k(mut self, top_k: usize) -> Self {
    116
    9
            self.top_k = top_k;
    117
    9
            self
    118
    9
        }
    119
    120
        /// Disable GPU acceleration
    121
        #[must_use]
    122
    5
        pub fn without_gpu(mut self) -> Self {
    123
    5
            self.no_gpu = true;
    124
    5
            self
    125
    5
        }
    126
    127
        /// Enable verbose output
    128
        #[must_use]
    129
    9
        pub fn with_verbose(mut self, verbose: bool) -> Self {
    130
    9
            self.verbose = verbose;
    131
    9
            self
    132
    9
        }
    133
    134
        /// Enable inference tracing
    135
        #[must_use]
    136
    7
        pub fn with_trace(mut self, trace: bool) -> Self {
    137
    7
            self.trace = trace;
    138
    7
            self
    139
    7
        }
    140
    }
    141
    142
    /// Result from inference
    143
    #[derive(Debug, Clone)]
    144
    pub struct InferenceResult {
    145
        /// Generated text (decoded from tokens)
    146
        pub text: String,
    147
        /// All tokens (input + generated)
    148
        pub tokens: Vec<u32>,
    149
        /// Number of input tokens
    150
        pub input_token_count: usize,
    151
        /// Number of generated tokens
    152
        pub generated_token_count: usize,
    153
        /// Inference time in milliseconds
    154
        pub inference_ms: f64,
    155
        /// Tokens per second
    156
        pub tok_per_sec: f64,
    157
        /// Model load time in milliseconds
    158
        pub load_ms: f64,
    159
        /// Model format that was loaded
    160
        pub format: String,
    161
        /// Whether GPU was used
    162
        pub used_gpu: bool,
    163
    }
    164
    165
    /// Run inference on a model
    166
    ///
    167
    /// This is the main entry point for inference. It handles:
    168
    /// - Model format detection (GGUF, APR, SafeTensors)
    169
    /// - Tokenization (using embedded tokenizer for GGUF)
    170
    /// - Generation with configurable sampling
    171
    /// - GPU acceleration when available
    172
    /// - Inference tracing (APR-TRACE-001)
    173
    ///
    174
    /// # Errors
    175
    ///
    176
    /// Returns error if:
    177
    /// - Model file cannot be read
    178
    /// - Model format is unsupported
    179
    /// - Generation fails
    180
    13
    pub fn run_inference(config: &InferenceConfig) -> Result<InferenceResult> {
    181
        // Read model file header for format detection
    182
    13
        let data8 = std::fs::read(&config.model_path).map_err(|e| RealizarError::IoError {
    183
    5
            message: format!("Failed to read model: {}", e),
    184
    5
        })?;
    185
    186
    8
        if data.len() < 8 {
    187
    2
            return Err(RealizarError::FormatError {
    188
    2
                reason: "File too small for format detection".to_string(),
    189
    2
            });
    190
    6
        }
    191
    192
        // Detect format
    193
    6
        let format4 = detect_format(&data[..8]).map_err(|e| RealizarError::FormatError {
    194
    2
            reason: format!("Format detection failed: {}", e),
    195
    2
        })?;
    196
    197
    4
        match format {
    198
    2
            ModelFormat::Gguf => run_gguf_inference(config),
    199
    0
            ModelFormat::Apr => run_apr_inference(config),
    200
    2
            ModelFormat::SafeTensors => run_safetensors_inference(config),
    201
        }
    202
    13
    }
    203
    204
    /// Run GGUF model inference
    205
    2
    fn run_gguf_inference(config: &InferenceConfig) -> Result<InferenceResult> {
    206
        use crate::chat_template::{format_messages, ChatMessage};
    207
        use crate::gguf::{MappedGGUFModel, OwnedQuantizedModel, QuantizedGenerateConfig};
    208
    209
        // Verbose: Show loading message BEFORE loading (NOISY-GUARD F-UX-27)
    210
    2
        if config.verbose {
    211
    0
            eprintln!("Loading model: {}", config.model_path.display());
    212
    2
        }
    213
    214
    2
        let load_start = Instant::now();
    215
    216
        // Load GGUF via mmap
    217
    2
        let mapped = MappedGGUFModel::from_path(&config.model_path)?0;
    218
    219
        // Pre-fault mmap pages (PAR-200: avoid page faults during inference)
    220
    2
        prefault_mmap(mapped.data());
    221
    222
        // Create quantized model
    223
    2
        let model0 = OwnedQuantizedModel::from_mapped(&mapped)?;
    224
    0
        let load_ms = load_start.elapsed().as_secs_f64() * 1000.0;
    225
    226
        // Extract architecture from model name
    227
    0
        let arch = config
    228
    0
            .model_path
    229
    0
            .file_stem()
    230
    0
            .and_then(|s| s.to_str())
    231
    0
            .map_or("Transformer", |s| {
    232
    0
                if s.to_lowercase().contains("qwen") {
    233
    0
                    "Qwen2"
    234
    0
                } else if s.to_lowercase().contains("llama") {
    235
    0
                    "LLaMA"
    236
    0
                } else if s.to_lowercase().contains("mistral") {
    237
    0
                    "Mistral"
    238
    0
                } else if s.to_lowercase().contains("phi") {
    239
    0
                    "Phi"
    240
                } else {
    241
    0
                    "Transformer"
    242
                }
    243
    0
            });
    244
    245
    0
        if config.verbose {
    246
    0
            eprintln!(
    247
    0
                "Architecture: {} ({} layers, vocab_size={})",
    248
    0
                arch, model.config.num_layers, model.config.vocab_size
    249
    0
            );
    250
    0
            eprintln!("Model loaded in {:.1}ms", load_ms);
    251
    0
        }
    252
    253
        // Get input tokens
    254
    0
        let input_tokens = if let Some(ref tokens) = config.input_tokens {
    255
    0
            tokens.clone()
    256
    0
        } else if let Some(ref prompt) = config.prompt {
    257
            // Detect instruct model and apply chat template
    258
    0
            let model_name = config
    259
    0
                .model_path
    260
    0
                .file_name()
    261
    0
                .and_then(|n| n.to_str())
    262
    0
                .unwrap_or("");
    263
    0
            let is_instruct = model_name.to_lowercase().contains("instruct");
    264
    265
    0
            let formatted_prompt = if is_instruct {
    266
    0
                let messages = vec![ChatMessage::user(prompt)];
    267
    0
                format_messages(&messages, Some(model_name)).unwrap_or_else(|_| prompt.clone())
    268
            } else {
    269
    0
                prompt.clone()
    270
            };
    271
    272
    0
            mapped
    273
    0
                .model
    274
    0
                .encode(&formatted_prompt)
    275
    0
                .unwrap_or_else(|| vec![1u32])
    276
        } else {
    277
    0
            vec![1u32] // BOS token
    278
        };
    279
    280
    0
        let input_token_count = input_tokens.len();
    281
    282
        // Configure generation
    283
    0
        let gen_config = QuantizedGenerateConfig {
    284
    0
            max_tokens: config.max_tokens.min(128),
    285
    0
            temperature: config.temperature,
    286
    0
            top_k: config.top_k,
    287
    0
            ..Default::default()
    288
    0
        };
    289
    290
        // Run inference (GPU or CPU)
    291
    0
        let infer_start = Instant::now();
    292
    0
        let (tokens, used_gpu) = run_gguf_generate(model, &input_tokens, &gen_config, config)?;
    293
    0
        let inference_ms = infer_start.elapsed().as_secs_f64() * 1000.0;
    294
    295
        // Decode output
    296
    0
        let generated_tokens = &tokens[input_token_count..];
    297
    0
        let text = mapped.model.decode(generated_tokens);
    298
    299
        // Clean output (strip ChatML markers)
    300
    0
        let text = clean_model_output(&text);
    301
    302
    0
        let generated_token_count = generated_tokens.len();
    303
    0
        let tok_per_sec = if inference_ms > 0.0 {
    304
    0
            generated_token_count as f64 / (inference_ms / 1000.0)
    305
        } else {
    306
    0
            0.0
    307
        };
    308
    309
    0
        Ok(InferenceResult {
    310
    0
            text,
    311
    0
            tokens,
    312
    0
            input_token_count,
    313
    0
            generated_token_count,
    314
    0
            inference_ms,
    315
    0
            tok_per_sec,
    316
    0
            load_ms,
    317
    0
            format: "GGUF".to_string(),
    318
    0
            used_gpu,
    319
    0
        })
    320
    2
    }
    321
    322
    /// Run GGUF generation with GPU or CPU
    323
    #[allow(unused_variables)] // config used only in CUDA feature
    324
    0
    fn run_gguf_generate(
    325
    0
        model: crate::gguf::OwnedQuantizedModel,
    326
    0
        input_tokens: &[u32],
    327
    0
        gen_config: &crate::gguf::QuantizedGenerateConfig,
    328
    0
        config: &InferenceConfig,
    329
    0
    ) -> Result<(Vec<u32>, bool)> {
    330
        #[cfg(feature = "cuda")]
    331
        if !config.no_gpu {
    332
            use crate::gguf::OwnedQuantizedModelCuda;
    333
    334
            match OwnedQuantizedModelCuda::new(model.clone(), 0) {
    335
                Ok(mut cuda_model) => {
    336
                    if config.verbose {
    337
                        eprintln!(
    338
                            "Backend: GPU ({}, {} MB VRAM)",
    339
                            cuda_model.device_name(),
    340
                            cuda_model.vram_mb()
    341
                        );
    342
                    }
    343
                    let tokens = cuda_model
    344
                        .generate_gpu_resident(input_tokens, gen_config)
    345
                        .map_err(|e| {
    346
                            RealizarError::InferenceError(format!("GPU generation failed: {}", e))
    347
                        })?;
    348
                    return Ok((tokens, true));
    349
                },
    350
                Err(e) => {
    351
                    if config.verbose {
    352
                        eprintln!("Backend: CPU (GPU unavailable: {})", e);
    353
                    }
    354
                },
    355
            }
    356
        }
    357
    358
        // CPU fallback
    359
    0
        if config.verbose {
    360
    0
            eprintln!("Backend: CPU (SIMD-accelerated)");
    361
    0
        }
    362
    0
        let tokens = model
    363
    0
            .generate_with_cache(input_tokens, gen_config)
    364
    0
            .map_err(|e| RealizarError::InferenceError(format!("CPU generation failed: {}", e)))?;
    365
    0
        Ok((tokens, false))
    366
    0
    }
    367
    368
    /// Run APR model inference (PAR-302)
    369
    ///
    370
    /// Uses AprTransformer with proper RoPE and SwiGLU for correct inference.
    371
    0
    fn run_apr_inference(config: &InferenceConfig) -> Result<InferenceResult> {
    372
        use crate::apr::AprV2Model;
    373
        use crate::apr_transformer::AprTransformer;
    374
    375
        // Verbose: Show loading message BEFORE loading
    376
    0
        if config.verbose {
    377
    0
            eprintln!("Loading APR model: {}", config.model_path.display());
    378
    0
        }
    379
    380
    0
        let load_start = Instant::now();
    381
    382
        // Load APR into AprTransformer for proper inference with RoPE and SwiGLU
    383
    0
        let transformer = AprTransformer::from_apr_file(&config.model_path)?;
    384
    0
        let load_ms = load_start.elapsed().as_secs_f64() * 1000.0;
    385
    386
    0
        let arch = &transformer.config.architecture;
    387
    388
    0
        if config.verbose {
    389
    0
            eprintln!(
    390
    0
                "Architecture: {} ({} layers, vocab_size={})",
    391
    0
                arch, transformer.config.num_layers, transformer.config.vocab_size
    392
    0
            );
    393
    0
            eprintln!("Model loaded in {:.1}ms", load_ms);
    394
    0
        }
    395
    396
        // Get input tokens (use sibling tokenizer.json)
    397
    0
        let input_tokens = if let Some(ref tokens) = config.input_tokens {
    398
    0
            tokens.clone()
    399
    0
        } else if let Some(ref prompt) = config.prompt {
    400
            // Load tokenizer from sibling tokenizer.json
    401
    0
            AprV2Model::encode_text(&config.model_path, prompt).unwrap_or_else(|| vec![1u32])
    402
        } else {
    403
    0
            vec![1u32] // BOS token
    404
        };
    405
    406
    0
        let input_token_count = input_tokens.len();
    407
    408
        // PMAT-103 FIX: Use KV-cache for O(n) generation instead of O(n²)
    409
    0
        let infer_start = Instant::now();
    410
    0
        let mut all_tokens = input_tokens.clone();
    411
    412
        // Create KV cache
    413
    0
        let mut cache = crate::apr_transformer::AprKVCache::new(&transformer.config);
    414
    415
        // Prefill: process each input token to populate KV cache
    416
    0
        for (pos, &token) in input_tokens.iter().enumerate() {
    417
    0
            let _ = transformer.forward_with_cache(token, &mut cache, pos)?;
    418
        }
    419
    420
        // Generate new tokens with KV cache (O(1) per token)
    421
    0
        let mut position = input_tokens.len();
    422
    0
        for _ in 0..config.max_tokens.min(128) {
    423
            // Forward pass with KV cache
    424
    0
            let last_token = *all_tokens.last().unwrap_or(&1);
    425
    0
            let logits = transformer.forward_with_cache(last_token, &mut cache, position)?;
    426
    427
            // Greedy sampling (argmax)
    428
    0
            let next_token = logits
    429
    0
                .iter()
    430
    0
                .enumerate()
    431
    0
                .max_by(|a, b| a.1.partial_cmp(b.1).unwrap_or(std::cmp::Ordering::Equal))
    432
    0
                .map_or(0, |(i, _)| i as u32);
    433
    434
            // Check for EOS (Qwen2 EOS=151645, BOS=151643, standard=2)
    435
    0
            if next_token == 151645 || next_token == 151643 || next_token == 2 {
    436
    0
                break;
    437
    0
            }
    438
    439
    0
            all_tokens.push(next_token);
    440
    0
            position += 1;
    441
        }
    442
    443
    0
        let inference_ms = infer_start.elapsed().as_secs_f64() * 1000.0;
    444
    445
        // Decode output tokens
    446
        // GH-156: Try multiple tokenizer sources for APR models
    447
    0
        let generated_tokens = &all_tokens[input_token_count..];
    448
    0
        let text = if let Some(tokenizer) = AprV2Model::load_tokenizer(&config.model_path) {
    449
    0
            tokenizer.decode(generated_tokens)
    450
    0
        } else if let Some(tokenizer) = find_fallback_tokenizer(&config.model_path) {
    451
    0
            tokenizer.decode(generated_tokens)
    452
        } else {
    453
    0
            format!(
    454
    0
                "[{} tokens generated, tokenizer not found]",
    455
    0
                generated_tokens.len()
    456
            )
    457
        };
    458
    459
        // Clean output
    460
    0
        let text = clean_model_output(&text);
    461
    462
    0
        let generated_token_count = generated_tokens.len();
    463
    0
        let tok_per_sec = if inference_ms > 0.0 {
    464
    0
            generated_token_count as f64 / (inference_ms / 1000.0)
    465
        } else {
    466
    0
            0.0
    467
        };
    468
    469
    0
        Ok(InferenceResult {
    470
    0
            text,
    471
    0
            tokens: all_tokens,
    472
    0
            input_token_count,
    473
    0
            generated_token_count,
    474
    0
            inference_ms,
    475
    0
            tok_per_sec,
    476
    0
            load_ms,
    477
    0
            format: "APR".to_string(),
    478
    0
            used_gpu: false,
    479
    0
        })
    480
    0
    }
    481
    482
    /// Run SafeTensors model inference (PAR-301)
    483
    2
    fn run_safetensors_inference(config: &InferenceConfig) -> Result<InferenceResult> {
    484
        use crate::apr::AprV2Model;
    485
        use crate::apr_transformer::AprKVCache;
    486
        use crate::safetensors_infer::SafetensorsToAprConverter;
    487
    488
        // Verbose: Show loading message BEFORE loading
    489
    2
        if config.verbose {
    490
    0
            eprintln!("Loading SafeTensors model: {}", config.model_path.display());
    491
    2
        }
    492
    493
    2
        let load_start = Instant::now();
    494
    495
        // Convert SafeTensors to AprTransformer
    496
    2
        let transformer0 = SafetensorsToAprConverter::convert(&config.model_path)?;
    497
    0
        let load_ms = load_start.elapsed().as_secs_f64() * 1000.0;
    498
    499
    0
        let arch = &transformer.config.architecture;
    500
    501
    0
        if config.verbose {
    502
    0
            eprintln!(
    503
    0
                "Architecture: {} ({} layers, vocab_size={})",
    504
    0
                arch, transformer.config.num_layers, transformer.config.vocab_size
    505
    0
            );
    506
    0
            eprintln!("Model loaded in {:.1}ms", load_ms);
    507
    0
        }
    508
    509
        // Get input tokens (use sibling tokenizer.json)
    510
    0
        let input_tokens = if let Some(ref tokens) = config.input_tokens {
    511
    0
            tokens.clone()
    512
    0
        } else if let Some(ref prompt) = config.prompt {
    513
            // Load tokenizer from sibling tokenizer.json
    514
    0
            AprV2Model::encode_text(&config.model_path, prompt).unwrap_or_else(|| vec![1u32])
    515
        } else {
    516
    0
            vec![1u32] // BOS token
    517
        };
    518
    519
    0
        let input_token_count = input_tokens.len();
    520
    521
        // PMAT-103: Use KV-cached generation for O(n) instead of O(n²) complexity
    522
        // Previous code used forward() in a loop which recomputed all tokens each time
    523
    0
        let infer_start = Instant::now();
    524
    0
        let mut cache = AprKVCache::new(&transformer.config);
    525
    0
        let mut all_tokens = input_tokens.clone();
    526
    527
        // Prefill phase: process all prompt tokens, get logits from last token
    528
    0
        let mut logits = Vec::new();
    529
    0
        for (pos, &token) in input_tokens.iter().enumerate() {
    530
    0
            logits = transformer.forward_with_cache(token, &mut cache, pos)?;
    531
        }
    532
    533
        // Decode phase: sample and generate new tokens
    534
    0
        let max_gen = config.max_tokens.min(128);
    535
    0
        for _ in 0..max_gen {
    536
            // Greedy sampling (argmax) from current logits
    537
    0
            let next_token = logits
    538
    0
                .iter()
    539
    0
                .enumerate()
    540
    0
                .max_by(|a, b| a.1.partial_cmp(b.1).unwrap_or(std::cmp::Ordering::Equal))
    541
    0
                .map_or(0, |(i, _)| i as u32);
    542
    543
            // Check for EOS (Qwen2 EOS=151645, BOS=151643, standard=2)
    544
    0
            if next_token == 151645 || next_token == 151643 || next_token == 2 {
    545
    0
                break;
    546
    0
            }
    547
    548
    0
            all_tokens.push(next_token);
    549
    550
            // Process newly generated token to get next logits
    551
    0
            let pos = all_tokens.len() - 1; // Position of the just-added token
    552
    0
            logits = transformer.forward_with_cache(next_token, &mut cache, pos)?;
    553
        }
    554
    555
    0
        let inference_ms = infer_start.elapsed().as_secs_f64() * 1000.0;
    556
    557
        // Decode output tokens
    558
    0
        let generated_tokens = &all_tokens[input_token_count..];
    559
    0
        let text = if let Some(tokenizer) = AprV2Model::load_tokenizer(&config.model_path) {
    560
    0
            tokenizer.decode(generated_tokens)
    561
        } else {
    562
    0
            format!(
    563
    0
                "[{} tokens generated, tokenizer not found]",
    564
    0
                generated_tokens.len()
    565
            )
    566
        };
    567
    568
        // Clean output
    569
    0
        let text = clean_model_output(&text);
    570
    571
    0
        let generated_token_count = generated_tokens.len();
    572
    0
        let tok_per_sec = if inference_ms > 0.0 {
    573
    0
            generated_token_count as f64 / (inference_ms / 1000.0)
    574
        } else {
    575
    0
            0.0
    576
        };
    577
    578
    0
        Ok(InferenceResult {
    579
    0
            text,
    580
    0
            tokens: all_tokens,
    581
    0
            input_token_count,
    582
    0
            generated_token_count,
    583
    0
            inference_ms,
    584
    0
            tok_per_sec,
    585
    0
            load_ms,
    586
    0
            format: "SafeTensors".to_string(),
    587
    0
            used_gpu: false, // SafeTensors currently CPU-only
    588
    0
        })
    589
    2
    }
    590
    591
    /// Pre-fault mmap pages to avoid page faults during inference
    592
    15
    fn prefault_mmap(data: &[u8]) {
    593
    15
        let page_size = 4096;
    594
    15
        let mut checksum: u8 = 0;
    595
    39
        for i in (0..data.len())15.step_by15(page_size15) {
    596
    39
            checksum = checksum.wrapping_add(data[i]);
    597
    39
        }
    598
    15
        std::hint::black_box(checksum);
    599
    15
    }
    600
    601
    /// Find a fallback tokenizer for APR models (GH-156)
    602
    ///
    603
    /// This function tries to load the embedded tokenizer from the APR model.
    604
    /// APR files can contain the vocabulary in metadata, so we don't need
    605
    /// a sibling tokenizer.json file.
    606
    ///
    607
    /// # Arguments
    608
    /// * `model_path` - Path to the APR model file
    609
    ///
    610
    /// # Returns
    611
    /// * `Some(BpeTokenizer)` - If embedded tokenizer found and converted
    612
    /// * `None` - If no embedded tokenizer available
    613
    0
    fn find_fallback_tokenizer(model_path: &std::path::Path) -> Option<crate::apr::BpeTokenizer> {
    614
        use crate::apr::AprV2Model;
    615
    616
        // Try to load the APR model and extract embedded tokenizer
    617
    0
        let model = AprV2Model::load(model_path).ok()?;
    618
    0
        let simple_tokenizer = model.load_embedded_tokenizer()?;
    619
    620
        // Convert SimpleTokenizer to BpeTokenizer for compatibility
    621
        // SimpleTokenizer is decode-only, but BpeTokenizer has encode support
    622
        // For fallback purposes, we only need decode, so this is fine
    623
        Some(crate::apr::BpeTokenizer {
    624
    0
            token_to_id: simple_tokenizer
    625
    0
                .id_to_token
    626
    0
                .iter()
    627
    0
                .enumerate()
    628
    0
                .map(|(id, token)| (token.clone(), id as u32))
    629
    0
                .collect(),
    630
    0
            id_to_token: simple_tokenizer.id_to_token,
    631
    0
            merge_rules: Vec::new(), // No merge rules for embedded tokenizer
    632
    0
            bos_id: simple_tokenizer.bos_token_id,
    633
    0
            eos_id: simple_tokenizer.eos_token_id,
    634
        })
    635
    0
    }
    636
    637
    /// Clean model output by stripping ChatML markers
    638
    26
    fn clean_model_output(raw: &str) -> String {
    639
    26
        let mut cleaned = raw.to_string();
    640
    26
        let markers = [
    641
    26
            "<|im_start|>assistant\n",
    642
    26
            "<|im_start|>assistant",
    643
    26
            "<|im_end|>",
    644
    26
            "<|im_start|>",
    645
    26
            "<|endoftext|>",
    646
    26
        ];
    647
    156
        for marker130 in markers {
    648
    130
            cleaned = cleaned.replace(marker, "");
    649
    130
        }
    650
    26
        cleaned.trim().to_string()
    651
    26
    }
    652
    653
    // Tests extracted to tests.rs (PMAT-802)
    654
    #[cfg(test)]
    655
    #[path = "tests.rs"]
    656
    mod infer_tests;