/home/noah/src/realizar/src/bench/runtime.rs

Source
//! Runtime backend abstraction for benchmark comparison
//!
//! Extracted from bench/mod.rs (PMAT-802) to reduce module size.
//! Contains:
//! - BENCH-002: Runtime Backend Abstraction
//! - LlamaCppBackend, VllmBackend, OllamaBackend implementations

#![allow(clippy::cast_precision_loss)]

use std::collections::HashMap;

use serde::{Deserialize, Serialize};

use crate::error::RealizarError;

#[cfg(feature = "bench-http")]
use crate::http_client::{CompletionRequest, ModelHttpClient, OllamaOptions, OllamaRequest};

/// Supported runtime types for inference benchmarking
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum RuntimeType {
    /// Native Realizar runtime (.apr format)
    Realizar,
    /// llama.cpp (GGUF format)
    LlamaCpp,
    /// vLLM (safetensors, HuggingFace)
    Vllm,
    /// Ollama (wraps llama.cpp)
    Ollama,
}

impl RuntimeType {
    /// Get string representation
    #[must_use]
    pub fn as_str(&self) -> &'static str {
        match self {
            Self::Realizar => "realizar",
            Self::LlamaCpp => "llama-cpp",
            Self::Vllm => "vllm",
            Self::Ollama => "ollama",
        }
    }

    /// Parse from string
    #[must_use]
    pub fn parse(s: &str) -> Option<Self> {
        match s.to_lowercase().as_str() {
            "realizar" => Some(Self::Realizar)1,
            "llama-cpp" | "llama.cpp"4 | "llamacpp"3 => Some(Self::LlamaCpp)2,
            "vllm" => Some(Self::Vllm)1,
            "ollama" => Some(Self::Ollama)1,
            _ => None,
        }
    }
}

/// Request for inference
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct InferenceRequest {
    /// Input prompt
    pub prompt: String,
    /// Maximum tokens to generate
    pub max_tokens: usize,
    /// Sampling temperature
    pub temperature: f64,
    /// Optional stop sequences
    pub stop: Vec<String>,
}

impl Default for InferenceRequest {
    fn default() -> Self {
        Self {
            prompt: String::new(),
            max_tokens: 100,
            temperature: 0.7,
            stop: Vec::new(),
        }
    }
}

impl InferenceRequest {
    /// Create new request with prompt
    #[must_use]
    pub fn new(prompt: &str) -> Self {
        Self {
            prompt: prompt.to_string(),
            ..Default::default()
        }
    }

    /// Set max tokens
    #[must_use]
    pub fn with_max_tokens(mut self, max_tokens: usize) -> Self {
        self.max_tokens = max_tokens;
        self
    }

    /// Set temperature
    #[must_use]
    pub fn with_temperature(mut self, temperature: f64) -> Self {
        self.temperature = temperature;
        self
    }
}

/// Response from inference
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct InferenceResponse {
    /// Generated text
    pub text: String,
    /// Number of tokens generated
    pub tokens_generated: usize,
    /// Time to first token (ms)
    pub ttft_ms: f64,
    /// Total generation time (ms)
    pub total_time_ms: f64,
    /// Inter-token latencies (ms)
    pub itl_ms: Vec<f64>,
}

impl InferenceResponse {
    /// Calculate tokens per second
    #[must_use]
    pub fn tokens_per_second(&self) -> f64 {
        if self.total_time_ms <= 0.0 {
            return 0.0;
        }
        (self.tokens_generated as f64) / (self.total_time_ms / 1000.0)
    }
}

/// Runtime backend information
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BackendInfo {
    /// Runtime type
    pub runtime_type: RuntimeType,
    /// Version string
    pub version: String,
    /// Whether streaming is supported
    pub supports_streaming: bool,
    /// Model currently loaded (if any)
    pub loaded_model: Option<String>,
}

/// Trait for inference runtime backends
pub trait RuntimeBackend: Send + Sync {
    /// Get backend information
    fn info(&self) -> BackendInfo;

    /// Run inference
    ///
    /// # Errors
    ///
    /// Returns `RealizarError` if inference fails due to:
    /// - Model not loaded
    /// - Backend communication failure
    /// - Invalid request parameters
    fn inference(&self, request: &InferenceRequest) -> Result<InferenceResponse, RealizarError>;

    /// Load a model (if applicable)
    ///
    /// # Errors
    ///
    /// Returns `RealizarError` if model loading fails due to:
    /// - Model file not found
    /// - Invalid model format
    /// - Insufficient memory
    fn load_model(&mut self, _model_path: &str) -> Result<(), RealizarError> {
        Ok(()) // Default: no-op
    }
}

/// Mock backend for testing
pub struct MockBackend {
    ttft_ms: f64,
    tokens_per_second: f64,
}

impl MockBackend {
    /// Create a new mock backend with specified latencies
    #[must_use]
    pub fn new(ttft_ms: f64, tokens_per_second: f64) -> Self {
        Self {
            ttft_ms,
            tokens_per_second,
        }
    }
}

impl RuntimeBackend for MockBackend {
    fn info(&self) -> BackendInfo {
        BackendInfo {
            runtime_type: RuntimeType::Realizar,
            version: env!("CARGO_PKG_VERSION").to_string(),
            supports_streaming: true,
            loaded_model: None,
        }
    }

    fn inference(&self, request: &InferenceRequest) -> Result<InferenceResponse, RealizarError> {
        let tokens = request.max_tokens.min(100);
        let gen_time_ms = (tokens as f64) / self.tokens_per_second * 1000.0;

        Ok(InferenceResponse {
            text: "Mock response".to_string(),
            tokens_generated: tokens,
            ttft_ms: self.ttft_ms,
            total_time_ms: self.ttft_ms + gen_time_ms,
            itl_ms: vec![gen_time_ms / tokens as f64; tokens],
        })
    }
}

/// Registry of available backends
pub struct BackendRegistry {
    backends: HashMap<RuntimeType, Box<dyn RuntimeBackend>>,
}

impl BackendRegistry {
    /// Create empty registry
    #[must_use]
    pub fn new() -> Self {
        Self {
            backends: HashMap::new(),
        }
    }

    /// Register a backend
    pub fn register(&mut self, runtime: RuntimeType, backend: Box<dyn RuntimeBackend>) {
        self.backends.insert(runtime, backend);
    }

    /// Get a backend by type
    #[must_use]
    pub fn get(&self, runtime: RuntimeType) -> Option<&dyn RuntimeBackend> {
        self.backends.get(&runtime).map(AsRef::as_ref)
    }

    /// List registered runtimes
    #[must_use]
    pub fn list(&self) -> Vec<RuntimeType> {
        self.backends.keys().copied().collect()
    }
}

impl Default for BackendRegistry {
    fn default() -> Self {
        Self::new()
    }
}

/// Configuration for llama.cpp backend
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LlamaCppConfig {
    /// Path to llama-cli binary
    pub binary_path: String,
    /// Path to model file
    pub model_path: Option<String>,
    /// Number of GPU layers to offload
    pub n_gpu_layers: u32,
    /// Context size
    pub ctx_size: usize,
    /// Number of threads
    pub threads: usize,
}

impl Default for LlamaCppConfig {
    fn default() -> Self {
        Self {
            binary_path: "llama-cli".to_string(),
            model_path: None,
            n_gpu_layers: 0,
            ctx_size: 2048,
            threads: 4,
        }
    }
}

impl LlamaCppConfig {
    /// Create new config with binary path
    #[must_use]
    pub fn new(binary_path: &str) -> Self {
        Self {
            binary_path: binary_path.to_string(),
            ..Default::default()
        }
    }

    /// Set model path
    #[must_use]
    pub fn with_model(mut self, model_path: &str) -> Self {
        self.model_path = Some(model_path.to_string());
        self
    }

    /// Set GPU layers
    #[must_use]
    pub fn with_gpu_layers(mut self, layers: u32) -> Self {
        self.n_gpu_layers = layers;
        self
    }

    /// Set context size
    #[must_use]
    pub fn with_ctx_size(mut self, ctx_size: usize) -> Self {
        self.ctx_size = ctx_size;
        self
    }
}

/// Configuration for vLLM backend
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VllmConfig {
    /// Base URL for vLLM server
    pub base_url: String,
    /// API version
    pub api_version: String,
    /// Model name/path
    pub model: Option<String>,
    /// API key (if required)
    pub api_key: Option<String>,
}

impl Default for VllmConfig {
    fn default() -> Self {
        Self {
            base_url: "http://localhost:8000".to_string(),
            api_version: "v1".to_string(),
            model: None,
            api_key: None,
        }
    }
}

impl VllmConfig {
    /// Create new config with base URL
    #[must_use]
    pub fn new(base_url: &str) -> Self {
        Self {
            base_url: base_url.to_string(),
            ..Default::default()
        }
    }

    /// Set model
    #[must_use]
    pub fn with_model(mut self, model: &str) -> Self {
        self.model = Some(model.to_string());
        self
    }

    /// Set API key
    #[must_use]
    pub fn with_api_key(mut self, api_key: &str) -> Self {
        self.api_key = Some(api_key.to_string());
        self
    }
}

// ============================================================================
// LlamaCppBackend Implementation (BENCH-002)
// ============================================================================

/// llama.cpp backend for GGUF model inference via subprocess
pub struct LlamaCppBackend {
    config: LlamaCppConfig,
}

impl LlamaCppBackend {
    /// Create new llama.cpp backend
    #[must_use]
    pub fn new(config: LlamaCppConfig) -> Self {
        Self { config }
    }

    /// Build CLI arguments for llama-cli invocation
    #[must_use]
    pub fn build_cli_args(&self, request: &InferenceRequest) -> Vec<String> {
        let mut args = Vec::new();

        // Model path
        if let Some(ref model_path) = self.config.model_path {
            args.push("-m".to_string());
            args.push(model_path.clone());
        }

        // Prompt
        args.push("-p".to_string());
        args.push(request.prompt.clone());

        // Number of tokens to generate
        args.push("-n".to_string());
        args.push(request.max_tokens.to_string());

        // GPU layers
        args.push("-ngl".to_string());
        args.push(self.config.n_gpu_layers.to_string());

        // Context size
        args.push("-c".to_string());
        args.push(self.config.ctx_size.to_string());

        // Threads
        args.push("-t".to_string());
        args.push(self.config.threads.to_string());

        // Temperature (if non-default)
        if (request.temperature - 0.8).abs() > 0.01 {
            args.push("--temp".to_string());
            args.push(format!("{:.2}", request.temperature));
        }

        args
    }

    /// Parse a timing line from llama-cli output
    ///
    /// Example: `llama_perf_context_print: prompt eval time =      12.34 ms /    10 tokens`
    /// Returns: `Some((12.34, 10))`
    #[must_use]
    pub fn parse_timing_line(output: &str, metric_name: &str) -> Option<(f64, usize)> {
        for line in output.lines() {
            // For "eval time", we need to exclude "prompt eval time"
            let matches = if metric_name == "eval time" {
                line.contains(metric_name) && !line.contains("prompt eval time")
            } else {
                line.contains(metric_name)
            };

            if matches && line.contains('=') {
                // Extract the value after "=" and before "ms"
                // Format: "metric_name =      12.34 ms /    10 tokens"
                if let Some(eq_pos) = line.find('=') {
                    let after_eq = &line[eq_pos + 1..];
                    // Find ms position
                    if let Some(ms_pos) = after_eq.find("ms") {
                        let value_str = after_eq[..ms_pos].trim();
                        if let Ok(value) = value_str.parse::<f64>() {
                            // Find the count after "/"
                            if let Some(slash_pos) = after_eq.find('/') {
                                let after_slash = &after_eq[slash_pos + 1..];
                                // Extract number before "tokens" or "runs"
                                let count_str =
                                    after_slash.split_whitespace().next().unwrap_or("0");
                                if let Ok(count) = count_str.parse::<usize>() {
                                    return Some((value, count));
                                }
                            }
                        }
                    }
                }
            }
        }
        None
    }

    /// Extract generated text from llama-cli output (before timing lines)
    #[must_use]
    pub fn extract_generated_text(output: &str) -> String {
        let mut text_lines = Vec::new();
        for line in output.lines() {
            // Stop when we hit timing/performance lines
            if line.contains("llama_perf_") || line.contains("sampler") {
                break;
            }
            text_lines.push(line);
        }
        text_lines.join("\n").trim().to_string()
    }

    /// Parse full CLI output into InferenceResponse
    ///
    /// # Errors
    ///
    /// Returns error if timing information cannot be parsed from output.
    pub fn parse_cli_output(output: &str) -> Result<InferenceResponse, RealizarError> {
        // Extract generated text
        let text = Self::extract_generated_text(output);

        // Parse timing metrics
        let ttft_ms = Self::parse_timing_line(output, "prompt eval time").map_or(0.0, |(ms, _)| ms);

        let (total_time_ms, _) = Self::parse_timing_line(output, "total time").unwrap_or((0.0, 0));

        let (_, tokens_generated) =
            Self::parse_timing_line(output, "eval time").unwrap_or((0.0, 0));

        // ITL is not directly available from CLI output, estimate from eval time
        let eval_time = Self::parse_timing_line(output, "eval time").map_or(0.0, |(ms, _)| ms);

        let itl_ms = if tokens_generated > 1 {
            let avg_itl = eval_time / (tokens_generated as f64);
            vec![avg_itl; tokens_generated.saturating_sub(1)]
        } else {
            vec![]
        };

        Ok(InferenceResponse {
            text,
            tokens_generated,
            ttft_ms,
            total_time_ms,
            itl_ms,
        })
    }
}

impl RuntimeBackend for LlamaCppBackend {
    fn info(&self) -> BackendInfo {
        BackendInfo {
            runtime_type: RuntimeType::LlamaCpp,
            version: "b2345".to_string(), // Would be detected from binary
            supports_streaming: false,    // CLI mode doesn't stream
            loaded_model: self.config.model_path.clone(),
        }
    }

    fn inference(&self, request: &InferenceRequest) -> Result<InferenceResponse, RealizarError> {
        use std::process::Command;

        // Require model path
        let model_path = self.config.model_path.as_ref().ok_or_else(|| {
            RealizarError::InvalidConfiguration("model_path is required".to_string())
        })?;

        // Build CLI arguments
        let args = self.build_cli_args(request);

        // Execute llama-cli
        let output = Command::new(&self.config.binary_path)
            .args(&args)
            .output()
            .map_err(|e| {
                RealizarError::ModelNotFound(format!(
                    "Failed to execute {}: {}",
                    self.config.binary_path, e
                ))
            })?;

        if !output.status.success() {
            let stderr = String::from_utf8_lossy(&output.stderr);
            return Err(RealizarError::InferenceError(format!(
                "llama-cli failed: {} (model: {})",
                stderr, model_path
            )));
        }

        // Parse stdout for response and timing
        let stdout = String::from_utf8_lossy(&output.stdout);
        let stderr = String::from_utf8_lossy(&output.stderr);

        // Timing info is often in stderr, combine both
        let combined_output = format!("{}\n{}", stdout, stderr);
        Self::parse_cli_output(&combined_output)
    }
}

// ============================================================================
// VllmBackend Implementation (BENCH-003) - REAL HTTP CALLS
// ============================================================================

/// vLLM backend for inference via HTTP API
///
/// **REAL IMPLEMENTATION** - makes actual HTTP requests to vLLM servers.
/// No mock data. Measures real latency and throughput.
#[cfg(feature = "bench-http")]
pub struct VllmBackend {
    config: VllmConfig,
    http_client: ModelHttpClient,
}

#[cfg(feature = "bench-http")]
impl VllmBackend {
    /// Create new vLLM backend with default HTTP client
    #[must_use]
    pub fn new(config: VllmConfig) -> Self {
        Self {
            config,
            http_client: ModelHttpClient::new(),
        }
    }

    /// Create new vLLM backend with custom HTTP client
    #[must_use]
    pub fn with_client(config: VllmConfig, client: ModelHttpClient) -> Self {
        Self {
            config,
            http_client: client,
        }
    }
}

#[cfg(feature = "bench-http")]
impl RuntimeBackend for VllmBackend {
    fn info(&self) -> BackendInfo {
        BackendInfo {
            runtime_type: RuntimeType::Vllm,
            version: "0.4.0".to_string(), // Would be detected from API
            supports_streaming: true,
            loaded_model: self.config.model.clone(),
        }
    }

    fn inference(&self, request: &InferenceRequest) -> Result<InferenceResponse, RealizarError> {
        // Parse URL to check for invalid port
        let url = &self.config.base_url;
        if let Some(port_str) = url.split(':').next_back() {
            if let Ok(port) = port_str.parse::<u32>() {
                if port > 65535 {
                    return Err(RealizarError::ConnectionError(format!(
                        "Invalid port in URL: {}",
                        url
                    )));
                }
            }
        }

        // REAL HTTP request to vLLM server via OpenAI-compatible API
        #[allow(clippy::cast_possible_truncation)]
        let completion_request = CompletionRequest {
            model: self
                .config
                .model
                .clone()
                .unwrap_or_else(|| "default".to_string()),
            prompt: request.prompt.clone(),
            max_tokens: request.max_tokens,
            temperature: Some(request.temperature as f32),
            stream: false,
        };

        let timing = self.http_client.openai_completion(
            &self.config.base_url,
            &completion_request,
            self.config.api_key.as_deref(),
        )?;

        Ok(InferenceResponse {
            text: timing.text,
            tokens_generated: timing.tokens_generated,
            ttft_ms: timing.ttft_ms,
            total_time_ms: timing.total_time_ms,
            itl_ms: vec![], // ITL requires streaming, not available in blocking mode
        })
    }
}

// ============================================================================
// OllamaBackend Implementation - REAL HTTP CALLS
// ============================================================================

/// Configuration for Ollama backend
#[cfg(feature = "bench-http")]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OllamaConfig {
    /// Base URL for Ollama server
    pub base_url: String,
    /// Model name
    pub model: String,
}

#[cfg(feature = "bench-http")]
impl Default for OllamaConfig {
    fn default() -> Self {
        Self {
            base_url: "http://localhost:11434".to_string(),
            model: "llama2".to_string(),
        }
    }
}

/// Ollama backend for inference via HTTP API
///
/// **REAL IMPLEMENTATION** - makes actual HTTP requests to Ollama servers.
/// No mock data. Measures real latency and throughput.
#[cfg(feature = "bench-http")]
pub struct OllamaBackend {
    config: OllamaConfig,
    http_client: ModelHttpClient,
}

#[cfg(feature = "bench-http")]
impl OllamaBackend {
    /// Create new Ollama backend with default HTTP client
    #[must_use]
    pub fn new(config: OllamaConfig) -> Self {
        Self {
            config,
            http_client: ModelHttpClient::new(),
        }
    }

    /// Create new Ollama backend with custom HTTP client
    #[must_use]
    pub fn with_client(config: OllamaConfig, client: ModelHttpClient) -> Self {
        Self {
            config,
            http_client: client,
        }
    }
}

#[cfg(feature = "bench-http")]
impl RuntimeBackend for OllamaBackend {
    fn info(&self) -> BackendInfo {
        BackendInfo {
            runtime_type: RuntimeType::Ollama,
            version: "0.1.0".to_string(), // Would be detected from API
            supports_streaming: true,
            loaded_model: Some(self.config.model.clone()),
        }
    }

    fn inference(&self, request: &InferenceRequest) -> Result<InferenceResponse, RealizarError> {
        // REAL HTTP request to Ollama server
        #[allow(clippy::cast_possible_truncation)]
        let ollama_request = OllamaRequest {
            model: self.config.model.clone(),
            prompt: request.prompt.clone(),
            stream: false,
            options: Some(OllamaOptions {
                num_predict: Some(request.max_tokens),
                temperature: Some(request.temperature as f32),
            }),
        };

        let timing = self
            .http_client
            .ollama_generate(&self.config.base_url, &ollama_request)?;

        Ok(InferenceResponse {
            text: timing.text,
            tokens_generated: timing.tokens_generated,
            ttft_ms: timing.ttft_ms,
            total_time_ms: timing.total_time_ms,
            itl_ms: vec![], // ITL requires streaming, not available in blocking mode
        })
    }
}


Coverage Report

Created: 2026-01-25 15:05

Line	Count	Source
1		//! Runtime backend abstraction for benchmark comparison
2		//!
3		//! Extracted from bench/mod.rs (PMAT-802) to reduce module size.
4		//! Contains:
5		//! - BENCH-002: Runtime Backend Abstraction
6		//! - LlamaCppBackend, VllmBackend, OllamaBackend implementations
7
8		#![allow(clippy::cast_precision_loss)]
9
10		use std::collections::HashMap;
11
12		use serde::{Deserialize, Serialize};
13
14		use crate::error::RealizarError;
15
16		#[cfg(feature = "bench-http")]
17		use crate::http_client::{CompletionRequest, ModelHttpClient, OllamaOptions, OllamaRequest};
18
19		/// Supported runtime types for inference benchmarking
20		#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
21		pub enum RuntimeType {
22		/// Native Realizar runtime (.apr format)
23		Realizar,
24		/// llama.cpp (GGUF format)
25		LlamaCpp,
26		/// vLLM (safetensors, HuggingFace)
27		Vllm,
28		/// Ollama (wraps llama.cpp)
29		Ollama,
30		}
31
32		impl RuntimeType {
33		/// Get string representation
34		#[must_use]
35	12	pub fn as_str(&self) -> &'static str {
36	12	match self {
37	4	Self::Realizar => "realizar",
38	3	Self::LlamaCpp => "llama-cpp",
39	2	Self::Vllm => "vllm",
40	3	Self::Ollama => "ollama",
41		}
42	12	}
43
44		/// Parse from string
45		#[must_use]
46	6	pub fn parse(s: &str) -> Option<Self> {
47	6	match s.to_lowercase().as_str() {
48	6	"realizar" => Some(Self::Realizar)1 ,
49	5	"llama-cpp" \| "llama.cpp"4 \| "llamacpp"3 => Some(Self::LlamaCpp)2 ,
50	3	"vllm" => Some(Self::Vllm)1 ,
51	2	"ollama" => Some(Self::Ollama)1 ,
52	1	_ => None,
53		}
54	6	}
55		}
56
57		/// Request for inference
58		#[derive(Debug, Clone, Serialize, Deserialize)]
59		pub struct InferenceRequest {
60		/// Input prompt
61		pub prompt: String,
62		/// Maximum tokens to generate
63		pub max_tokens: usize,
64		/// Sampling temperature
65		pub temperature: f64,
66		/// Optional stop sequences
67		pub stop: Vec<String>,
68		}
69
70		impl Default for InferenceRequest {
71	0	fn default() -> Self {
72	0	Self {
73	0	prompt: String::new(),
74	0	max_tokens: 100,
75	0	temperature: 0.7,
76	0	stop: Vec::new(),
77	0	}
78	0	}
79		}
80
81		impl InferenceRequest {
82		/// Create new request with prompt
83		#[must_use]
84	0	pub fn new(prompt: &str) -> Self {
85	0	Self {
86	0	prompt: prompt.to_string(),
87	0	..Default::default()
88	0	}
89	0	}
90
91		/// Set max tokens
92		#[must_use]
93	0	pub fn with_max_tokens(mut self, max_tokens: usize) -> Self {
94	0	self.max_tokens = max_tokens;
95	0	self
96	0	}
97
98		/// Set temperature
99		#[must_use]
100	0	pub fn with_temperature(mut self, temperature: f64) -> Self {
101	0	self.temperature = temperature;
102	0	self
103	0	}
104		}
105
106		/// Response from inference
107		#[derive(Debug, Clone, Serialize, Deserialize)]
108		pub struct InferenceResponse {
109		/// Generated text
110		pub text: String,
111		/// Number of tokens generated
112		pub tokens_generated: usize,
113		/// Time to first token (ms)
114		pub ttft_ms: f64,
115		/// Total generation time (ms)
116		pub total_time_ms: f64,
117		/// Inter-token latencies (ms)
118		pub itl_ms: Vec<f64>,
119		}
120
121		impl InferenceResponse {
122		/// Calculate tokens per second
123		#[must_use]
124	0	pub fn tokens_per_second(&self) -> f64 {
125	0	if self.total_time_ms <= 0.0 {
126	0	return 0.0;
127	0	}
128	0	(self.tokens_generated as f64) / (self.total_time_ms / 1000.0)
129	0	}
130		}
131
132		/// Runtime backend information
133		#[derive(Debug, Clone, Serialize, Deserialize)]
134		pub struct BackendInfo {
135		/// Runtime type
136		pub runtime_type: RuntimeType,
137		/// Version string
138		pub version: String,
139		/// Whether streaming is supported
140		pub supports_streaming: bool,
141		/// Model currently loaded (if any)
142		pub loaded_model: Option<String>,
143		}
144
145		/// Trait for inference runtime backends
146		pub trait RuntimeBackend: Send + Sync {
147		/// Get backend information
148		fn info(&self) -> BackendInfo;
149
150		/// Run inference
151		///
152		/// # Errors
153		///
154		/// Returns `RealizarError` if inference fails due to:
155		/// - Model not loaded
156		/// - Backend communication failure
157		/// - Invalid request parameters
158		fn inference(&self, request: &InferenceRequest) -> Result<InferenceResponse, RealizarError>;
159
160		/// Load a model (if applicable)
161		///
162		/// # Errors
163		///
164		/// Returns `RealizarError` if model loading fails due to:
165		/// - Model file not found
166		/// - Invalid model format
167		/// - Insufficient memory
168	0	fn load_model(&mut self, _model_path: &str) -> Result<(), RealizarError> {
169	0	Ok(()) // Default: no-op
170	0	}
171		}
172
173		/// Mock backend for testing
174		pub struct MockBackend {
175		ttft_ms: f64,
176		tokens_per_second: f64,
177		}
178
179		impl MockBackend {
180		/// Create a new mock backend with specified latencies
181		#[must_use]
182	0	pub fn new(ttft_ms: f64, tokens_per_second: f64) -> Self {
183	0	Self {
184	0	ttft_ms,
185	0	tokens_per_second,
186	0	}
187	0	}
188		}
189
190		impl RuntimeBackend for MockBackend {
191	0	fn info(&self) -> BackendInfo {
192	0	BackendInfo {
193	0	runtime_type: RuntimeType::Realizar,
194	0	version: env!("CARGO_PKG_VERSION").to_string(),
195	0	supports_streaming: true,
196	0	loaded_model: None,
197	0	}
198	0	}
199
200	0	fn inference(&self, request: &InferenceRequest) -> Result<InferenceResponse, RealizarError> {
201	0	let tokens = request.max_tokens.min(100);
202	0	let gen_time_ms = (tokens as f64) / self.tokens_per_second * 1000.0;
203
204	0	Ok(InferenceResponse {
205	0	text: "Mock response".to_string(),
206	0	tokens_generated: tokens,
207	0	ttft_ms: self.ttft_ms,
208	0	total_time_ms: self.ttft_ms + gen_time_ms,
209	0	itl_ms: vec![gen_time_ms / tokens as f64; tokens],
210	0	})
211	0	}
212		}
213
214		/// Registry of available backends
215		pub struct BackendRegistry {
216		backends: HashMap<RuntimeType, Box<dyn RuntimeBackend>>,
217		}
218
219		impl BackendRegistry {
220		/// Create empty registry
221		#[must_use]
222	0	pub fn new() -> Self {
223	0	Self {
224	0	backends: HashMap::new(),
225	0	}
226	0	}
227
228		/// Register a backend
229	0	pub fn register(&mut self, runtime: RuntimeType, backend: Box<dyn RuntimeBackend>) {
230	0	self.backends.insert(runtime, backend);
231	0	}
232
233		/// Get a backend by type
234		#[must_use]
235	0	pub fn get(&self, runtime: RuntimeType) -> Option<&dyn RuntimeBackend> {
236	0	self.backends.get(&runtime).map(AsRef::as_ref)
237	0	}
238
239		/// List registered runtimes
240		#[must_use]
241	0	pub fn list(&self) -> Vec<RuntimeType> {
242	0	self.backends.keys().copied().collect()
243	0	}
244		}
245
246		impl Default for BackendRegistry {
247	0	fn default() -> Self {
248	0	Self::new()
249	0	}
250		}
251
252		/// Configuration for llama.cpp backend
253		#[derive(Debug, Clone, Serialize, Deserialize)]
254		pub struct LlamaCppConfig {
255		/// Path to llama-cli binary
256		pub binary_path: String,
257		/// Path to model file
258		pub model_path: Option<String>,
259		/// Number of GPU layers to offload
260		pub n_gpu_layers: u32,
261		/// Context size
262		pub ctx_size: usize,
263		/// Number of threads
264		pub threads: usize,
265		}
266
267		impl Default for LlamaCppConfig {
268	0	fn default() -> Self {
269	0	Self {
270	0	binary_path: "llama-cli".to_string(),
271	0	model_path: None,
272	0	n_gpu_layers: 0,
273	0	ctx_size: 2048,
274	0	threads: 4,
275	0	}
276	0	}
277		}
278
279		impl LlamaCppConfig {
280		/// Create new config with binary path
281		#[must_use]
282	0	pub fn new(binary_path: &str) -> Self {
283	0	Self {
284	0	binary_path: binary_path.to_string(),
285	0	..Default::default()
286	0	}
287	0	}
288
289		/// Set model path
290		#[must_use]
291	0	pub fn with_model(mut self, model_path: &str) -> Self {
292	0	self.model_path = Some(model_path.to_string());
293	0	self
294	0	}
295
296		/// Set GPU layers
297		#[must_use]
298	0	pub fn with_gpu_layers(mut self, layers: u32) -> Self {
299	0	self.n_gpu_layers = layers;
300	0	self
301	0	}
302
303		/// Set context size
304		#[must_use]
305	0	pub fn with_ctx_size(mut self, ctx_size: usize) -> Self {
306	0	self.ctx_size = ctx_size;
307	0	self
308	0	}
309		}
310
311		/// Configuration for vLLM backend
312		#[derive(Debug, Clone, Serialize, Deserialize)]
313		pub struct VllmConfig {
314		/// Base URL for vLLM server
315		pub base_url: String,
316		/// API version
317		pub api_version: String,
318		/// Model name/path
319		pub model: Option<String>,
320		/// API key (if required)
321		pub api_key: Option<String>,
322		}
323
324		impl Default for VllmConfig {
325	0	fn default() -> Self {
326	0	Self {
327	0	base_url: "http://localhost:8000".to_string(),
328	0	api_version: "v1".to_string(),
329	0	model: None,
330	0	api_key: None,
331	0	}
332	0	}
333		}
334
335		impl VllmConfig {
336		/// Create new config with base URL
337		#[must_use]
338	0	pub fn new(base_url: &str) -> Self {
339	0	Self {
340	0	base_url: base_url.to_string(),
341	0	..Default::default()
342	0	}
343	0	}
344
345		/// Set model
346		#[must_use]
347	0	pub fn with_model(mut self, model: &str) -> Self {
348	0	self.model = Some(model.to_string());
349	0	self
350	0	}
351
352		/// Set API key
353		#[must_use]
354	0	pub fn with_api_key(mut self, api_key: &str) -> Self {
355	0	self.api_key = Some(api_key.to_string());
356	0	self
357	0	}
358		}
359
360		// ============================================================================
361		// LlamaCppBackend Implementation (BENCH-002)
362		// ============================================================================
363
364		/// llama.cpp backend for GGUF model inference via subprocess
365		pub struct LlamaCppBackend {
366		config: LlamaCppConfig,
367		}
368
369		impl LlamaCppBackend {
370		/// Create new llama.cpp backend
371		#[must_use]
372	0	pub fn new(config: LlamaCppConfig) -> Self {
373	0	Self { config }
374	0	}
375
376		/// Build CLI arguments for llama-cli invocation
377		#[must_use]
378	0	pub fn build_cli_args(&self, request: &InferenceRequest) -> Vec<String> {
379	0	let mut args = Vec::new();
380
381		// Model path
382	0	if let Some(ref model_path) = self.config.model_path {
383	0	args.push("-m".to_string());
384	0	args.push(model_path.clone());
385	0	}
386
387		// Prompt
388	0	args.push("-p".to_string());
389	0	args.push(request.prompt.clone());
390
391		// Number of tokens to generate
392	0	args.push("-n".to_string());
393	0	args.push(request.max_tokens.to_string());
394
395		// GPU layers
396	0	args.push("-ngl".to_string());
397	0	args.push(self.config.n_gpu_layers.to_string());
398
399		// Context size
400	0	args.push("-c".to_string());
401	0	args.push(self.config.ctx_size.to_string());
402
403		// Threads
404	0	args.push("-t".to_string());
405	0	args.push(self.config.threads.to_string());
406
407		// Temperature (if non-default)
408	0	if (request.temperature - 0.8).abs() > 0.01 {
409	0	args.push("--temp".to_string());
410	0	args.push(format!("{:.2}", request.temperature));
411	0	}
412
413	0	args
414	0	}
415
416		/// Parse a timing line from llama-cli output
417		///
418		/// Example: `llama_perf_context_print: prompt eval time = 12.34 ms / 10 tokens`
419		/// Returns: `Some((12.34, 10))`
420		#[must_use]
421	0	pub fn parse_timing_line(output: &str, metric_name: &str) -> Option<(f64, usize)> {
422	0	for line in output.lines() {
423		// For "eval time", we need to exclude "prompt eval time"
424	0	let matches = if metric_name == "eval time" {
425	0	line.contains(metric_name) && !line.contains("prompt eval time")
426		} else {
427	0	line.contains(metric_name)
428		};
429
430	0	if matches && line.contains('=') {
431		// Extract the value after "=" and before "ms"
432		// Format: "metric_name = 12.34 ms / 10 tokens"
433	0	if let Some(eq_pos) = line.find('=') {
434	0	let after_eq = &line[eq_pos + 1..];
435		// Find ms position
436	0	if let Some(ms_pos) = after_eq.find("ms") {
437	0	let value_str = after_eq[..ms_pos].trim();
438	0	if let Ok(value) = value_str.parse::<f64>() {
439		// Find the count after "/"
440	0	if let Some(slash_pos) = after_eq.find('/') {
441	0	let after_slash = &after_eq[slash_pos + 1..];
442		// Extract number before "tokens" or "runs"
443	0	let count_str =
444	0	after_slash.split_whitespace().next().unwrap_or("0");
445	0	if let Ok(count) = count_str.parse::<usize>() {
446	0	return Some((value, count));
447	0	}
448	0	}
449	0	}
450	0	}
451	0	}
452	0	}
453		}
454	0	None
455	0	}
456
457		/// Extract generated text from llama-cli output (before timing lines)
458		#[must_use]
459	0	pub fn extract_generated_text(output: &str) -> String {
460	0	let mut text_lines = Vec::new();
461	0	for line in output.lines() {
462		// Stop when we hit timing/performance lines
463	0	if line.contains("llama_perf_") \|\| line.contains("sampler") {
464	0	break;
465	0	}
466	0	text_lines.push(line);
467		}
468	0	text_lines.join("\n").trim().to_string()
469	0	}
470
471		/// Parse full CLI output into InferenceResponse
472		///
473		/// # Errors
474		///
475		/// Returns error if timing information cannot be parsed from output.
476	0	pub fn parse_cli_output(output: &str) -> Result<InferenceResponse, RealizarError> {
477		// Extract generated text
478	0	let text = Self::extract_generated_text(output);
479
480		// Parse timing metrics
481	0	let ttft_ms = Self::parse_timing_line(output, "prompt eval time").map_or(0.0, \|(ms, _)\| ms);
482
483	0	let (total_time_ms, _) = Self::parse_timing_line(output, "total time").unwrap_or((0.0, 0));
484
485	0	let (_, tokens_generated) =
486	0	Self::parse_timing_line(output, "eval time").unwrap_or((0.0, 0));
487
488		// ITL is not directly available from CLI output, estimate from eval time
489	0	let eval_time = Self::parse_timing_line(output, "eval time").map_or(0.0, \|(ms, _)\| ms);
490
491	0	let itl_ms = if tokens_generated > 1 {
492	0	let avg_itl = eval_time / (tokens_generated as f64);
493	0	vec![avg_itl; tokens_generated.saturating_sub(1)]
494		} else {
495	0	vec![]
496		};
497
498	0	Ok(InferenceResponse {
499	0	text,
500	0	tokens_generated,
501	0	ttft_ms,
502	0	total_time_ms,
503	0	itl_ms,
504	0	})
505	0	}
506		}
507
508		impl RuntimeBackend for LlamaCppBackend {
509	0	fn info(&self) -> BackendInfo {
510	0	BackendInfo {
511	0	runtime_type: RuntimeType::LlamaCpp,
512	0	version: "b2345".to_string(), // Would be detected from binary
513	0	supports_streaming: false, // CLI mode doesn't stream
514	0	loaded_model: self.config.model_path.clone(),
515	0	}
516	0	}
517
518	0	fn inference(&self, request: &InferenceRequest) -> Result<InferenceResponse, RealizarError> {
519		use std::process::Command;
520
521		// Require model path
522	0	let model_path = self.config.model_path.as_ref().ok_or_else(\|\| {
523	0	RealizarError::InvalidConfiguration("model_path is required".to_string())
524	0	})?;
525
526		// Build CLI arguments
527	0	let args = self.build_cli_args(request);
528
529		// Execute llama-cli
530	0	let output = Command::new(&self.config.binary_path)
531	0	.args(&args)
532	0	.output()
533	0	.map_err(\|e\| {
534	0	RealizarError::ModelNotFound(format!(
535	0	"Failed to execute {}: {}",
536	0	self.config.binary_path, e
537	0	))
538	0	})?;
539
540	0	if !output.status.success() {
541	0	let stderr = String::from_utf8_lossy(&output.stderr);
542	0	return Err(RealizarError::InferenceError(format!(
543	0	"llama-cli failed: {} (model: {})",
544	0	stderr, model_path
545	0	)));
546	0	}
547
548		// Parse stdout for response and timing
549	0	let stdout = String::from_utf8_lossy(&output.stdout);
550	0	let stderr = String::from_utf8_lossy(&output.stderr);
551
552		// Timing info is often in stderr, combine both
553	0	let combined_output = format!("{}\n{}", stdout, stderr);
554	0	Self::parse_cli_output(&combined_output)
555	0	}
556		}
557
558		// ============================================================================
559		// VllmBackend Implementation (BENCH-003) - REAL HTTP CALLS
560		// ============================================================================
561
562		/// vLLM backend for inference via HTTP API
563		///
564		/// REAL IMPLEMENTATION - makes actual HTTP requests to vLLM servers.
565		/// No mock data. Measures real latency and throughput.
566		#[cfg(feature = "bench-http")]
567		pub struct VllmBackend {
568		config: VllmConfig,
569		http_client: ModelHttpClient,
570		}
571
572		#[cfg(feature = "bench-http")]
573		impl VllmBackend {
574		/// Create new vLLM backend with default HTTP client
575		#[must_use]
576		pub fn new(config: VllmConfig) -> Self {
577		Self {
578		config,
579		http_client: ModelHttpClient::new(),
580		}
581		}
582
583		/// Create new vLLM backend with custom HTTP client
584		#[must_use]
585		pub fn with_client(config: VllmConfig, client: ModelHttpClient) -> Self {
586		Self {
587		config,
588		http_client: client,
589		}
590		}
591		}
592
593		#[cfg(feature = "bench-http")]
594		impl RuntimeBackend for VllmBackend {
595		fn info(&self) -> BackendInfo {
596		BackendInfo {
597		runtime_type: RuntimeType::Vllm,
598		version: "0.4.0".to_string(), // Would be detected from API
599		supports_streaming: true,
600		loaded_model: self.config.model.clone(),
601		}
602		}
603
604		fn inference(&self, request: &InferenceRequest) -> Result<InferenceResponse, RealizarError> {
605		// Parse URL to check for invalid port
606		let url = &self.config.base_url;
607		if let Some(port_str) = url.split(':').next_back() {
608		if let Ok(port) = port_str.parse::<u32>() {
609		if port > 65535 {
610		return Err(RealizarError::ConnectionError(format!(
611		"Invalid port in URL: {}",
612		url
613		)));
614		}
615		}
616		}
617
618		// REAL HTTP request to vLLM server via OpenAI-compatible API
619		#[allow(clippy::cast_possible_truncation)]
620		let completion_request = CompletionRequest {
621		model: self
622		.config
623		.model
624		.clone()
625		.unwrap_or_else(\|\| "default".to_string()),
626		prompt: request.prompt.clone(),
627		max_tokens: request.max_tokens,
628		temperature: Some(request.temperature as f32),
629		stream: false,
630		};
631
632		let timing = self.http_client.openai_completion(
633		&self.config.base_url,
634		&completion_request,
635		self.config.api_key.as_deref(),
636		)?;
637
638		Ok(InferenceResponse {
639		text: timing.text,
640		tokens_generated: timing.tokens_generated,
641		ttft_ms: timing.ttft_ms,
642		total_time_ms: timing.total_time_ms,
643		itl_ms: vec![], // ITL requires streaming, not available in blocking mode
644		})
645		}
646		}
647
648		// ============================================================================
649		// OllamaBackend Implementation - REAL HTTP CALLS
650		// ============================================================================
651
652		/// Configuration for Ollama backend
653		#[cfg(feature = "bench-http")]
654		#[derive(Debug, Clone, Serialize, Deserialize)]
655		pub struct OllamaConfig {
656		/// Base URL for Ollama server
657		pub base_url: String,
658		/// Model name
659		pub model: String,
660		}
661
662		#[cfg(feature = "bench-http")]
663		impl Default for OllamaConfig {
664		fn default() -> Self {
665		Self {
666		base_url: "http://localhost:11434".to_string(),
667		model: "llama2".to_string(),
668		}
669		}
670		}
671
672		/// Ollama backend for inference via HTTP API
673		///
674		/// REAL IMPLEMENTATION - makes actual HTTP requests to Ollama servers.
675		/// No mock data. Measures real latency and throughput.
676		#[cfg(feature = "bench-http")]
677		pub struct OllamaBackend {
678		config: OllamaConfig,
679		http_client: ModelHttpClient,
680		}
681
682		#[cfg(feature = "bench-http")]
683		impl OllamaBackend {
684		/// Create new Ollama backend with default HTTP client
685		#[must_use]
686		pub fn new(config: OllamaConfig) -> Self {
687		Self {
688		config,
689		http_client: ModelHttpClient::new(),
690		}
691		}
692
693		/// Create new Ollama backend with custom HTTP client
694		#[must_use]
695		pub fn with_client(config: OllamaConfig, client: ModelHttpClient) -> Self {
696		Self {
697		config,
698		http_client: client,
699		}
700		}
701		}
702
703		#[cfg(feature = "bench-http")]
704		impl RuntimeBackend for OllamaBackend {
705		fn info(&self) -> BackendInfo {
706		BackendInfo {
707		runtime_type: RuntimeType::Ollama,
708		version: "0.1.0".to_string(), // Would be detected from API
709		supports_streaming: true,
710		loaded_model: Some(self.config.model.clone()),
711		}
712		}
713
714		fn inference(&self, request: &InferenceRequest) -> Result<InferenceResponse, RealizarError> {
715		// REAL HTTP request to Ollama server
716		#[allow(clippy::cast_possible_truncation)]
717		let ollama_request = OllamaRequest {
718		model: self.config.model.clone(),
719		prompt: request.prompt.clone(),
720		stream: false,
721		options: Some(OllamaOptions {
722		num_predict: Some(request.max_tokens),
723		temperature: Some(request.temperature as f32),
724		}),
725		};
726
727		let timing = self
728		.http_client
729		.ollama_generate(&self.config.base_url, &ollama_request)?;
730
731		Ok(InferenceResponse {
732		text: timing.text,
733		tokens_generated: timing.tokens_generated,
734		ttft_ms: timing.ttft_ms,
735		total_time_ms: timing.total_time_ms,
736		itl_ms: vec![], // ITL requires streaming, not available in blocking mode
737		})
738		}
739		}
740