/home/noah/src/realizar/src/gguf/model.rs

Source
//! GGUF model types
//!
//! This module contains the core model structures for GGUF inference:
//!
//! - `MappedGGUFModel`: Memory-mapped GGUF file with zero-copy access
//! - `GGUFTransformer`: F32 transformer weights (dequantized from GGUF)
//! - `GGUFTransformerLayer`: Per-layer transformer weights
//! - `OwnedQuantizedModel`: Quantized model with owned weight data
//!
//! ## Design Philosophy
//!
//! Per Wulf & McKee (1995) "Hitting the Memory Wall", memory bandwidth is the
//! bottleneck for LLM inference. These types support:
//! - Zero-copy mmap loading (`MappedGGUFModel`)
//! - Quantized weights for 8x bandwidth reduction (`OwnedQuantizedModel`)
//! - Lazy dequantization during computation

use std::fs::File;
use std::path::Path;

use memmap2::Mmap;

use super::config::GGUFConfig;
use super::quantized::{OwnedQuantizedLayer, OwnedQuantizedTensor};
use super::types::GGUFModel;
use crate::error::{RealizarError, Result};

// ============================================================================
// MappedGGUFModel - Zero-copy memory-mapped model
// ============================================================================

/// Memory-mapped GGUF model for zero-copy tensor access
///
/// Uses `memmap2` for efficient large model loading without copying
/// entire file contents into heap memory.
///
/// # Example
///
/// ```rust,ignore
/// let model = MappedGGUFModel::from_path("phi-2-q4_k_m.gguf")?;
/// println!("Loaded {} tensors", model.model.tensors.len());
/// ```
pub struct MappedGGUFModel {
    /// Parsed model metadata (header, tensors, etc.)
    pub model: GGUFModel,
    /// Memory-mapped file contents
    pub(crate) mmap: Mmap,
}

impl MappedGGUFModel {
    /// Load GGUF model via memory mapping (zero-copy)
    ///
    /// # Arguments
    ///
    /// * `path` - Path to GGUF model file
    ///
    /// # Errors
    ///
    /// Returns error if:
    /// - File cannot be opened
    /// - Memory mapping fails
    /// - GGUF parsing fails (invalid format)
    ///
    /// # Performance
    ///
    /// Memory-mapped loading is faster than `std::fs::read` for large models:
    /// - No file content copy to heap memory
    /// - Kernel handles page management
    /// - Model remains accessible even if larger than RAM (via swap)
    ///
    /// # Examples
    ///
    /// ```rust,ignore
    /// let model = MappedGGUFModel::from_path("phi-2-q4_k_m.gguf")?;
    /// println!("Loaded {} tensors", model.model.tensors.len());
    /// ```
    pub fn from_path<P: AsRef<Path>>(path: P) -> Result<Self> {
        let file2 = File::open(path.as_ref()).map_err(|e| RealizarError::UnsupportedOperation {
            operation: "open_model_file".to_string(),
            reason: format!("Failed to open {}: {}", path.as_ref().display(), e),
        })?;

        // SAFETY: Memory mapping is safe as long as the file isn't modified
        // while mapped. We only read from the mapping, never write.
        let mmap = unsafe {
            Mmap::map(&file).map_err(|e| RealizarError::UnsupportedOperation {
                operation: "mmap_model_file".to_string(),
                reason: format!("Failed to mmap {}: {}", path.as_ref().display(), e),
            })?
        };

        // Parse the memory-mapped data
        let model = GGUFModel::from_bytes(&mmap)?0;

        Ok(Self { model, mmap })
    }

    /// Get the raw memory-mapped file data
    ///
    /// This provides direct access to the file contents without copying.
    /// Use this with tensor offsets to read quantized weights directly.
    #[must_use]
    pub fn data(&self) -> &[u8] {
        &self.mmap
    }

    /// Get tensor data slice by offset and size
    ///
    /// Returns a slice pointing directly into the memory-mapped file.
    /// No data is copied.
    ///
    /// # Arguments
    ///
    /// * `offset` - Byte offset from start of file
    /// * `size` - Size in bytes
    ///
    /// # Returns
    ///
    /// Slice of tensor data, or None if out of bounds
    #[must_use]
    pub fn tensor_slice(&self, offset: usize, size: usize) -> Option<&[u8]> {
        let end = offset.checked_add(size)?;
        if end <= self.mmap.len() {
            Some(&self.mmap[offset..end])
        } else {
            None
        }
    }

    /// Get the size of the memory-mapped file
    #[must_use]
    pub fn file_size(&self) -> usize {
        self.mmap.len()
    }

    /// Advise kernel to prefetch model data sequentially
    ///
    /// Per llama.cpp: Use madvise(MADV_SEQUENTIAL) to hint that the model
    /// will be read sequentially during loading. This improves prefetching.
    #[cfg(unix)]
    pub fn advise_sequential(&self) {
        // SAFETY: Memory safety ensured by mmap lifetime
        unsafe {
            libc::madvise(
                self.mmap.as_ptr().cast_mut().cast::<libc::c_void>(),
                self.mmap.len(),
                libc::MADV_SEQUENTIAL,
            );
        }
    }

    /// Advise kernel for random access pattern during inference
    ///
    /// Per llama.cpp: Use madvise(MADV_RANDOM) during inference when
    /// accessing weights non-sequentially.
    #[cfg(unix)]
    pub fn advise_random(&self) {
        // SAFETY: Memory safety ensured by mmap lifetime
        unsafe {
            libc::madvise(
                self.mmap.as_ptr().cast_mut().cast::<libc::c_void>(),
                self.mmap.len(),
                libc::MADV_RANDOM,
            );
        }
    }

    /// Advise kernel to keep model in memory (reduce swap pressure)
    ///
    /// Per llama.cpp: Use madvise(MADV_WILLNEED) to hint that the model
    /// will be needed soon, triggering prefetch.
    #[cfg(unix)]
    pub fn advise_willneed(&self) {
        // SAFETY: Memory safety ensured by mmap lifetime
        unsafe {
            libc::madvise(
                self.mmap.as_ptr().cast_mut().cast::<libc::c_void>(),
                self.mmap.len(),
                libc::MADV_WILLNEED,
            );
        }
    }

    /// Lock model in memory to prevent swapping (requires privileges)
    ///
    /// Per llama.cpp: Use mlock() to ensure model stays in RAM.
    /// Returns true if successful, false if failed (often due to ulimit).
    #[cfg(unix)]
    pub fn lock_memory(&self) -> bool {
        // SAFETY: Memory safety ensured by mmap lifetime
        unsafe { libc::mlock(self.mmap.as_ptr().cast::<libc::c_void>(), self.mmap.len()) == 0 }
    }
}

// ============================================================================
// GGUFTransformer - F32 transformer weights
// ============================================================================

/// F32 transformer weights loaded from GGUF
///
/// This struct holds dequantized weights in F32 format.
/// Used for reference implementations and debugging.
/// For production inference, use `OwnedQuantizedModel` instead.
pub struct GGUFTransformer {
    /// Model configuration
    pub config: GGUFConfig,
    /// Token embedding weights [vocab_size, hidden_dim]
    pub token_embedding: Vec<f32>,
    /// Attention weights per layer
    pub layers: Vec<GGUFTransformerLayer>,
    /// Output norm weight
    pub output_norm_weight: Vec<f32>,
    /// Output norm bias (optional)
    pub output_norm_bias: Option<Vec<f32>>,
    /// LM head / output projection weight
    pub lm_head_weight: Vec<f32>,
    /// LM head bias (optional)
    pub lm_head_bias: Option<Vec<f32>>,
}

// ============================================================================
// GGUFTransformerLayer - Per-layer F32 weights
// ============================================================================

/// Weights for a single transformer layer
pub struct GGUFTransformerLayer {
    /// Attention norm weight
    pub attn_norm_weight: Vec<f32>,
    /// Attention norm bias
    pub attn_norm_bias: Option<Vec<f32>>,
    /// QKV projection weights (combined for phi-2, concatenated Q+K+V for llama)
    pub qkv_weight: Vec<f32>,
    /// QKV bias (phi-2 has bias, llama doesn't)
    pub qkv_bias: Option<Vec<f32>>,
    /// Attention output projection weight
    pub attn_output_weight: Vec<f32>,
    /// Attention output projection bias
    pub attn_output_bias: Option<Vec<f32>>,
    /// FFN gate projection weight (SwiGLU models like llama)
    pub ffn_gate_weight: Option<Vec<f32>>,
    /// FFN gate projection bias
    pub ffn_gate_bias: Option<Vec<f32>>,
    /// FFN up projection weight
    pub ffn_up_weight: Vec<f32>,
    /// FFN up projection bias
    pub ffn_up_bias: Option<Vec<f32>>,
    /// FFN down projection weight
    pub ffn_down_weight: Vec<f32>,
    /// FFN down projection bias
    pub ffn_down_bias: Option<Vec<f32>>,
    /// FFN norm weight (for models with separate FFN normalization)
    pub ffn_norm_weight: Option<Vec<f32>>,
    /// FFN norm bias
    pub ffn_norm_bias: Option<Vec<f32>>,
}

// ============================================================================
// OwnedQuantizedModel - Quantized model with owned data
// ============================================================================

/// Owned quantized model with all weight data
///
/// IMP-100: This struct owns all quantized weight data, allowing storage
/// in `Arc` without lifetime parameters. Essential for async handlers.
///
/// # Memory Layout
///
/// - Token embedding: F32 for fast lookup
/// - Layer weights: Quantized (Q4_K, Q6_K, etc.)
/// - Output norm: F32 (small)
/// - LM head: Quantized
///
/// # GPU Acceleration
///
/// When the `cuda` feature is enabled, this struct includes a
/// `CudaExecutor` for GPU-accelerated matmul operations.
pub struct OwnedQuantizedModel {
    /// Model configuration
    pub config: GGUFConfig,
    /// Token embedding (f32 for fast lookup)
    pub token_embedding: Vec<f32>,
    /// Owned quantized layers
    pub layers: Vec<OwnedQuantizedLayer>,
    /// Output norm weight (f32)
    pub output_norm_weight: Vec<f32>,
    /// Output norm bias (optional)
    pub output_norm_bias: Option<Vec<f32>>,
    /// LM head weight (owned quantized)
    pub lm_head_weight: OwnedQuantizedTensor,
    /// LM head bias (optional, f32)
    pub lm_head_bias: Option<Vec<f32>>,
    /// PARITY-113: Optional CUDA executor for GPU acceleration
    /// When present, fused_matmul routes to CUDA GEMM kernels
    /// Uses Mutex for thread-safety in async handlers
    #[cfg(feature = "cuda")]
    pub(crate) cuda_executor: Option<std::sync::Mutex<crate::cuda::CudaExecutor>>,
    /// Track CUDA kernel execution count for metrics
    /// Uses AtomicU64 for thread-safe counting
    #[cfg(feature = "cuda")]
    pub(crate) cuda_kernel_count: std::sync::atomic::AtomicU64,
    /// PARITY-003: Set of weight names that have been cached on GPU
    /// Used to avoid repeated dequantization for the same weight
    #[cfg(feature = "cuda")]
    pub(crate) cached_weight_names: std::sync::Mutex<std::collections::HashSet<String>>,
}

// Manual Debug implementation (skip CUDA executor which doesn't impl Debug)
impl std::fmt::Debug for OwnedQuantizedModel {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        let mut s = f.debug_struct("OwnedQuantizedModel");
        s.field("config", &self.config)
            .field("token_embedding_len", &self.token_embedding.len())
            .field("layers_count", &self.layers.len())
            .field("output_norm_weight_len", &self.output_norm_weight.len())
            .field("has_output_norm_bias", &self.output_norm_bias.is_some())
            .field("lm_head_weight", &self.lm_head_weight)
            .field("has_lm_head_bias", &self.lm_head_bias.is_some());

        #[cfg(feature = "cuda")]
        s.field("cuda_enabled", &self.cuda_executor.is_some())
            .field(
                "cuda_kernel_count",
                &self
                    .cuda_kernel_count
                    .load(std::sync::atomic::Ordering::Relaxed),
            )
            .field(
                "cached_weight_count",
                &self
                    .cached_weight_names
                    .lock()
                    .map(|g| g.len())
                    .unwrap_or(0),
            );

        s.finish()
    }
}

// Manual Clone implementation due to Mutex
impl Clone for OwnedQuantizedModel {
    fn clone(&self) -> Self {
        Self {
            config: self.config.clone(),
            token_embedding: self.token_embedding.clone(),
            layers: self.layers.clone(),
            output_norm_weight: self.output_norm_weight.clone(),
            output_norm_bias: self.output_norm_bias.clone(),
            lm_head_weight: self.lm_head_weight.clone(),
            lm_head_bias: self.lm_head_bias.clone(),
            // CUDA executor is not cloned - new instance must enable CUDA separately
            #[cfg(feature = "cuda")]
            cuda_executor: None,
            #[cfg(feature = "cuda")]
            cuda_kernel_count: std::sync::atomic::AtomicU64::new(0),
            #[cfg(feature = "cuda")]
            cached_weight_names: std::sync::Mutex::new(std::collections::HashSet::new()),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_gguf_transformer_struct() {
        let config = GGUFConfig {
            architecture: "llama".to_string(),
            hidden_dim: 256,
            num_layers: 2,
            num_heads: 4,
            num_kv_heads: 4,
            vocab_size: 1000,
            intermediate_dim: 512,
            context_length: 512,
            rope_theta: 10000.0,
            eps: 1e-5,
            rope_type: 0,
        };

        let transformer = GGUFTransformer {
            config,
            token_embedding: vec![0.0; 256000], // 1000 vocab * 256 hidden
            layers: vec![],
            output_norm_weight: vec![1.0; 256],
            output_norm_bias: None,
            lm_head_weight: vec![0.0; 256000],
            lm_head_bias: None,
        };

        assert_eq!(transformer.config.architecture, "llama");
        assert_eq!(transformer.token_embedding.len(), 256000);
        assert!(transformer.layers.is_empty());
    }

    #[test]
    fn test_gguf_transformer_layer_struct() {
        let layer = GGUFTransformerLayer {
            attn_norm_weight: vec![1.0; 256],
            attn_norm_bias: None,
            qkv_weight: vec![0.0; 256 * 768], // 3 * hidden_dim
            qkv_bias: None,
            attn_output_weight: vec![0.0; 256 * 256],
            attn_output_bias: None,
            ffn_gate_weight: Some(vec![0.0; 256 * 512]),
            ffn_gate_bias: None,
            ffn_up_weight: vec![0.0; 256 * 512],
            ffn_up_bias: None,
            ffn_down_weight: vec![0.0; 512 * 256],
            ffn_down_bias: None,
            ffn_norm_weight: Some(vec![1.0; 256]),
            ffn_norm_bias: None,
        };

        assert_eq!(layer.attn_norm_weight.len(), 256);
        assert!(layer.ffn_gate_weight.is_some());
        assert!(layer.ffn_norm_weight.is_some());
    }

    #[test]
    fn test_owned_quantized_model_clone() {
        use super::super::quantized::OwnedQuantizedTensor;
        use super::super::types::GGUF_TYPE_Q4_K;

        let config = GGUFConfig {
            architecture: "test".to_string(),
            hidden_dim: 64,
            num_layers: 1,
            num_heads: 2,
            num_kv_heads: 2,
            vocab_size: 100,
            intermediate_dim: 128,
            context_length: 256,
            rope_theta: 10000.0,
            eps: 1e-5,
            rope_type: 0,
        };

        let model = OwnedQuantizedModel {
            config,
            token_embedding: vec![0.1; 6400],
            layers: vec![],
            output_norm_weight: vec![1.0; 64],
            output_norm_bias: None,
            lm_head_weight: OwnedQuantizedTensor {
                data: vec![0u8; 128],
                in_dim: 64,
                out_dim: 100,
                qtype: GGUF_TYPE_Q4_K,
            },
            lm_head_bias: None,
            #[cfg(feature = "cuda")]
            cuda_executor: None,
            #[cfg(feature = "cuda")]
            cuda_kernel_count: std::sync::atomic::AtomicU64::new(5),
            #[cfg(feature = "cuda")]
            cached_weight_names: std::sync::Mutex::new(std::collections::HashSet::new()),
        };

        let cloned = model.clone();
        assert_eq!(cloned.config.architecture, "test");
        assert_eq!(cloned.token_embedding.len(), 6400);

        // CUDA executor is not cloned
        #[cfg(feature = "cuda")]
        {
            assert!(cloned.cuda_executor.is_none());
            assert_eq!(
                cloned
                    .cuda_kernel_count
                    .load(std::sync::atomic::Ordering::Relaxed),
                0
            );
        }
    }

    #[test]
    fn test_owned_quantized_model_debug() {
        use super::super::quantized::OwnedQuantizedTensor;
        use super::super::types::GGUF_TYPE_Q4_K;

        let config = GGUFConfig {
            architecture: "debug_test".to_string(),
            hidden_dim: 32,
            num_layers: 1,
            num_heads: 1,
            num_kv_heads: 1,
            vocab_size: 50,
            intermediate_dim: 64,
            context_length: 128,
            rope_theta: 10000.0,
            eps: 1e-5,
            rope_type: 0,
        };

        let model = OwnedQuantizedModel {
            config,
            token_embedding: vec![0.0; 1600],
            layers: vec![],
            output_norm_weight: vec![1.0; 32],
            output_norm_bias: Some(vec![0.0; 32]),
            lm_head_weight: OwnedQuantizedTensor {
                data: vec![],
                in_dim: 32,
                out_dim: 50,
                qtype: GGUF_TYPE_Q4_K,
            },
            lm_head_bias: None,
            #[cfg(feature = "cuda")]
            cuda_executor: None,
            #[cfg(feature = "cuda")]
            cuda_kernel_count: std::sync::atomic::AtomicU64::new(0),
            #[cfg(feature = "cuda")]
            cached_weight_names: std::sync::Mutex::new(std::collections::HashSet::new()),
        };

        let debug_str = format!("{:?}", model);
        assert!(debug_str.contains("debug_test"));
        assert!(debug_str.contains("token_embedding_len"));
        assert!(debug_str.contains("1600"));
    }
}

Coverage Report

Created: 2026-01-25 15:05

Line	Count	Source
1		//! GGUF model types
2		//!
3		//! This module contains the core model structures for GGUF inference:
4		//!
5		//! - `MappedGGUFModel`: Memory-mapped GGUF file with zero-copy access
6		//! - `GGUFTransformer`: F32 transformer weights (dequantized from GGUF)
7		//! - `GGUFTransformerLayer`: Per-layer transformer weights
8		//! - `OwnedQuantizedModel`: Quantized model with owned weight data
9		//!
10		//! ## Design Philosophy
11		//!
12		//! Per Wulf & McKee (1995) "Hitting the Memory Wall", memory bandwidth is the
13		//! bottleneck for LLM inference. These types support:
14		//! - Zero-copy mmap loading (`MappedGGUFModel`)
15		//! - Quantized weights for 8x bandwidth reduction (`OwnedQuantizedModel`)
16		//! - Lazy dequantization during computation
17
18		use std::fs::File;
19		use std::path::Path;
20
21		use memmap2::Mmap;
22
23		use super::config::GGUFConfig;
24		use super::quantized::{OwnedQuantizedLayer, OwnedQuantizedTensor};
25		use super::types::GGUFModel;
26		use crate::error::{RealizarError, Result};
27
28		// ============================================================================
29		// MappedGGUFModel - Zero-copy memory-mapped model
30		// ============================================================================
31
32		/// Memory-mapped GGUF model for zero-copy tensor access
33		///
34		/// Uses `memmap2` for efficient large model loading without copying
35		/// entire file contents into heap memory.
36		///
37		/// # Example
38		///
39		/// ```rust,ignore
40		/// let model = MappedGGUFModel::from_path("phi-2-q4_k_m.gguf")?;
41		/// println!("Loaded {} tensors", model.model.tensors.len());
42		/// ```
43		pub struct MappedGGUFModel {
44		/// Parsed model metadata (header, tensors, etc.)
45		pub model: GGUFModel,
46		/// Memory-mapped file contents
47		pub(crate) mmap: Mmap,
48		}
49
50		impl MappedGGUFModel {
51		/// Load GGUF model via memory mapping (zero-copy)
52		///
53		/// # Arguments
54		///
55		/// * `path` - Path to GGUF model file
56		///
57		/// # Errors
58		///
59		/// Returns error if:
60		/// - File cannot be opened
61		/// - Memory mapping fails
62		/// - GGUF parsing fails (invalid format)
63		///
64		/// # Performance
65		///
66		/// Memory-mapped loading is faster than `std::fs::read` for large models:
67		/// - No file content copy to heap memory
68		/// - Kernel handles page management
69		/// - Model remains accessible even if larger than RAM (via swap)
70		///
71		/// # Examples
72		///
73		/// ```rust,ignore
74		/// let model = MappedGGUFModel::from_path("phi-2-q4_k_m.gguf")?;
75		/// println!("Loaded {} tensors", model.model.tensors.len());
76		/// ```
77	6	pub fn from_path<P: AsRef<Path>>(path: P) -> Result<Self> {
78	6	let file2 = File::open(path.as_ref()).map_err(\|e\| RealizarError::UnsupportedOperation {
79	4	operation: "open_model_file".to_string(),
80	4	reason: format!("Failed to open {}: {}", path.as_ref().display(), e),
81	4	})?;
82
83		// SAFETY: Memory mapping is safe as long as the file isn't modified
84		// while mapped. We only read from the mapping, never write.
85	2	let mmap = unsafe {
86	2	Mmap::map(&file).map_err(\|e\| RealizarError::UnsupportedOperation {
87	0	operation: "mmap_model_file".to_string(),
88	0	reason: format!("Failed to mmap {}: {}", path.as_ref().display(), e),
89	0	})?
90		};
91
92		// Parse the memory-mapped data
93	2	let model = GGUFModel::from_bytes(&mmap) ?0 ;
94
95	2	Ok(Self { model, mmap })
96	6	}
97
98		/// Get the raw memory-mapped file data
99		///
100		/// This provides direct access to the file contents without copying.
101		/// Use this with tensor offsets to read quantized weights directly.
102		#[must_use]
103	4	pub fn data(&self) -> &[u8] {
104	4	&self.mmap
105	4	}
106
107		/// Get tensor data slice by offset and size
108		///
109		/// Returns a slice pointing directly into the memory-mapped file.
110		/// No data is copied.
111		///
112		/// # Arguments
113		///
114		/// * `offset` - Byte offset from start of file
115		/// * `size` - Size in bytes
116		///
117		/// # Returns
118		///
119		/// Slice of tensor data, or None if out of bounds
120		#[must_use]
121	0	pub fn tensor_slice(&self, offset: usize, size: usize) -> Option<&[u8]> {
122	0	let end = offset.checked_add(size)?;
123	0	if end <= self.mmap.len() {
124	0	Some(&self.mmap[offset..end])
125		} else {
126	0	None
127		}
128	0	}
129
130		/// Get the size of the memory-mapped file
131		#[must_use]
132	0	pub fn file_size(&self) -> usize {
133	0	self.mmap.len()
134	0	}
135
136		/// Advise kernel to prefetch model data sequentially
137		///
138		/// Per llama.cpp: Use madvise(MADV_SEQUENTIAL) to hint that the model
139		/// will be read sequentially during loading. This improves prefetching.
140		#[cfg(unix)]
141	0	pub fn advise_sequential(&self) {
142		// SAFETY: Memory safety ensured by mmap lifetime
143	0	unsafe {
144	0	libc::madvise(
145	0	self.mmap.as_ptr().cast_mut().cast::<libc::c_void>(),
146	0	self.mmap.len(),
147	0	libc::MADV_SEQUENTIAL,
148	0	);
149	0	}
150	0	}
151
152		/// Advise kernel for random access pattern during inference
153		///
154		/// Per llama.cpp: Use madvise(MADV_RANDOM) during inference when
155		/// accessing weights non-sequentially.
156		#[cfg(unix)]
157	0	pub fn advise_random(&self) {
158		// SAFETY: Memory safety ensured by mmap lifetime
159	0	unsafe {
160	0	libc::madvise(
161	0	self.mmap.as_ptr().cast_mut().cast::<libc::c_void>(),
162	0	self.mmap.len(),
163	0	libc::MADV_RANDOM,
164	0	);
165	0	}
166	0	}
167
168		/// Advise kernel to keep model in memory (reduce swap pressure)
169		///
170		/// Per llama.cpp: Use madvise(MADV_WILLNEED) to hint that the model
171		/// will be needed soon, triggering prefetch.
172		#[cfg(unix)]
173	0	pub fn advise_willneed(&self) {
174		// SAFETY: Memory safety ensured by mmap lifetime
175	0	unsafe {
176	0	libc::madvise(
177	0	self.mmap.as_ptr().cast_mut().cast::<libc::c_void>(),
178	0	self.mmap.len(),
179	0	libc::MADV_WILLNEED,
180	0	);
181	0	}
182	0	}
183
184		/// Lock model in memory to prevent swapping (requires privileges)
185		///
186		/// Per llama.cpp: Use mlock() to ensure model stays in RAM.
187		/// Returns true if successful, false if failed (often due to ulimit).
188		#[cfg(unix)]
189	0	pub fn lock_memory(&self) -> bool {
190		// SAFETY: Memory safety ensured by mmap lifetime
191	0	unsafe { libc::mlock(self.mmap.as_ptr().cast::<libc::c_void>(), self.mmap.len()) == 0 }
192	0	}
193		}
194
195		// ============================================================================
196		// GGUFTransformer - F32 transformer weights
197		// ============================================================================
198
199		/// F32 transformer weights loaded from GGUF
200		///
201		/// This struct holds dequantized weights in F32 format.
202		/// Used for reference implementations and debugging.
203		/// For production inference, use `OwnedQuantizedModel` instead.
204		pub struct GGUFTransformer {
205		/// Model configuration
206		pub config: GGUFConfig,
207		/// Token embedding weights [vocab_size, hidden_dim]
208		pub token_embedding: Vec<f32>,
209		/// Attention weights per layer
210		pub layers: Vec<GGUFTransformerLayer>,
211		/// Output norm weight
212		pub output_norm_weight: Vec<f32>,
213		/// Output norm bias (optional)
214		pub output_norm_bias: Option<Vec<f32>>,
215		/// LM head / output projection weight
216		pub lm_head_weight: Vec<f32>,
217		/// LM head bias (optional)
218		pub lm_head_bias: Option<Vec<f32>>,
219		}
220
221		// ============================================================================
222		// GGUFTransformerLayer - Per-layer F32 weights
223		// ============================================================================
224
225		/// Weights for a single transformer layer
226		pub struct GGUFTransformerLayer {
227		/// Attention norm weight
228		pub attn_norm_weight: Vec<f32>,
229		/// Attention norm bias
230		pub attn_norm_bias: Option<Vec<f32>>,
231		/// QKV projection weights (combined for phi-2, concatenated Q+K+V for llama)
232		pub qkv_weight: Vec<f32>,
233		/// QKV bias (phi-2 has bias, llama doesn't)
234		pub qkv_bias: Option<Vec<f32>>,
235		/// Attention output projection weight
236		pub attn_output_weight: Vec<f32>,
237		/// Attention output projection bias
238		pub attn_output_bias: Option<Vec<f32>>,
239		/// FFN gate projection weight (SwiGLU models like llama)
240		pub ffn_gate_weight: Option<Vec<f32>>,
241		/// FFN gate projection bias
242		pub ffn_gate_bias: Option<Vec<f32>>,
243		/// FFN up projection weight
244		pub ffn_up_weight: Vec<f32>,
245		/// FFN up projection bias
246		pub ffn_up_bias: Option<Vec<f32>>,
247		/// FFN down projection weight
248		pub ffn_down_weight: Vec<f32>,
249		/// FFN down projection bias
250		pub ffn_down_bias: Option<Vec<f32>>,
251		/// FFN norm weight (for models with separate FFN normalization)
252		pub ffn_norm_weight: Option<Vec<f32>>,
253		/// FFN norm bias
254		pub ffn_norm_bias: Option<Vec<f32>>,
255		}
256
257		// ============================================================================
258		// OwnedQuantizedModel - Quantized model with owned data
259		// ============================================================================
260
261		/// Owned quantized model with all weight data
262		///
263		/// IMP-100: This struct owns all quantized weight data, allowing storage
264		/// in `Arc` without lifetime parameters. Essential for async handlers.
265		///
266		/// # Memory Layout
267		///
268		/// - Token embedding: F32 for fast lookup
269		/// - Layer weights: Quantized (Q4_K, Q6_K, etc.)
270		/// - Output norm: F32 (small)
271		/// - LM head: Quantized
272		///
273		/// # GPU Acceleration
274		///
275		/// When the `cuda` feature is enabled, this struct includes a
276		/// `CudaExecutor` for GPU-accelerated matmul operations.
277		pub struct OwnedQuantizedModel {
278		/// Model configuration
279		pub config: GGUFConfig,
280		/// Token embedding (f32 for fast lookup)
281		pub token_embedding: Vec<f32>,
282		/// Owned quantized layers
283		pub layers: Vec<OwnedQuantizedLayer>,
284		/// Output norm weight (f32)
285		pub output_norm_weight: Vec<f32>,
286		/// Output norm bias (optional)
287		pub output_norm_bias: Option<Vec<f32>>,
288		/// LM head weight (owned quantized)
289		pub lm_head_weight: OwnedQuantizedTensor,
290		/// LM head bias (optional, f32)
291		pub lm_head_bias: Option<Vec<f32>>,
292		/// PARITY-113: Optional CUDA executor for GPU acceleration
293		/// When present, fused_matmul routes to CUDA GEMM kernels
294		/// Uses Mutex for thread-safety in async handlers
295		#[cfg(feature = "cuda")]
296		pub(crate) cuda_executor: Option<std::sync::Mutex<crate::cuda::CudaExecutor>>,
297		/// Track CUDA kernel execution count for metrics
298		/// Uses AtomicU64 for thread-safe counting
299		#[cfg(feature = "cuda")]
300		pub(crate) cuda_kernel_count: std::sync::atomic::AtomicU64,
301		/// PARITY-003: Set of weight names that have been cached on GPU
302		/// Used to avoid repeated dequantization for the same weight
303		#[cfg(feature = "cuda")]
304		pub(crate) cached_weight_names: std::sync::Mutex<std::collections::HashSet<String>>,
305		}
306
307		// Manual Debug implementation (skip CUDA executor which doesn't impl Debug)
308		impl std::fmt::Debug for OwnedQuantizedModel {
309	1	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
310	1	let mut s = f.debug_struct("OwnedQuantizedModel");
311	1	s.field("config", &self.config)
312	1	.field("token_embedding_len", &self.token_embedding.len())
313	1	.field("layers_count", &self.layers.len())
314	1	.field("output_norm_weight_len", &self.output_norm_weight.len())
315	1	.field("has_output_norm_bias", &self.output_norm_bias.is_some())
316	1	.field("lm_head_weight", &self.lm_head_weight)
317	1	.field("has_lm_head_bias", &self.lm_head_bias.is_some());
318
319		#[cfg(feature = "cuda")]
320		s.field("cuda_enabled", &self.cuda_executor.is_some())
321		.field(
322		"cuda_kernel_count",
323		&self
324		.cuda_kernel_count
325		.load(std::sync::atomic::Ordering::Relaxed),
326		)
327		.field(
328		"cached_weight_count",
329		&self
330		.cached_weight_names
331		.lock()
332		.map(\|g\| g.len())
333		.unwrap_or(0),
334		);
335
336	1	s.finish()
337	1	}
338		}
339
340		// Manual Clone implementation due to Mutex
341		impl Clone for OwnedQuantizedModel {
342	4	fn clone(&self) -> Self {
343	4	Self {
344	4	config: self.config.clone(),
345	4	token_embedding: self.token_embedding.clone(),
346	4	layers: self.layers.clone(),
347	4	output_norm_weight: self.output_norm_weight.clone(),
348	4	output_norm_bias: self.output_norm_bias.clone(),
349	4	lm_head_weight: self.lm_head_weight.clone(),
350	4	lm_head_bias: self.lm_head_bias.clone(),
351	4	// CUDA executor is not cloned - new instance must enable CUDA separately
352	4	#[cfg(feature = "cuda")]
353	4	cuda_executor: None,
354	4	#[cfg(feature = "cuda")]
355	4	cuda_kernel_count: std::sync::atomic::AtomicU64::new(0),
356	4	#[cfg(feature = "cuda")]
357	4	cached_weight_names: std::sync::Mutex::new(std::collections::HashSet::new()),
358	4	}
359	4	}
360		}
361
362		#[cfg(test)]
363		mod tests {
364		use super::*;
365
366		#[test]
367	1	fn test_gguf_transformer_struct() {
368	1	let config = GGUFConfig {
369	1	architecture: "llama".to_string(),
370	1	hidden_dim: 256,
371	1	num_layers: 2,
372	1	num_heads: 4,
373	1	num_kv_heads: 4,
374	1	vocab_size: 1000,
375	1	intermediate_dim: 512,
376	1	context_length: 512,
377	1	rope_theta: 10000.0,
378	1	eps: 1e-5,
379	1	rope_type: 0,
380	1	};
381
382	1	let transformer = GGUFTransformer {
383	1	config,
384	1	token_embedding: vec![0.0; 256000], // 1000 vocab * 256 hidden
385	1	layers: vec![],
386	1	output_norm_weight: vec![1.0; 256],
387	1	output_norm_bias: None,
388	1	lm_head_weight: vec![0.0; 256000],
389	1	lm_head_bias: None,
390	1	};
391
392	1	assert_eq!(transformer.config.architecture, "llama");
393	1	assert_eq!(transformer.token_embedding.len(), 256000);
394	1	assert!(transformer.layers.is_empty());
395	1	}
396
397		#[test]
398	1	fn test_gguf_transformer_layer_struct() {
399	1	let layer = GGUFTransformerLayer {
400	1	attn_norm_weight: vec![1.0; 256],
401	1	attn_norm_bias: None,
402	1	qkv_weight: vec![0.0; 256 * 768], // 3 * hidden_dim
403	1	qkv_bias: None,
404	1	attn_output_weight: vec![0.0; 256 * 256],
405	1	attn_output_bias: None,
406	1	ffn_gate_weight: Some(vec![0.0; 256 * 512]),
407	1	ffn_gate_bias: None,
408	1	ffn_up_weight: vec![0.0; 256 * 512],
409	1	ffn_up_bias: None,
410	1	ffn_down_weight: vec![0.0; 512 * 256],
411	1	ffn_down_bias: None,
412	1	ffn_norm_weight: Some(vec![1.0; 256]),
413	1	ffn_norm_bias: None,
414	1	};
415
416	1	assert_eq!(layer.attn_norm_weight.len(), 256);
417	1	assert!(layer.ffn_gate_weight.is_some());
418	1	assert!(layer.ffn_norm_weight.is_some());
419	1	}
420
421		#[test]
422	1	fn test_owned_quantized_model_clone() {
423		use super::super::quantized::OwnedQuantizedTensor;
424		use super::super::types::GGUF_TYPE_Q4_K;
425
426	1	let config = GGUFConfig {
427	1	architecture: "test".to_string(),
428	1	hidden_dim: 64,
429	1	num_layers: 1,
430	1	num_heads: 2,
431	1	num_kv_heads: 2,
432	1	vocab_size: 100,
433	1	intermediate_dim: 128,
434	1	context_length: 256,
435	1	rope_theta: 10000.0,
436	1	eps: 1e-5,
437	1	rope_type: 0,
438	1	};
439
440	1	let model = OwnedQuantizedModel {
441	1	config,
442	1	token_embedding: vec![0.1; 6400],
443	1	layers: vec![],
444	1	output_norm_weight: vec![1.0; 64],
445	1	output_norm_bias: None,
446	1	lm_head_weight: OwnedQuantizedTensor {
447	1	data: vec![0u8; 128],
448	1	in_dim: 64,
449	1	out_dim: 100,
450	1	qtype: GGUF_TYPE_Q4_K,
451	1	},
452	1	lm_head_bias: None,
453	1	#[cfg(feature = "cuda")]
454	1	cuda_executor: None,
455	1	#[cfg(feature = "cuda")]
456	1	cuda_kernel_count: std::sync::atomic::AtomicU64::new(5),
457	1	#[cfg(feature = "cuda")]
458	1	cached_weight_names: std::sync::Mutex::new(std::collections::HashSet::new()),
459	1	};
460
461	1	let cloned = model.clone();
462	1	assert_eq!(cloned.config.architecture, "test");
463	1	assert_eq!(cloned.token_embedding.len(), 6400);
464
465		// CUDA executor is not cloned
466		#[cfg(feature = "cuda")]
467		{
468		assert!(cloned.cuda_executor.is_none());
469		assert_eq!(
470		cloned
471		.cuda_kernel_count
472		.load(std::sync::atomic::Ordering::Relaxed),
473		0
474		);
475		}
476	1	}
477
478		#[test]
479	1	fn test_owned_quantized_model_debug() {
480		use super::super::quantized::OwnedQuantizedTensor;
481		use super::super::types::GGUF_TYPE_Q4_K;
482
483	1	let config = GGUFConfig {
484	1	architecture: "debug_test".to_string(),
485	1	hidden_dim: 32,
486	1	num_layers: 1,
487	1	num_heads: 1,
488	1	num_kv_heads: 1,
489	1	vocab_size: 50,
490	1	intermediate_dim: 64,
491	1	context_length: 128,
492	1	rope_theta: 10000.0,
493	1	eps: 1e-5,
494	1	rope_type: 0,
495	1	};
496
497	1	let model = OwnedQuantizedModel {
498	1	config,
499	1	token_embedding: vec![0.0; 1600],
500	1	layers: vec![],
501	1	output_norm_weight: vec![1.0; 32],
502	1	output_norm_bias: Some(vec![0.0; 32]),
503	1	lm_head_weight: OwnedQuantizedTensor {
504	1	data: vec![],
505	1	in_dim: 32,
506	1	out_dim: 50,
507	1	qtype: GGUF_TYPE_Q4_K,
508	1	},
509	1	lm_head_bias: None,
510	1	#[cfg(feature = "cuda")]
511	1	cuda_executor: None,
512	1	#[cfg(feature = "cuda")]
513	1	cuda_kernel_count: std::sync::atomic::AtomicU64::new(0),
514	1	#[cfg(feature = "cuda")]
515	1	cached_weight_names: std::sync::Mutex::new(std::collections::HashSet::new()),
516	1	};
517
518	1	let debug_str = format!("{:?}", model);
519	1	assert!(debug_str.contains("debug_test"));
520	1	assert!(debug_str.contains("token_embedding_len"));
521	1	assert!(debug_str.contains("1600"));
522	1	}
523		}