/home/noah/src/realizar/src/convert/mod.rs
Line | Count | Source |
1 | | //! GGUF to APR Transformer Converter |
2 | | //! |
3 | | //! Converts GGUF models to APR Transformer format for fair comparison. |
4 | | //! All weights are dequantized to F32 for WASM compatibility. |
5 | | //! |
6 | | //! ## Example |
7 | | //! |
8 | | //! ```rust,ignore |
9 | | //! use realizar::convert::GgufToAprConverter; |
10 | | //! |
11 | | //! let gguf_data = std::fs::read("model.gguf")?; |
12 | | //! let apr_transformer = GgufToAprConverter::convert(&gguf_data)?; |
13 | | //! |
14 | | //! // Save to APR format |
15 | | //! let apr_bytes = apr_transformer.to_apr_bytes()?; |
16 | | //! std::fs::write("model.apr_transformer", apr_bytes)?; |
17 | | //! ``` |
18 | | |
19 | | use crate::apr::{AprHeader, TensorEntry, ALIGNMENT, HEADER_SIZE, MAGIC}; |
20 | | use crate::apr_transformer::{AprTransformer, AprTransformerConfig, AprTransformerLayer}; |
21 | | use crate::error::{RealizarError, Result}; |
22 | | use crate::gguf::{GGUFModel, GGUFTransformer}; |
23 | | |
24 | | /// GGUF to APR Transformer converter |
25 | | /// |
26 | | /// Converts GGUF models with quantized weights to APR format with F32 weights. |
27 | | /// This enables fair comparison between GGUF and APR serving performance. |
28 | | pub struct GgufToAprConverter; |
29 | | |
30 | | impl GgufToAprConverter { |
31 | | /// Convert GGUF file bytes to APR Transformer |
32 | | /// |
33 | | /// # Arguments |
34 | | /// |
35 | | /// * `gguf_data` - Raw GGUF file bytes |
36 | | /// |
37 | | /// # Returns |
38 | | /// |
39 | | /// `AprTransformer` with dequantized F32 weights |
40 | | /// |
41 | | /// # Errors |
42 | | /// |
43 | | /// Returns error if GGUF parsing or conversion fails |
44 | 0 | pub fn convert(gguf_data: &[u8]) -> Result<AprTransformer> { |
45 | | // Parse GGUF model |
46 | 0 | let gguf_model = GGUFModel::from_bytes(gguf_data)?; |
47 | | |
48 | | // Load transformer weights (dequantizes to F32) |
49 | 0 | let gguf_transformer = GGUFTransformer::from_gguf(&gguf_model, gguf_data)?; |
50 | | |
51 | | // Convert to APR format |
52 | 0 | Ok(Self::from_gguf_transformer(&gguf_transformer)) |
53 | 0 | } |
54 | | |
55 | | /// Convert from existing `GGUFTransformer` to `AprTransformer` |
56 | | /// |
57 | | /// # Arguments |
58 | | /// |
59 | | /// * `gguf` - Loaded GGUF transformer with dequantized weights |
60 | | /// |
61 | | /// # Returns |
62 | | /// |
63 | | /// `AprTransformer` with the same weights |
64 | 18 | pub fn from_gguf_transformer(gguf: &GGUFTransformer) -> AprTransformer { |
65 | 18 | let config = AprTransformerConfig { |
66 | 18 | architecture: gguf.config.architecture.clone(), |
67 | 18 | hidden_dim: gguf.config.hidden_dim, |
68 | 18 | num_layers: gguf.config.num_layers, |
69 | 18 | num_heads: gguf.config.num_heads, |
70 | 18 | num_kv_heads: gguf.config.num_kv_heads, |
71 | 18 | vocab_size: gguf.config.vocab_size, |
72 | 18 | intermediate_dim: gguf.config.intermediate_dim, |
73 | 18 | context_length: gguf.config.context_length, |
74 | 18 | rope_theta: gguf.config.rope_theta, |
75 | 18 | eps: gguf.config.eps, |
76 | 18 | }; |
77 | | |
78 | 18 | let layers = gguf |
79 | 18 | .layers |
80 | 18 | .iter() |
81 | 18 | .map(|l| AprTransformerLayer { |
82 | 36 | attn_norm_weight: l.attn_norm_weight.clone(), |
83 | 36 | attn_norm_bias: l.attn_norm_bias.clone(), |
84 | 36 | qkv_weight: l.qkv_weight.clone(), |
85 | 36 | qkv_bias: l.qkv_bias.clone(), |
86 | 36 | attn_output_weight: l.attn_output_weight.clone(), |
87 | 36 | attn_output_bias: l.attn_output_bias.clone(), |
88 | 36 | ffn_gate_weight: l.ffn_gate_weight.clone(), |
89 | 36 | ffn_gate_bias: l.ffn_gate_bias.clone(), |
90 | 36 | ffn_up_weight: l.ffn_up_weight.clone(), |
91 | 36 | ffn_up_bias: l.ffn_up_bias.clone(), |
92 | 36 | ffn_down_weight: l.ffn_down_weight.clone(), |
93 | 36 | ffn_down_bias: l.ffn_down_bias.clone(), |
94 | 36 | ffn_norm_weight: l.ffn_norm_weight.clone(), |
95 | 36 | ffn_norm_bias: l.ffn_norm_bias.clone(), |
96 | 36 | }) |
97 | 18 | .collect(); |
98 | | |
99 | 18 | AprTransformer { |
100 | 18 | config, |
101 | 18 | token_embedding: gguf.token_embedding.clone(), |
102 | 18 | layers, |
103 | 18 | output_norm_weight: gguf.output_norm_weight.clone(), |
104 | 18 | output_norm_bias: gguf.output_norm_bias.clone(), |
105 | 18 | lm_head_weight: gguf.lm_head_weight.clone(), |
106 | 18 | lm_head_bias: gguf.lm_head_bias.clone(), |
107 | 18 | q4k_layers: None, |
108 | 18 | lm_head_weight_q6k: None, |
109 | 18 | lm_head_weight_q4k: None, |
110 | 18 | } |
111 | 18 | } |
112 | | |
113 | | /// Convert APR Transformer to serialized APR v2 bytes |
114 | | /// |
115 | | /// Creates a valid .apr v2 file with: |
116 | | /// - APR v2 header (64 bytes) |
117 | | /// - JSON metadata (padded to 64-byte boundary) |
118 | | /// - Tensor index (JSON array) |
119 | | /// - Tensor data (each 64-byte aligned) |
120 | | /// |
121 | | /// # Arguments |
122 | | /// |
123 | | /// * `transformer` - APR Transformer to serialize |
124 | | /// |
125 | | /// # Returns |
126 | | /// |
127 | | /// Raw bytes in APR v2 format |
128 | | /// |
129 | | /// # Errors |
130 | | /// |
131 | | /// Returns error if serialization fails |
132 | | #[allow(clippy::cast_possible_truncation)] |
133 | 14 | pub fn to_apr_bytes(transformer: &AprTransformer) -> Result<Vec<u8>> { |
134 | | // Serialize metadata |
135 | 14 | let metadata = serde_json::json!({ |
136 | 14 | "model_type": "transformer_lm", |
137 | 14 | "architecture": transformer.config.architecture, |
138 | 14 | "hidden_size": transformer.config.hidden_dim, |
139 | 14 | "num_layers": transformer.config.num_layers, |
140 | 14 | "num_heads": transformer.config.num_heads, |
141 | 14 | "num_kv_heads": transformer.config.num_kv_heads, |
142 | 14 | "vocab_size": transformer.config.vocab_size, |
143 | 14 | "intermediate_dim": transformer.config.intermediate_dim, |
144 | 14 | "context_length": transformer.config.context_length, |
145 | 14 | "rope_theta": transformer.config.rope_theta, |
146 | 14 | "eps": transformer.config.eps, |
147 | | }); |
148 | 14 | let metadata_bytes = |
149 | 14 | serde_json::to_vec(&metadata).map_err(|e| RealizarError::FormatError { |
150 | 0 | reason: format!("Failed to serialize metadata: {e}"), |
151 | 0 | })?; |
152 | | |
153 | | // Pad metadata to 64-byte boundary |
154 | 14 | let metadata_padded_len = metadata_bytes.len().div_ceil(ALIGNMENT) * ALIGNMENT; |
155 | | |
156 | | // Serialize weights as single tensor (JSON payload for now) |
157 | 14 | let payload_bytes = |
158 | 14 | serde_json::to_vec(transformer).map_err(|e| RealizarError::FormatError { |
159 | 0 | reason: format!("Failed to serialize weights: {e}"), |
160 | 0 | })?; |
161 | | |
162 | | // Create tensor index with single entry for the full payload |
163 | 14 | let tensor_entries = vec![TensorEntry { |
164 | 14 | name: "weights".to_string(), |
165 | 14 | dtype: "json".to_string(), |
166 | 14 | shape: vec![payload_bytes.len()], |
167 | 14 | offset: 0, |
168 | 14 | size: payload_bytes.len() as u64, |
169 | 14 | }]; |
170 | 14 | let tensor_index_bytes = |
171 | 14 | serde_json::to_vec(&tensor_entries).map_err(|e| RealizarError::FormatError { |
172 | 0 | reason: format!("Failed to serialize tensor index: {e}"), |
173 | 0 | })?; |
174 | | |
175 | | // Calculate offsets |
176 | 14 | let metadata_offset = HEADER_SIZE as u64; |
177 | 14 | let tensor_index_offset = metadata_offset + metadata_padded_len as u64; |
178 | 14 | let data_offset = tensor_index_offset + tensor_index_bytes.len() as u64; |
179 | | |
180 | | // Build APR v2 header (64 bytes) |
181 | 14 | let mut header = vec![0u8; HEADER_SIZE]; |
182 | 14 | header[0..4].copy_from_slice(&MAGIC); |
183 | 14 | header[4] = 2; // version major |
184 | 14 | header[5] = 0; // version minor |
185 | 14 | header[6..8].copy_from_slice(&0u16.to_le_bytes()); // flags |
186 | 14 | header[8..12].copy_from_slice(&1u32.to_le_bytes()); // tensor_count |
187 | 14 | header[12..20].copy_from_slice(&metadata_offset.to_le_bytes()); |
188 | 14 | header[20..24].copy_from_slice(&(metadata_bytes.len() as u32).to_le_bytes()); |
189 | 14 | header[24..32].copy_from_slice(&tensor_index_offset.to_le_bytes()); |
190 | 14 | header[32..40].copy_from_slice(&data_offset.to_le_bytes()); |
191 | 14 | header[40..44].copy_from_slice(&0u32.to_le_bytes()); // checksum: reserved for future use |
192 | | // bytes 44-63 reserved |
193 | | |
194 | | // Combine all parts |
195 | 14 | let total_size = |
196 | 14 | HEADER_SIZE + metadata_padded_len + tensor_index_bytes.len() + payload_bytes.len(); |
197 | 14 | let mut result = Vec::with_capacity(total_size); |
198 | 14 | result.extend_from_slice(&header); |
199 | 14 | result.extend_from_slice(&metadata_bytes); |
200 | 14 | result.resize(HEADER_SIZE + metadata_padded_len, 0); // pad metadata |
201 | 14 | result.extend_from_slice(&tensor_index_bytes); |
202 | 14 | result.extend_from_slice(&payload_bytes); |
203 | | |
204 | 14 | Ok(result) |
205 | 14 | } |
206 | | |
207 | | /// Load APR Transformer from APR v2 bytes |
208 | | /// |
209 | | /// # Arguments |
210 | | /// |
211 | | /// * `data` - Raw APR v2 file bytes |
212 | | /// |
213 | | /// # Returns |
214 | | /// |
215 | | /// Loaded `AprTransformer` |
216 | | /// |
217 | | /// # Errors |
218 | | /// |
219 | | /// Returns error if parsing fails |
220 | 19 | pub fn from_apr_bytes(data: &[u8]) -> Result<AprTransformer> { |
221 | | // Parse header |
222 | 19 | let header15 = AprHeader::from_bytes(data)?4 ; |
223 | | |
224 | | // Get tensor index to find the weights tensor |
225 | 15 | let index_start = header.tensor_index_offset as usize; |
226 | 15 | let index_end = header.data_offset as usize; |
227 | | |
228 | 15 | if data.len() < index_end { |
229 | 2 | return Err(RealizarError::FormatError { |
230 | 2 | reason: format!( |
231 | 2 | "APR file truncated: expected {} bytes for tensor index, got {}", |
232 | 2 | index_end, |
233 | 2 | data.len() |
234 | 2 | ), |
235 | 2 | }); |
236 | 13 | } |
237 | | |
238 | 4 | let tensor_entries: Vec<TensorEntry> = |
239 | 13 | serde_json::from_slice(&data[index_start..index_end]).map_err(|e| {9 |
240 | 9 | RealizarError::FormatError { |
241 | 9 | reason: format!("Failed to parse tensor index: {e}"), |
242 | 9 | } |
243 | 9 | })?; |
244 | | |
245 | | // Find the weights tensor |
246 | 4 | let weights_entry3 = tensor_entries |
247 | 4 | .iter() |
248 | 4 | .find(|e| e.name3 == "weights"3 ) |
249 | 4 | .ok_or_else(|| RealizarError::FormatError { |
250 | 1 | reason: "No 'weights' tensor found in APR file".to_string(), |
251 | 1 | })?; |
252 | | |
253 | | // Extract weights data |
254 | 3 | let data_start = header.data_offset as usize + weights_entry.offset as usize; |
255 | 3 | let data_end = data_start + weights_entry.size as usize; |
256 | | |
257 | 3 | if data.len() < data_end { |
258 | 0 | return Err(RealizarError::FormatError { |
259 | 0 | reason: format!( |
260 | 0 | "APR file truncated: expected {} bytes for tensor data, got {}", |
261 | 0 | data_end, |
262 | 0 | data.len() |
263 | 0 | ), |
264 | 0 | }); |
265 | 3 | } |
266 | | |
267 | 3 | let payload_bytes = &data[data_start..data_end]; |
268 | | |
269 | | // Deserialize transformer |
270 | 3 | let transformer: AprTransformer = |
271 | 3 | serde_json::from_slice(payload_bytes).map_err(|e| RealizarError::FormatError { |
272 | 0 | reason: format!("Failed to deserialize transformer: {e}"), |
273 | 0 | })?; |
274 | | |
275 | 3 | Ok(transformer) |
276 | 19 | } |
277 | | |
278 | | /// Get conversion statistics |
279 | | /// |
280 | | /// # Arguments |
281 | | /// |
282 | | /// * `transformer` - APR Transformer to analyze |
283 | | /// |
284 | | /// # Returns |
285 | | /// |
286 | | /// Statistics about the conversion |
287 | 6 | pub fn stats(transformer: &AprTransformer) -> ConversionStats { |
288 | 6 | let params = transformer.num_parameters(); |
289 | 6 | let memory_bytes = transformer.memory_size(); |
290 | | |
291 | 6 | ConversionStats { |
292 | 6 | total_parameters: params, |
293 | 6 | memory_bytes_f32: memory_bytes, |
294 | 6 | num_layers: transformer.config.num_layers, |
295 | 6 | hidden_dim: transformer.config.hidden_dim, |
296 | 6 | vocab_size: transformer.config.vocab_size, |
297 | 6 | architecture: transformer.config.architecture.clone(), |
298 | 6 | } |
299 | 6 | } |
300 | | } |
301 | | |
302 | | /// Statistics about a converted model |
303 | | #[derive(Debug, Clone)] |
304 | | pub struct ConversionStats { |
305 | | /// Total number of parameters |
306 | | pub total_parameters: usize, |
307 | | /// Memory size in bytes (F32) |
308 | | pub memory_bytes_f32: usize, |
309 | | /// Number of transformer layers |
310 | | pub num_layers: usize, |
311 | | /// Hidden dimension |
312 | | pub hidden_dim: usize, |
313 | | /// Vocabulary size |
314 | | pub vocab_size: usize, |
315 | | /// Model architecture name |
316 | | pub architecture: String, |
317 | | } |
318 | | |
319 | | impl ConversionStats { |
320 | | /// Memory size in MB |
321 | | #[must_use] |
322 | 7 | pub fn memory_mb(&self) -> f64 { |
323 | 7 | self.memory_bytes_f32 as f64 / (1024.0 * 1024.0) |
324 | 7 | } |
325 | | |
326 | | /// Memory size in GB |
327 | | #[must_use] |
328 | 7 | pub fn memory_gb(&self) -> f64 { |
329 | 7 | self.memory_bytes_f32 as f64 / (1024.0 * 1024.0 * 1024.0) |
330 | 7 | } |
331 | | |
332 | | /// Parameters in millions |
333 | | #[must_use] |
334 | 6 | pub fn parameters_m(&self) -> f64 { |
335 | 6 | self.total_parameters as f64 / 1_000_000.0 |
336 | 6 | } |
337 | | |
338 | | /// Parameters in billions |
339 | | #[must_use] |
340 | 8 | pub fn parameters_b(&self) -> f64 { |
341 | 8 | self.total_parameters as f64 / 1_000_000_000.0 |
342 | 8 | } |
343 | | } |
344 | | |
345 | | // ============================================================================= |
346 | | // Q4K APR Converter (preserves GGUF quantization for GPU inference) |
347 | | // ============================================================================= |
348 | | |
349 | | /// Raw tensor with preserved quantization |
350 | | #[derive(Debug, Clone)] |
351 | | pub struct RawTensor { |
352 | | /// Tensor name |
353 | | pub name: String, |
354 | | /// Raw bytes (Q4K super-blocks or F32/F16 data) |
355 | | pub data: Vec<u8>, |
356 | | /// Tensor shape (logical elements, not bytes) |
357 | | pub shape: Vec<usize>, |
358 | | /// GGML dtype: 0=F32, 1=F16, 6=Q4_K, 7=Q5_K, 8=Q8_0, 12=Q6_K |
359 | | pub dtype: u32, |
360 | | } |
361 | | |
362 | | /// GGUF to APR Q4K converter (preserves quantization) |
363 | | /// |
364 | | /// Unlike `GgufToAprConverter` which dequantizes to F32, this converter |
365 | | /// preserves Q4K/Q6K quantization for GPU inference with batched GEMV. |
366 | | /// |
367 | | /// This is essential for achieving 2X Ollama performance. |
368 | | pub struct GgufToAprQ4KConverter; |
369 | | |
370 | | impl GgufToAprQ4KConverter { |
371 | | /// Helper to extract string from GGUF metadata |
372 | 9 | fn get_string( |
373 | 9 | metadata: &std::collections::HashMap<String, crate::gguf::GGUFValue>, |
374 | 9 | key: &str, |
375 | 9 | ) -> Option<String> { |
376 | 9 | match metadata.get(key) { |
377 | 3 | Some(crate::gguf::GGUFValue::String(s)) => Some(s.clone()), |
378 | 6 | _ => None, |
379 | | } |
380 | 9 | } |
381 | | |
382 | | /// Helper to extract u32 from GGUF metadata |
383 | 17 | fn get_u32( |
384 | 17 | metadata: &std::collections::HashMap<String, crate::gguf::GGUFValue>, |
385 | 17 | key: &str, |
386 | 17 | ) -> Option<u32> { |
387 | 17 | match metadata.get(key) { |
388 | 3 | Some(crate::gguf::GGUFValue::UInt32(v)) => Some(*v), |
389 | 4 | Some(crate::gguf::GGUFValue::Int32(v)) => Some(*v as u32), |
390 | 4 | Some(crate::gguf::GGUFValue::UInt64(v)) => Some(*v as u32), |
391 | 6 | _ => None, |
392 | | } |
393 | 17 | } |
394 | | |
395 | | /// Helper to extract f32 from GGUF metadata |
396 | 13 | fn get_f32( |
397 | 13 | metadata: &std::collections::HashMap<String, crate::gguf::GGUFValue>, |
398 | 13 | key: &str, |
399 | 13 | ) -> Option<f32> { |
400 | 13 | match metadata.get(key) { |
401 | 3 | Some(crate::gguf::GGUFValue::Float32(v)) => Some(*v), |
402 | 4 | Some(crate::gguf::GGUFValue::Float64(v)) => Some(*v as f32), |
403 | 6 | _ => None, |
404 | | } |
405 | 13 | } |
406 | | |
407 | | /// Convert GGUF file to APR v2 with preserved Q4K quantization |
408 | | /// |
409 | | /// # Arguments |
410 | | /// |
411 | | /// * `gguf_path` - Path to GGUF file |
412 | | /// * `output_path` - Path to write APR v2 file |
413 | | /// |
414 | | /// # Returns |
415 | | /// |
416 | | /// Statistics about the conversion |
417 | | #[allow(clippy::cast_possible_truncation)] |
418 | 0 | pub fn convert( |
419 | 0 | gguf_path: &std::path::Path, |
420 | 0 | output_path: &std::path::Path, |
421 | 0 | ) -> Result<Q4KConversionStats> { |
422 | | use std::io::Write; |
423 | | |
424 | | // Load GGUF with raw quantized tensors |
425 | 0 | let gguf_data = std::fs::read(gguf_path).map_err(|e| RealizarError::IoError { |
426 | 0 | message: format!("Failed to read GGUF: {e}"), |
427 | 0 | })?; |
428 | | |
429 | 0 | let gguf_model = crate::gguf::GGUFModel::from_bytes(&gguf_data)?; |
430 | | |
431 | | // Extract model config from metadata |
432 | 0 | let architecture = Self::get_string(&gguf_model.metadata, "general.architecture") |
433 | 0 | .unwrap_or_else(|| "unknown".to_string()); |
434 | 0 | let hidden_size = Self::get_u32( |
435 | 0 | &gguf_model.metadata, |
436 | 0 | &format!("{architecture}.embedding_length"), |
437 | | ) |
438 | 0 | .unwrap_or(0); |
439 | 0 | let num_layers = |
440 | 0 | Self::get_u32(&gguf_model.metadata, &format!("{architecture}.block_count")) |
441 | 0 | .unwrap_or(0); |
442 | 0 | let num_heads = Self::get_u32( |
443 | 0 | &gguf_model.metadata, |
444 | 0 | &format!("{architecture}.attention.head_count"), |
445 | | ) |
446 | 0 | .unwrap_or(0); |
447 | 0 | let num_kv_heads = Self::get_u32( |
448 | 0 | &gguf_model.metadata, |
449 | 0 | &format!("{architecture}.attention.head_count_kv"), |
450 | | ) |
451 | 0 | .unwrap_or(num_heads); |
452 | 0 | let vocab_size = Self::get_u32(&gguf_model.metadata, &format!("{architecture}.vocab_size")) |
453 | 0 | .or_else(|| Self::get_u32(&gguf_model.metadata, "tokenizer.ggml.vocab_size")) |
454 | 0 | .unwrap_or_else(|| { |
455 | | // Infer from embedding tensor shape if metadata not available |
456 | 0 | gguf_model |
457 | 0 | .tensors |
458 | 0 | .iter() |
459 | 0 | .find(|t| { |
460 | 0 | t.name.contains("token_embd") |
461 | 0 | || t.name.contains("embed_tokens") |
462 | 0 | || t.name.contains("tok_embeddings") |
463 | 0 | }) |
464 | 0 | .and_then(|t| t.dims.first().copied().map(|d| d as u32)) |
465 | 0 | .unwrap_or(0) |
466 | 0 | }) as usize; |
467 | 0 | let intermediate_size = Self::get_u32( |
468 | 0 | &gguf_model.metadata, |
469 | 0 | &format!("{architecture}.feed_forward_length"), |
470 | | ) |
471 | 0 | .unwrap_or(0); |
472 | 0 | let context_length = Self::get_u32( |
473 | 0 | &gguf_model.metadata, |
474 | 0 | &format!("{architecture}.context_length"), |
475 | | ) |
476 | 0 | .unwrap_or(2048); |
477 | 0 | let rope_theta = Self::get_f32( |
478 | 0 | &gguf_model.metadata, |
479 | 0 | &format!("{architecture}.rope.freq_base"), |
480 | | ) |
481 | 0 | .unwrap_or(10000.0); |
482 | 0 | let eps = Self::get_f32( |
483 | 0 | &gguf_model.metadata, |
484 | 0 | &format!("{architecture}.attention.layer_norm_rms_epsilon"), |
485 | | ) |
486 | 0 | .unwrap_or(1e-5); |
487 | | |
488 | | // Build metadata JSON |
489 | 0 | let metadata = serde_json::json!({ |
490 | 0 | "model_type": "transformer_lm_q4k", |
491 | 0 | "architecture": architecture, |
492 | 0 | "hidden_size": hidden_size, |
493 | 0 | "num_layers": num_layers, |
494 | 0 | "num_heads": num_heads, |
495 | 0 | "num_kv_heads": num_kv_heads, |
496 | 0 | "vocab_size": vocab_size, |
497 | 0 | "intermediate_dim": intermediate_size, |
498 | 0 | "context_length": context_length, |
499 | 0 | "rope_theta": rope_theta, |
500 | 0 | "eps": eps, |
501 | 0 | "quantization": "Q4_K_M", |
502 | | }); |
503 | 0 | let metadata_bytes = |
504 | 0 | serde_json::to_vec(&metadata).map_err(|e| RealizarError::FormatError { |
505 | 0 | reason: format!("Failed to serialize metadata: {e}"), |
506 | 0 | })?; |
507 | 0 | let metadata_padded_len = metadata_bytes.len().div_ceil(ALIGNMENT) * ALIGNMENT; |
508 | | |
509 | | // Extract raw tensors from GGUF |
510 | 0 | let mut raw_tensors: Vec<RawTensor> = Vec::new(); |
511 | 0 | let mut q4k_count = 0usize; |
512 | 0 | let mut total_bytes = 0usize; |
513 | | |
514 | 0 | for tensor_meta in &gguf_model.tensors { |
515 | 0 | let name = tensor_meta.name.clone(); |
516 | 0 | let shape: Vec<usize> = tensor_meta.dims.iter().map(|&d| d as usize).collect(); |
517 | 0 | let num_elements: usize = shape.iter().product(); |
518 | 0 | let qtype = tensor_meta.qtype; |
519 | | |
520 | | // Calculate byte size based on qtype (GGML dtype) |
521 | | // GGML types: 0=F32, 1=F16, 8=Q8_0, 12=Q4_K, 13=Q5_K, 14=Q6_K |
522 | 0 | let byte_size = match qtype { |
523 | 0 | 0 => num_elements * 4, // F32 |
524 | 0 | 1 => num_elements * 2, // F16 |
525 | 0 | 8 => (num_elements / 32) * 34, // Q8_0: 32 elements = 2 (scale) + 32 (quants) |
526 | 0 | 12 => (num_elements / 256) * 144, // Q4_K: 256 elements = 144 bytes |
527 | 0 | 13 => (num_elements / 256) * 176, // Q5_K: 256 elements = 176 bytes |
528 | 0 | 14 => (num_elements / 256) * 210, // Q6_K: 256 elements = 210 bytes |
529 | 0 | _ => num_elements * 4, // Default to F32 |
530 | | }; |
531 | | |
532 | | // Extract raw bytes |
533 | 0 | let tensor_start = gguf_model.tensor_data_start + tensor_meta.offset as usize; |
534 | 0 | if tensor_start + byte_size > gguf_data.len() { |
535 | 0 | return Err(RealizarError::FormatError { |
536 | 0 | reason: format!( |
537 | 0 | "Tensor '{}' exceeds file bounds (start={}, size={}, file_len={})", |
538 | 0 | name, |
539 | 0 | tensor_start, |
540 | 0 | byte_size, |
541 | 0 | gguf_data.len() |
542 | 0 | ), |
543 | 0 | }); |
544 | 0 | } |
545 | | |
546 | 0 | let data = gguf_data[tensor_start..tensor_start + byte_size].to_vec(); |
547 | | |
548 | | // Q4_K is GGML type 12 |
549 | 0 | if qtype == 12 { |
550 | 0 | q4k_count += 1; |
551 | 0 | } |
552 | 0 | total_bytes += byte_size; |
553 | | |
554 | 0 | raw_tensors.push(RawTensor { |
555 | 0 | name, |
556 | 0 | data, |
557 | 0 | shape, |
558 | 0 | dtype: qtype, |
559 | 0 | }); |
560 | | } |
561 | | |
562 | | // Build binary tensor index |
563 | 0 | let mut tensor_index_bytes: Vec<u8> = Vec::new(); |
564 | 0 | let mut current_offset = 0u64; |
565 | | |
566 | 0 | for tensor in &raw_tensors { |
567 | | // name_len (2 bytes) + name |
568 | 0 | let name_bytes = tensor.name.as_bytes(); |
569 | 0 | tensor_index_bytes.extend_from_slice(&(name_bytes.len() as u16).to_le_bytes()); |
570 | 0 | tensor_index_bytes.extend_from_slice(name_bytes); |
571 | | |
572 | | // dtype (1 byte) - map GGML dtype to APR dtype |
573 | | // GGML: 0=F32, 1=F16, 8=Q8_0, 12=Q4_K, 13=Q5_K, 14=Q6_K |
574 | | // APR: 0=F32, 1=F16, 8=Q4_K, 9=Q6_K, 10=Q8_0 |
575 | 0 | let apr_dtype = match tensor.dtype { |
576 | 0 | 0 => 0u8, // F32 -> F32 |
577 | 0 | 1 => 1u8, // F16 -> F16 |
578 | 0 | 8 => 10u8, // Q8_0 -> APR dtype 10 |
579 | 0 | 12 => 8u8, // Q4_K -> APR dtype 8 |
580 | 0 | 13 => 8u8, // Q5_K -> treat as Q4_K for now |
581 | 0 | 14 => 9u8, // Q6_K -> APR dtype 9 |
582 | 0 | _ => 0u8, |
583 | | }; |
584 | 0 | tensor_index_bytes.push(apr_dtype); |
585 | | |
586 | | // ndim (1 byte) + dims (8 bytes each) |
587 | 0 | tensor_index_bytes.push(tensor.shape.len() as u8); |
588 | 0 | for &dim in &tensor.shape { |
589 | 0 | tensor_index_bytes.extend_from_slice(&(dim as u64).to_le_bytes()); |
590 | 0 | } |
591 | | |
592 | | // offset (8 bytes) |
593 | 0 | tensor_index_bytes.extend_from_slice(¤t_offset.to_le_bytes()); |
594 | | |
595 | | // size (8 bytes) |
596 | 0 | let size = tensor.data.len() as u64; |
597 | 0 | tensor_index_bytes.extend_from_slice(&size.to_le_bytes()); |
598 | | |
599 | | // Align next tensor to 64 bytes |
600 | 0 | current_offset += size; |
601 | 0 | let aligned = current_offset.div_ceil(ALIGNMENT as u64) * ALIGNMENT as u64; |
602 | 0 | current_offset = aligned; |
603 | | } |
604 | | |
605 | | // Calculate offsets |
606 | 0 | let metadata_offset = HEADER_SIZE as u64; |
607 | 0 | let tensor_index_offset = metadata_offset + metadata_padded_len as u64; |
608 | 0 | let data_offset = tensor_index_offset + tensor_index_bytes.len() as u64; |
609 | | // Align data offset |
610 | 0 | let data_offset_aligned = data_offset.div_ceil(ALIGNMENT as u64) * ALIGNMENT as u64; |
611 | | |
612 | | // Build header (64 bytes) |
613 | 0 | let mut header = vec![0u8; HEADER_SIZE]; |
614 | 0 | header[0..4].copy_from_slice(&MAGIC); |
615 | 0 | header[4] = 2; // version major |
616 | 0 | header[5] = 0; // version minor |
617 | 0 | header[6..8].copy_from_slice(&0x0020u16.to_le_bytes()); // flags: QUANTIZED=0x0020 |
618 | 0 | header[8..12].copy_from_slice(&(raw_tensors.len() as u32).to_le_bytes()); |
619 | 0 | header[12..20].copy_from_slice(&metadata_offset.to_le_bytes()); |
620 | 0 | header[20..24].copy_from_slice(&(metadata_bytes.len() as u32).to_le_bytes()); |
621 | 0 | header[24..32].copy_from_slice(&tensor_index_offset.to_le_bytes()); |
622 | 0 | header[32..40].copy_from_slice(&data_offset_aligned.to_le_bytes()); |
623 | | |
624 | | // Write file |
625 | 0 | let mut file = std::fs::File::create(output_path).map_err(|e| RealizarError::IoError { |
626 | 0 | message: format!("Failed to create output file: {e}"), |
627 | 0 | })?; |
628 | | |
629 | | // Header |
630 | 0 | file.write_all(&header) |
631 | 0 | .map_err(|e| RealizarError::IoError { |
632 | 0 | message: format!("Failed to write header: {e}"), |
633 | 0 | })?; |
634 | | |
635 | | // Metadata (padded) |
636 | 0 | file.write_all(&metadata_bytes) |
637 | 0 | .map_err(|e| RealizarError::IoError { |
638 | 0 | message: format!("Failed to write metadata: {e}"), |
639 | 0 | })?; |
640 | 0 | let padding = metadata_padded_len - metadata_bytes.len(); |
641 | 0 | if padding > 0 { |
642 | 0 | file.write_all(&vec![0u8; padding]) |
643 | 0 | .map_err(|e| RealizarError::IoError { |
644 | 0 | message: format!("Failed to write padding: {e}"), |
645 | 0 | })?; |
646 | 0 | } |
647 | | |
648 | | // Tensor index |
649 | 0 | file.write_all(&tensor_index_bytes) |
650 | 0 | .map_err(|e| RealizarError::IoError { |
651 | 0 | message: format!("Failed to write tensor index: {e}"), |
652 | 0 | })?; |
653 | | |
654 | | // Alignment padding before data |
655 | 0 | let pre_data_padding = (data_offset_aligned - data_offset) as usize; |
656 | 0 | if pre_data_padding > 0 { |
657 | 0 | file.write_all(&vec![0u8; pre_data_padding]) |
658 | 0 | .map_err(|e| RealizarError::IoError { |
659 | 0 | message: format!("Failed to write data alignment: {e}"), |
660 | 0 | })?; |
661 | 0 | } |
662 | | |
663 | | // Tensor data (with alignment) |
664 | 0 | for tensor in &raw_tensors { |
665 | 0 | file.write_all(&tensor.data) |
666 | 0 | .map_err(|e| RealizarError::IoError { |
667 | 0 | message: format!("Failed to write tensor '{}': {e}", tensor.name), |
668 | 0 | })?; |
669 | | |
670 | | // Align to 64 bytes |
671 | 0 | let pad = (ALIGNMENT - (tensor.data.len() % ALIGNMENT)) % ALIGNMENT; |
672 | 0 | if pad > 0 { |
673 | 0 | file.write_all(&vec![0u8; pad]) |
674 | 0 | .map_err(|e| RealizarError::IoError { |
675 | 0 | message: format!("Failed to write tensor padding: {e}"), |
676 | 0 | })?; |
677 | 0 | } |
678 | | } |
679 | | |
680 | 0 | Ok(Q4KConversionStats { |
681 | 0 | tensor_count: raw_tensors.len(), |
682 | 0 | q4k_tensor_count: q4k_count, |
683 | 0 | total_bytes, |
684 | 0 | architecture: architecture.clone(), |
685 | 0 | num_layers: num_layers as usize, |
686 | 0 | hidden_size: hidden_size as usize, |
687 | 0 | }) |
688 | 0 | } |
689 | | } |
690 | | |
691 | | /// Statistics from Q4K conversion |
692 | | #[derive(Debug, Clone)] |
693 | | pub struct Q4KConversionStats { |
694 | | /// Total number of tensors |
695 | | pub tensor_count: usize, |
696 | | /// Number of Q4K quantized tensors |
697 | | pub q4k_tensor_count: usize, |
698 | | /// Total bytes written |
699 | | pub total_bytes: usize, |
700 | | /// Model architecture |
701 | | pub architecture: String, |
702 | | /// Number of layers |
703 | | pub num_layers: usize, |
704 | | /// Hidden size |
705 | | pub hidden_size: usize, |
706 | | } |
707 | | |
708 | | // Tests extracted to tests.rs (PMAT-802) |
709 | | #[cfg(test)] |
710 | | #[path = "tests.rs"] |
711 | | mod convert_tests; |