/home/noah/src/realizar/src/gguf/format_factory.rs
Line | Count | Source |
1 | | //! Rosetta Format Factory - Synthetic Model Files for All Formats |
2 | | //! |
3 | | //! Following aprender's Rosetta Stone spec, this module provides builders |
4 | | //! for synthesizing valid model files in ALL supported formats: |
5 | | //! |
6 | | //! | Format | Builder | Magic | |
7 | | //! |--------|---------|-------| |
8 | | //! | GGUF | `GGUFBuilder` | "GGUF" | |
9 | | //! | SafeTensors | `SafetensorsBuilder` | JSON header | |
10 | | //! | APR | `AprBuilder` | "APR\0" | |
11 | | //! |
12 | | //! # Conversion Matrix (6 Direct Paths) |
13 | | //! |
14 | | //! ```text |
15 | | //! GGUF ←──────→ APR ←──────→ SafeTensors |
16 | | //! ↑ ↑ |
17 | | //! └──────────────────────────────┘ |
18 | | //! ``` |
19 | | //! |
20 | | //! # Example |
21 | | //! |
22 | | //! ```ignore |
23 | | //! use realizar::gguf::format_factory::{GGUFBuilder, SafetensorsBuilder, AprBuilder}; |
24 | | //! |
25 | | //! // Create a minimal model in each format |
26 | | //! let gguf_data = GGUFBuilder::minimal_llama(100, 64); |
27 | | //! let st_data = SafetensorsBuilder::minimal_model(100, 64); |
28 | | //! let apr_data = AprBuilder::minimal_model(100, 64); |
29 | | //! ``` |
30 | | |
31 | | use serde::Serialize; |
32 | | use std::collections::BTreeMap; |
33 | | |
34 | | // Re-export GGUFBuilder from test_factory |
35 | | pub use super::test_factory::{ |
36 | | build_minimal_llama_gguf, build_minimal_phi2_gguf, create_f32_embedding_data, |
37 | | create_f32_norm_weights, create_q4_0_data, create_q4_k_data, create_q5_k_data, |
38 | | create_q6_k_data, create_q8_0_data, GGUFBuilder, |
39 | | }; |
40 | | |
41 | | // ============================================================================= |
42 | | // SafeTensors Builder |
43 | | // ============================================================================= |
44 | | |
45 | | /// SafeTensors tensor metadata |
46 | | #[derive(Debug, Clone, Serialize)] |
47 | | struct SafetensorsTensorMeta { |
48 | | dtype: String, |
49 | | shape: Vec<usize>, |
50 | | data_offsets: [usize; 2], |
51 | | } |
52 | | |
53 | | /// Builder for creating valid SafeTensors files in memory |
54 | | /// |
55 | | /// SafeTensors format: |
56 | | /// - 8 bytes: JSON header length (little-endian u64) |
57 | | /// - N bytes: JSON header with tensor metadata |
58 | | /// - Tensor data (contiguous, aligned) |
59 | | pub struct SafetensorsBuilder { |
60 | | tensors: Vec<(String, String, Vec<usize>, Vec<u8>)>, // name, dtype, shape, data |
61 | | } |
62 | | |
63 | | impl Default for SafetensorsBuilder { |
64 | 0 | fn default() -> Self { |
65 | 0 | Self::new() |
66 | 0 | } |
67 | | } |
68 | | |
69 | | impl SafetensorsBuilder { |
70 | | /// Create a new SafeTensors builder |
71 | | #[must_use] |
72 | 6 | pub fn new() -> Self { |
73 | 6 | Self { |
74 | 6 | tensors: Vec::new(), |
75 | 6 | } |
76 | 6 | } |
77 | | |
78 | | /// Add an F32 tensor |
79 | | #[must_use] |
80 | 8 | pub fn add_f32_tensor(mut self, name: &str, shape: &[usize], data: &[f32]) -> Self { |
81 | 19.5k | let bytes8 : Vec<u8>8 = data8 .iter8 ().flat_map8 (|f| f.to_le_bytes()).collect8 (); |
82 | 8 | self.tensors |
83 | 8 | .push((name.to_string(), "F32".to_string(), shape.to_vec(), bytes)); |
84 | 8 | self |
85 | 8 | } |
86 | | |
87 | | /// Add an F16 tensor |
88 | | #[must_use] |
89 | 0 | pub fn add_f16_tensor(mut self, name: &str, shape: &[usize], data: &[u8]) -> Self { |
90 | 0 | self.tensors |
91 | 0 | .push((name.to_string(), "F16".to_string(), shape.to_vec(), data.to_vec())); |
92 | 0 | self |
93 | 0 | } |
94 | | |
95 | | /// Add a BF16 tensor |
96 | | #[must_use] |
97 | 0 | pub fn add_bf16_tensor(mut self, name: &str, shape: &[usize], data: &[u8]) -> Self { |
98 | 0 | self.tensors |
99 | 0 | .push((name.to_string(), "BF16".to_string(), shape.to_vec(), data.to_vec())); |
100 | 0 | self |
101 | 0 | } |
102 | | |
103 | | /// Build the SafeTensors file as a byte vector |
104 | | #[must_use] |
105 | 6 | pub fn build(self) -> Vec<u8> { |
106 | | // Calculate offsets and build metadata |
107 | 6 | let mut metadata: BTreeMap<String, SafetensorsTensorMeta> = BTreeMap::new(); |
108 | 6 | let mut current_offset = 0usize; |
109 | | |
110 | 14 | for (name8 , dtype8 , shape8 , data8 ) in &self.tensors { |
111 | 8 | let end_offset = current_offset + data.len(); |
112 | 8 | metadata.insert( |
113 | 8 | name.clone(), |
114 | 8 | SafetensorsTensorMeta { |
115 | 8 | dtype: dtype.clone(), |
116 | 8 | shape: shape.clone(), |
117 | 8 | data_offsets: [current_offset, end_offset], |
118 | 8 | }, |
119 | 8 | ); |
120 | 8 | current_offset = end_offset; |
121 | 8 | } |
122 | | |
123 | | // Serialize metadata to JSON |
124 | 6 | let json = serde_json::to_string(&metadata).expect("JSON serialization"); |
125 | 6 | let json_bytes = json.as_bytes(); |
126 | | |
127 | | // Build final file |
128 | 6 | let mut data = Vec::new(); |
129 | | |
130 | | // Header: JSON length as u64 |
131 | 6 | data.extend_from_slice(&(json_bytes.len() as u64).to_le_bytes()); |
132 | | |
133 | | // JSON metadata |
134 | 6 | data.extend_from_slice(json_bytes); |
135 | | |
136 | | // Tensor data |
137 | 14 | for (_, _, _, tensor_data8 ) in &self.tensors { |
138 | 8 | data.extend_from_slice(tensor_data); |
139 | 8 | } |
140 | | |
141 | 6 | data |
142 | 6 | } |
143 | | |
144 | | /// Build a minimal SafeTensors model for testing |
145 | | #[must_use] |
146 | 3 | pub fn minimal_model(vocab_size: usize, hidden_dim: usize) -> Vec<u8> { |
147 | 3 | let embed_data = create_f32_embedding_data(vocab_size, hidden_dim); |
148 | 3 | let norm_data = create_f32_norm_weights(hidden_dim); |
149 | | |
150 | 3 | Self::new() |
151 | 3 | .add_f32_tensor("model.embed_tokens.weight", &[vocab_size, hidden_dim], &embed_data) |
152 | 3 | .add_f32_tensor("model.norm.weight", &[hidden_dim], &norm_data) |
153 | 3 | .build() |
154 | 3 | } |
155 | | } |
156 | | |
157 | | // ============================================================================= |
158 | | // APR Builder |
159 | | // ============================================================================= |
160 | | |
161 | | /// APR v2 format constants |
162 | | const APR_MAGIC: &[u8; 4] = b"APR\0"; |
163 | | const APR_VERSION_MAJOR: u8 = 2; |
164 | | const APR_VERSION_MINOR: u8 = 0; |
165 | | const APR_HEADER_SIZE: usize = 64; |
166 | | const APR_ALIGNMENT: usize = 64; |
167 | | |
168 | | /// Builder for creating valid APR v2 files in memory |
169 | | /// |
170 | | /// APR v2 format (64-byte header): |
171 | | /// - 4 bytes: Magic "APR\0" |
172 | | /// - 2 bytes: Version (major.minor) |
173 | | /// - 2 bytes: Flags |
174 | | /// - 4 bytes: Tensor count |
175 | | /// - 8 bytes: Metadata offset |
176 | | /// - 4 bytes: Metadata size |
177 | | /// - 8 bytes: Tensor index offset |
178 | | /// - 8 bytes: Data offset |
179 | | /// - 4 bytes: Checksum |
180 | | /// - 20 bytes: Reserved |
181 | | pub struct AprBuilder { |
182 | | metadata: BTreeMap<String, serde_json::Value>, |
183 | | tensors: Vec<(String, Vec<usize>, u32, Vec<u8>)>, // name, shape, dtype, data |
184 | | } |
185 | | |
186 | | impl Default for AprBuilder { |
187 | 0 | fn default() -> Self { |
188 | 0 | Self::new() |
189 | 0 | } |
190 | | } |
191 | | |
192 | | /// APR dtype codes |
193 | | pub const APR_DTYPE_F32: u32 = 0; |
194 | | pub const APR_DTYPE_F16: u32 = 1; |
195 | | pub const APR_DTYPE_Q4_0: u32 = 2; |
196 | | pub const APR_DTYPE_Q8_0: u32 = 8; |
197 | | |
198 | | impl AprBuilder { |
199 | | /// Create a new APR builder |
200 | | #[must_use] |
201 | 7 | pub fn new() -> Self { |
202 | 7 | Self { |
203 | 7 | metadata: BTreeMap::new(), |
204 | 7 | tensors: Vec::new(), |
205 | 7 | } |
206 | 7 | } |
207 | | |
208 | | /// Set architecture metadata |
209 | | #[must_use] |
210 | 4 | pub fn architecture(mut self, arch: &str) -> Self { |
211 | 4 | self.metadata |
212 | 4 | .insert("architecture".to_string(), serde_json::json!(arch)); |
213 | 4 | self |
214 | 4 | } |
215 | | |
216 | | /// Set hidden dimension metadata |
217 | | #[must_use] |
218 | 4 | pub fn hidden_dim(mut self, dim: usize) -> Self { |
219 | 4 | self.metadata |
220 | 4 | .insert("hidden_dim".to_string(), serde_json::json!(dim)); |
221 | 4 | self |
222 | 4 | } |
223 | | |
224 | | /// Set number of layers metadata |
225 | | #[must_use] |
226 | 4 | pub fn num_layers(mut self, count: usize) -> Self { |
227 | 4 | self.metadata |
228 | 4 | .insert("num_layers".to_string(), serde_json::json!(count)); |
229 | 4 | self |
230 | 4 | } |
231 | | |
232 | | /// Add an F32 tensor |
233 | | #[must_use] |
234 | 8 | pub fn add_f32_tensor(mut self, name: &str, shape: &[usize], data: &[f32]) -> Self { |
235 | 19.5k | let bytes8 : Vec<u8>8 = data8 .iter8 ().flat_map8 (|f| f.to_le_bytes()).collect8 (); |
236 | 8 | self.tensors |
237 | 8 | .push((name.to_string(), shape.to_vec(), APR_DTYPE_F32, bytes)); |
238 | 8 | self |
239 | 8 | } |
240 | | |
241 | | /// Add a Q4_0 tensor |
242 | | #[must_use] |
243 | 0 | pub fn add_q4_0_tensor(mut self, name: &str, shape: &[usize], data: &[u8]) -> Self { |
244 | 0 | self.tensors |
245 | 0 | .push((name.to_string(), shape.to_vec(), APR_DTYPE_Q4_0, data.to_vec())); |
246 | 0 | self |
247 | 0 | } |
248 | | |
249 | | /// Add a Q8_0 tensor |
250 | | #[must_use] |
251 | 0 | pub fn add_q8_0_tensor(mut self, name: &str, shape: &[usize], data: &[u8]) -> Self { |
252 | 0 | self.tensors |
253 | 0 | .push((name.to_string(), shape.to_vec(), APR_DTYPE_Q8_0, data.to_vec())); |
254 | 0 | self |
255 | 0 | } |
256 | | |
257 | | /// Build the APR v2 file as a byte vector |
258 | | #[must_use] |
259 | 7 | pub fn build(self) -> Vec<u8> { |
260 | 7 | let mut data = Vec::new(); |
261 | | |
262 | | // Serialize metadata to JSON |
263 | 7 | let json = serde_json::to_string(&self.metadata).expect("JSON serialization"); |
264 | 7 | let json_bytes = json.as_bytes(); |
265 | | |
266 | | // Pad JSON to 64-byte boundary |
267 | 7 | let json_padded_len = json_bytes.len().div_ceil(APR_ALIGNMENT) * APR_ALIGNMENT; |
268 | | |
269 | | // Build tensor index |
270 | 7 | let mut tensor_index = Vec::new(); |
271 | 7 | let mut tensor_data_offset = 0u64; |
272 | | |
273 | 15 | for (name8 , shape8 , dtype8 , tensor_bytes8 ) in &self.tensors { |
274 | | // Tensor index entry: name_len(4) + name + ndims(4) + dims + dtype(4) + offset(8) + size(8) |
275 | 8 | let name_bytes = name.as_bytes(); |
276 | 8 | tensor_index.extend_from_slice(&(name_bytes.len() as u32).to_le_bytes()); |
277 | 8 | tensor_index.extend_from_slice(name_bytes); |
278 | 8 | tensor_index.extend_from_slice(&(shape.len() as u32).to_le_bytes()); |
279 | 21 | for dim13 in shape { |
280 | 13 | tensor_index.extend_from_slice(&(*dim as u64).to_le_bytes()); |
281 | 13 | } |
282 | 8 | tensor_index.extend_from_slice(&dtype.to_le_bytes()); |
283 | 8 | tensor_index.extend_from_slice(&tensor_data_offset.to_le_bytes()); |
284 | 8 | tensor_index.extend_from_slice(&(tensor_bytes.len() as u64).to_le_bytes()); |
285 | | |
286 | | // Align tensor data to 64 bytes |
287 | 8 | let aligned_size = tensor_bytes.len().div_ceil(APR_ALIGNMENT) * APR_ALIGNMENT; |
288 | 8 | tensor_data_offset += aligned_size as u64; |
289 | | } |
290 | | |
291 | | // Pad tensor index to 64-byte boundary |
292 | 7 | let index_padded_len = tensor_index.len().div_ceil(APR_ALIGNMENT) * APR_ALIGNMENT; |
293 | | |
294 | | // Calculate offsets |
295 | 7 | let metadata_offset = APR_HEADER_SIZE as u64; |
296 | 7 | let tensor_index_offset = metadata_offset + json_padded_len as u64; |
297 | 7 | let data_offset = tensor_index_offset + index_padded_len as u64; |
298 | | |
299 | | // Write header (64 bytes) |
300 | 7 | data.extend_from_slice(APR_MAGIC); |
301 | 7 | data.push(APR_VERSION_MAJOR); |
302 | 7 | data.push(APR_VERSION_MINOR); |
303 | 7 | data.extend_from_slice(&0u16.to_le_bytes()); // flags |
304 | 7 | data.extend_from_slice(&(self.tensors.len() as u32).to_le_bytes()); |
305 | 7 | data.extend_from_slice(&metadata_offset.to_le_bytes()); |
306 | 7 | data.extend_from_slice(&(json_bytes.len() as u32).to_le_bytes()); |
307 | 7 | data.extend_from_slice(&tensor_index_offset.to_le_bytes()); |
308 | 7 | data.extend_from_slice(&data_offset.to_le_bytes()); |
309 | 7 | data.extend_from_slice(&0u32.to_le_bytes()); // checksum (placeholder) |
310 | 7 | data.extend([0u8; 20]); // reserved |
311 | | |
312 | 7 | assert_eq!(data.len(), APR_HEADER_SIZE); |
313 | | |
314 | | // Write metadata (padded) |
315 | 7 | data.extend_from_slice(json_bytes); |
316 | 7 | data.resize(APR_HEADER_SIZE + json_padded_len, 0); |
317 | | |
318 | | // Write tensor index (padded) |
319 | 7 | data.extend_from_slice(&tensor_index); |
320 | 7 | data.resize(APR_HEADER_SIZE + json_padded_len + index_padded_len, 0); |
321 | | |
322 | | // Write tensor data (each aligned to 64 bytes) |
323 | 15 | for (_, _, _, tensor_bytes8 ) in &self.tensors { |
324 | 8 | let start = data.len(); |
325 | 8 | data.extend_from_slice(tensor_bytes); |
326 | 8 | let aligned_end = start + tensor_bytes.len().div_ceil(APR_ALIGNMENT) * APR_ALIGNMENT; |
327 | 8 | data.resize(aligned_end, 0); |
328 | 8 | } |
329 | | |
330 | 7 | data |
331 | 7 | } |
332 | | |
333 | | /// Build a minimal APR model for testing |
334 | | #[must_use] |
335 | 3 | pub fn minimal_model(vocab_size: usize, hidden_dim: usize) -> Vec<u8> { |
336 | 3 | let embed_data = create_f32_embedding_data(vocab_size, hidden_dim); |
337 | 3 | let norm_data = create_f32_norm_weights(hidden_dim); |
338 | | |
339 | 3 | Self::new() |
340 | 3 | .architecture("llama") |
341 | 3 | .hidden_dim(hidden_dim) |
342 | 3 | .num_layers(1) |
343 | 3 | .add_f32_tensor("token_embd.weight", &[vocab_size, hidden_dim], &embed_data) |
344 | 3 | .add_f32_tensor("output_norm.weight", &[hidden_dim], &norm_data) |
345 | 3 | .build() |
346 | 3 | } |
347 | | } |
348 | | |
349 | | // ============================================================================= |
350 | | // Format Detection |
351 | | // ============================================================================= |
352 | | |
353 | | /// Detected model format |
354 | | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
355 | | pub enum FormatType { |
356 | | /// GGUF format (llama.cpp) |
357 | | Gguf, |
358 | | /// SafeTensors format (HuggingFace) |
359 | | SafeTensors, |
360 | | /// APR format (Aprender) |
361 | | Apr, |
362 | | /// Unknown format |
363 | | Unknown, |
364 | | } |
365 | | |
366 | | impl FormatType { |
367 | | /// Detect format from magic bytes (Genchi Genbutsu) |
368 | | #[must_use] |
369 | 17 | pub fn from_magic(data: &[u8]) -> Self { |
370 | 17 | if data.len() < 8 { |
371 | 1 | return Self::Unknown; |
372 | 16 | } |
373 | | |
374 | | // GGUF: "GGUF" magic |
375 | 16 | if &data[0..4] == b"GGUF" { |
376 | 3 | return Self::Gguf; |
377 | 13 | } |
378 | | |
379 | | // APR: "APR\0" magic |
380 | 13 | if &data[0..4] == b"APR\0" || &data[0..4] == b"APR2"6 { |
381 | 7 | return Self::Apr; |
382 | 6 | } |
383 | | |
384 | | // SafeTensors: u64 header length followed by '{"' |
385 | 6 | if data.len() >= 10 { |
386 | 6 | let header_len = u64::from_le_bytes(data[0..8].try_into().unwrap_or([0; 8])); |
387 | 6 | if header_len < 100_000_000 && &data[8..10] == b"{\"" { |
388 | 5 | return Self::SafeTensors; |
389 | 1 | } |
390 | 0 | } |
391 | | |
392 | 1 | Self::Unknown |
393 | 17 | } |
394 | | } |
395 | | |
396 | | // ============================================================================= |
397 | | // Tests |
398 | | // ============================================================================= |
399 | | |
400 | | #[cfg(test)] |
401 | | mod tests { |
402 | | use super::*; |
403 | | |
404 | | // ========================================================================= |
405 | | // SafeTensors Builder Tests |
406 | | // ========================================================================= |
407 | | |
408 | | #[test] |
409 | 1 | fn test_safetensors_builder_empty() { |
410 | 1 | let data = SafetensorsBuilder::new().build(); |
411 | | |
412 | | // Should have valid header (8 bytes length + empty JSON "{}") |
413 | 1 | assert!(data.len() >= 10); |
414 | | |
415 | | // First 8 bytes are header length |
416 | 1 | let header_len = u64::from_le_bytes(data[0..8].try_into().unwrap()); |
417 | 1 | assert_eq!(header_len, 2); // "{}" |
418 | | |
419 | | // Empty SafeTensors is technically valid but format detection |
420 | | // requires "{" at byte 8, which "{}" satisfies |
421 | | // Note: An empty model isn't useful but the format is valid |
422 | 1 | assert!(data[8] == b'{'); |
423 | 1 | } |
424 | | |
425 | | #[test] |
426 | 1 | fn test_safetensors_builder_with_tensor() { |
427 | 1 | let data = SafetensorsBuilder::new() |
428 | 1 | .add_f32_tensor("test.weight", &[4, 8], &vec![0.0f32; 32]) |
429 | 1 | .build(); |
430 | | |
431 | 1 | assert!(data.len() > 10); |
432 | 1 | assert_eq!(FormatType::from_magic(&data), FormatType::SafeTensors); |
433 | | |
434 | | // Verify JSON header contains tensor metadata |
435 | 1 | let header_len = u64::from_le_bytes(data[0..8].try_into().unwrap()) as usize; |
436 | 1 | let json_str = std::str::from_utf8(&data[8..8 + header_len]).expect("valid UTF-8"); |
437 | 1 | assert!(json_str.contains("test.weight")); |
438 | 1 | assert!(json_str.contains("F32")); |
439 | 1 | } |
440 | | |
441 | | #[test] |
442 | 1 | fn test_safetensors_minimal_model() { |
443 | 1 | let data = SafetensorsBuilder::minimal_model(100, 64); |
444 | | |
445 | 1 | assert_eq!(FormatType::from_magic(&data), FormatType::SafeTensors); |
446 | | |
447 | 1 | let header_len = u64::from_le_bytes(data[0..8].try_into().unwrap()) as usize; |
448 | 1 | let json_str = std::str::from_utf8(&data[8..8 + header_len]).expect("valid UTF-8"); |
449 | 1 | assert!(json_str.contains("model.embed_tokens.weight")); |
450 | 1 | assert!(json_str.contains("model.norm.weight")); |
451 | 1 | } |
452 | | |
453 | | // ========================================================================= |
454 | | // APR Builder Tests |
455 | | // ========================================================================= |
456 | | |
457 | | #[test] |
458 | 1 | fn test_apr_builder_empty() { |
459 | 1 | let data = AprBuilder::new().build(); |
460 | | |
461 | | // Should have valid header (64 bytes minimum) |
462 | 1 | assert!(data.len() >= APR_HEADER_SIZE); |
463 | | |
464 | | // Check magic |
465 | 1 | assert_eq!(&data[0..4], b"APR\0"); |
466 | | |
467 | | // Detect format |
468 | 1 | assert_eq!(FormatType::from_magic(&data), FormatType::Apr); |
469 | 1 | } |
470 | | |
471 | | #[test] |
472 | 1 | fn test_apr_builder_with_metadata() { |
473 | 1 | let data = AprBuilder::new() |
474 | 1 | .architecture("llama") |
475 | 1 | .hidden_dim(64) |
476 | 1 | .num_layers(2) |
477 | 1 | .build(); |
478 | | |
479 | 1 | assert_eq!(FormatType::from_magic(&data), FormatType::Apr); |
480 | | |
481 | | // Verify version |
482 | 1 | assert_eq!(data[4], APR_VERSION_MAJOR); |
483 | 1 | assert_eq!(data[5], APR_VERSION_MINOR); |
484 | 1 | } |
485 | | |
486 | | #[test] |
487 | 1 | fn test_apr_builder_with_tensor() { |
488 | 1 | let embed_data = create_f32_embedding_data(10, 8); |
489 | 1 | let data = AprBuilder::new() |
490 | 1 | .add_f32_tensor("token_embd.weight", &[10, 8], &embed_data) |
491 | 1 | .build(); |
492 | | |
493 | 1 | assert_eq!(FormatType::from_magic(&data), FormatType::Apr); |
494 | | |
495 | | // Verify tensor count in header |
496 | 1 | let tensor_count = u32::from_le_bytes(data[8..12].try_into().unwrap()); |
497 | 1 | assert_eq!(tensor_count, 1); |
498 | 1 | } |
499 | | |
500 | | #[test] |
501 | 1 | fn test_apr_minimal_model() { |
502 | 1 | let data = AprBuilder::minimal_model(100, 64); |
503 | | |
504 | 1 | assert_eq!(FormatType::from_magic(&data), FormatType::Apr); |
505 | | |
506 | | // Verify tensor count |
507 | 1 | let tensor_count = u32::from_le_bytes(data[8..12].try_into().unwrap()); |
508 | 1 | assert_eq!(tensor_count, 2); // embed + norm |
509 | 1 | } |
510 | | |
511 | | // ========================================================================= |
512 | | // Format Detection Tests |
513 | | // ========================================================================= |
514 | | |
515 | | #[test] |
516 | 1 | fn test_format_detection_gguf() { |
517 | 1 | let data = build_minimal_llama_gguf(100, 64, 128, 4, 4); |
518 | 1 | assert_eq!(FormatType::from_magic(&data), FormatType::Gguf); |
519 | 1 | } |
520 | | |
521 | | #[test] |
522 | 1 | fn test_format_detection_safetensors() { |
523 | 1 | let data = SafetensorsBuilder::minimal_model(100, 64); |
524 | 1 | assert_eq!(FormatType::from_magic(&data), FormatType::SafeTensors); |
525 | 1 | } |
526 | | |
527 | | #[test] |
528 | 1 | fn test_format_detection_apr() { |
529 | 1 | let data = AprBuilder::minimal_model(100, 64); |
530 | 1 | assert_eq!(FormatType::from_magic(&data), FormatType::Apr); |
531 | 1 | } |
532 | | |
533 | | #[test] |
534 | 1 | fn test_format_detection_unknown() { |
535 | 1 | let data = vec![0u8; 100]; |
536 | 1 | assert_eq!(FormatType::from_magic(&data), FormatType::Unknown); |
537 | 1 | } |
538 | | |
539 | | #[test] |
540 | 1 | fn test_format_detection_too_short() { |
541 | 1 | let data = vec![0u8; 4]; |
542 | 1 | assert_eq!(FormatType::from_magic(&data), FormatType::Unknown); |
543 | 1 | } |
544 | | |
545 | | // ========================================================================= |
546 | | // Cross-Format Tensor Data Tests (Rosetta Parity) |
547 | | // ========================================================================= |
548 | | |
549 | | #[test] |
550 | 1 | fn test_rosetta_same_embedding_data() { |
551 | | // Same embedding data should produce same raw bytes in all formats |
552 | 1 | let embed_data = create_f32_embedding_data(10, 8); |
553 | | |
554 | 1 | let gguf = GGUFBuilder::new() |
555 | 1 | .add_f32_tensor("token_embd.weight", &[10, 8], &embed_data) |
556 | 1 | .build(); |
557 | | |
558 | 1 | let st = SafetensorsBuilder::new() |
559 | 1 | .add_f32_tensor("token_embd.weight", &[10, 8], &embed_data) |
560 | 1 | .build(); |
561 | | |
562 | 1 | let apr = AprBuilder::new() |
563 | 1 | .add_f32_tensor("token_embd.weight", &[10, 8], &embed_data) |
564 | 1 | .build(); |
565 | | |
566 | | // All formats should be valid |
567 | 1 | assert_eq!(FormatType::from_magic(&gguf), FormatType::Gguf); |
568 | 1 | assert_eq!(FormatType::from_magic(&st), FormatType::SafeTensors); |
569 | 1 | assert_eq!(FormatType::from_magic(&apr), FormatType::Apr); |
570 | | |
571 | | // The raw f32 bytes are stored somewhere in each format |
572 | 80 | let f32_bytes1 : Vec<u8>1 = embed_data.iter()1 .flat_map1 (|f| f.to_le_bytes()).collect1 (); |
573 | 1 | assert_eq!(f32_bytes.len(), 10 * 8 * 4); // 320 bytes |
574 | | |
575 | | // GGUF, SafeTensors, and APR all store raw F32 as little-endian |
576 | | // The exact offsets differ by format, but data integrity is preserved |
577 | 1 | } |
578 | | |
579 | | #[test] |
580 | 1 | fn test_rosetta_all_formats_valid() { |
581 | | // Generate all three formats and verify they're all valid |
582 | 1 | let vocab_size = 100; |
583 | 1 | let hidden_dim = 64; |
584 | | |
585 | 1 | let gguf = build_minimal_llama_gguf(vocab_size, hidden_dim, 128, 4, 4); |
586 | 1 | let st = SafetensorsBuilder::minimal_model(vocab_size, hidden_dim); |
587 | 1 | let apr = AprBuilder::minimal_model(vocab_size, hidden_dim); |
588 | | |
589 | | // All should be detected correctly |
590 | 1 | assert_eq!(FormatType::from_magic(&gguf), FormatType::Gguf); |
591 | 1 | assert_eq!(FormatType::from_magic(&st), FormatType::SafeTensors); |
592 | 1 | assert_eq!(FormatType::from_magic(&apr), FormatType::Apr); |
593 | | |
594 | | // All should have reasonable size (not empty) |
595 | 1 | assert!(gguf.len() > 1000, "GGUF too small: {}"0 , gguf0 .len0 ()); |
596 | 1 | assert!(st.len() > 100, "SafeTensors too small: {}"0 , st0 .len0 ()); |
597 | 1 | assert!(apr.len() > 100, "APR too small: {}"0 , apr0 .len0 ()); |
598 | 1 | } |
599 | | } |