## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/gguf/format_factory.rs

    Line
    Count
    Source
    1
    //! Rosetta Format Factory - Synthetic Model Files for All Formats
    2
    //!
    3
    //! Following aprender's Rosetta Stone spec, this module provides builders
    4
    //! for synthesizing valid model files in ALL supported formats:
    5
    //!
    6
    //! | Format | Builder | Magic |
    7
    //! |--------|---------|-------|
    8
    //! | GGUF | `GGUFBuilder` | "GGUF" |
    9
    //! | SafeTensors | `SafetensorsBuilder` | JSON header |
    10
    //! | APR | `AprBuilder` | "APR\0" |
    11
    //!
    12
    //! # Conversion Matrix (6 Direct Paths)
    13
    //!
    14
    //! ```text
    15
    //!     GGUF ←──────→ APR ←──────→ SafeTensors
    16
    //!       ↑                              ↑
    17
    //!       └──────────────────────────────┘
    18
    //! ```
    19
    //!
    20
    //! # Example
    21
    //!
    22
    //! ```ignore
    23
    //! use realizar::gguf::format_factory::{GGUFBuilder, SafetensorsBuilder, AprBuilder};
    24
    //!
    25
    //! // Create a minimal model in each format
    26
    //! let gguf_data = GGUFBuilder::minimal_llama(100, 64);
    27
    //! let st_data = SafetensorsBuilder::minimal_model(100, 64);
    28
    //! let apr_data = AprBuilder::minimal_model(100, 64);
    29
    //! ```
    30
    31
    use serde::Serialize;
    32
    use std::collections::BTreeMap;
    33
    34
    // Re-export GGUFBuilder from test_factory
    35
    pub use super::test_factory::{
    36
        build_minimal_llama_gguf, build_minimal_phi2_gguf, create_f32_embedding_data,
    37
        create_f32_norm_weights, create_q4_0_data, create_q4_k_data, create_q5_k_data,
    38
        create_q6_k_data, create_q8_0_data, GGUFBuilder,
    39
    };
    40
    41
    // =============================================================================
    42
    // SafeTensors Builder
    43
    // =============================================================================
    44
    45
    /// SafeTensors tensor metadata
    46
    #[derive(Debug, Clone, Serialize)]
    47
    struct SafetensorsTensorMeta {
    48
        dtype: String,
    49
        shape: Vec<usize>,
    50
        data_offsets: [usize; 2],
    51
    }
    52
    53
    /// Builder for creating valid SafeTensors files in memory
    54
    ///
    55
    /// SafeTensors format:
    56
    /// - 8 bytes: JSON header length (little-endian u64)
    57
    /// - N bytes: JSON header with tensor metadata
    58
    /// - Tensor data (contiguous, aligned)
    59
    pub struct SafetensorsBuilder {
    60
        tensors: Vec<(String, String, Vec<usize>, Vec<u8>)>, // name, dtype, shape, data
    61
    }
    62
    63
    impl Default for SafetensorsBuilder {
    64
    0
        fn default() -> Self {
    65
    0
            Self::new()
    66
    0
        }
    67
    }
    68
    69
    impl SafetensorsBuilder {
    70
        /// Create a new SafeTensors builder
    71
        #[must_use]
    72
    6
        pub fn new() -> Self {
    73
    6
            Self {
    74
    6
                tensors: Vec::new(),
    75
    6
            }
    76
    6
        }
    77
    78
        /// Add an F32 tensor
    79
        #[must_use]
    80
    8
        pub fn add_f32_tensor(mut self, name: &str, shape: &[usize], data: &[f32]) -> Self {
    81
    19.5k
            let bytes8: Vec<u8>8 = data8.iter8().flat_map8(|f| f.to_le_bytes()).collect8();
    82
    8
            self.tensors
    83
    8
                .push((name.to_string(), "F32".to_string(), shape.to_vec(), bytes));
    84
    8
            self
    85
    8
        }
    86
    87
        /// Add an F16 tensor
    88
        #[must_use]
    89
    0
        pub fn add_f16_tensor(mut self, name: &str, shape: &[usize], data: &[u8]) -> Self {
    90
    0
            self.tensors
    91
    0
                .push((name.to_string(), "F16".to_string(), shape.to_vec(), data.to_vec()));
    92
    0
            self
    93
    0
        }
    94
    95
        /// Add a BF16 tensor
    96
        #[must_use]
    97
    0
        pub fn add_bf16_tensor(mut self, name: &str, shape: &[usize], data: &[u8]) -> Self {
    98
    0
            self.tensors
    99
    0
                .push((name.to_string(), "BF16".to_string(), shape.to_vec(), data.to_vec()));
    100
    0
            self
    101
    0
        }
    102
    103
        /// Build the SafeTensors file as a byte vector
    104
        #[must_use]
    105
    6
        pub fn build(self) -> Vec<u8> {
    106
            // Calculate offsets and build metadata
    107
    6
            let mut metadata: BTreeMap<String, SafetensorsTensorMeta> = BTreeMap::new();
    108
    6
            let mut current_offset = 0usize;
    109
    110
    14
            for (name8, dtype8, shape8, data8) in &self.tensors {
    111
    8
                let end_offset = current_offset + data.len();
    112
    8
                metadata.insert(
    113
    8
                    name.clone(),
    114
    8
                    SafetensorsTensorMeta {
    115
    8
                        dtype: dtype.clone(),
    116
    8
                        shape: shape.clone(),
    117
    8
                        data_offsets: [current_offset, end_offset],
    118
    8
                    },
    119
    8
                );
    120
    8
                current_offset = end_offset;
    121
    8
            }
    122
    123
            // Serialize metadata to JSON
    124
    6
            let json = serde_json::to_string(&metadata).expect("JSON serialization");
    125
    6
            let json_bytes = json.as_bytes();
    126
    127
            // Build final file
    128
    6
            let mut data = Vec::new();
    129
    130
            // Header: JSON length as u64
    131
    6
            data.extend_from_slice(&(json_bytes.len() as u64).to_le_bytes());
    132
    133
            // JSON metadata
    134
    6
            data.extend_from_slice(json_bytes);
    135
    136
            // Tensor data
    137
    14
            for (_, _, _, tensor_data8) in &self.tensors {
    138
    8
                data.extend_from_slice(tensor_data);
    139
    8
            }
    140
    141
    6
            data
    142
    6
        }
    143
    144
        /// Build a minimal SafeTensors model for testing
    145
        #[must_use]
    146
    3
        pub fn minimal_model(vocab_size: usize, hidden_dim: usize) -> Vec<u8> {
    147
    3
            let embed_data = create_f32_embedding_data(vocab_size, hidden_dim);
    148
    3
            let norm_data = create_f32_norm_weights(hidden_dim);
    149
    150
    3
            Self::new()
    151
    3
                .add_f32_tensor("model.embed_tokens.weight", &[vocab_size, hidden_dim], &embed_data)
    152
    3
                .add_f32_tensor("model.norm.weight", &[hidden_dim], &norm_data)
    153
    3
                .build()
    154
    3
        }
    155
    }
    156
    157
    // =============================================================================
    158
    // APR Builder
    159
    // =============================================================================
    160
    161
    /// APR v2 format constants
    162
    const APR_MAGIC: &[u8; 4] = b"APR\0";
    163
    const APR_VERSION_MAJOR: u8 = 2;
    164
    const APR_VERSION_MINOR: u8 = 0;
    165
    const APR_HEADER_SIZE: usize = 64;
    166
    const APR_ALIGNMENT: usize = 64;
    167
    168
    /// Builder for creating valid APR v2 files in memory
    169
    ///
    170
    /// APR v2 format (64-byte header):
    171
    /// - 4 bytes: Magic "APR\0"
    172
    /// - 2 bytes: Version (major.minor)
    173
    /// - 2 bytes: Flags
    174
    /// - 4 bytes: Tensor count
    175
    /// - 8 bytes: Metadata offset
    176
    /// - 4 bytes: Metadata size
    177
    /// - 8 bytes: Tensor index offset
    178
    /// - 8 bytes: Data offset
    179
    /// - 4 bytes: Checksum
    180
    /// - 20 bytes: Reserved
    181
    pub struct AprBuilder {
    182
        metadata: BTreeMap<String, serde_json::Value>,
    183
        tensors: Vec<(String, Vec<usize>, u32, Vec<u8>)>, // name, shape, dtype, data
    184
    }
    185
    186
    impl Default for AprBuilder {
    187
    0
        fn default() -> Self {
    188
    0
            Self::new()
    189
    0
        }
    190
    }
    191
    192
    /// APR dtype codes
    193
    pub const APR_DTYPE_F32: u32 = 0;
    194
    pub const APR_DTYPE_F16: u32 = 1;
    195
    pub const APR_DTYPE_Q4_0: u32 = 2;
    196
    pub const APR_DTYPE_Q8_0: u32 = 8;
    197
    198
    impl AprBuilder {
    199
        /// Create a new APR builder
    200
        #[must_use]
    201
    7
        pub fn new() -> Self {
    202
    7
            Self {
    203
    7
                metadata: BTreeMap::new(),
    204
    7
                tensors: Vec::new(),
    205
    7
            }
    206
    7
        }
    207
    208
        /// Set architecture metadata
    209
        #[must_use]
    210
    4
        pub fn architecture(mut self, arch: &str) -> Self {
    211
    4
            self.metadata
    212
    4
                .insert("architecture".to_string(), serde_json::json!(arch));
    213
    4
            self
    214
    4
        }
    215
    216
        /// Set hidden dimension metadata
    217
        #[must_use]
    218
    4
        pub fn hidden_dim(mut self, dim: usize) -> Self {
    219
    4
            self.metadata
    220
    4
                .insert("hidden_dim".to_string(), serde_json::json!(dim));
    221
    4
            self
    222
    4
        }
    223
    224
        /// Set number of layers metadata
    225
        #[must_use]
    226
    4
        pub fn num_layers(mut self, count: usize) -> Self {
    227
    4
            self.metadata
    228
    4
                .insert("num_layers".to_string(), serde_json::json!(count));
    229
    4
            self
    230
    4
        }
    231
    232
        /// Add an F32 tensor
    233
        #[must_use]
    234
    8
        pub fn add_f32_tensor(mut self, name: &str, shape: &[usize], data: &[f32]) -> Self {
    235
    19.5k
            let bytes8: Vec<u8>8 = data8.iter8().flat_map8(|f| f.to_le_bytes()).collect8();
    236
    8
            self.tensors
    237
    8
                .push((name.to_string(), shape.to_vec(), APR_DTYPE_F32, bytes));
    238
    8
            self
    239
    8
        }
    240
    241
        /// Add a Q4_0 tensor
    242
        #[must_use]
    243
    0
        pub fn add_q4_0_tensor(mut self, name: &str, shape: &[usize], data: &[u8]) -> Self {
    244
    0
            self.tensors
    245
    0
                .push((name.to_string(), shape.to_vec(), APR_DTYPE_Q4_0, data.to_vec()));
    246
    0
            self
    247
    0
        }
    248
    249
        /// Add a Q8_0 tensor
    250
        #[must_use]
    251
    0
        pub fn add_q8_0_tensor(mut self, name: &str, shape: &[usize], data: &[u8]) -> Self {
    252
    0
            self.tensors
    253
    0
                .push((name.to_string(), shape.to_vec(), APR_DTYPE_Q8_0, data.to_vec()));
    254
    0
            self
    255
    0
        }
    256
    257
        /// Build the APR v2 file as a byte vector
    258
        #[must_use]
    259
    7
        pub fn build(self) -> Vec<u8> {
    260
    7
            let mut data = Vec::new();
    261
    262
            // Serialize metadata to JSON
    263
    7
            let json = serde_json::to_string(&self.metadata).expect("JSON serialization");
    264
    7
            let json_bytes = json.as_bytes();
    265
    266
            // Pad JSON to 64-byte boundary
    267
    7
            let json_padded_len = json_bytes.len().div_ceil(APR_ALIGNMENT) * APR_ALIGNMENT;
    268
    269
            // Build tensor index
    270
    7
            let mut tensor_index = Vec::new();
    271
    7
            let mut tensor_data_offset = 0u64;
    272
    273
    15
            for (name8, shape8, dtype8, tensor_bytes8) in &self.tensors {
    274
                // Tensor index entry: name_len(4) + name + ndims(4) + dims + dtype(4) + offset(8) + size(8)
    275
    8
                let name_bytes = name.as_bytes();
    276
    8
                tensor_index.extend_from_slice(&(name_bytes.len() as u32).to_le_bytes());
    277
    8
                tensor_index.extend_from_slice(name_bytes);
    278
    8
                tensor_index.extend_from_slice(&(shape.len() as u32).to_le_bytes());
    279
    21
                for dim13 in shape {
    280
    13
                    tensor_index.extend_from_slice(&(*dim as u64).to_le_bytes());
    281
    13
                }
    282
    8
                tensor_index.extend_from_slice(&dtype.to_le_bytes());
    283
    8
                tensor_index.extend_from_slice(&tensor_data_offset.to_le_bytes());
    284
    8
                tensor_index.extend_from_slice(&(tensor_bytes.len() as u64).to_le_bytes());
    285
    286
                // Align tensor data to 64 bytes
    287
    8
                let aligned_size = tensor_bytes.len().div_ceil(APR_ALIGNMENT) * APR_ALIGNMENT;
    288
    8
                tensor_data_offset += aligned_size as u64;
    289
            }
    290
    291
            // Pad tensor index to 64-byte boundary
    292
    7
            let index_padded_len = tensor_index.len().div_ceil(APR_ALIGNMENT) * APR_ALIGNMENT;
    293
    294
            // Calculate offsets
    295
    7
            let metadata_offset = APR_HEADER_SIZE as u64;
    296
    7
            let tensor_index_offset = metadata_offset + json_padded_len as u64;
    297
    7
            let data_offset = tensor_index_offset + index_padded_len as u64;
    298
    299
            // Write header (64 bytes)
    300
    7
            data.extend_from_slice(APR_MAGIC);
    301
    7
            data.push(APR_VERSION_MAJOR);
    302
    7
            data.push(APR_VERSION_MINOR);
    303
    7
            data.extend_from_slice(&0u16.to_le_bytes()); // flags
    304
    7
            data.extend_from_slice(&(self.tensors.len() as u32).to_le_bytes());
    305
    7
            data.extend_from_slice(&metadata_offset.to_le_bytes());
    306
    7
            data.extend_from_slice(&(json_bytes.len() as u32).to_le_bytes());
    307
    7
            data.extend_from_slice(&tensor_index_offset.to_le_bytes());
    308
    7
            data.extend_from_slice(&data_offset.to_le_bytes());
    309
    7
            data.extend_from_slice(&0u32.to_le_bytes()); // checksum (placeholder)
    310
    7
            data.extend([0u8; 20]); // reserved
    311
    312
    7
            assert_eq!(data.len(), APR_HEADER_SIZE);
    313
    314
            // Write metadata (padded)
    315
    7
            data.extend_from_slice(json_bytes);
    316
    7
            data.resize(APR_HEADER_SIZE + json_padded_len, 0);
    317
    318
            // Write tensor index (padded)
    319
    7
            data.extend_from_slice(&tensor_index);
    320
    7
            data.resize(APR_HEADER_SIZE + json_padded_len + index_padded_len, 0);
    321
    322
            // Write tensor data (each aligned to 64 bytes)
    323
    15
            for (_, _, _, tensor_bytes8) in &self.tensors {
    324
    8
                let start = data.len();
    325
    8
                data.extend_from_slice(tensor_bytes);
    326
    8
                let aligned_end = start + tensor_bytes.len().div_ceil(APR_ALIGNMENT) * APR_ALIGNMENT;
    327
    8
                data.resize(aligned_end, 0);
    328
    8
            }
    329
    330
    7
            data
    331
    7
        }
    332
    333
        /// Build a minimal APR model for testing
    334
        #[must_use]
    335
    3
        pub fn minimal_model(vocab_size: usize, hidden_dim: usize) -> Vec<u8> {
    336
    3
            let embed_data = create_f32_embedding_data(vocab_size, hidden_dim);
    337
    3
            let norm_data = create_f32_norm_weights(hidden_dim);
    338
    339
    3
            Self::new()
    340
    3
                .architecture("llama")
    341
    3
                .hidden_dim(hidden_dim)
    342
    3
                .num_layers(1)
    343
    3
                .add_f32_tensor("token_embd.weight", &[vocab_size, hidden_dim], &embed_data)
    344
    3
                .add_f32_tensor("output_norm.weight", &[hidden_dim], &norm_data)
    345
    3
                .build()
    346
    3
        }
    347
    }
    348
    349
    // =============================================================================
    350
    // Format Detection
    351
    // =============================================================================
    352
    353
    /// Detected model format
    354
    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
    355
    pub enum FormatType {
    356
        /// GGUF format (llama.cpp)
    357
        Gguf,
    358
        /// SafeTensors format (HuggingFace)
    359
        SafeTensors,
    360
        /// APR format (Aprender)
    361
        Apr,
    362
        /// Unknown format
    363
        Unknown,
    364
    }
    365
    366
    impl FormatType {
    367
        /// Detect format from magic bytes (Genchi Genbutsu)
    368
        #[must_use]
    369
    17
        pub fn from_magic(data: &[u8]) -> Self {
    370
    17
            if data.len() < 8 {
    371
    1
                return Self::Unknown;
    372
    16
            }
    373
    374
            // GGUF: "GGUF" magic
    375
    16
            if &data[0..4] == b"GGUF" {
    376
    3
                return Self::Gguf;
    377
    13
            }
    378
    379
            // APR: "APR\0" magic
    380
    13
            if &data[0..4] == b"APR\0" || &data[0..4] == b"APR2"6 {
    381
    7
                return Self::Apr;
    382
    6
            }
    383
    384
            // SafeTensors: u64 header length followed by '{"'
    385
    6
            if data.len() >= 10 {
    386
    6
                let header_len = u64::from_le_bytes(data[0..8].try_into().unwrap_or([0; 8]));
    387
    6
                if header_len < 100_000_000 && &data[8..10] == b"{\"" {
    388
    5
                    return Self::SafeTensors;
    389
    1
                }
    390
    0
            }
    391
    392
    1
            Self::Unknown
    393
    17
        }
    394
    }
    395
    396
    // =============================================================================
    397
    // Tests
    398
    // =============================================================================
    399
    400
    #[cfg(test)]
    401
    mod tests {
    402
        use super::*;
    403
    404
        // =========================================================================
    405
        // SafeTensors Builder Tests
    406
        // =========================================================================
    407
    408
        #[test]
    409
    1
        fn test_safetensors_builder_empty() {
    410
    1
            let data = SafetensorsBuilder::new().build();
    411
    412
            // Should have valid header (8 bytes length + empty JSON "{}")
    413
    1
            assert!(data.len() >= 10);
    414
    415
            // First 8 bytes are header length
    416
    1
            let header_len = u64::from_le_bytes(data[0..8].try_into().unwrap());
    417
    1
            assert_eq!(header_len, 2); // "{}"
    418
    419
            // Empty SafeTensors is technically valid but format detection
    420
            // requires "{" at byte 8, which "{}" satisfies
    421
            // Note: An empty model isn't useful but the format is valid
    422
    1
            assert!(data[8] == b'{');
    423
    1
        }
    424
    425
        #[test]
    426
    1
        fn test_safetensors_builder_with_tensor() {
    427
    1
            let data = SafetensorsBuilder::new()
    428
    1
                .add_f32_tensor("test.weight", &[4, 8], &vec![0.0f32; 32])
    429
    1
                .build();
    430
    431
    1
            assert!(data.len() > 10);
    432
    1
            assert_eq!(FormatType::from_magic(&data), FormatType::SafeTensors);
    433
    434
            // Verify JSON header contains tensor metadata
    435
    1
            let header_len = u64::from_le_bytes(data[0..8].try_into().unwrap()) as usize;
    436
    1
            let json_str = std::str::from_utf8(&data[8..8 + header_len]).expect("valid UTF-8");
    437
    1
            assert!(json_str.contains("test.weight"));
    438
    1
            assert!(json_str.contains("F32"));
    439
    1
        }
    440
    441
        #[test]
    442
    1
        fn test_safetensors_minimal_model() {
    443
    1
            let data = SafetensorsBuilder::minimal_model(100, 64);
    444
    445
    1
            assert_eq!(FormatType::from_magic(&data), FormatType::SafeTensors);
    446
    447
    1
            let header_len = u64::from_le_bytes(data[0..8].try_into().unwrap()) as usize;
    448
    1
            let json_str = std::str::from_utf8(&data[8..8 + header_len]).expect("valid UTF-8");
    449
    1
            assert!(json_str.contains("model.embed_tokens.weight"));
    450
    1
            assert!(json_str.contains("model.norm.weight"));
    451
    1
        }
    452
    453
        // =========================================================================
    454
        // APR Builder Tests
    455
        // =========================================================================
    456
    457
        #[test]
    458
    1
        fn test_apr_builder_empty() {
    459
    1
            let data = AprBuilder::new().build();
    460
    461
            // Should have valid header (64 bytes minimum)
    462
    1
            assert!(data.len() >= APR_HEADER_SIZE);
    463
    464
            // Check magic
    465
    1
            assert_eq!(&data[0..4], b"APR\0");
    466
    467
            // Detect format
    468
    1
            assert_eq!(FormatType::from_magic(&data), FormatType::Apr);
    469
    1
        }
    470
    471
        #[test]
    472
    1
        fn test_apr_builder_with_metadata() {
    473
    1
            let data = AprBuilder::new()
    474
    1
                .architecture("llama")
    475
    1
                .hidden_dim(64)
    476
    1
                .num_layers(2)
    477
    1
                .build();
    478
    479
    1
            assert_eq!(FormatType::from_magic(&data), FormatType::Apr);
    480
    481
            // Verify version
    482
    1
            assert_eq!(data[4], APR_VERSION_MAJOR);
    483
    1
            assert_eq!(data[5], APR_VERSION_MINOR);
    484
    1
        }
    485
    486
        #[test]
    487
    1
        fn test_apr_builder_with_tensor() {
    488
    1
            let embed_data = create_f32_embedding_data(10, 8);
    489
    1
            let data = AprBuilder::new()
    490
    1
                .add_f32_tensor("token_embd.weight", &[10, 8], &embed_data)
    491
    1
                .build();
    492
    493
    1
            assert_eq!(FormatType::from_magic(&data), FormatType::Apr);
    494
    495
            // Verify tensor count in header
    496
    1
            let tensor_count = u32::from_le_bytes(data[8..12].try_into().unwrap());
    497
    1
            assert_eq!(tensor_count, 1);
    498
    1
        }
    499
    500
        #[test]
    501
    1
        fn test_apr_minimal_model() {
    502
    1
            let data = AprBuilder::minimal_model(100, 64);
    503
    504
    1
            assert_eq!(FormatType::from_magic(&data), FormatType::Apr);
    505
    506
            // Verify tensor count
    507
    1
            let tensor_count = u32::from_le_bytes(data[8..12].try_into().unwrap());
    508
    1
            assert_eq!(tensor_count, 2); // embed + norm
    509
    1
        }
    510
    511
        // =========================================================================
    512
        // Format Detection Tests
    513
        // =========================================================================
    514
    515
        #[test]
    516
    1
        fn test_format_detection_gguf() {
    517
    1
            let data = build_minimal_llama_gguf(100, 64, 128, 4, 4);
    518
    1
            assert_eq!(FormatType::from_magic(&data), FormatType::Gguf);
    519
    1
        }
    520
    521
        #[test]
    522
    1
        fn test_format_detection_safetensors() {
    523
    1
            let data = SafetensorsBuilder::minimal_model(100, 64);
    524
    1
            assert_eq!(FormatType::from_magic(&data), FormatType::SafeTensors);
    525
    1
        }
    526
    527
        #[test]
    528
    1
        fn test_format_detection_apr() {
    529
    1
            let data = AprBuilder::minimal_model(100, 64);
    530
    1
            assert_eq!(FormatType::from_magic(&data), FormatType::Apr);
    531
    1
        }
    532
    533
        #[test]
    534
    1
        fn test_format_detection_unknown() {
    535
    1
            let data = vec![0u8; 100];
    536
    1
            assert_eq!(FormatType::from_magic(&data), FormatType::Unknown);
    537
    1
        }
    538
    539
        #[test]
    540
    1
        fn test_format_detection_too_short() {
    541
    1
            let data = vec![0u8; 4];
    542
    1
            assert_eq!(FormatType::from_magic(&data), FormatType::Unknown);
    543
    1
        }
    544
    545
        // =========================================================================
    546
        // Cross-Format Tensor Data Tests (Rosetta Parity)
    547
        // =========================================================================
    548
    549
        #[test]
    550
    1
        fn test_rosetta_same_embedding_data() {
    551
            // Same embedding data should produce same raw bytes in all formats
    552
    1
            let embed_data = create_f32_embedding_data(10, 8);
    553
    554
    1
            let gguf = GGUFBuilder::new()
    555
    1
                .add_f32_tensor("token_embd.weight", &[10, 8], &embed_data)
    556
    1
                .build();
    557
    558
    1
            let st = SafetensorsBuilder::new()
    559
    1
                .add_f32_tensor("token_embd.weight", &[10, 8], &embed_data)
    560
    1
                .build();
    561
    562
    1
            let apr = AprBuilder::new()
    563
    1
                .add_f32_tensor("token_embd.weight", &[10, 8], &embed_data)
    564
    1
                .build();
    565
    566
            // All formats should be valid
    567
    1
            assert_eq!(FormatType::from_magic(&gguf), FormatType::Gguf);
    568
    1
            assert_eq!(FormatType::from_magic(&st), FormatType::SafeTensors);
    569
    1
            assert_eq!(FormatType::from_magic(&apr), FormatType::Apr);
    570
    571
            // The raw f32 bytes are stored somewhere in each format
    572
    80
            let f32_bytes1: Vec<u8>1 = embed_data.iter()1.flat_map1(|f| f.to_le_bytes()).collect1();
    573
    1
            assert_eq!(f32_bytes.len(), 10 * 8 * 4); // 320 bytes
    574
    575
            // GGUF, SafeTensors, and APR all store raw F32 as little-endian
    576
            // The exact offsets differ by format, but data integrity is preserved
    577
    1
        }
    578
    579
        #[test]
    580
    1
        fn test_rosetta_all_formats_valid() {
    581
            // Generate all three formats and verify they're all valid
    582
    1
            let vocab_size = 100;
    583
    1
            let hidden_dim = 64;
    584
    585
    1
            let gguf = build_minimal_llama_gguf(vocab_size, hidden_dim, 128, 4, 4);
    586
    1
            let st = SafetensorsBuilder::minimal_model(vocab_size, hidden_dim);
    587
    1
            let apr = AprBuilder::minimal_model(vocab_size, hidden_dim);
    588
    589
            // All should be detected correctly
    590
    1
            assert_eq!(FormatType::from_magic(&gguf), FormatType::Gguf);
    591
    1
            assert_eq!(FormatType::from_magic(&st), FormatType::SafeTensors);
    592
    1
            assert_eq!(FormatType::from_magic(&apr), FormatType::Apr);
    593
    594
            // All should have reasonable size (not empty)
    595
    1
            assert!(gguf.len() > 1000, "GGUF too small: {}"0, gguf0.len0());
    596
    1
            assert!(st.len() > 100, "SafeTensors too small: {}"0, st0.len0());
    597
    1
            assert!(apr.len() > 100, "APR too small: {}"0, apr0.len0());
    598
    1
        }
    599
    }