## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/format.rs

    Line
    Count
    Source
    1
    //! Unified Model Format Detection and Loading
    2
    //!
    3
    //! Per spec §3: Format Support Matrix - auto-detect APR, GGUF, SafeTensors from magic bytes.
    4
    //!
    5
    //! ## Jidoka (Built-in Quality)
    6
    //!
    7
    //! - CRC32 verification for APR format
    8
    //! - Header size validation for SafeTensors (DOS protection)
    9
    //! - Magic byte validation for GGUF
    10
    //!
    11
    //! ## Supported Formats
    12
    //!
    13
    //! | Format | Magic | Extension |
    14
    //! |--------|-------|-----------|
    15
    //! | APR    | `APR\0` | `.apr` |
    16
    //! | GGUF   | `GGUF` | `.gguf` |
    17
    //! | SafeTensors | (u64 header size) | `.safetensors` |
    18
    19
    use std::path::Path;
    20
    21
    /// Detected model format
    22
    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
    23
    pub enum ModelFormat {
    24
        /// Aprender native format (first-class support)
    25
        Apr,
    26
        /// GGUF format (llama.cpp compatible)
    27
        Gguf,
    28
        /// SafeTensors format (HuggingFace compatible)
    29
        SafeTensors,
    30
    }
    31
    32
    impl std::fmt::Display for ModelFormat {
    33
    10
        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
    34
    10
            match self {
    35
    2
                Self::Apr => write!(f, "APR"),
    36
    6
                Self::Gguf => write!(f, "GGUF"),
    37
    2
                Self::SafeTensors => write!(f, "SafeTensors"),
    38
            }
    39
    10
        }
    40
    }
    41
    42
    /// Errors during format detection
    43
    #[derive(Debug, Clone, PartialEq, Eq)]
    44
    pub enum FormatError {
    45
        /// Data too short for format detection (need at least 8 bytes)
    46
        TooShort {
    47
            /// Actual length
    48
            len: usize,
    49
        },
    50
        /// Unknown format (no magic bytes matched)
    51
        UnknownFormat,
    52
        /// SafeTensors header too large (DOS protection per spec §7.1)
    53
        HeaderTooLarge {
    54
            /// Header size in bytes
    55
            size: u64,
    56
        },
    57
        /// File extension doesn't match detected format
    58
        ExtensionMismatch {
    59
            /// Detected format
    60
            detected: ModelFormat,
    61
            /// Extension from filename
    62
            extension: String,
    63
        },
    64
    }
    65
    66
    impl std::fmt::Display for FormatError {
    67
    13
        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
    68
    13
            match self {
    69
    5
                Self::TooShort { len } => {
    70
    5
                    write!(
    71
    5
                        f,
    72
    5
                        "Data too short for format detection: {len} bytes (need 8)"
    73
                    )
    74
                },
    75
    5
                Self::UnknownFormat => write!(f, "Unknown model format (no magic bytes matched)"),
    76
    1
                Self::HeaderTooLarge { size } => write!(
    77
    1
                    f,
    78
    1
                    "SafeTensors header too large: {size} bytes (max 100MB for DOS protection)"
    79
                ),
    80
                Self::ExtensionMismatch {
    81
    2
                    detected,
    82
    2
                    extension,
    83
                } => {
    84
    2
                    write!(
    85
    2
                        f,
    86
    2
                        "Extension mismatch: detected {detected} but file has extension .{extension}"
    87
                    )
    88
                },
    89
            }
    90
    13
        }
    91
    }
    92
    93
    impl std::error::Error for FormatError {}
    94
    95
    /// APR format magic bytes (first 3 bytes, 4th is version)
    96
    ///
    97
    /// APR v1: `APR1` (0x41505231)
    98
    /// APR v2: `APR2` (0x41505232)
    99
    /// Legacy: `APR\0` (0x41505200)
    100
    pub const APR_MAGIC: &[u8; 3] = b"APR";
    101
    102
    /// GGUF format magic bytes
    103
    pub const GGUF_MAGIC: &[u8; 4] = b"GGUF";
    104
    105
    /// Maximum SafeTensors header size (100MB for DOS protection per spec §7.1)
    106
    pub const MAX_SAFETENSORS_HEADER: u64 = 100_000_000;
    107
    108
    /// Detect model format from magic bytes (Jidoka: fail-fast)
    109
    ///
    110
    /// Per spec §3.2: Format Detection
    111
    ///
    112
    /// # Arguments
    113
    ///
    114
    /// * `data` - First 8+ bytes of the model file
    115
    ///
    116
    /// # Returns
    117
    ///
    118
    /// Detected format or error
    119
    ///
    120
    /// # Errors
    121
    ///
    122
    /// Returns error if:
    123
    /// - Data is too short (<8 bytes)
    124
    /// - No known magic bytes detected
    125
    /// - SafeTensors header size exceeds limit (DOS protection)
    126
    ///
    127
    /// # Example
    128
    ///
    129
    /// ```
    130
    /// use realizar::format::{detect_format, ModelFormat};
    131
    ///
    132
    /// // APR format
    133
    /// let apr_data = b"APR\0xxxxxxxxxxxx";
    134
    /// assert_eq!(detect_format(apr_data).expect("test"), ModelFormat::Apr);
    135
    ///
    136
    /// // GGUF format
    137
    /// let gguf_data = b"GGUFxxxxxxxxxxxx";
    138
    /// assert_eq!(detect_format(gguf_data).expect("test"), ModelFormat::Gguf);
    139
    /// ```
    140
    61
    pub fn detect_format(data: &[u8]) -> Result<ModelFormat, FormatError> {
    141
    61
        if data.len() < 8 {
    142
    6
            return Err(FormatError::TooShort { len: data.len() });
    143
    55
        }
    144
    145
        // Check APR magic - first 3 bytes are "APR", 4th byte is version (1, 2, or 0 for legacy)
    146
    55
        if &data[0..3] == APR_MAGIC {
    147
    34
            let version = data[3];
    148
            // Accept version 1, 2, or legacy (0)
    149
    34
            if version == b'1' || version == b'2'33 || version == 032 {
    150
    33
                return Ok(ModelFormat::Apr);
    151
    1
            }
    152
    21
        }
    153
    154
        // Check GGUF magic
    155
    22
        if &data[0..4] == GGUF_MAGIC {
    156
    9
            return Ok(ModelFormat::Gguf);
    157
    13
        }
    158
    159
        // SafeTensors: first 8 bytes are header size (little-endian u64)
    160
        // If it's a reasonable size, assume SafeTensors
    161
    13
        let header_size = u64::from_le_bytes(data[0..8].try_into().expect("slice is exactly 8 bytes"));
    162
    163
        // SafeTensors header should be reasonable size
    164
        // Very large values indicate this isn't SafeTensors
    165
    13
        if header_size < MAX_SAFETENSORS_HEADER && header_size > 011 {
    166
    7
            return Ok(ModelFormat::SafeTensors);
    167
    6
        }
    168
    169
        // Check if header size looks like SafeTensors but is too large
    170
    6
        if header_size >= MAX_SAFETENSORS_HEADER {
    171
    2
            return Err(FormatError::HeaderTooLarge { size: header_size });
    172
    4
        }
    173
    174
    4
        Err(FormatError::UnknownFormat)
    175
    61
    }
    176
    177
    /// Detect format from file path (using extension as hint, then verify magic)
    178
    ///
    179
    /// # Arguments
    180
    ///
    181
    /// * `path` - Path to model file
    182
    ///
    183
    /// # Returns
    184
    ///
    185
    /// Detected format (verified against magic bytes if data provided)
    186
    ///
    187
    /// # Errors
    188
    ///
    189
    /// Returns `FormatError::UnknownFormat` if extension is not recognized.
    190
    ///
    191
    /// # Example
    192
    ///
    193
    /// ```
    194
    /// use realizar::format::{detect_format_from_path, ModelFormat};
    195
    /// use std::path::Path;
    196
    ///
    197
    /// assert_eq!(
    198
    ///     detect_format_from_path(Path::new("model.apr")).expect("test"),
    199
    ///     ModelFormat::Apr
    200
    /// );
    201
    /// ```
    202
    8
    pub fn detect_format_from_path(path: &Path) -> Result<ModelFormat, FormatError> {
    203
    8
        let extension = path.extension().and_then(|e| e.to_str()).unwrap_or("");
    204
    205
    8
        match extension.to_lowercase().as_str() {
    206
    8
            "apr" => Ok(ModelFormat::Apr)4,
    207
    4
            "gguf" => Ok(ModelFormat::Gguf)1,
    208
    3
            "safetensors" => Ok(ModelFormat::SafeTensors)1,
    209
    2
            _ => Err(FormatError::UnknownFormat),
    210
        }
    211
    8
    }
    212
    213
    /// Detect format from path and verify against data magic bytes
    214
    ///
    215
    /// Per Jidoka: stop immediately if extension doesn't match magic
    216
    ///
    217
    /// # Arguments
    218
    ///
    219
    /// * `path` - Path to model file
    220
    /// * `data` - First 8+ bytes of model data
    221
    ///
    222
    /// # Returns
    223
    ///
    224
    /// Verified format or error if mismatch
    225
    ///
    226
    /// # Errors
    227
    ///
    228
    /// Returns error if:
    229
    /// - Format cannot be detected from magic bytes
    230
    /// - File extension doesn't match detected format
    231
    3
    pub fn detect_and_verify_format(path: &Path, data: &[u8]) -> Result<ModelFormat, FormatError> {
    232
    3
        let from_data = detect_format(data)?0;
    233
    3
        let from_path = detect_format_from_path(path);
    234
    235
        // If path detection succeeded, verify it matches data
    236
    3
        if let Ok(path_format2) = from_path {
    237
    2
            if path_format != from_data {
    238
                return Err(FormatError::ExtensionMismatch {
    239
    1
                    detected: from_data,
    240
    1
                    extension: path
    241
    1
                        .extension()
    242
    1
                        .and_then(|e| e.to_str())
    243
    1
                        .unwrap_or("unknown")
    244
    1
                        .to_string(),
    245
                });
    246
    1
            }
    247
    1
        }
    248
    249
        // Data-based detection is authoritative
    250
    2
        Ok(from_data)
    251
    3
    }
    252
    253
    #[cfg(test)]
    254
    mod tests {
    255
        use super::*;
    256
    257
        // ===== EXTREME TDD: Format Detection Tests =====
    258
    259
        #[test]
    260
    1
        fn test_detect_apr_format_legacy() {
    261
    1
            let data = b"APR\0xxxxxxxxxxxxxxxx";
    262
    1
            assert_eq!(detect_format(data).expect("test"), ModelFormat::Apr);
    263
    1
        }
    264
    265
        #[test]
    266
    1
        fn test_detect_apr_format_v1() {
    267
    1
            let data = b"APR1xxxxxxxxxxxxxxxx";
    268
    1
            assert_eq!(detect_format(data).expect("test"), ModelFormat::Apr);
    269
    1
        }
    270
    271
        #[test]
    272
    1
        fn test_detect_apr_format_v2() {
    273
    1
            let data = b"APR2xxxxxxxxxxxxxxxx";
    274
    1
            assert_eq!(detect_format(data).expect("test"), ModelFormat::Apr);
    275
    1
        }
    276
    277
        #[test]
    278
    1
        fn test_detect_gguf_format() {
    279
    1
            let data = b"GGUFxxxxxxxxxxxxxxxx";
    280
    1
            assert_eq!(detect_format(data).expect("test"), ModelFormat::Gguf);
    281
    1
        }
    282
    283
        #[test]
    284
    1
        fn test_detect_safetensors_format() {
    285
            // SafeTensors: first 8 bytes are header size (little-endian)
    286
            // A reasonable header size like 1000 bytes
    287
    1
            let mut data = vec![0u8; 16];
    288
    1
            let header_size: u64 = 1000;
    289
    1
            data[0..8].copy_from_slice(&header_size.to_le_bytes());
    290
    1
            assert_eq!(
    291
    1
                detect_format(&data).expect("test"),
    292
                ModelFormat::SafeTensors
    293
            );
    294
    1
        }
    295
    296
        #[test]
    297
    1
        fn test_detect_format_too_short() {
    298
    1
            let data = b"APR"; // Only 3 bytes
    299
    1
            let result = detect_format(data);
    300
    1
            assert!(matches!0(result, Err(FormatError::TooShort { len: 3 })));
    301
    1
        }
    302
    303
        #[test]
    304
    1
        fn test_detect_format_empty() {
    305
    1
            let data: &[u8] = &[];
    306
    1
            let result = detect_format(data);
    307
    1
            assert!(matches!0(result, Err(FormatError::TooShort { len: 0 })));
    308
    1
        }
    309
    310
        #[test]
    311
    1
        fn test_detect_safetensors_header_too_large() {
    312
            // Header size > 100MB should fail (DOS protection)
    313
    1
            let mut data = vec![0u8; 16];
    314
    1
            let header_size: u64 = 200_000_000; // 200MB
    315
    1
            data[0..8].copy_from_slice(&header_size.to_le_bytes());
    316
    1
            let result = detect_format(&data);
    317
    1
            assert!(matches!0(
    318
    1
                result,
    319
                Err(FormatError::HeaderTooLarge { size: 200_000_000 })
    320
            ));
    321
    1
        }
    322
    323
        #[test]
    324
    1
        fn test_detect_unknown_format() {
    325
            // Random bytes that don't match any format
    326
            // Zero header size means not SafeTensors either
    327
    1
            let data = b"\x00\x00\x00\x00\x00\x00\x00\x00xxxx";
    328
    1
            let result = detect_format(data);
    329
    1
            assert!(matches!0(result, Err(FormatError::UnknownFormat)));
    330
    1
        }
    331
    332
        #[test]
    333
    1
        fn test_detect_format_from_path_apr() {
    334
    1
            let path = Path::new("model.apr");
    335
    1
            assert_eq!(
    336
    1
                detect_format_from_path(path).expect("test"),
    337
                ModelFormat::Apr
    338
            );
    339
    1
        }
    340
    341
        #[test]
    342
    1
        fn test_detect_format_from_path_gguf() {
    343
    1
            let path = Path::new("llama-7b-q4.gguf");
    344
    1
            assert_eq!(
    345
    1
                detect_format_from_path(path).expect("test"),
    346
                ModelFormat::Gguf
    347
            );
    348
    1
        }
    349
    350
        #[test]
    351
    1
        fn test_detect_format_from_path_safetensors() {
    352
    1
            let path = Path::new("model.safetensors");
    353
    1
            assert_eq!(
    354
    1
                detect_format_from_path(path).expect("test"),
    355
                ModelFormat::SafeTensors
    356
            );
    357
    1
        }
    358
    359
        #[test]
    360
    1
        fn test_detect_format_from_path_unknown() {
    361
    1
            let path = Path::new("model.bin");
    362
    1
            let result = detect_format_from_path(path);
    363
    1
            assert!(matches!0(result, Err(FormatError::UnknownFormat)));
    364
    1
        }
    365
    366
        #[test]
    367
    1
        fn test_detect_format_from_path_uppercase() {
    368
            // Extension comparison should be case-insensitive
    369
    1
            let path = Path::new("MODEL.APR");
    370
    1
            assert_eq!(
    371
    1
                detect_format_from_path(path).expect("test"),
    372
                ModelFormat::Apr
    373
            );
    374
    1
        }
    375
    376
        #[test]
    377
    1
        fn test_detect_and_verify_format_match() {
    378
    1
            let path = Path::new("model.apr");
    379
    1
            let data = b"APR\0xxxxxxxxxxxxxxxx";
    380
    1
            assert_eq!(
    381
    1
                detect_and_verify_format(path, data).expect("test"),
    382
                ModelFormat::Apr
    383
            );
    384
    1
        }
    385
    386
        #[test]
    387
    1
        fn test_detect_and_verify_format_mismatch() {
    388
    1
            let path = Path::new("model.apr"); // Says APR
    389
    1
            let data = b"GGUFxxxxxxxxxxxxxxxx"; // But data is GGUF
    390
    1
            let result = detect_and_verify_format(path, data);
    391
    1
            assert!(matches!0(
    392
    1
                result,
    393
                Err(FormatError::ExtensionMismatch {
    394
                    detected: ModelFormat::Gguf,
    395
                    ..
    396
                })
    397
            ));
    398
    1
        }
    399
    400
        #[test]
    401
    1
        fn test_detect_and_verify_unknown_extension_ok() {
    402
            // Unknown extension but valid magic should work
    403
    1
            let path = Path::new("model.bin");
    404
    1
            let data = b"APR\0xxxxxxxxxxxxxxxx";
    405
    1
            assert_eq!(
    406
    1
                detect_and_verify_format(path, data).expect("test"),
    407
                ModelFormat::Apr
    408
            );
    409
    1
        }
    410
    411
        #[test]
    412
    1
        fn test_model_format_display() {
    413
    1
            assert_eq!(format!("{}", ModelFormat::Apr), "APR");
    414
    1
            assert_eq!(format!("{}", ModelFormat::Gguf), "GGUF");
    415
    1
            assert_eq!(format!("{}", ModelFormat::SafeTensors), "SafeTensors");
    416
    1
        }
    417
    418
        #[test]
    419
    1
        fn test_format_error_display() {
    420
    1
            let err = FormatError::TooShort { len: 5 };
    421
    1
            assert!(err.to_string().contains("5 bytes"));
    422
    423
    1
            let err = FormatError::UnknownFormat;
    424
    1
            assert!(err.to_string().contains("Unknown"));
    425
    426
    1
            let err = FormatError::HeaderTooLarge { size: 999 };
    427
    1
            assert!(err.to_string().contains("999 bytes"));
    428
    429
    1
            let err = FormatError::ExtensionMismatch {
    430
    1
                detected: ModelFormat::Gguf,
    431
    1
                extension: "apr".to_string(),
    432
    1
            };
    433
    1
            assert!(err.to_string().contains("GGUF"));
    434
    1
            assert!(err.to_string().contains(".apr"));
    435
    1
        }
    436
    437
        #[test]
    438
    1
        fn test_magic_constants() {
    439
            // APR_MAGIC is now 3 bytes, version in 4th byte
    440
    1
            assert_eq!(APR_MAGIC, b"APR");
    441
    1
            assert_eq!(GGUF_MAGIC, b"GGUF");
    442
    1
            assert_eq!(MAX_SAFETENSORS_HEADER, 100_000_000);
    443
    1
        }
    444
    445
        // ===== Property-based edge cases =====
    446
    447
        #[test]
    448
    1
        fn test_exactly_8_bytes_safetensors() {
    449
            // Exactly 8 bytes with valid header size
    450
    1
            let header_size: u64 = 500;
    451
    1
            let data = header_size.to_le_bytes();
    452
    1
            assert_eq!(
    453
    1
                detect_format(&data).expect("test"),
    454
                ModelFormat::SafeTensors
    455
            );
    456
    1
        }
    457
    458
        #[test]
    459
    1
        fn test_apr_with_trailing_data() {
    460
            // APR magic followed by lots of other data
    461
    1
            let mut data = b"APR\0".to_vec();
    462
    1
            data.extend_from_slice(&[0u8; 1000]);
    463
    1
            assert_eq!(detect_format(&data).expect("test"), ModelFormat::Apr);
    464
    1
        }
    465
    466
        #[test]
    467
    1
        fn test_gguf_with_trailing_data() {
    468
            // GGUF magic followed by lots of other data
    469
    1
            let mut data = b"GGUF".to_vec();
    470
    1
            data.extend_from_slice(&[0u8; 1000]);
    471
    1
            assert_eq!(detect_format(&data).expect("test"), ModelFormat::Gguf);
    472
    1
        }
    473
    474
        #[test]
    475
    1
        fn test_safetensors_boundary_header_size() {
    476
            // Just under the limit
    477
    1
            let mut data = vec![0u8; 16];
    478
    1
            let header_size: u64 = MAX_SAFETENSORS_HEADER - 1;
    479
    1
            data[0..8].copy_from_slice(&header_size.to_le_bytes());
    480
    1
            assert_eq!(
    481
    1
                detect_format(&data).expect("test"),
    482
                ModelFormat::SafeTensors
    483
            );
    484
    1
        }
    485
    486
        #[test]
    487
    1
        fn test_safetensors_exactly_at_limit() {
    488
            // Exactly at limit should fail
    489
    1
            let mut data = vec![0u8; 16];
    490
    1
            let header_size: u64 = MAX_SAFETENSORS_HEADER;
    491
    1
            data[0..8].copy_from_slice(&header_size.to_le_bytes());
    492
    1
            let result = detect_format(&data);
    493
    1
            assert!(matches!0(result, Err(FormatError::HeaderTooLarge { .. })));
    494
    1
        }
    495
    }