## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/gguf/utils.rs

    Line
    Count
    Source
    1
    //! GGUF utility functions
    2
    //!
    3
    //! Private helpers used across GGUF modules for parsing, tensor manipulation,
    4
    //! and inference operations.
    5
    6
    use std::sync::OnceLock;
    7
    8
    // ============================================================================
    9
    // Verbose mode helper
    10
    // ============================================================================
    11
    12
    /// Check if verbose mode is enabled (REALIZAR_VERBOSE=1)
    13
    /// Default is quiet - only errors are printed
    14
    1
    pub(crate) fn verbose() -> bool {
    15
        static VERBOSE: OnceLock<bool> = OnceLock::new();
    16
    1
        *VERBOSE.get_or_init(|| std::env::var("REALIZAR_VERBOSE").is_ok())
    17
    1
    }
    18
    19
    // ============================================================================
    20
    // GPT-2 BPE Unicode utilities
    21
    // ============================================================================
    22
    23
    /// Convert GPT-2 style byte-level BPE unicode character back to raw byte.
    24
    ///
    25
    /// GPT-2's byte-level BPE uses a mapping where:
    26
    /// - Printable ASCII (0x21-0x7E) and Latin-1 (0xA1-0xAC, 0xAE-0xFF) map to themselves
    27
    /// - Other bytes (0x00-0x20, 0x7F-0xA0, 0xAD) map to U+0100-U+0143
    28
    ///
    29
    /// This function returns the original byte value for a GPT-2 BPE token character.
    30
    #[inline]
    31
    14
    pub(crate) fn gpt2_unicode_to_byte(c: char) -> Option<u8> {
    32
    14
        let cp = c as u32;
    33
    34
        // Special encoded bytes: U+0100 to U+0143 map back to non-printable/special bytes
    35
    14
        if (0x0100..=0x0143).contains(&cp) {
    36
    4
            let offset = (cp - 0x0100) as u8;
    37
            // The special bytes in order: 0x00-0x20 (0-32), then 0x7F (33), then 0x80-0xA0 (34-66), then 0xAD (67)
    38
    4
            let byte = if offset <= 32 {
    39
    3
                offset // 0x00-0x20
    40
    1
            } else if offset == 33 {
    41
    1
                0x7F // DEL
    42
    0
            } else if offset <= 66 {
    43
    0
                0x80 + (offset - 34) // 0x80-0xA0
    44
            } else {
    45
    0
                0xAD // Soft hyphen
    46
            };
    47
    4
            Some(byte)
    48
    10
        } else if cp <= 0xFF {
    49
            // Direct mapping for printable chars
    50
    10
            Some(cp as u8)
    51
        } else {
    52
    0
            None
    53
        }
    54
    14
    }
    55
    56
    /// Decode a GPT-2 style byte-level BPE token to raw bytes.
    57
    ///
    58
    /// Each character in the token may represent either a direct byte (printable ASCII/Latin-1)
    59
    /// or an encoded byte (using Unicode codepoints U+0100-U+0143).
    60
    2
    pub(crate) fn decode_gpt2_token_to_bytes(token: &str) -> Vec<u8> {
    61
    2
        token.chars().filter_map(gpt2_unicode_to_byte).collect()
    62
    2
    }
    63
    64
    #[cfg(test)]
    65
    mod tests {
    66
        use super::*;
    67
    68
        #[test]
    69
    1
        fn test_verbose_default_false() {
    70
            // Unless REALIZAR_VERBOSE is set, should be false
    71
            // Note: This may be true if env var is set during testing
    72
    1
            let _ = verbose(); // Just verify it doesn't panic
    73
    1
        }
    74
    75
        #[test]
    76
    1
        fn test_gpt2_unicode_to_byte_printable() {
    77
            // Printable ASCII maps to itself
    78
    1
            assert_eq!(gpt2_unicode_to_byte('A'), Some(0x41));
    79
    1
            assert_eq!(gpt2_unicode_to_byte('z'), Some(0x7A));
    80
    1
            assert_eq!(gpt2_unicode_to_byte('!'), Some(0x21));
    81
    1
        }
    82
    83
        #[test]
    84
    1
        fn test_gpt2_unicode_to_byte_special() {
    85
            // Special encoded bytes U+0100-U+0143
    86
    1
            assert_eq!(gpt2_unicode_to_byte('\u{0100}'), Some(0x00)); // NUL
    87
    1
            assert_eq!(gpt2_unicode_to_byte('\u{0120}'), Some(0x20)); // Space (offset 32)
    88
    1
            assert_eq!(gpt2_unicode_to_byte('\u{0121}'), Some(0x7F)); // DEL (offset 33)
    89
    1
        }
    90
    91
        #[test]
    92
    1
        fn test_decode_gpt2_token() {
    93
    1
            let token = "Hello";
    94
    1
            let bytes = decode_gpt2_token_to_bytes(token);
    95
    1
            assert_eq!(bytes, b"Hello");
    96
    1
        }
    97
    98
        #[test]
    99
    1
        fn test_decode_gpt2_token_with_special() {
    100
            // Token with encoded space character
    101
    1
            let bytes = decode_gpt2_token_to_bytes("A\u{0120}B");
    102
    1
            assert_eq!(bytes, vec![0x41, 0x20, 0x42]); // A, space, B
    103
    1
        }
    104
    }