## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/tokenizer.rs

    Line
    Count
    Source
    1
    //! Tokenizer for text encoding and decoding
    2
    //!
    3
    //! Implements tokenization for transformer models:
    4
    //! - BPE (Byte Pair Encoding) - Used by GPT models
    5
    //! - Vocabulary management
    6
    //! - Special token handling
    7
    //!
    8
    //! ## Example
    9
    //!
    10
    //! ```rust,ignore
    11
    //! use realizar::Tokenizer;
    12
    //!
    13
    //! let tokenizer = Tokenizer::from_vocab(vocab);
    14
    //! let token_ids = tokenizer.encode("Hello, world!")?;
    15
    //! let text = tokenizer.decode(&token_ids)?;
    16
    //! ```
    17
    18
    use std::collections::HashMap;
    19
    20
    use crate::error::{RealizarError, Result};
    21
    22
    /// Vocabulary mapping between tokens and IDs
    23
    #[derive(Debug, Clone)]
    24
    pub struct Vocabulary {
    25
        /// Token to ID mapping
    26
        token_to_id: HashMap<String, u32>,
    27
        /// ID to token mapping
    28
        id_to_token: HashMap<u32, String>,
    29
    }
    30
    31
    impl Vocabulary {
    32
        /// Create a new vocabulary from token list
    33
        ///
    34
        /// # Arguments
    35
        ///
    36
        /// * `tokens` - List of tokens in order (index = token ID)
    37
        ///
    38
        /// # Errors
    39
        ///
    40
        /// Returns error if tokens list is empty or contains duplicates
    41
        ///
    42
        /// # Examples
    43
        ///
    44
        /// ```rust,ignore
    45
        /// let vocab = Vocabulary::from_tokens(vec![
    46
        ///     "<unk>".to_string(),
    47
        ///     "hello".to_string(),
    48
        ///     "world".to_string(),
    49
        /// ])?;
    50
        /// ```
    51
    11
        pub fn from_tokens(tokens: Vec<String>) -> Result<Self> {
    52
    11
            if tokens.is_empty() {
    53
    1
                return Err(RealizarError::UnsupportedOperation {
    54
    1
                    operation: "create_vocabulary".to_string(),
    55
    1
                    reason: "Vocabulary cannot be empty".to_string(),
    56
    1
                });
    57
    10
            }
    58
    59
    10
            let mut token_to_id = HashMap::new();
    60
    10
            let mut id_to_token = HashMap::new();
    61
    62
    26
            for (id, token) in tokens10.into_iter10().enumerate10() {
    63
    26
                let id = u32::try_from(id).map_err(|_| RealizarError::UnsupportedOperation {
    64
    0
                    operation: "convert_token_id".to_string(),
    65
    0
                    reason: format!("Token ID {id} exceeds u32 limit"),
    66
    0
                })?;
    67
    68
    26
                if token_to_id.contains_key(&token) {
    69
    1
                    return Err(RealizarError::UnsupportedOperation {
    70
    1
                        operation: "create_vocabulary".to_string(),
    71
    1
                        reason: format!("Duplicate token: {token}"),
    72
    1
                    });
    73
    25
                }
    74
    75
    25
                token_to_id.insert(token.clone(), id);
    76
    25
                id_to_token.insert(id, token);
    77
            }
    78
    79
    9
            Ok(Self {
    80
    9
                token_to_id,
    81
    9
                id_to_token,
    82
    9
            })
    83
    11
        }
    84
    85
        /// Get token ID for a token
    86
        ///
    87
        /// Returns `None` if token not in vocabulary
    88
        #[must_use]
    89
    33
        pub fn get_id(&self, token: &str) -> Option<u32> {
    90
    33
            self.token_to_id.get(token).copied()
    91
    33
        }
    92
    93
        /// Get token for a token ID
    94
        ///
    95
        /// Returns `None` if ID not in vocabulary
    96
        #[must_use]
    97
    20
        pub fn get_token(&self, id: u32) -> Option<&str> {
    98
    20
            self.id_to_token.get(&id).map(String::as_str)
    99
    20
        }
    100
    101
        /// Get vocabulary size
    102
        #[must_use]
    103
    2
        pub fn size(&self) -> usize {
    104
    2
            self.token_to_id.len()
    105
    2
        }
    106
    }
    107
    108
    /// Tokenizer for encoding and decoding text
    109
    #[derive(Debug, Clone)]
    110
    pub struct Tokenizer {
    111
        /// Vocabulary
    112
        vocab: Vocabulary,
    113
        /// Unknown token ID
    114
        unk_token_id: u32,
    115
    }
    116
    117
    /// BPE (Byte Pair Encoding) tokenizer
    118
    ///
    119
    /// Implements subword tokenization using byte pair encoding algorithm.
    120
    /// Used by GPT-2, GPT-3, and many other models.
    121
    #[derive(Debug, Clone)]
    122
    pub struct BPETokenizer {
    123
        /// Token to ID mapping
    124
        token_to_id: HashMap<String, u32>,
    125
        /// ID to token mapping
    126
        id_to_token: HashMap<u32, String>,
    127
        /// Merge rules: pairs to merge in order of priority
    128
        merges: Vec<(String, String)>,
    129
        /// Unknown token ID
    130
        unk_token_id: u32,
    131
    }
    132
    133
    impl BPETokenizer {
    134
        /// Create a new BPE tokenizer
    135
        ///
    136
        /// # Arguments
    137
        ///
    138
        /// * `vocab` - List of tokens (index = token ID)
    139
        /// * `merges` - List of merge pairs in priority order
    140
        /// * `unk_token` - Unknown token string
    141
        ///
    142
        /// # Errors
    143
        ///
    144
        /// Returns error if vocabulary is empty or unknown token not found
    145
    196
        pub fn new(vocab: Vec<String>, merges: Vec<(String, String)>, unk_token: &str) -> Result<Self> {
    146
    196
            if vocab.is_empty() {
    147
    1
                return Err(RealizarError::UnsupportedOperation {
    148
    1
                    operation: "create_bpe_tokenizer".to_string(),
    149
    1
                    reason: "Vocabulary cannot be empty".to_string(),
    150
    1
                });
    151
    195
            }
    152
    153
    195
            let mut token_to_id = HashMap::new();
    154
    195
            let mut id_to_token = HashMap::new();
    155
    156
    16.3k
            for (id, token) in vocab195.into_iter195().enumerate195() {
    157
    16.3k
                let id = u32::try_from(id).map_err(|_| RealizarError::UnsupportedOperation {
    158
    0
                    operation: "convert_token_id".to_string(),
    159
    0
                    reason: format!("Token ID {id} exceeds u32 limit"),
    160
    0
                })?;
    161
    16.3k
                token_to_id.insert(token.clone(), id);
    162
    16.3k
                id_to_token.insert(id, token);
    163
            }
    164
    165
    194
            let unk_token_id =
    166
    195
                *token_to_id
    167
    195
                    .get(unk_token)
    168
    195
                    .ok_or_else(|| RealizarError::UnsupportedOperation {
    169
    1
                        operation: "create_bpe_tokenizer".to_string(),
    170
    1
                        reason: format!("Unknown token '{unk_token}' not in vocabulary"),
    171
    1
                    })?;
    172
    173
    194
            Ok(Self {
    174
    194
                token_to_id,
    175
    194
                id_to_token,
    176
    194
                merges,
    177
    194
                unk_token_id,
    178
    194
            })
    179
    196
        }
    180
    181
        /// Encode text to token IDs using greedy longest match
    182
        ///
    183
        /// Uses GPT-2 style encoding where spaces become Ġ (U+0120) and
    184
        /// newlines become Ċ (U+010A).
    185
        ///
    186
        /// # Arguments
    187
        ///
    188
        /// * `text` - Input text to encode
    189
        ///
    190
        /// # Returns
    191
        ///
    192
        /// Vector of token IDs
    193
        #[must_use]
    194
    74
        pub fn encode(&self, text: &str) -> Vec<u32> {
    195
    74
            if text.is_empty() {
    196
    8
                return Vec::new();
    197
    66
            }
    198
    199
            // Convert to GPT-2 encoding: space -> Ġ, newline -> Ċ
    200
    66
            let processed: String = text
    201
    66
                .chars()
    202
    412
                .map66(|c| match c {
    203
    16
                    ' ' => 'Ġ',  // U+0120
    204
    0
                    '\n' => 'Ċ', // U+010A
    205
    0
                    '\r' => 'Ḃ', // U+1E02
    206
    396
                    _ => c,
    207
    412
                })
    208
    66
                .collect();
    209
    210
    66
            let mut tokens = Vec::new();
    211
    66
            let mut remaining = processed.as_str();
    212
    213
    344
            while !remaining.is_empty() {
    214
                // Greedy longest match
    215
    278
                let mut best_len = 0;
    216
    278
                let mut best_id = None;
    217
    218
                // Collect character byte offsets for proper slicing
    219
    278
                let char_indices: Vec<usize> = remaining
    220
    278
                    .char_indices()
    221
    278
                    .map(|(i, _)| i)
    222
    278
                    .chain(std::iter::once(remaining.len()))
    223
    278
                    .collect();
    224
    225
                // Try all prefixes up to 32 chars
    226
    1.87k
                for char_count in 1..=char_indices278.len278().saturating_sub278(1).min278(32) {
    227
    1.87k
                    let byte_end = char_indices[char_count];
    228
    1.87k
                    let prefix = &remaining[..byte_end];
    229
    1.87k
                    if let Some(&id56) = self.token_to_id.get(prefix) {
    230
    56
                        best_len = byte_end;
    231
    56
                        best_id = Some(id);
    232
    1.81k
                    }
    233
                }
    234
    235
    278
                if let Some(id44) = best_id {
    236
    44
                    tokens.push(id);
    237
    44
                    remaining = &remaining[best_len..];
    238
    44
                } else {
    239
                    // No match - try byte tokens like <0x48>
    240
    234
                    let ch = remaining.chars().next().expect("non-empty");
    241
    234
                    let ch_len = ch.len_utf8();
    242
    243
    249
                    for byte in remaining[..ch_len]234.bytes234() {
    244
    249
                        let byte_token = format!("<0x{byte:02X}>");
    245
    249
                        if let Some(&id0) = self.token_to_id.get(&byte_token) {
    246
    0
                            tokens.push(id);
    247
    249
                        } else {
    248
    249
                            tokens.push(self.unk_token_id);
    249
    249
                        }
    250
                    }
    251
    234
                    remaining = &remaining[ch_len..];
    252
                }
    253
            }
    254
    255
    66
            tokens
    256
    74
        }
    257
    258
        /// Apply a single merge rule to token list
    259
    0
        fn apply_merge(tokens: &[String], first: &str, second: &str) -> Vec<String> {
    260
    0
            if tokens.len() < 2 {
    261
    0
                return tokens.to_vec();
    262
    0
            }
    263
    264
    0
            let mut result = Vec::new();
    265
    0
            let mut i = 0;
    266
    267
    0
            while i < tokens.len() {
    268
    0
                if i + 1 < tokens.len() && tokens[i] == first && tokens[i + 1] == second {
    269
    0
                    // Merge the pair
    270
    0
                    result.push(format!("{first}{second}"));
    271
    0
                    i += 2;
    272
    0
                } else {
    273
    0
                    result.push(tokens[i].clone());
    274
    0
                    i += 1;
    275
    0
                }
    276
            }
    277
    278
    0
            result
    279
    0
        }
    280
    281
        /// Decode token IDs to text
    282
        ///
    283
        /// # Arguments
    284
        ///
    285
        /// * `token_ids` - Token IDs to decode
    286
        ///
    287
        /// # Errors
    288
        ///
    289
        /// Returns error if any token ID is invalid
    290
    47
        pub fn decode(&self, token_ids: &[u32]) -> Result<String> {
    291
    47
            let mut bytes: Vec<u8> = Vec::new();
    292
    293
    870
            for &id824 in token_ids {
    294
    823
                let token =
    295
    824
                    self.id_to_token
    296
    824
                        .get(&id)
    297
    824
                        .ok_or_else(|| RealizarError::UnsupportedOperation {
    298
    1
                            operation: "decode_bpe_token".to_string(),
    299
    1
                            reason: format!("Invalid token ID: {id}"),
    300
    1
                        })?;
    301
    302
                // Skip special tokens
    303
    823
                if token.starts_with("<|") && token.ends_with("|>")1 {
    304
    1
                    continue;
    305
    822
                }
    306
    822
                if token == "<s>" || token == "</s>"821 || token == "<unk>"820 || token == "<pad>"314 {
    307
    509
                    continue;
    308
    313
                }
    309
    310
                // Handle byte tokens like <0xE6>
    311
    313
                if token.starts_with("<0x") && token.ends_with('>')4 && token.len() == 64 {
    312
    4
                    if let Ok(byte_val3) = u8::from_str_radix(&token[3..5], 16) {
    313
    3
                        bytes.push(byte_val);
    314
    3
                        continue;
    315
    1
                    }
    316
    309
                }
    317
    318
                // Decode GPT-2 style byte-level BPE
    319
    2.01k
                for c in token310.chars310() {
    320
    2.01k
                    match c {
    321
    3
                        'Ġ' => bytes.push(b' '),  // U+0120 -> space
    322
    2
                        'Ċ' => bytes.push(b'\n'), // U+010A -> newline
    323
    1
                        'ċ' => bytes.push(b'\n'), // lowercase variant
    324
    1
                        'Ḃ' => bytes.push(b'\r'), // U+1E02 -> carriage return
    325
    1
                        '▁' => bytes.push(b' '),  // U+2581 SentencePiece -> space
    326
                        _ => {
    327
                            // Try GPT-2 unicode-to-byte mapping
    328
    2.01k
                            if let Some(byte2.00k) = Self::gpt2_char_to_byte(c) {
    329
    2.00k
                                bytes.push(byte);
    330
    2.00k
                            } else {
    331
    5
                                // Regular UTF-8 character
    332
    5
                                let mut buf = [0u8; 4];
    333
    5
                                let encoded = c.encode_utf8(&mut buf);
    334
    5
                                bytes.extend_from_slice(encoded.as_bytes());
    335
    5
                            }
    336
                        },
    337
                    }
    338
                }
    339
            }
    340
    341
            // Decode as UTF-8, replacing invalid sequences
    342
    46
            Ok(String::from_utf8_lossy(&bytes).into_owned())
    343
    47
        }
    344
    345
        /// Convert GPT-2 unicode character to original byte value
    346
    2.01k
        fn gpt2_char_to_byte(c: char) -> Option<u8> {
    347
            // GPT-2 maps bytes 0-255 to unicode characters
    348
            // Printable ASCII (33-126) maps to itself
    349
            // Other bytes map to unicode range starting at U+0100
    350
    2.01k
            let code = c as u32;
    351
    2.01k
            if (33..=126).contains(&code) || code == 329 {
    352
    2.00k
                Some(code as u8)
    353
    8
            } else if (0x100..=0x100 + 255).contains(&code) {
    354
                // GPT-2 remapped bytes
    355
    3
                let byte = (code - 0x100) as u8;
    356
                // Map back based on GPT-2's byte_encoder
    357
    3
                match byte {
    358
    3
                    0..=32 => Some(byte)0,    // Control chars + space
    359
    3
                    127..=160 => Some(byte)2, // DEL + extended ASCII
    360
    1
                    173 => Some(173),        // Soft hyphen
    361
    0
                    _ => None,
    362
                }
    363
            } else {
    364
    5
                None
    365
            }
    366
    2.01k
        }
    367
    368
        /// Get vocabulary size
    369
        #[must_use]
    370
    2
        pub fn vocab_size(&self) -> usize {
    371
    2
            self.token_to_id.len()
    372
    2
        }
    373
    374
        /// Get token ID for a token
    375
        #[must_use]
    376
    2
        pub fn get_token_id(&self, token: &str) -> Option<u32> {
    377
    2
            self.token_to_id.get(token).copied()
    378
    2
        }
    379
    380
        /// Get token for a token ID
    381
        #[must_use]
    382
    2
        pub fn get_token(&self, id: u32) -> Option<&str> {
    383
    2
            self.id_to_token.get(&id).map(String::as_str)
    384
    2
        }
    385
    }
    386
    387
    /// Viterbi algorithm state: `(best_score, best_token)` at each position
    388
    type ViterbiState = (Vec<f32>, Vec<Option<String>>);
    389
    390
    /// `SentencePiece` tokenizer (Unigram model)
    391
    ///
    392
    /// Implements subword tokenization using unigram language model.
    393
    /// Used by `LLaMA`, T5, ALBERT, and many other models.
    394
    ///
    395
    /// Unlike BPE which uses greedy merges, `SentencePiece` finds the
    396
    /// most likely segmentation using token scores (log probabilities).
    397
    #[derive(Debug, Clone)]
    398
    pub struct SentencePieceTokenizer {
    399
        /// Token to ID mapping
    400
        token_to_id: HashMap<String, u32>,
    401
        /// ID to token mapping
    402
        id_to_token: HashMap<u32, String>,
    403
        /// Token scores (log probabilities)
    404
        scores: HashMap<String, f32>,
    405
        /// Unknown token ID
    406
        unk_token_id: u32,
    407
    }
    408
    409
    impl SentencePieceTokenizer {
    410
        /// Create a new `SentencePiece` tokenizer
    411
        ///
    412
        /// # Arguments
    413
        ///
    414
        /// * `vocab` - List of (token, score) pairs where score is log probability
    415
        /// * `unk_token` - Unknown token string
    416
        ///
    417
        /// # Errors
    418
        ///
    419
        /// Returns error if vocabulary is empty or unknown token not found
    420
    15
        pub fn new(vocab: Vec<(String, f32)>, unk_token: &str) -> Result<Self> {
    421
    15
            if vocab.is_empty() {
    422
    1
                return Err(RealizarError::UnsupportedOperation {
    423
    1
                    operation: "create_sentencepiece_tokenizer".to_string(),
    424
    1
                    reason: "Vocabulary cannot be empty".to_string(),
    425
    1
                });
    426
    14
            }
    427
    428
    14
            let mut token_to_id = HashMap::new();
    429
    14
            let mut id_to_token = HashMap::new();
    430
    14
            let mut scores = HashMap::new();
    431
    432
    46
            for (id, (token, score)) in vocab14.into_iter14().enumerate14() {
    433
    46
                let id = u32::try_from(id).map_err(|_| RealizarError::UnsupportedOperation {
    434
    0
                    operation: "convert_token_id".to_string(),
    435
    0
                    reason: format!("Token ID {id} exceeds u32 limit"),
    436
    0
                })?;
    437
    46
                token_to_id.insert(token.clone(), id);
    438
    46
                id_to_token.insert(id, token.clone());
    439
    46
                scores.insert(token, score);
    440
            }
    441
    442
    13
            let unk_token_id =
    443
    14
                *token_to_id
    444
    14
                    .get(unk_token)
    445
    14
                    .ok_or_else(|| RealizarError::UnsupportedOperation {
    446
    1
                        operation: "create_sentencepiece_tokenizer".to_string(),
    447
    1
                        reason: format!("Unknown token '{unk_token}' not in vocabulary"),
    448
    1
                    })?;
    449
    450
    13
            Ok(Self {
    451
    13
                token_to_id,
    452
    13
                id_to_token,
    453
    13
                scores,
    454
    13
                unk_token_id,
    455
    13
            })
    456
    15
        }
    457
    458
        /// Encode text to token IDs using Viterbi algorithm
    459
        ///
    460
        /// Finds the most likely segmentation based on token scores.
    461
        ///
    462
        /// # Arguments
    463
        ///
    464
        /// * `text` - Input text to encode
    465
        ///
    466
        /// # Returns
    467
        ///
    468
        /// Vector of token IDs
    469
        #[must_use]
    470
    7
        pub fn encode(&self, text: &str) -> Vec<u32> {
    471
    7
            if text.is_empty() {
    472
    1
                return Vec::new();
    473
    6
            }
    474
    475
    6
            let chars: Vec<char> = text.chars().collect();
    476
    6
            let (_best_score, best_token) = self.viterbi_forward(&chars);
    477
    6
            let tokens = Self::viterbi_backtrack(&chars, &best_token);
    478
    479
            // Convert tokens to IDs
    480
    6
            tokens
    481
    6
                .into_iter()
    482
    11
                .map6(|t| {
    483
    11
                    self.token_to_id
    484
    11
                        .get(&t)
    485
    11
                        .copied()
    486
    11
                        .unwrap_or(self.unk_token_id)
    487
    11
                })
    488
    6
                .collect()
    489
    7
        }
    490
    491
        /// Viterbi forward pass: find best score and token at each position
    492
        ///
    493
        /// # Arguments
    494
        ///
    495
        /// * `chars` - Character array of input text
    496
        ///
    497
        /// # Returns
    498
        ///
    499
        /// Tuple of `(best_score, best_token)` vectors
    500
    6
        fn viterbi_forward(&self, chars: &[char]) -> ViterbiState {
    501
    6
            let n = chars.len();
    502
    6
            let mut best_score = vec![f32::NEG_INFINITY; n + 1];
    503
    6
            let mut best_token: Vec<Option<String>> = vec![None; n + 1];
    504
    6
            best_score[0] = 0.0;
    505
    506
    34
            for end in 1..=n6 {
    507
                // Try all possible start positions for token ending at `end`
    508
    132
                for start in 0..end34 {
    509
    132
                    let substr: String = chars[start..end].iter().collect();
    510
    132
                    if let Some(&score27) = self.scores.get(&substr) {
    511
    27
                        let new_score = best_score[start] + score;
    512
    27
                        if new_score > best_score[end] {
    513
    21
                            best_score[end] = new_score;
    514
    21
                            best_token[end] = Some(substr);
    515
    21
                        }6
    516
    105
                    }
    517
                }
    518
    519
                // If no token found ending at this position, use single character as unknown
    520
    34
                if best_token[end].is_none() && best_score[end - 1] > f32::NEG_INFINITY13 {
    521
    13
                    let char_str: String = chars[end - 1..end].iter().collect();
    522
    13
                    best_score[end] = best_score[end - 1] - 100.0; // Penalty for unknown
    523
    13
                    best_token[end] = Some(char_str);
    524
    21
                }
    525
            }
    526
    527
    6
            (best_score, best_token)
    528
    6
        }
    529
    530
        /// Viterbi backtracking: reconstruct token sequence from `best_token`
    531
        ///
    532
        /// # Arguments
    533
        ///
    534
        /// * `chars` - Character array of input text
    535
        /// * `best_token` - Best token at each position (from forward pass)
    536
        ///
    537
        /// # Returns
    538
        ///
    539
        /// Vector of tokens in forward order
    540
    6
        fn viterbi_backtrack(chars: &[char], best_token: &[Option<String>]) -> Vec<String> {
    541
    6
            let n = chars.len();
    542
    6
            let mut tokens = Vec::new();
    543
    6
            let mut pos = n;
    544
    545
    17
            while pos > 0 {
    546
    11
                if let Some(token) = &best_token[pos] {
    547
    11
                    tokens.push(token.clone());
    548
    11
                    pos -= token.chars().count();
    549
    11
                } else {
    550
    0
                    // Fallback: single character
    551
    0
                    let char_str: String = chars[pos - 1..pos].iter().collect();
    552
    0
                    tokens.push(char_str);
    553
    0
                    pos -= 1;
    554
    0
                }
    555
            }
    556
    557
    6
            tokens.reverse();
    558
    6
            tokens
    559
    6
        }
    560
    561
        /// Decode token IDs to text
    562
        ///
    563
        /// # Arguments
    564
        ///
    565
        /// * `token_ids` - Token IDs to decode
    566
        ///
    567
        /// # Errors
    568
        ///
    569
        /// Returns error if any token ID is invalid
    570
    4
        pub fn decode(&self, token_ids: &[u32]) -> Result<String> {
    571
    4
            let mut result = String::new();
    572
    573
    8
            for &id5 in token_ids {
    574
    4
                let token =
    575
    5
                    self.id_to_token
    576
    5
                        .get(&id)
    577
    5
                        .ok_or_else(|| RealizarError::UnsupportedOperation {
    578
    1
                            operation: "decode_sentencepiece_token".to_string(),
    579
    1
                            reason: format!("Invalid token ID: {id}"),
    580
    1
                        })?;
    581
    4
                result.push_str(token);
    582
            }
    583
    584
    3
            Ok(result)
    585
    4
        }
    586
    587
        /// Get vocabulary size
    588
        #[must_use]
    589
    2
        pub fn vocab_size(&self) -> usize {
    590
    2
            self.token_to_id.len()
    591
    2
        }
    592
    593
        /// Get token ID for a token
    594
        #[must_use]
    595
    2
        pub fn get_token_id(&self, token: &str) -> Option<u32> {
    596
    2
            self.token_to_id.get(token).copied()
    597
    2
        }
    598
    599
        /// Get token for a token ID
    600
        #[must_use]
    601
    2
        pub fn get_token(&self, id: u32) -> Option<&str> {
    602
    2
            self.id_to_token.get(&id).map(String::as_str)
    603
    2
        }
    604
    605
        /// Get score for a token
    606
        #[must_use]
    607
    2
        pub fn get_score(&self, token: &str) -> Option<f32> {
    608
    2
            self.scores.get(token).copied()
    609
    2
        }
    610
    }
    611
    612
    impl Tokenizer {
    613
        /// Create a new tokenizer from vocabulary
    614
        ///
    615
        /// # Arguments
    616
        ///
    617
        /// * `vocab` - Vocabulary mapping
    618
        /// * `unk_token` - Unknown token (default: `"<unk>"`)
    619
        ///
    620
        /// # Errors
    621
        ///
    622
        /// Returns error if unknown token not in vocabulary
    623
        ///
    624
        /// # Examples
    625
        ///
    626
        /// ```rust,ignore
    627
        /// let vocab = Vocabulary::from_tokens(vec!["<unk>".to_string(), "hello".to_string()])?;
    628
        /// let tokenizer = Tokenizer::new(vocab, "<unk>")?;
    629
        /// ```
    630
    7
        pub fn new(vocab: Vocabulary, unk_token: &str) -> Result<Self> {
    631
    6
            let unk_token_id =
    632
    7
                vocab
    633
    7
                    .get_id(unk_token)
    634
    7
                    .ok_or_else(|| RealizarError::UnsupportedOperation {
    635
    1
                        operation: "create_tokenizer".to_string(),
    636
    1
                        reason: format!("Unknown token '{unk_token}' not in vocabulary"),
    637
    1
                    })?;
    638
    639
    6
            Ok(Self {
    640
    6
                vocab,
    641
    6
                unk_token_id,
    642
    6
            })
    643
    7
        }
    644
    645
        /// Encode text to token IDs (simple word-level tokenization)
    646
        ///
    647
        /// # Arguments
    648
        ///
    649
        /// * `text` - Input text to encode
    650
        ///
    651
        /// # Returns
    652
        ///
    653
        /// Vector of token IDs
    654
        ///
    655
        /// # Examples
    656
        ///
    657
        /// ```rust,ignore
    658
        /// let token_ids = tokenizer.encode("hello world")?;
    659
        /// assert_eq!(token_ids, vec![1, 2]);
    660
        /// ```
    661
        #[must_use]
    662
    6
        pub fn encode(&self, text: &str) -> Vec<u32> {
    663
    6
            text.split_whitespace()
    664
    22
                .map6(|word| self.vocab.get_id(word).unwrap_or(self.unk_token_id))
    665
    6
                .collect()
    666
    6
        }
    667
    668
        /// Decode token IDs to text
    669
        ///
    670
        /// # Arguments
    671
        ///
    672
        /// * `token_ids` - Token IDs to decode
    673
        ///
    674
        /// # Returns
    675
        ///
    676
        /// Decoded text string
    677
        ///
    678
        /// # Errors
    679
        ///
    680
        /// Returns error if any token ID is invalid
    681
        ///
    682
        /// # Examples
    683
        ///
    684
        /// ```rust,ignore
    685
        /// let text = tokenizer.decode(&[1, 2])?;
    686
        /// assert_eq!(text, "hello world");
    687
        /// ```
    688
    5
        pub fn decode(&self, token_ids: &[u32]) -> Result<String> {
    689
    5
            let tokens: Result<Vec<&str>> = token_ids
    690
    5
                .iter()
    691
    16
                .map5(|&id| {
    692
    16
                    self.vocab
    693
    16
                        .get_token(id)
    694
    16
                        .ok_or_else(|| RealizarError::UnsupportedOperation {
    695
    1
                            operation: "decode_token".to_string(),
    696
    1
                            reason: format!("Invalid token ID: {id}"),
    697
    1
                        })
    698
    16
                })
    699
    5
                .collect();
    700
    701
    5
            Ok(tokens?1.join4(" "4))
    702
    5
        }
    703
    704
        /// Get vocabulary size
    705
        #[must_use]
    706
    1
        pub fn vocab_size(&self) -> usize {
    707
    1
            self.vocab.size()
    708
    1
        }
    709
    }
    710
    711
    #[cfg(test)]
    712
    mod tests {
    713
        use super::*;
    714
    715
        #[test]
    716
    1
        fn test_vocabulary_from_tokens() {
    717
    1
            let tokens = vec![
    718
    1
                "<unk>".to_string(),
    719
    1
                "hello".to_string(),
    720
    1
                "world".to_string(),
    721
            ];
    722
    723
    1
            let vocab = Vocabulary::from_tokens(tokens).expect("test");
    724
    1
            assert_eq!(vocab.size(), 3);
    725
    1
            assert_eq!(vocab.get_id("<unk>"), Some(0));
    726
    1
            assert_eq!(vocab.get_id("hello"), Some(1));
    727
    1
            assert_eq!(vocab.get_id("world"), Some(2));
    728
    1
            assert_eq!(vocab.get_token(0), Some("<unk>"));
    729
    1
            assert_eq!(vocab.get_token(1), Some("hello"));
    730
    1
            assert_eq!(vocab.get_token(2), Some("world"));
    731
    1
        }
    732
    733
        #[test]
    734
    1
        fn test_vocabulary_empty_error() {
    735
    1
            let result = Vocabulary::from_tokens(vec![]);
    736
    1
            assert!(result.is_err());
    737
    1
        }
    738
    739
        #[test]
    740
    1
        fn test_vocabulary_duplicate_error() {
    741
    1
            let tokens = vec![
    742
    1
                "hello".to_string(),
    743
    1
                "world".to_string(),
    744
    1
                "hello".to_string(), // Duplicate
    745
            ];
    746
    1
            let result = Vocabulary::from_tokens(tokens);
    747
    1
            assert!(result.is_err());
    748
    1
        }
    749
    750
        #[test]
    751
    1
        fn test_vocabulary_get_missing() {
    752
    1
            let tokens = vec!["hello".to_string()];
    753
    1
            let vocab = Vocabulary::from_tokens(tokens).expect("test");
    754
    1
            assert_eq!(vocab.get_id("world"), None);
    755
    1
            assert_eq!(vocab.get_token(999), None);
    756
    1
        }
    757
    758
        #[test]
    759
    1
        fn test_tokenizer_encode_decode() {
    760
    1
            let tokens = vec![
    761
    1
                "<unk>".to_string(),
    762
    1
                "hello".to_string(),
    763
    1
                "world".to_string(),
    764
            ];
    765
    1
            let vocab = Vocabulary::from_tokens(tokens).expect("test");
    766
    1
            let tokenizer = Tokenizer::new(vocab, "<unk>").expect("test");
    767
    768
            // Encode known tokens
    769
    1
            let encoded = tokenizer.encode("hello world");
    770
    1
            assert_eq!(encoded, vec![1, 2]);
    771
    772
            // Decode back
    773
    1
            let decoded = tokenizer.decode(&encoded).expect("test");
    774
    1
            assert_eq!(decoded, "hello world");
    775
    1
        }
    776
    777
        #[test]
    778
    1
        fn test_tokenizer_unknown_token() {
    779
    1
            let tokens = vec!["<unk>".to_string(), "hello".to_string()];
    780
    1
            let vocab = Vocabulary::from_tokens(tokens).expect("test");
    781
    1
            let tokenizer = Tokenizer::new(vocab, "<unk>").expect("test");
    782
    783
            // Unknown token should map to <unk> (ID 0)
    784
    1
            let encoded = tokenizer.encode("hello foo");
    785
    1
            assert_eq!(encoded, vec![1, 0]);
    786
    1
        }
    787
    788
        #[test]
    789
    1
        fn test_tokenizer_invalid_unk_token() {
    790
    1
            let tokens = vec!["hello".to_string()];
    791
    1
            let vocab = Vocabulary::from_tokens(tokens).expect("test");
    792
    1
            let result = Tokenizer::new(vocab, "<unk>");
    793
    1
            assert!(result.is_err());
    794
    1
        }
    795
    796
        #[test]
    797
    1
        fn test_tokenizer_decode_invalid_id() {
    798
    1
            let tokens = vec!["<unk>".to_string(), "hello".to_string()];
    799
    1
            let vocab = Vocabulary::from_tokens(tokens).expect("test");
    800
    1
            let tokenizer = Tokenizer::new(vocab, "<unk>").expect("test");
    801
    802
    1
            let result = tokenizer.decode(&[1, 999]); // 999 is invalid
    803
    1
            assert!(result.is_err());
    804
    1
        }
    805
    806
        #[test]
    807
    1
        fn test_tokenizer_empty_string() {
    808
    1
            let tokens = vec!["<unk>".to_string()];
    809
    1
            let vocab = Vocabulary::from_tokens(tokens).expect("test");
    810
    1
            let tokenizer = Tokenizer::new(vocab, "<unk>").expect("test");
    811
    812
    1
            let encoded = tokenizer.encode("");
    813
    1
            assert_eq!(encoded, Vec::<u32>::new());
    814
    815
    1
            let decoded = tokenizer.decode(&[]).expect("test");
    816
    1
            assert_eq!(decoded, "");
    817
    1
        }
    818
    819
        #[test]
    820
    1
        fn test_tokenizer_vocab_size() {
    821
    1
            let tokens = vec![
    822
    1
                "<unk>".to_string(),
    823
    1
                "hello".to_string(),
    824
    1
                "world".to_string(),
    825
            ];
    826
    1
            let vocab = Vocabulary::from_tokens(tokens).expect("test");
    827
    1
            let tokenizer = Tokenizer::new(vocab, "<unk>").expect("test");
    828
    829
    1
            assert_eq!(tokenizer.vocab_size(), 3);
    830
    1
        }
    831
    832
        // BPE Tokenizer tests
    833
    834
        #[test]
    835
    1
        fn test_bpe_tokenizer_creation() {
    836
    1
            let vocab = vec![
    837
    1
                "<unk>".to_string(),
    838
    1
                "h".to_string(),
    839
    1
                "e".to_string(),
    840
    1
                "l".to_string(),
    841
    1
                "o".to_string(),
    842
    1
                "he".to_string(),
    843
    1
                "ll".to_string(),
    844
    1
                "hel".to_string(),
    845
    1
                "hello".to_string(),
    846
            ];
    847
    1
            let merges = vec![
    848
    1
                ("h".to_string(), "e".to_string()),
    849
    1
                ("l".to_string(), "l".to_string()),
    850
    1
                ("he".to_string(), "l".to_string()),
    851
    1
                ("hel".to_string(), "lo".to_string()),
    852
            ];
    853
    854
    1
            let tokenizer = BPETokenizer::new(vocab, merges, "<unk>").expect("test");
    855
    1
            assert_eq!(tokenizer.vocab_size(), 9);
    856
    1
        }
    857
    858
        #[test]
    859
    1
        fn test_bpe_tokenizer_empty_vocab_error() {
    860
    1
            let result = BPETokenizer::new(vec![], vec![], "<unk>");
    861
    1
            assert!(result.is_err());
    862
    1
        }
    863
    864
        #[test]
    865
    1
        fn test_bpe_tokenizer_invalid_unk_token_error() {
    866
    1
            let vocab = vec!["hello".to_string()];
    867
    1
            let result = BPETokenizer::new(vocab, vec![], "<unk>");
    868
    1
            assert!(result.is_err());
    869
    1
        }
    870
    871
        #[test]
    872
    1
        fn test_bpe_encode_no_merges() {
    873
            // Simple character-level tokenization without merges
    874
    1
            let vocab = vec!["<unk>".to_string(), "h".to_string(), "i".to_string()];
    875
    1
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>").expect("test");
    876
    877
    1
            let encoded = tokenizer.encode("hi");
    878
    1
            assert_eq!(encoded, vec![1, 2]); // h=1, i=2
    879
    1
        }
    880
    881
        #[test]
    882
    1
        fn test_bpe_encode_with_merges() {
    883
    1
            let vocab = vec![
    884
    1
                "<unk>".to_string(),
    885
    1
                "h".to_string(),
    886
    1
                "e".to_string(),
    887
    1
                "l".to_string(),
    888
    1
                "o".to_string(),
    889
    1
                "he".to_string(),
    890
    1
                "ll".to_string(),
    891
            ];
    892
    1
            let merges = vec![
    893
    1
                ("h".to_string(), "e".to_string()),
    894
    1
                ("l".to_string(), "l".to_string()),
    895
            ];
    896
    897
    1
            let tokenizer = BPETokenizer::new(vocab, merges, "<unk>").expect("test");
    898
    1
            let encoded = tokenizer.encode("hello");
    899
            // h+e -> he, l+l -> ll, o stays
    900
            // so: he, ll, o = [5, 6, 4]
    901
    1
            assert_eq!(encoded, vec![5, 6, 4]);
    902
    1
        }
    903
    904
        #[test]
    905
    1
        fn test_bpe_encode_unknown_char() {
    906
    1
            let vocab = vec!["<unk>".to_string(), "h".to_string(), "i".to_string()];
    907
    1
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>").expect("test");
    908
    909
            // 'x' is not in vocab, should map to <unk>
    910
    1
            let encoded = tokenizer.encode("hix");
    911
    1
            assert_eq!(encoded, vec![1, 2, 0]);
    912
    1
        }
    913
    914
        #[test]
    915
    1
        fn test_bpe_encode_empty_string() {
    916
    1
            let vocab = vec!["<unk>".to_string()];
    917
    1
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>").expect("test");
    918
    919
    1
            let encoded = tokenizer.encode("");
    920
    1
            assert!(encoded.is_empty());
    921
    1
        }
    922
    923
        #[test]
    924
    1
        fn test_bpe_encode_multiple_words() {
    925
            // BPE uses GPT-2 encoding: space -> Ġ (U+0120)
    926
    1
            let vocab = vec![
    927
    1
                "<unk>".to_string(),
    928
    1
                "h".to_string(),
    929
    1
                "i".to_string(),
    930
    1
                "Ġ".to_string(),  // GPT-2 space encoding
    931
    1
                "Ġh".to_string(), // GPT-2 space + h
    932
            ];
    933
    1
            let merges = vec![("Ġ".to_string(), "h".to_string())];
    934
    935
    1
            let tokenizer = BPETokenizer::new(vocab, merges, "<unk>").expect("test");
    936
            // "hi hi" -> "hi" + " hi" (space becomes Ġ)
    937
            // "hi" -> h, i
    938
            // "Ġhi" -> "Ġ" + "h" -> "Ġh", then "i"
    939
    1
            let encoded = tokenizer.encode("hi hi");
    940
    1
            assert_eq!(encoded, vec![1, 2, 4, 2]); // h, i, "Ġh", i
    941
    1
        }
    942
    943
        #[test]
    944
    1
        fn test_bpe_decode() {
    945
    1
            let vocab = vec!["<unk>".to_string(), "hel".to_string(), "lo".to_string()];
    946
    1
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>").expect("test");
    947
    948
    1
            let decoded = tokenizer.decode(&[1, 2]).expect("test");
    949
    1
            assert_eq!(decoded, "hello");
    950
    1
        }
    951
    952
        #[test]
    953
    1
        fn test_bpe_decode_empty() {
    954
    1
            let vocab = vec!["<unk>".to_string()];
    955
    1
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>").expect("test");
    956
    957
    1
            let decoded = tokenizer.decode(&[]).expect("test");
    958
    1
            assert_eq!(decoded, "");
    959
    1
        }
    960
    961
        #[test]
    962
    1
        fn test_bpe_decode_invalid_id_error() {
    963
    1
            let vocab = vec!["<unk>".to_string(), "hi".to_string()];
    964
    1
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>").expect("test");
    965
    966
    1
            let result = tokenizer.decode(&[1, 999]);
    967
    1
            assert!(result.is_err());
    968
    1
        }
    969
    970
        #[test]
    971
    1
        fn test_bpe_encode_decode_roundtrip() {
    972
    1
            let vocab = vec![
    973
    1
                "<unk>".to_string(),
    974
    1
                "h".to_string(),
    975
    1
                "e".to_string(),
    976
    1
                "l".to_string(),
    977
    1
                "o".to_string(),
    978
    1
                "he".to_string(),
    979
    1
                "ll".to_string(),
    980
    1
                "lo".to_string(),
    981
    1
                "hel".to_string(),
    982
    1
                "hello".to_string(),
    983
            ];
    984
    1
            let merges = vec![
    985
    1
                ("h".to_string(), "e".to_string()),
    986
    1
                ("l".to_string(), "l".to_string()),
    987
    1
                ("l".to_string(), "o".to_string()),
    988
    1
                ("he".to_string(), "l".to_string()),
    989
    1
                ("hel".to_string(), "lo".to_string()),
    990
            ];
    991
    992
    1
            let tokenizer = BPETokenizer::new(vocab, merges, "<unk>").expect("test");
    993
    994
    1
            let encoded = tokenizer.encode("hello");
    995
    1
            let decoded = tokenizer.decode(&encoded).expect("test");
    996
    1
            assert_eq!(decoded, "hello");
    997
    1
        }
    998
    999
        #[test]
    1000
    1
        fn test_bpe_get_token_methods() {
    1001
    1
            let vocab = vec!["<unk>".to_string(), "hello".to_string()];
    1002
    1
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>").expect("test");
    1003
    1004
    1
            assert_eq!(tokenizer.get_token_id("hello"), Some(1));
    1005
    1
            assert_eq!(tokenizer.get_token_id("world"), None);
    1006
    1
            assert_eq!(tokenizer.get_token(1), Some("hello"));
    1007
    1
            assert_eq!(tokenizer.get_token(999), None);
    1008
    1
        }
    1009
    1010
        #[test]
    1011
    1
        fn test_bpe_multiple_consecutive_merges() {
    1012
            // Test that multiple merges are applied correctly
    1013
    1
            let vocab = vec![
    1014
    1
                "<unk>".to_string(),
    1015
    1
                "a".to_string(),
    1016
    1
                "b".to_string(),
    1017
    1
                "ab".to_string(),
    1018
    1
                "abab".to_string(),
    1019
            ];
    1020
    1
            let merges = vec![
    1021
    1
                ("a".to_string(), "b".to_string()),
    1022
    1
                ("ab".to_string(), "ab".to_string()),
    1023
            ];
    1024
    1025
    1
            let tokenizer = BPETokenizer::new(vocab, merges, "<unk>").expect("test");
    1026
    1
            let encoded = tokenizer.encode("abab");
    1027
            // First: a+b -> ab, a+b -> ab giving [ab, ab]
    1028
            // Then: ab+ab -> abab giving [abab]
    1029
    1
            assert_eq!(encoded, vec![4]);
    1030
    1
        }
    1031
    1032
        // SentencePiece Tokenizer tests
    1033
    1034
        #[test]
    1035
    1
        fn test_sentencepiece_tokenizer_creation() {
    1036
    1
            let vocab = vec![
    1037
    1
                ("<unk>".to_string(), 0.0),
    1038
    1
                ("hello".to_string(), -1.0),
    1039
    1
                ("world".to_string(), -1.5),
    1040
            ];
    1041
    1042
    1
            let tokenizer = SentencePieceTokenizer::new(vocab, "<unk>").expect("test");
    1043
    1
            assert_eq!(tokenizer.vocab_size(), 3);
    1044
    1
        }
    1045
    1046
        #[test]
    1047
    1
        fn test_sentencepiece_empty_vocab_error() {
    1048
    1
            let result = SentencePieceTokenizer::new(vec![], "<unk>");
    1049
    1
            assert!(result.is_err());
    1050
    1
        }
    1051
    1052
        #[test]
    1053
    1
        fn test_sentencepiece_invalid_unk_token_error() {
    1054
    1
            let vocab = vec![("hello".to_string(), -1.0)];
    1055
    1
            let result = SentencePieceTokenizer::new(vocab, "<unk>");
    1056
    1
            assert!(result.is_err());
    1057
    1
        }
    1058
    1059
        #[test]
    1060
    1
        fn test_sentencepiece_encode_empty() {
    1061
    1
            let vocab = vec![("<unk>".to_string(), 0.0)];
    1062
    1
            let tokenizer = SentencePieceTokenizer::new(vocab, "<unk>").expect("test");
    1063
    1064
    1
            let encoded = tokenizer.encode("");
    1065
    1
            assert!(encoded.is_empty());
    1066
    1
        }
    1067
    1068
        #[test]
    1069
    1
        fn test_sentencepiece_encode_single_token() {
    1070
    1
            let vocab = vec![("<unk>".to_string(), 0.0), ("hello".to_string(), -1.0)];
    1071
    1
            let tokenizer = SentencePieceTokenizer::new(vocab, "<unk>").expect("test");
    1072
    1073
    1
            let encoded = tokenizer.encode("hello");
    1074
    1
            assert_eq!(encoded, vec![1]);
    1075
    1
        }
    1076
    1077
        #[test]
    1078
    1
        fn test_sentencepiece_encode_prefers_higher_score() {
    1079
            // "hello" as single token has score -1.0
    1080
            // "hel" + "lo" would have score -2.0 + -2.0 = -4.0
    1081
            // So "hello" should be preferred
    1082
    1
            let vocab = vec![
    1083
    1
                ("<unk>".to_string(), 0.0),
    1084
    1
                ("h".to_string(), -5.0),
    1085
    1
                ("e".to_string(), -5.0),
    1086
    1
                ("l".to_string(), -5.0),
    1087
    1
                ("o".to_string(), -5.0),
    1088
    1
                ("hel".to_string(), -2.0),
    1089
    1
                ("lo".to_string(), -2.0),
    1090
    1
                ("hello".to_string(), -1.0),
    1091
            ];
    1092
    1
            let tokenizer = SentencePieceTokenizer::new(vocab, "<unk>").expect("test");
    1093
    1094
    1
            let encoded = tokenizer.encode("hello");
    1095
            // Should prefer single "hello" token (score -1.0) over subwords
    1096
    1
            assert_eq!(encoded, vec![7]);
    1097
    1
        }
    1098
    1099
        #[test]
    1100
    1
        fn test_sentencepiece_encode_subwords() {
    1101
            // Only subwords available, not full word
    1102
    1
            let vocab = vec![
    1103
    1
                ("<unk>".to_string(), 0.0),
    1104
    1
                ("h".to_string(), -1.0),
    1105
    1
                ("e".to_string(), -1.0),
    1106
    1
                ("l".to_string(), -1.0),
    1107
    1
                ("o".to_string(), -1.0),
    1108
    1
                ("he".to_string(), -0.5),
    1109
    1
                ("llo".to_string(), -0.5),
    1110
            ];
    1111
    1
            let tokenizer = SentencePieceTokenizer::new(vocab, "<unk>").expect("test");
    1112
    1113
    1
            let encoded = tokenizer.encode("hello");
    1114
            // "he" (-0.5) + "llo" (-0.5) = -1.0 is better than "h" + "e" + "l" + "l" + "o" = -5.0
    1115
    1
            assert_eq!(encoded, vec![5, 6]);
    1116
    1
        }
    1117
    1118
        #[test]
    1119
    1
        fn test_sentencepiece_decode() {
    1120
    1
            let vocab = vec![
    1121
    1
                ("<unk>".to_string(), 0.0),
    1122
    1
                ("hel".to_string(), -1.0),
    1123
    1
                ("lo".to_string(), -1.0),
    1124
            ];
    1125
    1
            let tokenizer = SentencePieceTokenizer::new(vocab, "<unk>").expect("test");
    1126
    1127
    1
            let decoded = tokenizer.decode(&[1, 2]).expect("test");
    1128
    1
            assert_eq!(decoded, "hello");
    1129
    1
        }
    1130
    1131
        #[test]
    1132
    1
        fn test_sentencepiece_decode_empty() {
    1133
    1
            let vocab = vec![("<unk>".to_string(), 0.0)];
    1134
    1
            let tokenizer = SentencePieceTokenizer::new(vocab, "<unk>").expect("test");
    1135
    1136
    1
            let decoded = tokenizer.decode(&[]).expect("test");
    1137
    1
            assert_eq!(decoded, "");
    1138
    1
        }
    1139
    1140
        #[test]
    1141
    1
        fn test_sentencepiece_decode_invalid_id_error() {
    1142
    1
            let vocab = vec![("<unk>".to_string(), 0.0), ("hi".to_string(), -1.0)];
    1143
    1
            let tokenizer = SentencePieceTokenizer::new(vocab, "<unk>").expect("test");
    1144
    1145
    1
            let result = tokenizer.decode(&[1, 999]);
    1146
    1
            assert!(result.is_err());
    1147
    1
        }
    1148
    1149
        #[test]
    1150
    1
        fn test_sentencepiece_encode_decode_roundtrip() {
    1151
    1
            let vocab = vec![
    1152
    1
                ("<unk>".to_string(), 0.0),
    1153
    1
                ("h".to_string(), -2.0),
    1154
    1
                ("e".to_string(), -2.0),
    1155
    1
                ("l".to_string(), -2.0),
    1156
    1
                ("o".to_string(), -2.0),
    1157
    1
                ("hello".to_string(), -1.0),
    1158
            ];
    1159
    1
            let tokenizer = SentencePieceTokenizer::new(vocab, "<unk>").expect("test");
    1160
    1161
    1
            let encoded = tokenizer.encode("hello");
    1162
    1
            let decoded = tokenizer.decode(&encoded).expect("test");
    1163
    1
            assert_eq!(decoded, "hello");
    1164
    1
        }
    1165
    1166
        #[test]
    1167
    1
        fn test_sentencepiece_get_methods() {
    1168
    1
            let vocab = vec![("<unk>".to_string(), 0.0), ("hello".to_string(), -1.5)];
    1169
    1
            let tokenizer = SentencePieceTokenizer::new(vocab, "<unk>").expect("test");
    1170
    1171
    1
            assert_eq!(tokenizer.get_token_id("hello"), Some(1));
    1172
    1
            assert_eq!(tokenizer.get_token_id("world"), None);
    1173
    1
            assert_eq!(tokenizer.get_token(1), Some("hello"));
    1174
    1
            assert_eq!(tokenizer.get_token(999), None);
    1175
    1
            assert!((tokenizer.get_score("hello").expect("test") - (-1.5)).abs() < 1e-6);
    1176
    1
            assert_eq!(tokenizer.get_score("world"), None);
    1177
    1
        }
    1178
    1179
        #[test]
    1180
    1
        fn test_sentencepiece_unknown_character() {
    1181
            // Character not in vocabulary should use unknown penalty
    1182
    1
            let vocab = vec![
    1183
    1
                ("<unk>".to_string(), 0.0),
    1184
    1
                ("h".to_string(), -1.0),
    1185
    1
                ("i".to_string(), -1.0),
    1186
            ];
    1187
    1
            let tokenizer = SentencePieceTokenizer::new(vocab, "<unk>").expect("test");
    1188
    1189
            // 'x' is not in vocab, should be tokenized with penalty
    1190
    1
            let encoded = tokenizer.encode("hix");
    1191
    1
            assert_eq!(encoded.len(), 3);
    1192
    1
            assert_eq!(encoded[0], 1); // h
    1193
    1
            assert_eq!(encoded[1], 2); // i
    1194
                                       // x should map to unk
    1195
    1
            assert_eq!(encoded[2], 0);
    1196
    1
        }
    1197
    1198
        #[test]
    1199
    1
        fn test_sentencepiece_multiple_words() {
    1200
    1
            let vocab = vec![
    1201
    1
                ("<unk>".to_string(), 0.0),
    1202
    1
                ("hello".to_string(), -1.0),
    1203
    1
                (" ".to_string(), -0.5),
    1204
    1
                ("world".to_string(), -1.0),
    1205
            ];
    1206
    1
            let tokenizer = SentencePieceTokenizer::new(vocab, "<unk>").expect("test");
    1207
    1208
    1
            let encoded = tokenizer.encode("hello world");
    1209
    1
            assert_eq!(encoded, vec![1, 2, 3]); // hello, space, world
    1210
    1
        }
    1211
    1212
        // -------------------------------------------------------------------------
    1213
        // Additional BPE Decode Tests (95% coverage push)
    1214
        // -------------------------------------------------------------------------
    1215
    1216
        #[test]
    1217
    1
        fn test_bpe_decode_special_tokens() {
    1218
    1
            let vocab = vec![
    1219
    1
                "<unk>".to_string(),
    1220
    1
                "<|endoftext|>".to_string(),
    1221
    1
                "<s>".to_string(),
    1222
    1
                "</s>".to_string(),
    1223
    1
                "<pad>".to_string(),
    1224
    1
                "hello".to_string(),
    1225
            ];
    1226
    1
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>").expect("test");
    1227
    1228
            // Special tokens should be skipped in decode
    1229
    1
            let decoded = tokenizer.decode(&[1, 2, 3, 4, 5]).expect("test");
    1230
    1
            assert_eq!(decoded, "hello"); // Only "hello" should remain
    1231
    1
        }
    1232
    1233
        #[test]
    1234
    1
        fn test_bpe_decode_byte_tokens() {
    1235
    1
            let vocab = vec![
    1236
    1
                "<unk>".to_string(),
    1237
    1
                "<0xE6>".to_string(), // UTF-8 first byte of some CJK chars
    1238
    1
                "<0x97>".to_string(),
    1239
    1
                "<0xA5>".to_string(),
    1240
            ];
    1241
    1
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>").expect("test");
    1242
    1243
            // These three bytes form the UTF-8 sequence for "日" (U+65E5)
    1244
    1
            let decoded = tokenizer.decode(&[1, 2, 3]).expect("test");
    1245
    1
            assert_eq!(decoded, "日");
    1246
    1
        }
    1247
    1248
        #[test]
    1249
    1
        fn test_bpe_decode_gpt2_space() {
    1250
    1
            let vocab = vec![
    1251
    1
                "<unk>".to_string(),
    1252
    1
                "Ġhello".to_string(), // Ġ = space prefix in GPT-2
    1253
            ];
    1254
    1
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>").expect("test");
    1255
    1256
    1
            let decoded = tokenizer.decode(&[1]).expect("test");
    1257
    1
            assert_eq!(decoded, " hello");
    1258
    1
        }
    1259
    1260
        #[test]
    1261
    1
        fn test_bpe_decode_gpt2_newline() {
    1262
    1
            let vocab = vec![
    1263
    1
                "<unk>".to_string(),
    1264
    1
                "Ċ".to_string(), // newline in GPT-2
    1265
    1
                "ċ".to_string(), // lowercase variant
    1266
            ];
    1267
    1
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>").expect("test");
    1268
    1269
    1
            let decoded = tokenizer.decode(&[1, 2]).expect("test");
    1270
    1
            assert_eq!(decoded, "\n\n");
    1271
    1
        }
    1272
    1273
        #[test]
    1274
    1
        fn test_bpe_decode_sentencepiece_space() {
    1275
    1
            let vocab = vec![
    1276
    1
                "<unk>".to_string(),
    1277
    1
                "▁hello".to_string(), // SentencePiece space
    1278
            ];
    1279
    1
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>").expect("test");
    1280
    1281
    1
            let decoded = tokenizer.decode(&[1]).expect("test");
    1282
    1
            assert_eq!(decoded, " hello");
    1283
    1
        }
    1284
    1285
        #[test]
    1286
    1
        fn test_bpe_decode_gpt2_carriage_return() {
    1287
    1
            let vocab = vec![
    1288
    1
                "<unk>".to_string(),
    1289
    1
                "Ḃ".to_string(), // carriage return in GPT-2
    1290
    1
                "a".to_string(),
    1291
            ];
    1292
    1
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>").expect("test");
    1293
    1294
    1
            let decoded = tokenizer.decode(&[1, 2]).expect("test");
    1295
    1
            assert_eq!(decoded, "\ra");
    1296
    1
        }
    1297
    1298
        #[test]
    1299
    1
        fn test_bpe_decode_regular_utf8() {
    1300
    1
            let vocab = vec![
    1301
    1
                "<unk>".to_string(),
    1302
    1
                "こんにちは".to_string(), // Japanese
    1303
            ];
    1304
    1
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>").expect("test");
    1305
    1306
    1
            let decoded = tokenizer.decode(&[1]).expect("test");
    1307
    1
            assert_eq!(decoded, "こんにちは");
    1308
    1
        }
    1309
    1310
        #[test]
    1311
    1
        fn test_bpe_decode_invalid_byte_token() {
    1312
    1
            let vocab = vec![
    1313
    1
                "<unk>".to_string(),
    1314
    1
                "<0xGG>".to_string(), // Invalid hex
    1315
            ];
    1316
    1
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>").expect("test");
    1317
    1318
            // Should not panic, just treat as regular token
    1319
    1
            let decoded = tokenizer.decode(&[1]).expect("test");
    1320
    1
            assert!(decoded.contains("<0xGG>"));
    1321
    1
        }
    1322
    1323
        #[test]
    1324
    1
        fn test_bpe_decode_mixed_tokens() {
    1325
    1
            let vocab = vec![
    1326
    1
                "<unk>".to_string(),
    1327
    1
                "Ġhello".to_string(), // GPT-2 space + word
    1328
    1
                "Ċ".to_string(),      // newline
    1329
    1
                "world".to_string(),
    1330
            ];
    1331
    1
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>").expect("test");
    1332
    1333
    1
            let decoded = tokenizer.decode(&[1, 2, 3]).expect("test");
    1334
    1
            assert_eq!(decoded, " hello\nworld");
    1335
    1
        }
    1336
    1337
        #[test]
    1338
    1
        fn test_bpe_gpt2_char_to_byte_printable_ascii() {
    1339
            // Test that printable ASCII is preserved
    1340
    1
            let vocab = vec!["<unk>".to_string(), "a".to_string(), "!".to_string()];
    1341
    1
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>").expect("test");
    1342
    1343
    1
            let decoded = tokenizer.decode(&[1, 2]).expect("test");
    1344
    1
            assert_eq!(decoded, "a!");
    1345
    1
        }
    1346
    1347
        #[test]
    1348
    1
        fn test_bpe_gpt2_char_to_byte_space() {
    1349
            // Space (ASCII 32) should be preserved
    1350
    1
            let vocab = vec!["<unk>".to_string(), " ".to_string(), "a".to_string()];
    1351
    1
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>").expect("test");
    1352
    1353
    1
            let decoded = tokenizer.decode(&[1, 2]).expect("test");
    1354
    1
            assert_eq!(decoded, " a");
    1355
    1
        }
    1356
    1357
        #[test]
    1358
    1
        fn test_sentencepiece_vocab_size() {
    1359
    1
            let vocab = vec![
    1360
    1
                ("<unk>".to_string(), 0.0),
    1361
    1
                ("a".to_string(), -1.0),
    1362
    1
                ("b".to_string(), -1.0),
    1363
            ];
    1364
    1
            let tokenizer = SentencePieceTokenizer::new(vocab, "<unk>").expect("test");
    1365
    1366
    1
            assert_eq!(tokenizer.vocab_size(), 3);
    1367
    1
        }
    1368
    1369
        // =========================================================================
    1370
        // Additional 95% Coverage Tests
    1371
        // =========================================================================
    1372
    1373
        #[test]
    1374
    1
        fn test_cov95_bpe_apply_merge_single_token() {
    1375
            // Test apply_merge with single token (early return path)
    1376
    1
            let vocab = vec![
    1377
    1
                "<unk>".to_string(),
    1378
    1
                "a".to_string(),
    1379
    1
                "b".to_string(),
    1380
    1
                "ab".to_string(),
    1381
            ];
    1382
    1
            let merges = vec![("a".to_string(), "b".to_string())];
    1383
    1
            let tokenizer = BPETokenizer::new(vocab, merges, "<unk>").expect("test");
    1384
    1385
            // Encode single char - no merge possible
    1386
    1
            let encoded = tokenizer.encode("a");
    1387
    1
            assert_eq!(encoded, vec![1]);
    1388
    1
        }
    1389
    1390
        #[test]
    1391
    1
        fn test_cov95_bpe_apply_merge_no_match() {
    1392
            // Test apply_merge when merge pair doesn't match
    1393
    1
            let vocab = vec![
    1394
    1
                "<unk>".to_string(),
    1395
    1
                "x".to_string(),
    1396
    1
                "y".to_string(),
    1397
    1
                "z".to_string(),
    1398
    1
                "ab".to_string(),
    1399
            ];
    1400
    1
            let merges = vec![("a".to_string(), "b".to_string())]; // merge won't match "xyz"
    1401
    1
            let tokenizer = BPETokenizer::new(vocab, merges, "<unk>").expect("test");
    1402
    1403
            // "xyz" has no matching merge pairs
    1404
    1
            let encoded = tokenizer.encode("xyz");
    1405
    1
            assert_eq!(encoded, vec![1, 2, 3]);
    1406
    1
        }
    1407
    1408
        #[test]
    1409
    1
        fn test_cov95_bpe_apply_merge_consecutive() {
    1410
            // Test apply_merge with consecutive matches
    1411
    1
            let vocab = vec![
    1412
    1
                "<unk>".to_string(),
    1413
    1
                "a".to_string(),
    1414
    1
                "b".to_string(),
    1415
    1
                "ab".to_string(),
    1416
            ];
    1417
    1
            let merges = vec![("a".to_string(), "b".to_string())];
    1418
    1
            let tokenizer = BPETokenizer::new(vocab, merges, "<unk>").expect("test");
    1419
    1420
            // "abab" should merge to "ab" + "ab"
    1421
    1
            let encoded = tokenizer.encode("abab");
    1422
    1
            assert_eq!(encoded, vec![3, 3]); // "ab", "ab"
    1423
    1
        }
    1424
    1425
        #[test]
    1426
    1
        fn test_cov95_bpe_decode_gpt2_remapped_bytes() {
    1427
            // Test decode with GPT-2 remapped byte tokens (0x100+ range)
    1428
            // GPT-2 uses chars starting at U+0100 for raw bytes
    1429
    1
            let vocab = vec![
    1430
    1
                "<unk>".to_string(),
    1431
    1
                "\u{0100}".to_string(), // maps to byte 0
    1432
    1
                "\u{0101}".to_string(), // maps to byte 1
    1433
    1
                "\u{0120}".to_string(), // maps to byte 32 (space in GPT-2)
    1434
            ];
    1435
    1
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>").expect("test");
    1436
    1437
            // Decode tokens with GPT-2 byte remapping
    1438
    1
            let decoded = tokenizer.decode(&[3]).expect("test");
    1439
            // U+0120 should decode to space (byte 32)
    1440
    1
            assert!(decoded.contains('\u{0120}') || decoded.contains(' '));
    1441
    1
        }
    1442
    1443
        #[test]
    1444
    1
        fn test_cov95_bpe_decode_high_unicode_byte() {
    1445
            // Test decode with high unicode that maps to byte via GPT-2 encoding
    1446
            // GPT-2 remaps bytes 127-160 to U+0100 + offset
    1447
    1
            let vocab = vec![
    1448
    1
                "<unk>".to_string(),
    1449
    1
                "\u{017F}".to_string(), // Should map to byte 127 in GPT-2 encoding
    1450
    1
                "\u{01A0}".to_string(), // Should map to byte 160 in GPT-2 encoding
    1451
            ];
    1452
    1
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>").expect("test");
    1453
    1454
            // These tokens should decode (possibly with replacement chars)
    1455
    1
            let decoded = tokenizer.decode(&[1, 2]).expect("test");
    1456
    1
            assert!(!decoded.is_empty());
    1457
    1
        }
    1458
    1459
        #[test]
    1460
    1
        fn test_cov95_bpe_decode_soft_hyphen() {
    1461
            // Test decode with soft hyphen (byte 173)
    1462
    1
            let vocab = vec![
    1463
    1
                "<unk>".to_string(),
    1464
    1
                "\u{01AD}".to_string(), // U+0100 + 173 = U+01AD for soft hyphen in GPT-2
    1465
            ];
    1466
    1
            let tokenizer = BPETokenizer::new(vocab, vec![], "<unk>").expect("test");
    1467
    1468
    1
            let decoded = tokenizer.decode(&[1]).expect("test");
    1469
    1
            assert!(!decoded.is_empty());
    1470
    1
        }
    1471
    1472
        #[test]
    1473
    1
        fn test_cov95_bpe_encode_with_multiple_merges() {
    1474
            // Test encoding that exercises multiple merge iterations
    1475
    1
            let vocab = vec![
    1476
    1
                "<unk>".to_string(),
    1477
    1
                "a".to_string(),
    1478
    1
                "b".to_string(),
    1479
    1
                "c".to_string(),
    1480
    1
                "ab".to_string(),
    1481
    1
                "abc".to_string(),
    1482
            ];
    1483
    1
            let merges = vec![
    1484
    1
                ("a".to_string(), "b".to_string()),
    1485
    1
                ("ab".to_string(), "c".to_string()),
    1486
            ];
    1487
    1
            let tokenizer = BPETokenizer::new(vocab, merges, "<unk>").expect("test");
    1488
    1489
    1
            let encoded = tokenizer.encode("abc");
    1490
            // First merge: a+b -> ab, then ab+c -> abc
    1491
    1
            assert_eq!(encoded, vec![5]); // "abc"
    1492
    1
        }
    1493
    }