## Coverage Report

#### Created: 2026-01-25 15:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)

    /home/noah/src/realizar/src/gguf/loader.rs

    Line
    Count
    Source
    1
    //! GGUF model loading and parsing
    2
    //!
    3
    //! Contains GGUFModel and GGUFTransformer parsing implementations extracted from monolith.
    4
    //! This handles the binary format parsing and tensor info extraction.
    5
    6
    use crate::error::{RealizarError, Result};
    7
    use crate::gguf::utils::gpt2_unicode_to_byte;
    8
    use crate::gguf::{
    9
        GGUFConfig, GGUFHeader, GGUFModel, GGUFTransformer, GGUFTransformerLayer, GGUFValue, TensorInfo,
    10
        GGUF_ALIGNMENT, GGUF_MAGIC, GGUF_TYPE_F16, GGUF_TYPE_F32, GGUF_TYPE_Q2_K, GGUF_TYPE_Q4_0,
    11
        GGUF_TYPE_Q4_1, GGUF_TYPE_Q4_K, GGUF_TYPE_Q5_0, GGUF_TYPE_Q5_1, GGUF_TYPE_Q5_K, GGUF_TYPE_Q6_K,
    12
        GGUF_TYPE_Q8_0, GGUF_VERSION_V3,
    13
    };
    14
    use std::collections::HashMap;
    15
    use std::io::{Cursor, Read};
    16
    17
    impl GGUFModel {
    18
        /// Parse GGUF file from bytes
    19
        ///
    20
        /// # Arguments
    21
        ///
    22
        /// * `data` - Raw GGUF file bytes
    23
        ///
    24
        /// # Errors
    25
        ///
    26
        /// Returns error if:
    27
        /// - Invalid magic number
    28
        /// - Unsupported version
    29
        /// - Malformed data
    30
        ///
    31
        /// # Examples
    32
        ///
    33
        /// ```rust,ignore
    34
        /// let data = std::fs::read("model.gguf")?;
    35
        /// let model = GGUFModel::from_bytes(&data)?;
    36
        /// println!("Loaded {} tensors", model.tensors.len());
    37
        /// ```
    38
    96
        pub fn from_bytes(data: &[u8]) -> Result<Self> {
    39
    96
            let mut cursor = Cursor::new(data);
    40
    41
            // Parse header
    42
    96
            let header82 = Self::parse_header(&mut cursor)?14;
    43
    44
            // Parse metadata
    45
    82
            let metadata81 = Self::parse_metadata(&mut cursor, header.metadata_count)?1;
    46
    47
            // Parse tensor info
    48
    81
            let tensors = Self::parse_tensor_info(&mut cursor, header.tensor_count)?0;
    49
    50
            // Calculate tensor data start with 32-byte alignment
    51
    81
            let current_pos = cursor.position() as usize;
    52
    81
            let tensor_data_start = current_pos.div_ceil(GGUF_ALIGNMENT) * GGUF_ALIGNMENT;
    53
    54
    81
            Ok(Self {
    55
    81
                header,
    56
    81
                metadata,
    57
    81
                tensors,
    58
    81
                tensor_data_start,
    59
    81
            })
    60
    96
        }
    61
    62
        /// Parse GGUF header
    63
    96
        fn parse_header(cursor: &mut Cursor<&[u8]>) -> Result<GGUFHeader> {
    64
    96
            let mut buf = [0u8; 4];
    65
    66
            // Read magic
    67
    96
            cursor
    68
    96
                .read_exact(&mut buf)
    69
    96
                .map_err(|e| RealizarError::UnsupportedOperation {
    70
    3
                    operation: "read_magic".to_string(),
    71
    3
                    reason: e.to_string(),
    72
    3
                })?;
    73
    93
            let magic = u32::from_le_bytes(buf);
    74
    75
    93
            if magic != GGUF_MAGIC {
    76
    6
                return Err(RealizarError::InvalidShape {
    77
    6
                    reason: format!("Invalid GGUF magic: 0x{magic:08X}, expected 0x{GGUF_MAGIC:08X}"),
    78
    6
                });
    79
    87
            }
    80
    81
            // Read version
    82
    87
            cursor
    83
    87
                .read_exact(&mut buf)
    84
    87
                .map_err(|e| RealizarError::UnsupportedOperation {
    85
    3
                    operation: "read_version".to_string(),
    86
    3
                    reason: e.to_string(),
    87
    3
                })?;
    88
    84
            let version = u32::from_le_bytes(buf);
    89
    90
    84
            if version != GGUF_VERSION_V3 {
    91
    1
                return Err(RealizarError::UnsupportedOperation {
    92
    1
                    operation: "parse_gguf".to_string(),
    93
    1
                    reason: format!("Unsupported GGUF version: {version}, only v3 supported"),
    94
    1
                });
    95
    83
            }
    96
    97
            // Read tensor_count
    98
    83
            let mut buf8 = [0u8; 8];
    99
    83
            cursor
    100
    83
                .read_exact(&mut buf8)
    101
    83
                .map_err(|e| RealizarError::UnsupportedOperation {
    102
    1
                    operation: "read_tensor_count".to_string(),
    103
    1
                    reason: e.to_string(),
    104
    1
                })?;
    105
    82
            let tensor_count = u64::from_le_bytes(buf8);
    106
    107
            // Read metadata_count
    108
    82
            cursor
    109
    82
                .read_exact(&mut buf8)
    110
    82
                .map_err(|e| RealizarError::UnsupportedOperation {
    111
    0
                    operation: "read_metadata_count".to_string(),
    112
    0
                    reason: e.to_string(),
    113
    0
                })?;
    114
    82
            let metadata_count = u64::from_le_bytes(buf8);
    115
    116
    82
            Ok(GGUFHeader {
    117
    82
                magic,
    118
    82
                version,
    119
    82
                tensor_count,
    120
    82
                metadata_count,
    121
    82
            })
    122
    96
        }
    123
    124
        /// Parse metadata key-value pairs
    125
    82
        fn parse_metadata(
    126
    82
            cursor: &mut Cursor<&[u8]>,
    127
    82
            count: u64,
    128
    82
        ) -> Result<HashMap<String, GGUFValue>> {
    129
    82
            let mut metadata = HashMap::new();
    130
    131
    82
            for _ in 0..count {
    132
                // Read key (string: u64 length + bytes)
    133
    199
                let key = Self::read_string(cursor)?0;
    134
    135
                // Read value type (u32)
    136
    199
                let mut buf = [0u8; 4];
    137
    199
                cursor
    138
    199
                    .read_exact(&mut buf)
    139
    199
                    .map_err(|e| RealizarError::UnsupportedOperation {
    140
    0
                        operation: "read_metadata_type".to_string(),
    141
    0
                        reason: e.to_string(),
    142
    0
                    })?;
    143
    199
                let value_type = u32::from_le_bytes(buf);
    144
    145
                // Read value based on type
    146
    199
                let value198 = Self::read_value(cursor, value_type)?1;
    147
    148
    198
                metadata.insert(key, value);
    149
            }
    150
    151
    81
            Ok(metadata)
    152
    82
        }
    153
    154
        /// Read a string: u64 length + UTF-8 bytes
    155
    479
        fn read_string(cursor: &mut Cursor<&[u8]>) -> Result<String> {
    156
    479
            let mut buf8 = [0u8; 8];
    157
    479
            cursor
    158
    479
                .read_exact(&mut buf8)
    159
    479
                .map_err(|e| RealizarError::UnsupportedOperation {
    160
    0
                    operation: "read_string_length".to_string(),
    161
    0
                    reason: e.to_string(),
    162
    0
                })?;
    163
    479
            let len_u64 = u64::from_le_bytes(buf8);
    164
    479
            let len = usize::try_from(len_u64).map_err(|_| RealizarError::UnsupportedOperation {
    165
    0
                operation: "convert_string_length".to_string(),
    166
    0
                reason: format!("String length {len_u64} exceeds platform usize limit"),
    167
    0
            })?;
    168
    169
    479
            let mut string_bytes = vec![0u8; len];
    170
    479
            cursor
    171
    479
                .read_exact(&mut string_bytes)
    172
    479
                .map_err(|e| RealizarError::UnsupportedOperation {
    173
    0
                    operation: "read_string_data".to_string(),
    174
    0
                    reason: e.to_string(),
    175
    0
                })?;
    176
    177
    479
            String::from_utf8(string_bytes).map_err(|e| RealizarError::UnsupportedOperation {
    178
    0
                operation: "parse_utf8".to_string(),
    179
    0
                reason: e.to_string(),
    180
    0
            })
    181
    479
        }
    182
    183
        /// Read a value based on type
    184
    213
        fn read_value(cursor: &mut Cursor<&[u8]>, value_type: u32) -> Result<GGUFValue> {
    185
    213
            match value_type {
    186
    1
                0 => Ok(GGUFValue::UInt8(Self::read_u8(cursor)?0)),
    187
    1
                1 => Ok(GGUFValue::Int8(Self::read_i8(cursor)?0)),
    188
    1
                2 => Ok(GGUFValue::UInt16(Self::read_u16(cursor)?0)),
    189
    1
                3 => Ok(GGUFValue::Int16(Self::read_i16(cursor)?0)),
    190
    132
                4 => Ok(GGUFValue::UInt32(Self::read_u32(cursor)?0)),
    191
    1
                5 => Ok(GGUFValue::Int32(Self::read_i32(cursor)?0)),
    192
    25
                6 => Ok(GGUFValue::Float32(Self::read_f32(cursor)?0)),
    193
    2
                7 => Ok(GGUFValue::Bool(Self::read_bool(cursor)?0)),
    194
    40
                8 => Ok(GGUFValue::String(Self::read_string(cursor)?0)),
    195
                9 => {
    196
                    // Array: element_type (u32) + array_len (u64) + elements
    197
    5
                    let element_type = Self::read_u32(cursor)?0;
    198
    5
                    let array_len = Self::read_u64(cursor)?0;
    199
    200
                    // Safely convert array_len to usize
    201
    5
                    let len = usize::try_from(array_len).map_err(|_| RealizarError::InvalidShape {
    202
    0
                        reason: format!("Array length too large: {array_len}"),
    203
    0
                    })?;
    204
    205
    5
                    let mut elements = Vec::with_capacity(len);
    206
    5
                    for _ in 0..array_len {
    207
    14
                        elements.push(Self::read_value(cursor, element_type)?0);
    208
                    }
    209
    5
                    Ok(GGUFValue::Array(elements))
    210
                },
    211
    1
                10 => Ok(GGUFValue::UInt64(Self::read_u64(cursor)?0)),
    212
    1
                11 => Ok(GGUFValue::Int64(Self::read_i64(cursor)?0)),
    213
    1
                12 => Ok(GGUFValue::Float64(Self::read_f64(cursor)?0)),
    214
    1
                _ => Err(RealizarError::UnsupportedOperation {
    215
    1
                    operation: "read_value".to_string(),
    216
    1
                    reason: format!("Unsupported value type: {value_type}"),
    217
    1
                }),
    218
            }
    219
    213
        }
    220
    221
        /// Read u8
    222
    1
        fn read_u8(cursor: &mut Cursor<&[u8]>) -> Result<u8> {
    223
    1
            let mut buf = [0u8; 1];
    224
    1
            cursor
    225
    1
                .read_exact(&mut buf)
    226
    1
                .map_err(|e| RealizarError::UnsupportedOperation {
    227
    0
                    operation: "read_u8".to_string(),
    228
    0
                    reason: e.to_string(),
    229
    0
                })?;
    230
    1
            Ok(buf[0])
    231
    1
        }
    232
    233
        /// Read i8
    234
    1
        fn read_i8(cursor: &mut Cursor<&[u8]>) -> Result<i8> {
    235
    1
            let mut buf = [0u8; 1];
    236
    1
            cursor
    237
    1
                .read_exact(&mut buf)
    238
    1
                .map_err(|e| RealizarError::UnsupportedOperation {
    239
    0
                    operation: "read_i8".to_string(),
    240
    0
                    reason: e.to_string(),
    241
    0
                })?;
    242
    1
            Ok(i8::from_le_bytes(buf))
    243
    1
        }
    244
    245
        /// Read u16
    246
    1
        fn read_u16(cursor: &mut Cursor<&[u8]>) -> Result<u16> {
    247
    1
            let mut buf = [0u8; 2];
    248
    1
            cursor
    249
    1
                .read_exact(&mut buf)
    250
    1
                .map_err(|e| RealizarError::UnsupportedOperation {
    251
    0
                    operation: "read_u16".to_string(),
    252
    0
                    reason: e.to_string(),
    253
    0
                })?;
    254
    1
            Ok(u16::from_le_bytes(buf))
    255
    1
        }
    256
    257
        /// Read i16
    258
    1
        fn read_i16(cursor: &mut Cursor<&[u8]>) -> Result<i16> {
    259
    1
            let mut buf = [0u8; 2];
    260
    1
            cursor
    261
    1
                .read_exact(&mut buf)
    262
    1
                .map_err(|e| RealizarError::UnsupportedOperation {
    263
    0
                    operation: "read_i16".to_string(),
    264
    0
                    reason: e.to_string(),
    265
    0
                })?;
    266
    1
            Ok(i16::from_le_bytes(buf))
    267
    1
        }
    268
    269
        /// Read u32
    270
    617
        fn read_u32(cursor: &mut Cursor<&[u8]>) -> Result<u32> {
    271
    617
            let mut buf = [0u8; 4];
    272
    617
            cursor
    273
    617
                .read_exact(&mut buf)
    274
    617
                .map_err(|e| RealizarError::UnsupportedOperation {
    275
    0
                    operation: "read_u32".to_string(),
    276
    0
                    reason: e.to_string(),
    277
    0
                })?;
    278
    617
            Ok(u32::from_le_bytes(buf))
    279
    617
        }
    280
    281
        /// Read i32
    282
    1
        fn read_i32(cursor: &mut Cursor<&[u8]>) -> Result<i32> {
    283
    1
            let mut buf = [0u8; 4];
    284
    1
            cursor
    285
    1
                .read_exact(&mut buf)
    286
    1
                .map_err(|e| RealizarError::UnsupportedOperation {
    287
    0
                    operation: "read_i32".to_string(),
    288
    0
                    reason: e.to_string(),
    289
    0
                })?;
    290
    1
            Ok(i32::from_le_bytes(buf))
    291
    1
        }
    292
    293
        /// Read f32
    294
    25
        fn read_f32(cursor: &mut Cursor<&[u8]>) -> Result<f32> {
    295
    25
            let mut buf = [0u8; 4];
    296
    25
            cursor
    297
    25
                .read_exact(&mut buf)
    298
    25
                .map_err(|e| RealizarError::UnsupportedOperation {
    299
    0
                    operation: "read_f32".to_string(),
    300
    0
                    reason: e.to_string(),
    301
    0
                })?;
    302
    25
            Ok(f32::from_le_bytes(buf))
    303
    25
        }
    304
    305
        /// Read bool
    306
    2
        fn read_bool(cursor: &mut Cursor<&[u8]>) -> Result<bool> {
    307
    2
            let mut buf = [0u8; 1];
    308
    2
            cursor
    309
    2
                .read_exact(&mut buf)
    310
    2
                .map_err(|e| RealizarError::UnsupportedOperation {
    311
    0
                    operation: "read_bool".to_string(),
    312
    0
                    reason: e.to_string(),
    313
    0
                })?;
    314
    2
            Ok(buf[0] != 0)
    315
    2
        }
    316
    317
        /// Read u64
    318
    579
        fn read_u64(cursor: &mut Cursor<&[u8]>) -> Result<u64> {
    319
    579
            let mut buf = [0u8; 8];
    320
    579
            cursor
    321
    579
                .read_exact(&mut buf)
    322
    579
                .map_err(|e| RealizarError::UnsupportedOperation {
    323
    0
                    operation: "read_u64".to_string(),
    324
    0
                    reason: e.to_string(),
    325
    0
                })?;
    326
    579
            Ok(u64::from_le_bytes(buf))
    327
    579
        }
    328
    329
        /// Read i64
    330
    1
        fn read_i64(cursor: &mut Cursor<&[u8]>) -> Result<i64> {
    331
    1
            let mut buf = [0u8; 8];
    332
    1
            cursor
    333
    1
                .read_exact(&mut buf)
    334
    1
                .map_err(|e| RealizarError::UnsupportedOperation {
    335
    0
                    operation: "read_i64".to_string(),
    336
    0
                    reason: e.to_string(),
    337
    0
                })?;
    338
    1
            Ok(i64::from_le_bytes(buf))
    339
    1
        }
    340
    341
        /// Read f64
    342
    1
        fn read_f64(cursor: &mut Cursor<&[u8]>) -> Result<f64> {
    343
    1
            let mut buf = [0u8; 8];
    344
    1
            cursor
    345
    1
                .read_exact(&mut buf)
    346
    1
                .map_err(|e| RealizarError::UnsupportedOperation {
    347
    0
                    operation: "read_f64".to_string(),
    348
    0
                    reason: e.to_string(),
    349
    0
                })?;
    350
    1
            Ok(f64::from_le_bytes(buf))
    351
    1
        }
    352
    353
        /// Parse tensor info
    354
    81
        fn parse_tensor_info(cursor: &mut Cursor<&[u8]>, count: u64) -> Result<Vec<TensorInfo>> {
    355
    81
            let mut tensors = Vec::new();
    356
    357
    81
            for _ in 0..count {
    358
                // Read tensor name (string)
    359
    240
                let name = Self::read_string(cursor)?0;
    360
    361
                // Read n_dims (u32)
    362
    240
                let n_dims = Self::read_u32(cursor)?0;
    363
    364
                // Read dimensions array
    365
                // GGUF stores dimensions in GGML order (reversed from standard row-major)
    366
                // We need to reverse them to get the correct shape [out_dim, in_dim]
    367
    240
                let mut dims = Vec::with_capacity(n_dims as usize);
    368
    240
                for _ in 0..n_dims {
    369
    333
                    dims.push(Self::read_u64(cursor)?0);
    370
                }
    371
    240
                dims.reverse();
    372
    373
                // Read quantization type (u32)
    374
    240
                let qtype = Self::read_u32(cursor)?0;
    375
    376
                // Read offset (u64)
    377
    240
                let offset = Self::read_u64(cursor)?0;
    378
    379
    240
                tensors.push(TensorInfo {
    380
    240
                    name,
    381
    240
                    n_dims,
    382
    240
                    dims,
    383
    240
                    qtype,
    384
    240
                    offset,
    385
    240
                });
    386
            }
    387
    388
    81
            Ok(tensors)
    389
    81
        }
    390
    391
        /// Extract tensor data by name with dequantization
    392
        ///
    393
        /// # Arguments
    394
        ///
    395
        /// * `name` - Tensor name to extract
    396
        /// * `file_data` - Complete GGUF file bytes
    397
        ///
    398
        /// # Returns
    399
        ///
    400
        /// Dequantized f32 tensor data
    401
        ///
    402
        /// # Errors
    403
        ///
    404
        /// Returns error if:
    405
        /// - Tensor not found
    406
        /// - Unsupported quantization type
    407
        /// - Invalid data at offset
    408
        ///
    409
        /// # Examples
    410
        ///
    411
        /// ```rust,ignore
    412
        /// let file_data = std::fs::read("model.gguf")?;
    413
        /// let model = GGUFModel::from_bytes(&file_data)?;
    414
        /// let weights = model.get_tensor_f32("layer.0.weight", &file_data)?;
    415
        /// ```
    416
    159
        pub fn get_tensor_f32(&self, name: &str, file_data: &[u8]) -> Result<Vec<f32>> {
    417
            // Find tensor info
    418
    159
            let tensor51 = self
    419
    159
                .tensors
    420
    159
                .iter()
    421
    1.55k
                .find159(|t| t.name == name)
    422
    159
                .ok_or_else(|| RealizarError::UnsupportedOperation {
    423
    108
                    operation: "get_tensor_f32".to_string(),
    424
    108
                    reason: format!("Tensor '{name}' not found"),
    425
    108
                })?;
    426
    427
            // Calculate tensor size in elements
    428
    51
            let size: usize = tensor
    429
    51
                .dims
    430
    51
                .iter()
    431
    60
                .try_fold51(1usize, |acc, &dim| {
    432
    60
                    usize::try_from(dim).ok().and_then(|d| acc.checked_mul(d))
    433
    60
                })
    434
    51
                .ok_or_else(|| RealizarError::InvalidShape {
    435
    0
                    reason: format!("Tensor dimensions overflow: {:?}", tensor.dims),
    436
    0
                })?;
    437
    438
            // Convert tensor offset to usize and add tensor data start
    439
    51
            let tensor_offset =
    440
    51
                usize::try_from(tensor.offset).map_err(|_| RealizarError::UnsupportedOperation {
    441
    0
                    operation: "convert_offset".to_string(),
    442
    0
                    reason: format!("Offset {} exceeds platform usize limit", tensor.offset),
    443
    0
                })?;
    444
    51
            let offset = self.tensor_data_start + tensor_offset;
    445
    446
            // Extract and dequantize based on qtype
    447
    51
            match tensor.qtype {
    448
                GGUF_TYPE_F32 => {
    449
                    // Unquantized F32 data
    450
    40
                    let byte_size = size * 4; // 4 bytes per f32
    451
    40
                    if offset + byte_size > file_data.len() {
    452
    1
                        return Err(RealizarError::UnsupportedOperation {
    453
    1
                            operation: "get_tensor_f32".to_string(),
    454
    1
                            reason: format!(
    455
    1
                                "Data range [{}, {}) exceeds file size {}",
    456
    1
                                offset,
    457
    1
                                offset + byte_size,
    458
    1
                                file_data.len()
    459
    1
                            ),
    460
    1
                        });
    461
    39
                    }
    462
    463
    39
                    let bytes = &file_data[offset..offset + byte_size];
    464
    39
                    let values = bytes
    465
    39
                        .chunks_exact(4)
    466
    99.0k
                        .map39(|chunk| f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]))
    467
    39
                        .collect();
    468
    39
                    Ok(values)
    469
                },
    470
                GGUF_TYPE_Q4_0 => {
    471
                    // Q4_0 quantized data
    472
                    use crate::quantize::dequantize_q4_0;
    473
    474
                    // Q4_0 block: 32 elements
    475
                    // Layout: 1×f16 scale (2 bytes) + 16 bytes (32×4-bit values) = 18 bytes
    476
                    const BLOCK_BYTES: usize = 18;
    477
                    const BLOCK_SIZE: usize = 32;
    478
    479
    1
                    let num_blocks = size.div_ceil(BLOCK_SIZE);
    480
    1
                    let byte_size = num_blocks * BLOCK_BYTES;
    481
    482
    1
                    if offset + byte_size > file_data.len() {
    483
    0
                        return Err(RealizarError::UnsupportedOperation {
    484
    0
                            operation: "get_tensor_f32".to_string(),
    485
    0
                            reason: format!(
    486
    0
                                "Data range [{}, {}) exceeds file size {}",
    487
    0
                                offset,
    488
    0
                                offset + byte_size,
    489
    0
                                file_data.len()
    490
    0
                            ),
    491
    0
                        });
    492
    1
                    }
    493
    494
    1
                    let bytes = &file_data[offset..offset + byte_size];
    495
    1
                    let mut values = dequantize_q4_0(bytes)?0;
    496
    497
                    // Trim to exact size (dequantization pads to block boundaries)
    498
    1
                    values.truncate(size);
    499
    1
                    Ok(values)
    500
                },
    501
                GGUF_TYPE_Q8_0 => {
    502
                    // Q8_0 quantized data - use SIMD-parallel for faster loading
    503
                    use crate::quantize::dequantize_q8_0_simd;
    504
    505
                    // Q8_0 block size: 34 bytes (2 for f16 scale + 32 for quants)
    506
                    const BLOCK_BYTES: usize = 34;
    507
                    const BLOCK_SIZE: usize = 32;
    508
    509
    1
                    let num_blocks = size.div_ceil(BLOCK_SIZE);
    510
    1
                    let byte_size = num_blocks * BLOCK_BYTES;
    511
    512
    1
                    if offset + byte_size > file_data.len() {
    513
    0
                        return Err(RealizarError::UnsupportedOperation {
    514
    0
                            operation: "get_tensor_f32".to_string(),
    515
    0
                            reason: format!(
    516
    0
                                "Data range [{}, {}) exceeds file size {}",
    517
    0
                                offset,
    518
    0
                                offset + byte_size,
    519
    0
                                file_data.len()
    520
    0
                            ),
    521
    0
                        });
    522
    1
                    }
    523
    524
    1
                    let bytes = &file_data[offset..offset + byte_size];
    525
    1
                    let mut values = dequantize_q8_0_simd(bytes)?0;
    526
    527
                    // Trim to exact size
    528
    1
                    values.truncate(size);
    529
    1
                    Ok(values)
    530
                },
    531
                GGUF_TYPE_Q2_K => {
    532
                    // Q2_K quantized data (K-quantization) - 2 bits per weight
    533
                    use crate::quantize::{dequantize_q2_k, QK_K};
    534
    535
                    // Q2_K super-block size: 84 bytes for 256 values
    536
                    const SUPER_BLOCK_BYTES: usize = 84;
    537
    538
    1
                    let num_super_blocks = size.div_ceil(QK_K);
    539
    1
                    let byte_size = num_super_blocks * SUPER_BLOCK_BYTES;
    540
    541
    1
                    if offset + byte_size > file_data.len() {
    542
    0
                        return Err(RealizarError::UnsupportedOperation {
    543
    0
                            operation: "get_tensor_f32".to_string(),
    544
    0
                            reason: format!(
    545
    0
                                "Data range [{}, {}) exceeds file size {}",
    546
    0
                                offset,
    547
    0
                                offset + byte_size,
    548
    0
                                file_data.len()
    549
    0
                            ),
    550
    0
                        });
    551
    1
                    }
    552
    553
    1
                    let bytes = &file_data[offset..offset + byte_size];
    554
    1
                    let mut values = dequantize_q2_k(bytes)?0;
    555
    556
                    // Trim to exact size
    557
    1
                    values.truncate(size);
    558
    1
                    Ok(values)
    559
                },
    560
                GGUF_TYPE_Q4_K => {
    561
                    // Q4_K quantized data (K-quantization) - use SIMD-parallel for faster loading
    562
                    use crate::quantize::{dequantize_q4_k_simd, QK_K};
    563
    564
                    // Q4_K super-block size: 144 bytes for 256 values
    565
                    const SUPER_BLOCK_BYTES: usize = 144;
    566
    567
    1
                    let num_super_blocks = size.div_ceil(QK_K);
    568
    1
                    let byte_size = num_super_blocks * SUPER_BLOCK_BYTES;
    569
    570
    1
                    if offset + byte_size > file_data.len() {
    571
    0
                        return Err(RealizarError::UnsupportedOperation {
    572
    0
                            operation: "get_tensor_f32".to_string(),
    573
    0
                            reason: format!(
    574
    0
                                "Data range [{}, {}) exceeds file size {}",
    575
    0
                                offset,
    576
    0
                                offset + byte_size,
    577
    0
                                file_data.len()
    578
    0
                            ),
    579
    0
                        });
    580
    1
                    }
    581
    582
    1
                    let bytes = &file_data[offset..offset + byte_size];
    583
    1
                    let mut values = dequantize_q4_k_simd(bytes)?0;
    584
    585
                    // Trim to exact size
    586
    1
                    values.truncate(size);
    587
    1
                    Ok(values)
    588
                },
    589
                GGUF_TYPE_Q5_K => {
    590
                    // Q5_K quantized data (K-quantization)
    591
                    use crate::quantize::{dequantize_q5_k, QK_K};
    592
    593
                    // Q5_K super-block size: 176 bytes for 256 values
    594
                    const SUPER_BLOCK_BYTES: usize = 176;
    595
    596
    1
                    let num_super_blocks = size.div_ceil(QK_K);
    597
    1
                    let byte_size = num_super_blocks * SUPER_BLOCK_BYTES;
    598
    599
    1
                    if offset + byte_size > file_data.len() {
    600
    0
                        return Err(RealizarError::UnsupportedOperation {
    601
    0
                            operation: "get_tensor_f32".to_string(),
    602
    0
                            reason: format!(
    603
    0
                                "Data range [{}, {}) exceeds file size {}",
    604
    0
                                offset,
    605
    0
                                offset + byte_size,
    606
    0
                                file_data.len()
    607
    0
                            ),
    608
    0
                        });
    609
    1
                    }
    610
    611
    1
                    let bytes = &file_data[offset..offset + byte_size];
    612
    1
                    let mut values = dequantize_q5_k(bytes)?0;
    613
    614
                    // Trim to exact size
    615
    1
                    values.truncate(size);
    616
    1
                    Ok(values)
    617
                },
    618
                GGUF_TYPE_Q6_K => {
    619
                    // Q6_K quantized data (K-quantization)
    620
                    use crate::quantize::{dequantize_q6_k, QK_K};
    621
    622
                    // Q6_K super-block size: 210 bytes for 256 values
    623
                    const SUPER_BLOCK_BYTES: usize = 210;
    624
    625
    1
                    let num_super_blocks = size.div_ceil(QK_K);
    626
    1
                    let byte_size = num_super_blocks * SUPER_BLOCK_BYTES;
    627
    628
    1
                    if offset + byte_size > file_data.len() {
    629
    0
                        return Err(RealizarError::UnsupportedOperation {
    630
    0
                            operation: "get_tensor_f32".to_string(),
    631
    0
                            reason: format!(
    632
    0
                                "Data range [{}, {}) exceeds file size {}",
    633
    0
                                offset,
    634
    0
                                offset + byte_size,
    635
    0
                                file_data.len()
    636
    0
                            ),
    637
    0
                        });
    638
    1
                    }
    639
    640
    1
                    let bytes = &file_data[offset..offset + byte_size];
    641
    1
                    let mut values = dequantize_q6_k(bytes)?0;
    642
    643
                    // Trim to exact size
    644
    1
                    values.truncate(size);
    645
    1
                    Ok(values)
    646
                },
    647
                GGUF_TYPE_F16 => {
    648
                    // F16 (half-precision float) data
    649
                    use crate::quantize::dequantize_f16;
    650
    651
    1
                    let byte_size = size * 2; // 2 bytes per f16
    652
    1
                    if offset + byte_size > file_data.len() {
    653
    0
                        return Err(RealizarError::UnsupportedOperation {
    654
    0
                            operation: "get_tensor_f32".to_string(),
    655
    0
                            reason: format!(
    656
    0
                                "Data range [{}, {}) exceeds file size {}",
    657
    0
                                offset,
    658
    0
                                offset + byte_size,
    659
    0
                                file_data.len()
    660
    0
                            ),
    661
    0
                        });
    662
    1
                    }
    663
    664
    1
                    let bytes = &file_data[offset..offset + byte_size];
    665
    1
                    let values = dequantize_f16(bytes)?0;
    666
    1
                    Ok(values)
    667
                },
    668
                GGUF_TYPE_Q4_1 => {
    669
                    // Q4_1 quantized data
    670
                    use crate::quantize::dequantize_q4_1;
    671
    672
                    // Q4_1 block size: 20 bytes (2 for scale + 2 for min + 16 for quants)
    673
                    const BLOCK_BYTES: usize = 20;
    674
                    const BLOCK_SIZE: usize = 32;
    675
    676
    1
                    let num_blocks = size.div_ceil(BLOCK_SIZE);
    677
    1
                    let byte_size = num_blocks * BLOCK_BYTES;
    678
    679
    1
                    if offset + byte_size > file_data.len() {
    680
    0
                        return Err(RealizarError::UnsupportedOperation {
    681
    0
                            operation: "get_tensor_f32".to_string(),
    682
    0
                            reason: format!(
    683
    0
                                "Data range [{}, {}) exceeds file size {}",
    684
    0
                                offset,
    685
    0
                                offset + byte_size,
    686
    0
                                file_data.len()
    687
    0
                            ),
    688
    0
                        });
    689
    1
                    }
    690
    691
    1
                    let bytes = &file_data[offset..offset + byte_size];
    692
    1
                    let mut values = dequantize_q4_1(bytes)?0;
    693
    694
                    // Trim to exact size
    695
    1
                    values.truncate(size);
    696
    1
                    Ok(values)
    697
                },
    698
                GGUF_TYPE_Q5_0 => {
    699
                    // Q5_0 quantized data
    700
                    use crate::quantize::dequantize_q5_0;
    701
    702
                    // Q5_0 block size: 22 bytes (2 for scale + 4 for high bits + 16 for quants)
    703
                    const BLOCK_BYTES: usize = 22;
    704
                    const BLOCK_SIZE: usize = 32;
    705
    706
    1
                    let num_blocks = size.div_ceil(BLOCK_SIZE);
    707
    1
                    let byte_size = num_blocks * BLOCK_BYTES;
    708
    709
    1
                    if offset + byte_size > file_data.len() {
    710
    0
                        return Err(RealizarError::UnsupportedOperation {
    711
    0
                            operation: "get_tensor_f32".to_string(),
    712
    0
                            reason: format!(
    713
    0
                                "Data range [{}, {}) exceeds file size {}",
    714
    0
                                offset,
    715
    0
                                offset + byte_size,
    716
    0
                                file_data.len()
    717
    0
                            ),
    718
    0
                        });
    719
    1
                    }
    720
    721
    1
                    let bytes = &file_data[offset..offset + byte_size];
    722
    1
                    let mut values = dequantize_q5_0(bytes)?0;
    723
    724
                    // Trim to exact size
    725
    1
                    values.truncate(size);
    726
    1
                    Ok(values)
    727
                },
    728
                GGUF_TYPE_Q5_1 => {
    729
                    // Q5_1 quantized data
    730
                    use crate::quantize::dequantize_q5_1;
    731
    732
                    // Q5_1 block size: 24 bytes (2 for scale + 2 for min + 4 for high bits + 16 for quants)
    733
                    const BLOCK_BYTES: usize = 24;
    734
                    const BLOCK_SIZE: usize = 32;
    735
    736
    1
                    let num_blocks = size.div_ceil(BLOCK_SIZE);
    737
    1
                    let byte_size = num_blocks * BLOCK_BYTES;
    738
    739
    1
                    if offset + byte_size > file_data.len() {
    740
    0
                        return Err(RealizarError::UnsupportedOperation {
    741
    0
                            operation: "get_tensor_f32".to_string(),
    742
    0
                            reason: format!(
    743
    0
                                "Data range [{}, {}) exceeds file size {}",
    744
    0
                                offset,
    745
    0
                                offset + byte_size,
    746
    0
                                file_data.len()
    747
    0
                            ),
    748
    0
                        });
    749
    1
                    }
    750
    751
    1
                    let bytes = &file_data[offset..offset + byte_size];
    752
    1
                    let mut values = dequantize_q5_1(bytes)?0;
    753
    754
                    // Trim to exact size
    755
    1
                    values.truncate(size);
    756
    1
                    Ok(values)
    757
                },
    758
    1
                _ => Err(RealizarError::UnsupportedOperation {
    759
    1
                    operation: "get_tensor_f32".to_string(),
    760
    1
                    reason: format!("Unsupported quantization type: {}", tensor.qtype),
    761
    1
                }),
    762
            }
    763
    159
        }
    764
    765
        /// Extract model architecture from metadata
    766
    111
        pub fn architecture(&self) -> Option<&str> {
    767
    111
            if let Some(GGUFValue::String(arch108)) = self.metadata.get("general.architecture") {
    768
    108
                Some(arch.as_str())
    769
            } else {
    770
    3
                None
    771
            }
    772
    111
        }
    773
    774
        /// Get embedding dimension from metadata
    775
    12
        pub fn embedding_dim(&self) -> Option<usize> {
    776
    12
            let arch = self.architecture()?0;
    777
    12
            let key = format!("{}.embedding_length", arch);
    778
    12
            if let Some(GGUFValue::UInt32(dim)) = self.metadata.get(&key) {
    779
    12
                Some(*dim as usize)
    780
            } else {
    781
    0
                None
    782
            }
    783
    12
        }
    784
    785
        /// Get number of layers from metadata
    786
    12
        pub fn num_layers(&self) -> Option<usize> {
    787
    12
            let arch = self.architecture()?0;
    788
    12
            let key = format!("{}.block_count", arch);
    789
    12
            if let Some(GGUFValue::UInt32(count)) = self.metadata.get(&key) {
    790
    12
                Some(*count as usize)
    791
            } else {
    792
    0
                None
    793
            }
    794
    12
        }
    795
    796
        /// Get number of attention heads from metadata
    797
    12
        pub fn num_heads(&self) -> Option<usize> {
    798
    12
            let arch = self.architecture()?0;
    799
    12
            let key = format!("{}.attention.head_count", arch);
    800
    12
            if let Some(GGUFValue::UInt32(count)) = self.metadata.get(&key) {
    801
    12
                Some(*count as usize)
    802
            } else {
    803
    0
                None
    804
            }
    805
    12
        }
    806
    807
        /// Get context length from metadata
    808
    11
        pub fn context_length(&self) -> Option<usize> {
    809
    11
            let arch = self.architecture()?0;
    810
    11
            let key = format!("{}.context_length", arch);
    811
    11
            if let Some(GGUFValue::UInt32(len)) = self.metadata.get(&key) {
    812
    11
                Some(*len as usize)
    813
            } else {
    814
    0
                None
    815
            }
    816
    11
        }
    817
    818
        /// Get number of key-value heads from metadata (for GQA)
    819
    11
        pub fn num_kv_heads(&self) -> Option<usize> {
    820
    11
            let arch = self.architecture()?0;
    821
    11
            let key = format!("{}.attention.head_count_kv", arch);
    822
    11
            if let Some(GGUFValue::UInt32(count)) = self.metadata.get(&key) {
    823
    11
                Some(*count as usize)
    824
            } else {
    825
    0
                None
    826
            }
    827
    11
        }
    828
    829
        /// Get RoPE frequency base from metadata
    830
        /// Different models use different bases (LLaMA: 10000, Qwen2: 1000000)
    831
    11
        pub fn rope_freq_base(&self) -> Option<f32> {
    832
    11
            let arch = self.architecture()?0;
    833
    11
            let key = format!("{}.rope.freq_base", arch);
    834
    11
            if let Some(GGUFValue::Float32(base10)) = self.metadata.get(&key) {
    835
    10
                Some(*base)
    836
            } else {
    837
    1
                None
    838
            }
    839
    11
        }
    840
    841
        /// Get RMSNorm epsilon from metadata
    842
        /// Different models use different values (LLaMA: 1e-5, Qwen2: 1e-6)
    843
    11
        pub fn rms_epsilon(&self) -> Option<f32> {
    844
    11
            let arch = self.architecture()?0;
    845
    11
            let key = format!("{}.attention.layer_norm_rms_epsilon", arch);
    846
    11
            if let Some(GGUFValue::Float32(eps10)) = self.metadata.get(&key) {
    847
    10
                Some(*eps)
    848
            } else {
    849
    1
                None
    850
            }
    851
    11
        }
    852
    853
        /// Get RoPE type from metadata or infer from architecture
    854
        /// Returns: 0 = NORM (adjacent pairs), 2 = NEOX (split halves)
    855
        /// Per llama.cpp: LLAMA_ROPE_TYPE_NORM = 0, LLAMA_ROPE_TYPE_NEOX = 2
    856
        ///
    857
        /// Architecture-based inference matches llama.cpp's llama-model.cpp:7763-7811
    858
    15
        pub fn rope_type(&self) -> Option<u32> {
    859
    15
            let arch = self.architecture()?0;
    860
    15
            let key = format!("{}.rope.scaling.type", arch);
    861
            // Try rope type from scaling type first
    862
    15
            if let Some(GGUFValue::String(s2)) = self.metadata.get(&key) {
    863
    2
                match s.as_str() {
    864
    2
                    "none" | "linear" => return Some(0)1, // NORM style
    865
    1
                    "yarn" | "neox"0 => return Some(2),   // NEOX style
    866
    0
                    _ => {},
    867
                }
    868
    13
            }
    869
            // Infer rope type from architecture (matches llama.cpp llama-model.cpp:7763-7811)
    870
            // NEOX style (type 2): pairs offset by n_rot/2
    871
    13
            let arch_lower = arch.to_lowercase();
    872
    13
            let neox_architectures = [
    873
    13
                "qwen",
    874
    13
                "qwen2",
    875
    13
                "qwen3",
    876
    13
                "stablelm",
    877
    13
                "phi2",
    878
    13
                "phi3",
    879
    13
                "gemma",
    880
    13
                "gemma2",
    881
    13
                "gemma3",
    882
    13
                "starcoder2",
    883
    13
                "gptneox",
    884
    13
                "falcon",
    885
    13
                "codeshell",
    886
    13
                "orion",
    887
    13
                "bert",
    888
    13
                "nomic-bert",
    889
    13
                "dbrx",
    890
    13
                "olmo2",
    891
    13
                "olmoe",
    892
    13
                "plamo",
    893
    13
                "plamo2",
    894
    13
                "openelm",
    895
    13
                "exaone",
    896
    13
                "minicpm3",
    897
    13
                "nemotron",
    898
    13
                "internlm2",
    899
    13
                "deepseek2",
    900
    13
            ];
    901
    292
            for neox_arch282 in neox_architectures {
    902
    282
                if arch_lower.contains(neox_arch) {
    903
    3
                    return Some(2); // NEOX style
    904
    279
                }
    905
            }
    906
            // NORM style (type 0): adjacent pairs - default for LLaMA, TinyLlama
    907
    10
            Some(0)
    908
    15
        }
    909
    910
        /// Get BOS (beginning of sentence) token ID
    911
        #[must_use]
    912
    2
        pub fn bos_token_id(&self) -> Option<u32> {
    913
    2
            if let Some(GGUFValue::UInt32(id1)) = self.metadata.get("tokenizer.ggml.bos_token_id") {
    914
    1
                Some(*id)
    915
            } else {
    916
    1
                None
    917
            }
    918
    2
        }
    919
    920
        /// Get EOS (end of sentence) token ID
    921
        #[must_use]
    922
    1
        pub fn eos_token_id(&self) -> Option<u32> {
    923
    1
            if let Some(GGUFValue::UInt32(id)) = self.metadata.get("tokenizer.ggml.eos_token_id") {
    924
    1
                Some(*id)
    925
            } else {
    926
    0
                None
    927
            }
    928
    1
        }
    929
    930
        /// Get vocabulary tokens from metadata
    931
        ///
    932
        /// Returns the token strings indexed by token ID.
    933
        /// Uses "tokenizer.ggml.tokens" key from GGUF metadata.
    934
        #[must_use]
    935
    7
        pub fn vocabulary(&self) -> Option<Vec<String>> {
    936
    7
            if let Some(GGUFValue::Array(arr4)) = self.metadata.get("tokenizer.ggml.tokens") {
    937
    4
                let tokens: Vec<String> = arr
    938
    4
                    .iter()
    939
    11
                    .filter_map4(|v| {
    940
    11
                        if let GGUFValue::String(s) = v {
    941
    11
                            Some(s.clone())
    942
                        } else {
    943
    0
                            None
    944
                        }
    945
    11
                    })
    946
    4
                    .collect();
    947
    4
                if tokens.is_empty() {
    948
    0
                    None
    949
                } else {
    950
    4
                    Some(tokens)
    951
                }
    952
            } else {
    953
    3
                None
    954
            }
    955
    7
        }
    956
    957
        /// Decode token IDs to text using vocabulary
    958
        ///
    959
        /// Returns decoded string. Unknown tokens are replaced with "�".
    960
        /// Handles BPE markers:
    961
        /// - GPT-2 style: Ġ (U+0120) → space, Ċ (U+010A) → newline
    962
        /// - SentencePiece: ▁ (U+2581) → space
    963
        /// - Byte tokens: <0xHH> → actual byte value
    964
        #[must_use]
    965
    3
        pub fn decode(&self, token_ids: &[u32]) -> String {
    966
    3
            if let Some(vocab2) = self.vocabulary() {
    967
                // Detect tokenizer type from metadata
    968
    2
                let is_gpt2_style = self
    969
    2
                    .metadata
    970
    2
                    .get("tokenizer.ggml.model")
    971
    2
                    .is_some_and(|v| matches!(v0, GGUFValue::String(s0) if s0 == "gpt2"0));
    972
    973
                // Collect raw tokens and convert byte tokens to actual bytes
    974
    2
                let mut bytes: Vec<u8> = Vec::new();
    975
    976
    7
                for &id5 in token_ids {
    977
    5
                    let token = vocab
    978
    5
                        .get(id as usize)
    979
    5
                        .map_or("�", std::string::String::as_str);
    980
    981
                    // Check if this is a byte token like <0xE6>
    982
    5
                    if token.starts_with("<0x") && token2.ends_with2('>') && token.len() == 62 {
    983
    2
                        if let Ok(byte_val) = u8::from_str_radix(&token[3..5], 16) {
    984
    2
                            bytes.push(byte_val);
    985
    2
                            continue;
    986
    0
                        }
    987
    3
                    }
    988
    989
                    // For GPT-2 style tokenizers, decode byte-level BPE properly
    990
                    // Each unicode character in the token represents a raw byte
    991
    3
                    if is_gpt2_style {
    992
    0
                        for c in token.chars() {
    993
    0
                            if let Some(byte) = gpt2_unicode_to_byte(c) {
    994
    0
                                bytes.push(byte);
    995
    0
                            }
    996
                        }
    997
    3
                    } else {
    998
    3
                        // SentencePiece style - tokens are regular strings
    999
    3
                        bytes.extend_from_slice(token.as_bytes());
    1000
    3
                    }
    1001
                }
    1002
    1003
                // Decode bytes as UTF-8 (lossy for invalid sequences)
    1004
    2
                let raw = String::from_utf8_lossy(&bytes).into_owned();
    1005
    1006
                // Post-process BPE markers (only for SentencePiece, GPT-2 already handled)
    1007
    2
                if !is_gpt2_style {
    1008
    2
                    raw.replace('▁', " ") // SentencePiece word boundary
    1009
                } else {
    1010
    0
                    raw
    1011
                }
    1012
            } else {
    1013
                // Fallback to ASCII if no vocabulary
    1014
    1
                token_ids
    1015
    1
                    .iter()
    1016
    3
                    .map1(|&t| char::from_u32(t.min(127)).unwrap_or('?'))
    1017
    1
                    .collect()
    1018
            }
    1019
    3
        }
    1020
    1021
        /// Encode text to token IDs using vocabulary
    1022
        ///
    1023
        /// Uses greedy longest-match tokenization with special token priority.
    1024
        /// Returns None if no vocabulary is available.
    1025
        ///
    1026
        /// Supports both tokenizer types:
    1027
        /// - SentencePiece (llama): Uses `▁` (U+2581) for word boundaries
    1028
        /// - GPT-2 (qwen2, gpt2): Uses `Ġ` (U+0120) for space prefixes
    1029
        #[must_use]
    1030
    2
        pub fn encode(&self, text: &str) -> Option<Vec<u32>> {
    1031
    2
            let vocab1 = self.vocabulary()?1;
    1032
    1033
            // Build reverse lookup: token string -> token ID
    1034
    1
            let token_to_id: std::collections::HashMap<&str, u32> = vocab
    1035
    1
                .iter()
    1036
    1
                .enumerate()
    1037
    3
                .map1(|(id, token)| (token.as_str(), id as u32))
    1038
    1
                .collect();
    1039
    1040
            // Identify special tokens (high-ID tokens with <|...|> pattern)
    1041
            // These need priority matching to avoid being split by greedy algorithm
    1042
    1
            let special_tokens: Vec<(&str, u32)> = vocab
    1043
    1
                .iter()
    1044
    1
                .enumerate()
    1045
    3
                .filter1(|(id, tok)| *id >= 151643 && tok.starts_with("<|")0 && tok.ends_with("|>")0)
    1046
    1
                .map(|(id, tok)| (tok0.as_str0(), id as u320))
    1047
    1
                .collect();
    1048
    1049
            // Detect tokenizer type from metadata
    1050
            // GPT-2 style uses Ġ (U+0120), SentencePiece uses ▁ (U+2581)
    1051
    1
            let is_gpt2_style = self
    1052
    1
                .metadata
    1053
    1
                .get("tokenizer.ggml.model")
    1054
    1
                .is_some_and(|v| matches!(v0, GGUFValue::String(s0) if s0 == "gpt2"0));
    1055
    1056
    1
            let space_char = if is_gpt2_style { '\u{0120}'0 } else { '▁' };
    1057
    1058
            // Split text on special tokens first, preserving them
    1059
    1
            let mut segments: Vec<(bool, &str)> = Vec::new(); // (is_special, text)
    1060
    1
            let mut text_remaining = text;
    1061
    1
            while !text_remaining.is_empty() {
    1062
                // Find earliest special token match
    1063
    1
                let mut earliest_match: Option<(usize, &str, u32)> = None;
    1064
    1
                for &(special_tok0, special_id0) in &special_tokens {
    1065
    0
                    if let Some(pos) = text_remaining.find(special_tok) {
    1066
    0
                        if earliest_match.is_none()
    1067
    0
                            || pos < earliest_match.as_ref().map_or(usize::MAX, |m| m.0)
    1068
    0
                        {
    1069
    0
                            earliest_match = Some((pos, special_tok, special_id));
    1070
    0
                        }
    1071
    0
                    }
    1072
                }
    1073
    1074
    1
                if let Some((pos0, special_tok0, _)) = earliest_match {
    1075
    0
                    if pos > 0 {
    1076
    0
                        segments.push((false, &text_remaining[..pos]));
    1077
    0
                    }
    1078
    0
                    segments.push((true, special_tok));
    1079
    0
                    text_remaining = &text_remaining[pos + special_tok.len()..];
    1080
                } else {
    1081
    1
                    segments.push((false, text_remaining));
    1082
    1
                    break;
    1083
                }
    1084
            }
    1085
    1086
    1
            let mut tokens = Vec::new();
    1087
    1088
    2
            for (is_special1, segment1) in segments {
    1089
    1
                if is_special {
    1090
                    // Direct lookup for special token
    1091
    0
                    if let Some(&id) = token_to_id.get(segment) {
    1092
    0
                        tokens.push(id);
    1093
    0
                    }
    1094
    0
                    continue;
    1095
    1
                }
    1096
    1097
                // Process non-special segment with character replacement
    1098
    1
                let text_with_prefix = if is_gpt2_style {
    1099
    0
                    segment.to_string()
    1100
    1
                } else if segment.starts_with(' ') {
    1101
    0
                    segment.to_string()
    1102
                } else {
    1103
    1
                    format!(" {}", segment)
    1104
                };
    1105
    1106
    1
                let processed = if is_gpt2_style {
    1107
    0
                    text_with_prefix
    1108
    0
                        .replace(' ', &space_char.to_string())
    1109
    0
                        .replace('\n', "\u{010A}") // Ċ = GPT-2 newline
    1110
                } else {
    1111
    1
                    text_with_prefix.replace(' ', &space_char.to_string())
    1112
                };
    1113
    1114
    1
                let mut remaining = processed.as_str();
    1115
    1116
    4
                while !remaining.is_empty() {
    1117
                    // Greedy longest match using character boundaries (not byte indices)
    1118
    3
                    let mut best_byte_len = 0;
    1119
    3
                    let mut best_id = None;
    1120
    1121
                    // Collect character byte offsets for proper slicing
    1122
    3
                    let char_indices: Vec<usize> = remaining
    1123
    3
                        .char_indices()
    1124
    3
                        .map(|(i, _)| i)
    1125
    3
                        .chain(std::iter::once(remaining.len()))
    1126
    3
                        .collect();
    1127
    1128
                    // Try all prefixes up to 32 chars (reasonable max token length)
    1129
    21
                    for char_count in 1..=char_indices3.len3().saturating_sub3(1).min3(32) {
    1130
    21
                        let byte_end = char_indices[char_count];
    1131
    21
                        let prefix = &remaining[..byte_end];
    1132
    21
                        if let Some(&id3) = token_to_id.get(prefix) {
    1133
    3
                            best_byte_len = byte_end;
    1134
    3
                            best_id = Some(id);
    1135
    18
                        }
    1136
                    }
    1137
    1138
    3
                    if let Some(id) = best_id {
    1139
    3
                        tokens.push(id);
    1140
    3
                        remaining = &remaining[best_byte_len..];
    1141
    3
                    } else {
    1142
                        // No match found - try single UTF-8 char as byte tokens
    1143
                        // SAFETY: remaining is non-empty (loop condition guarantees this)
    1144
    0
                        let ch = remaining
    1145
    0
                            .chars()
    1146
    0
                            .next()
    1147
    0
                            .expect("loop invariant: remaining non-empty");
    1148
    0
                        let ch_len = ch.len_utf8();
    1149
    1150
                        // Look for byte tokens like <0x48> for 'H'
    1151
    0
                        for byte in remaining[..ch_len].bytes() {
    1152
    0
                            let byte_token = format!("<0x{:02X}>", byte);
    1153
    0
                            if let Some(&id) = token_to_id.get(byte_token.as_str()) {
    1154
    0
                                tokens.push(id);
    1155
    0
                            } else {
    1156
    0
                                // Unknown byte - use a common unknown token ID (usually 0 or 1)
    1157
    0
                                tokens.push(0);
    1158
    0
                            }
    1159
                        }
    1160
    0
                        remaining = &remaining[ch_len..];
    1161
                    }
    1162
                }
    1163
            }
    1164
    1165
    1
            Some(tokens)
    1166
    2
        }
    1167
    }
    1168
    1169
    use crate::gguf::{OwnedQuantizedModel, OwnedQuantizedLayer, OwnedQuantizedTensor, OwnedQKVWeights, QuantizedGGUFTransformer};
    1170
    1171
    impl GGUFTransformer {
    1172
        /// Load transformer weights from GGUF model
    1173
        ///
    1174
        /// # Arguments
    1175
        ///
    1176
        /// * `model` - Parsed GGUF model
    1177
        /// * `file_data` - Original file bytes for tensor extraction
    1178
        ///
    1179
        /// # Errors
    1180
        ///
    1181
        /// Returns error if required tensors are missing or malformed
    1182
    0
        pub fn from_gguf(model: &GGUFModel, file_data: &[u8]) -> Result<Self> {
    1183
    0
            let config = GGUFConfig::from_gguf(model)?;
    1184
    1185
            // Load token embedding
    1186
    0
            let token_embedding = model.get_tensor_f32("token_embd.weight", file_data)?;
    1187
    1188
            // Load layers
    1189
    0
            let mut layers = Vec::with_capacity(config.num_layers);
    1190
    0
            for layer_idx in 0..config.num_layers {
    1191
    0
                let layer = Self::load_layer(model, file_data, layer_idx)?;
    1192
    0
                layers.push(layer);
    1193
            }
    1194
    1195
            // Load output norm (raw gamma values - no delta transformation needed)
    1196
    0
            let output_norm_weight = model.get_tensor_f32("output_norm.weight", file_data)?;
    1197
    0
            let output_norm_bias = model.get_tensor_f32("output_norm.bias", file_data).ok();
    1198
    1199
            // Load LM head (output projection)
    1200
            // Fall back to token_embd.weight for tied embeddings (Qwen2, some LLaMA variants)
    1201
    0
            let lm_head_weight = model
    1202
    0
                .get_tensor_f32("output.weight", file_data)
    1203
    0
                .or_else(|_| model.get_tensor_f32("token_embd.weight", file_data))?;
    1204
    0
            let lm_head_bias = model.get_tensor_f32("output.bias", file_data).ok();
    1205
    1206
    0
            Ok(Self {
    1207
    0
                config,
    1208
    0
                token_embedding,
    1209
    0
                layers,
    1210
    0
                output_norm_weight,
    1211
    0
                output_norm_bias,
    1212
    0
                lm_head_weight,
    1213
    0
                lm_head_bias,
    1214
    0
            })
    1215
    0
        }
    1216
    1217
        /// Load a single transformer layer
    1218
        ///
    1219
        /// Supports both tensor naming conventions:
    1220
        /// - phi-2 style: combined `attn_qkv.weight`
    1221
        /// - llama style: separate `attn_q.weight`, `attn_k.weight`, `attn_v.weight`
    1222
    0
        fn load_layer(
    1223
    0
            model: &GGUFModel,
    1224
    0
            file_data: &[u8],
    1225
    0
            layer_idx: usize,
    1226
    0
        ) -> Result<GGUFTransformerLayer> {
    1227
    0
            let prefix = format!("blk.{}", layer_idx);
    1228
    1229
            // Attention norm weights
    1230
    0
            let attn_norm_weight =
    1231
    0
                model.get_tensor_f32(&format!("{}.attn_norm.weight", prefix), file_data)?;
    1232
    0
            let attn_norm_bias = model
    1233
    0
                .get_tensor_f32(&format!("{}.attn_norm.bias", prefix), file_data)
    1234
    0
                .ok();
    1235
    1236
            // QKV weights - try combined first (phi-2), fall back to separate (llama)
    1237
    0
            let (qkv_weight, qkv_bias) = if let Ok(combined) =
    1238
    0
                model.get_tensor_f32(&format!("{}.attn_qkv.weight", prefix), file_data)
    1239
            {
    1240
                // phi-2 style: combined QKV tensor
    1241
    0
                let bias = model
    1242
    0
                    .get_tensor_f32(&format!("{}.attn_qkv.bias", prefix), file_data)
    1243
    0
                    .ok();
    1244
    0
                (combined, bias)
    1245
            } else {
    1246
                // llama style: separate Q, K, V tensors - concatenate them
    1247
    0
                let q_weight = model.get_tensor_f32(&format!("{}.attn_q.weight", prefix), file_data)?;
    1248
    0
                let k_weight = model.get_tensor_f32(&format!("{}.attn_k.weight", prefix), file_data)?;
    1249
    0
                let v_weight = model.get_tensor_f32(&format!("{}.attn_v.weight", prefix), file_data)?;
    1250
    1251
                // Concatenate Q, K, V weights
    1252
    0
                let mut qkv = Vec::with_capacity(q_weight.len() + k_weight.len() + v_weight.len());
    1253
    0
                qkv.extend_from_slice(&q_weight);
    1254
    0
                qkv.extend_from_slice(&k_weight);
    1255
    0
                qkv.extend_from_slice(&v_weight);
    1256
    1257
                // Try to get biases (llama usually doesn't have them)
    1258
    0
                let q_bias = model
    1259
    0
                    .get_tensor_f32(&format!("{}.attn_q.bias", prefix), file_data)
    1260
    0
                    .ok();
    1261
    0
                let k_bias = model
    1262
    0
                    .get_tensor_f32(&format!("{}.attn_k.bias", prefix), file_data)
    1263
    0
                    .ok();
    1264
    0
                let v_bias = model
    1265
    0
                    .get_tensor_f32(&format!("{}.attn_v.bias", prefix), file_data)
    1266
    0
                    .ok();
    1267
    1268
    0
                let bias = match (q_bias, k_bias, v_bias) {
    1269
    0
                    (Some(q), Some(k), Some(v)) => {
    1270
    0
                        let mut combined_bias = Vec::with_capacity(q.len() + k.len() + v.len());
    1271
    0
                        combined_bias.extend_from_slice(&q);
    1272
    0
                        combined_bias.extend_from_slice(&k);
    1273
    0
                        combined_bias.extend_from_slice(&v);
    1274
    0
                        Some(combined_bias)
    1275
                    },
    1276
    0
                    _ => None,
    1277
                };
    1278
    1279
    0
                (qkv, bias)
    1280
            };
    1281
    1282
            // Attention output
    1283
    0
            let attn_output_weight =
    1284
    0
                model.get_tensor_f32(&format!("{}.attn_output.weight", prefix), file_data)?;
    1285
    0
            let attn_output_bias = model
    1286
    0
                .get_tensor_f32(&format!("{}.attn_output.bias", prefix), file_data)
    1287
    0
                .ok();
    1288
    1289
            // FFN gate (SwiGLU models like llama have this)
    1290
    0
            let ffn_gate_weight = model
    1291
    0
                .get_tensor_f32(&format!("{}.ffn_gate.weight", prefix), file_data)
    1292
    0
                .ok();
    1293
    0
            let ffn_gate_bias = model
    1294
    0
                .get_tensor_f32(&format!("{}.ffn_gate.bias", prefix), file_data)
    1295
    0
                .ok();
    1296
    1297
            // FFN up/down projections
    1298
    0
            let ffn_up_weight =
    1299
    0
                model.get_tensor_f32(&format!("{}.ffn_up.weight", prefix), file_data)?;
    1300
    0
            let ffn_up_bias = model
    1301
    0
                .get_tensor_f32(&format!("{}.ffn_up.bias", prefix), file_data)
    1302
    0
                .ok();
    1303
    0
            let ffn_down_weight =
    1304
    0
                model.get_tensor_f32(&format!("{}.ffn_down.weight", prefix), file_data)?;
    1305
    0
            let ffn_down_bias = model
    1306
    0
                .get_tensor_f32(&format!("{}.ffn_down.bias", prefix), file_data)
    1307
    0
                .ok();
    1308
    1309
            // FFN norm (models with separate FFN normalization)
    1310
    0
            let ffn_norm_weight = model
    1311
    0
                .get_tensor_f32(&format!("{}.ffn_norm.weight", prefix), file_data)
    1312
    0
                .ok();
    1313
    0
            let ffn_norm_bias = model
    1314
    0
                .get_tensor_f32(&format!("{}.ffn_norm.bias", prefix), file_data)
    1315
    0
                .ok();
    1316
    1317
    0
            Ok(GGUFTransformerLayer {
    1318
    0
                attn_norm_weight,
    1319
    0
                attn_norm_bias,
    1320
    0
                qkv_weight,
    1321
    0
                qkv_bias,
    1322
    0
                attn_output_weight,
    1323
    0
                attn_output_bias,
    1324
    0
                ffn_gate_weight,
    1325
    0
                ffn_gate_bias,
    1326
    0
                ffn_up_weight,
    1327
    0
                ffn_up_bias,
    1328
    0
                ffn_down_weight,
    1329
    0
                ffn_down_bias,
    1330
    0
                ffn_norm_weight,
    1331
    0
                ffn_norm_bias,
    1332
    0
            })
    1333
    0
        }
    1334
    }
    1335
    1336
    impl OwnedQuantizedModel {
    1337
        /// Create owned model from memory-mapped GGUF file
    1338
        ///
    1339
        /// # Errors
    1340
        ///
    1341
        /// Returns error if model loading fails
    1342
    2
        pub fn from_mapped(mapped: &crate::gguf::MappedGGUFModel) -> Result<Self> {
    1343
    2
            let data = mapped.data();
    1344
    2
            let transformer0 = QuantizedGGUFTransformer::from_gguf(&mapped.model, data)?;
    1345
    1346
            // Get config for dimension calculations
    1347
    0
            let config = &transformer.config;
    1348
    0
            let hidden_dim = config.hidden_dim;
    1349
    0
            let vocab_size = config.vocab_size;
    1350
    1351
            // Convert layers to owned (passing config for dimensions)
    1352
    0
            let layers: Vec<OwnedQuantizedLayer> = transformer
    1353
    0
                .layers
    1354
    0
                .iter()
    1355
    0
                .map(|l| OwnedQuantizedLayer::from_borrowed(l, data, config))
    1356
    0
                .collect();
    1357
    1358
    0
            Ok(Self {
    1359
    0
                config: transformer.config.clone(),
    1360
    0
                token_embedding: transformer.token_embedding,
    1361
    0
                layers,
    1362
    0
                output_norm_weight: transformer.output_norm_weight,
    1363
    0
                output_norm_bias: transformer.output_norm_bias,
    1364
    0
                // LM head: [hidden_dim] -> [vocab_size]
    1365
    0
                lm_head_weight: OwnedQuantizedTensor::from_ref_with_dims(
    1366
    0
                    &transformer.lm_head_weight,
    1367
    0
                    data,
    1368
    0
                    hidden_dim,
    1369
    0
                    vocab_size,
    1370
    0
                ),
    1371
    0
                lm_head_bias: transformer.lm_head_bias,
    1372
    0
                #[cfg(feature = "cuda")]
    1373
    0
                cuda_executor: None,
    1374
    0
                #[cfg(feature = "cuda")]
    1375
    0
                cuda_kernel_count: std::sync::atomic::AtomicU64::new(0),
    1376
    0
                #[cfg(feature = "cuda")]
    1377
    0
                cached_weight_names: std::sync::Mutex::new(std::collections::HashSet::new()),
    1378
    0
            })
    1379
    2
        }
    1380
    1381
        /// Create a model for testing purposes
    1382
        ///
    1383
        /// This constructor handles the internal CUDA fields automatically,
    1384
        /// allowing external tests to construct models without accessing pub(crate) fields.
    1385
        ///
    1386
        /// # Arguments
    1387
        /// * `config` - Model configuration
    1388
        /// * `token_embedding` - Token embedding weights
    1389
        /// * `layers` - Quantized transformer layers
    1390
        /// * `output_norm_weight` - Output normalization weight
    1391
        /// * `output_norm_bias` - Optional output normalization bias
    1392
        /// * `lm_head_weight` - Language model head weight
    1393
        /// * `lm_head_bias` - Optional language model head bias
    1394
        #[must_use]
    1395
    0
        pub fn new_for_test(
    1396
    0
            config: GGUFConfig,
    1397
    0
            token_embedding: Vec<f32>,
    1398
    0
            layers: Vec<OwnedQuantizedLayer>,
    1399
    0
            output_norm_weight: Vec<f32>,
    1400
    0
            output_norm_bias: Option<Vec<f32>>,
    1401
    0
            lm_head_weight: OwnedQuantizedTensor,
    1402
    0
            lm_head_bias: Option<Vec<f32>>,
    1403
    0
        ) -> Self {
    1404
    0
            Self {
    1405
    0
                config,
    1406
    0
                token_embedding,
    1407
    0
                layers,
    1408
    0
                output_norm_weight,
    1409
    0
                output_norm_bias,
    1410
    0
                lm_head_weight,
    1411
    0
                lm_head_bias,
    1412
    0
                #[cfg(feature = "cuda")]
    1413
    0
                cuda_executor: None,
    1414
    0
                #[cfg(feature = "cuda")]
    1415
    0
                cuda_kernel_count: std::sync::atomic::AtomicU64::new(0),
    1416
    0
                #[cfg(feature = "cuda")]
    1417
    0
                cached_weight_names: std::sync::Mutex::new(std::collections::HashSet::new()),
    1418
    0
            }
    1419
    0
        }
    1420
    1421
        /// Create model from memory-mapped APR file (SHOWCASE-APR-GPU)
    1422
        ///
    1423
        /// Converts APR Q4K format to GGUF-compatible model for GPU inference.
    1424
        /// The raw Q4K tensor data is byte-compatible between formats.
    1425
        ///
    1426
        /// # Arguments
    1427
        /// * `apr` - Memory-mapped APR model
    1428
        ///
    1429
        /// # Errors
    1430
        /// Returns error if APR format is invalid or missing required tensors.
    1431
    0
        pub fn from_apr(apr: &crate::apr::MappedAprModel) -> Result<Self> {
    1432
            use crate::apr::MappedAprModel;
    1433
    1434
    0
            let data = apr.data();
    1435
    0
            let data_offset = apr.data_offset() as usize;
    1436
    1437
            // Build config from APR metadata
    1438
    0
            let hidden_dim = apr.metadata.hidden_size.unwrap_or(1536);
    1439
    0
            let num_layers = apr.metadata.num_layers.unwrap_or(28);
    1440
    0
            let num_heads = apr.metadata.num_heads.unwrap_or(12);
    1441
    0
            let num_kv_heads = apr.metadata.num_kv_heads.unwrap_or(2);
    1442
    0
            let intermediate_dim = apr.metadata.intermediate_size.unwrap_or(8960);
    1443
    0
            let eps = apr.metadata.rms_norm_eps.unwrap_or(1e-6);
    1444
    0
            let rope_theta = apr.metadata.rope_theta.unwrap_or(1_000_000.0);
    1445
    1446
            // Infer vocab_size from embedding tensor if metadata is 0 or missing
    1447
    0
            let vocab_size = match apr.metadata.vocab_size {
    1448
    0
                Some(v) if v > 0 => v,
    1449
                _ => {
    1450
                    // Try to infer from embedding tensor shape
    1451
    0
                    apr.tensors
    1452
    0
                        .iter()
    1453
    0
                        .find(|t| {
    1454
    0
                            t.name.contains("embed_tokens")
    1455
    0
                                || t.name.contains("tok_embeddings")
    1456
    0
                                || t.name.contains("token_embd")
    1457
    0
                        })
    1458
    0
                        .and_then(|t| t.shape.first().copied())
    1459
    0
                        .unwrap_or(151936)
    1460
                },
    1461
            };
    1462
    1463
    0
            let config = GGUFConfig {
    1464
    0
                architecture: apr
    1465
    0
                    .metadata
    1466
    0
                    .architecture
    1467
    0
                    .clone()
    1468
    0
                    .unwrap_or_else(|| "qwen2".to_string()),
    1469
    0
                vocab_size,
    1470
    0
                hidden_dim,
    1471
    0
                num_layers,
    1472
    0
                num_heads,
    1473
    0
                num_kv_heads,
    1474
    0
                intermediate_dim,
    1475
    0
                eps,
    1476
    0
                rope_theta,
    1477
                rope_type: 2, // NEOX style for Qwen2.5
    1478
                context_length: 32768,
    1479
            };
    1480
    1481
            // Helper to get tensor data
    1482
    0
            let get_tensor = |name: &str| -> Result<&[u8]> {
    1483
    0
                let tensor = apr
    1484
    0
                    .find_tensor(name)
    1485
    0
                    .ok_or_else(|| RealizarError::FormatError {
    1486
    0
                        reason: format!("APR: tensor not found: {name}"),
    1487
    0
                    })?;
    1488
    0
                let start = data_offset + tensor.offset as usize;
    1489
    0
                let end = start + tensor.size as usize;
    1490
    0
                if end > data.len() {
    1491
    0
                    return Err(RealizarError::FormatError {
    1492
    0
                        reason: format!("APR: tensor {name} extends past EOF"),
    1493
    0
                    });
    1494
    0
                }
    1495
    0
                Ok(&data[start..end])
    1496
    0
            };
    1497
    1498
            // Helper to get tensor qtype
    1499
    0
            let get_qtype = |name: &str| -> u32 {
    1500
    0
                apr.find_tensor(name)
    1501
    0
                    .map_or(0, |t| MappedAprModel::dtype_to_qtype(&t.dtype))
    1502
    0
            };
    1503
    1504
            // Helper to make OwnedQuantizedTensor
    1505
    0
            let make_tensor =
    1506
    0
                |name: &str, in_dim: usize, out_dim: usize| -> Result<OwnedQuantizedTensor> {
    1507
    0
                    let tensor_data = get_tensor(name)?;
    1508
    0
                    let qtype = get_qtype(name);
    1509
    0
                    Ok(OwnedQuantizedTensor {
    1510
    0
                        data: tensor_data.to_vec(),
    1511
    0
                        in_dim,
    1512
    0
                        out_dim,
    1513
    0
                        qtype,
    1514
    0
                    })
    1515
    0
                };
    1516
    1517
            // Load token embeddings (F32)
    1518
    0
            let embed_name = apr
    1519
    0
                .tensors
    1520
    0
                .iter()
    1521
    0
                .find(|t| {
    1522
    0
                    t.name.contains("embed_tokens")
    1523
    0
                        || t.name.contains("tok_embeddings")
    1524
    0
                        || t.name.contains("token_embd")
    1525
    0
                })
    1526
    0
                .map(|t| t.name.as_str())
    1527
    0
                .ok_or_else(|| RealizarError::FormatError {
    1528
    0
                    reason: "APR: embedding tensor not found".to_string(),
    1529
    0
                })?;
    1530
    1531
    0
            let embed_data = get_tensor(embed_name)?;
    1532
    0
            let embed_dtype = apr.find_tensor(embed_name).map(|t| t.dtype.as_str());
    1533
    0
            let token_embedding: Vec<f32> = match embed_dtype {
    1534
    0
                Some("F32") => embed_data
    1535
    0
                    .chunks_exact(4)
    1536
    0
                    .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
    1537
    0
                    .collect(),
    1538
    0
                Some("Q4_K") => {
    1539
                    // Dequantize Q4_K embeddings
    1540
    0
                    crate::quantize::dequantize_q4_k(embed_data)?
    1541
                },
    1542
    0
                Some(dtype) => {
    1543
    0
                    return Err(RealizarError::FormatError {
    1544
    0
                        reason: format!("APR: unsupported embedding dtype: {dtype}"),
    1545
    0
                    });
    1546
                },
    1547
                None => {
    1548
    0
                    return Err(RealizarError::FormatError {
    1549
    0
                        reason: "APR: embedding tensor dtype not found".to_string(),
    1550
    0
                    });
    1551
                },
    1552
            };
    1553
    1554
            // Build layers
    1555
    0
            let mut layers = Vec::with_capacity(num_layers);
    1556
    0
            let head_dim = hidden_dim / num_heads;
    1557
    0
            let kv_dim = num_kv_heads * head_dim;
    1558
    1559
    0
            for layer_idx in 0..num_layers {
    1560
                // Find layer tensors (try multiple naming conventions)
    1561
    0
                let q_name = format!("blk.{layer_idx}.attn_q.weight");
    1562
    0
                let k_name = format!("blk.{layer_idx}.attn_k.weight");
    1563
    0
                let v_name = format!("blk.{layer_idx}.attn_v.weight");
    1564
    0
                let o_name = format!("blk.{layer_idx}.attn_output.weight");
    1565
    1566
    0
                let gate_name = format!("blk.{layer_idx}.ffn_gate.weight");
    1567
    0
                let up_name = format!("blk.{layer_idx}.ffn_up.weight");
    1568
    0
                let down_name = format!("blk.{layer_idx}.ffn_down.weight");
    1569
    1570
    0
                let attn_norm_name = format!("blk.{layer_idx}.attn_norm.weight");
    1571
    0
                let ffn_norm_name = format!("blk.{layer_idx}.ffn_norm.weight");
    1572
    1573
                // Q/K/V weights
    1574
    0
                let q_weight = make_tensor(&q_name, hidden_dim, hidden_dim)?;
    1575
    0
                let k_weight = make_tensor(&k_name, hidden_dim, kv_dim)?;
    1576
    0
                let v_weight = make_tensor(&v_name, hidden_dim, kv_dim)?;
    1577
    1578
    0
                let qkv_weight = OwnedQKVWeights::Separate {
    1579
    0
                    q: q_weight,
    1580
    0
                    k: k_weight,
    1581
    0
                    v: v_weight,
    1582
    0
                };
    1583
    1584
                // O projection
    1585
    0
                let o_weight = make_tensor(&o_name, hidden_dim, hidden_dim)?;
    1586
    1587
                // FFN weights
    1588
    0
                let ffn_gate_weight = make_tensor(&gate_name, hidden_dim, intermediate_dim)?;
    1589
    0
                let ffn_up_weight = make_tensor(&up_name, hidden_dim, intermediate_dim)?;
    1590
    0
                let ffn_down_weight = make_tensor(&down_name, intermediate_dim, hidden_dim)?;
    1591
    1592
                // Norm weights (F32)
    1593
    0
                let attn_norm_data = get_tensor(&attn_norm_name)?;
    1594
    0
                let ffn_norm_data = get_tensor(&ffn_norm_name)?;
    1595
    1596
    0
                let attn_norm_weight: Vec<f32> = attn_norm_data
    1597
    0
                    .chunks_exact(4)
    1598
    0
                    .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
    1599
    0
                    .collect();
    1600
    0
                let ffn_norm_weight: Vec<f32> = ffn_norm_data
    1601
    0
                    .chunks_exact(4)
    1602
    0
                    .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
    1603
    0
                    .collect();
    1604
    1605
    0
                layers.push(OwnedQuantizedLayer {
    1606
    0
                    attn_norm_weight,
    1607
    0
                    attn_norm_bias: None,
    1608
    0
                    qkv_weight,
    1609
    0
                    qkv_bias: None,
    1610
    0
                    attn_output_weight: o_weight,
    1611
    0
                    attn_output_bias: None,
    1612
    0
                    ffn_norm_weight: Some(ffn_norm_weight),
    1613
    0
                    ffn_norm_bias: None,
    1614
    0
                    ffn_gate_weight: Some(ffn_gate_weight),
    1615
    0
                    ffn_gate_bias: None,
    1616
    0
                    ffn_up_weight,
    1617
    0
                    ffn_up_bias: None,
    1618
    0
                    ffn_down_weight,
    1619
    0
                    ffn_down_bias: None,
    1620
    0
                });
    1621
            }
    1622
    1623
            // Output norm
    1624
    0
            let output_norm_name = apr
    1625
    0
                .tensors
    1626
    0
                .iter()
    1627
    0
                .find(|t| t.name.contains("output_norm") || t.name.contains("norm.weight"))
    1628
    0
                .map_or("output_norm.weight", |t| t.name.as_str());
    1629
    1630
    0
            let output_norm_data = get_tensor(output_norm_name)?;
    1631
    0
            let output_norm_weight: Vec<f32> = output_norm_data
    1632
    0
                .chunks_exact(4)
    1633
    0
                .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
    1634
    0
                .collect();
    1635
    1636
            // LM head - prioritize exact match, then contains (excluding layer tensors)
    1637
    0
            let lm_head_name = apr
    1638
    0
                .tensors
    1639
    0
                .iter()
    1640
    0
                .find(|t| t.name == "output.weight" || t.name == "lm_head.weight")
    1641
    0
                .or_else(|| {
    1642
    0
                    apr.tensors.iter().find(|t| {
    1643
    0
                        !t.name.starts_with("blk.")
    1644
    0
                            && (t.name.contains("output.weight") || t.name.contains("lm_head"))
    1645
    0
                    })
    1646
    0
                })
    1647
    0
                .map_or("output.weight", |t| t.name.as_str());
    1648
    1649
    0
            let lm_head_weight = make_tensor(lm_head_name, hidden_dim, vocab_size)?;
    1650
    1651
    0
            Ok(Self {
    1652
    0
                config,
    1653
    0
                token_embedding,
    1654
    0
                layers,
    1655
    0
                output_norm_weight,
    1656
    0
                output_norm_bias: None,
    1657
    0
                lm_head_weight,
    1658
    0
                lm_head_bias: None,
    1659
    0
                #[cfg(feature = "cuda")]
    1660
    0
                cuda_executor: None,
    1661
    0
                #[cfg(feature = "cuda")]
    1662
    0
                cuda_kernel_count: std::sync::atomic::AtomicU64::new(0),
    1663
    0
                #[cfg(feature = "cuda")]
    1664
    0
                cached_weight_names: std::sync::Mutex::new(std::collections::HashSet::new()),
    1665
    0
            })
    1666
    0
        }
    1667
    1668
        /// Serialize model to APR format with quantized weights preserved
    1669
        ///
    1670
        /// Creates a valid .apr file that can be loaded via `from_apr()`.
    1671
        /// Quantization types (Q4_K, Q6_K, etc.) are preserved in the tensor dtypes.
    1672
        ///
    1673
        /// # Returns
    1674
        ///
    1675
        /// Raw bytes in APR v2 format
    1676
        ///
    1677
        /// # Errors
    1678
        ///
    1679
        /// Returns error if serialization fails
    1680
        #[allow(clippy::cast_possible_truncation)]
    1681
    0
        pub fn to_apr_bytes(&self) -> Result<Vec<u8>> {
    1682
            use crate::apr::{ALIGNMENT, HEADER_SIZE, MAGIC};
    1683
    1684
            // Helper to convert GGML qtype to APR dtype
    1685
    0
            fn qtype_to_dtype(qtype: u32) -> &'static str {
    1686
    0
                match qtype {
    1687
    0
                    0 => "F32",
    1688
    0
                    1 => "F16",
    1689
    0
                    2 => "Q4_0",
    1690
    0
                    3 => "Q4_1",
    1691
    0
                    6 => "Q5_0",
    1692
    0
                    7 => "Q5_1",
    1693
    0
                    8 => "Q8_0",
    1694
    0
                    9 => "Q8_1",
    1695
    0
                    10 => "Q2_K",
    1696
    0
                    11 => "Q3_K",
    1697
    0
                    12 => "Q4_K",
    1698
    0
                    13 => "Q5_K",
    1699
    0
                    14 => "Q6_K",
    1700
    0
                    16 => "IQ2_XXS",
    1701
    0
                    17 => "IQ2_XS",
    1702
    0
                    30 => "BF16",
    1703
    0
                    _ => "F32",
    1704
                }
    1705
    0
            }
    1706
    1707
            // Helper to convert dtype string to byte for binary tensor entry
    1708
    0
            fn dtype_to_byte(dtype: &str) -> u8 {
    1709
    0
                match dtype {
    1710
    0
                    "F32" => 0,
    1711
    0
                    "F16" => 1,
    1712
    0
                    "BF16" => 2,
    1713
    0
                    "I8" => 3,
    1714
    0
                    "I16" => 4,
    1715
    0
                    "I32" => 5,
    1716
    0
                    "I64" => 6,
    1717
    0
                    "U8" => 7,
    1718
    0
                    "Q4_K" => 8,
    1719
    0
                    "Q6_K" => 9,
    1720
    0
                    "Q8_0" => 10,
    1721
    0
                    "Q4_0" => 11,
    1722
    0
                    "Q5_K" => 12,
    1723
    0
                    "Q3_K" => 13,
    1724
    0
                    "Q2_K" => 14,
    1725
    0
                    _ => 0,
    1726
                }
    1727
    0
            }
    1728
    1729
            // Helper to write tensor entry to binary format
    1730
    0
            fn write_tensor_entry(
    1731
    0
                name: &str,
    1732
    0
                dtype: &str,
    1733
    0
                shape: &[usize],
    1734
    0
                offset: u64,
    1735
    0
                size: u64,
    1736
    0
            ) -> Vec<u8> {
    1737
    0
                let mut entry = Vec::new();
    1738
    1739
                // Name: 2-byte length + bytes
    1740
    0
                let name_bytes = name.as_bytes();
    1741
    0
                entry.extend_from_slice(&(name_bytes.len() as u16).to_le_bytes());
    1742
    0
                entry.extend_from_slice(name_bytes);
    1743
    1744
                // Dtype: 1 byte
    1745
    0
                entry.push(dtype_to_byte(dtype));
    1746
    1747
                // Shape: 1-byte ndim + 8-byte dims
    1748
    0
                entry.push(shape.len() as u8);
    1749
    0
                for &dim in shape {
    1750
    0
                    entry.extend_from_slice(&(dim as u64).to_le_bytes());
    1751
    0
                }
    1752
    1753
                // Offset and size: 8 bytes each
    1754
    0
                entry.extend_from_slice(&offset.to_le_bytes());
    1755
    0
                entry.extend_from_slice(&size.to_le_bytes());
    1756
    1757
    0
                entry
    1758
    0
            }
    1759
    1760
            // Collect all tensors
    1761
            struct TensorInfo {
    1762
                name: String,
    1763
                dtype: String,
    1764
                shape: Vec<usize>,
    1765
                data: Vec<u8>,
    1766
            }
    1767
    1768
    0
            let mut tensors: Vec<TensorInfo> = Vec::new();
    1769
    1770
            // Token embedding (F32)
    1771
    0
            let embed_bytes: Vec<u8> = self
    1772
    0
                .token_embedding
    1773
    0
                .iter()
    1774
    0
                .flat_map(|f| f.to_le_bytes())
    1775
    0
                .collect();
    1776
    0
            tensors.push(TensorInfo {
    1777
    0
                name: "token_embd.weight".to_string(),
    1778
    0
                dtype: "F32".to_string(),
    1779
    0
                shape: vec![self.config.vocab_size, self.config.hidden_dim],
    1780
    0
                data: embed_bytes,
    1781
    0
            });
    1782
    1783
            // Layers
    1784
    0
            let head_dim = self.config.hidden_dim / self.config.num_heads;
    1785
    0
            let kv_dim = self.config.num_kv_heads * head_dim;
    1786
    1787
    0
            for (layer_idx, layer) in self.layers.iter().enumerate() {
    1788
                // Attention norm (F32)
    1789
    0
                let norm_bytes: Vec<u8> = layer
    1790
    0
                    .attn_norm_weight
    1791
    0
                    .iter()
    1792
    0
                    .flat_map(|f| f.to_le_bytes())
    1793
    0
                    .collect();
    1794
    0
                tensors.push(TensorInfo {
    1795
    0
                    name: format!("blk.{layer_idx}.attn_norm.weight"),
    1796
    0
                    dtype: "F32".to_string(),
    1797
    0
                    shape: vec![self.config.hidden_dim],
    1798
    0
                    data: norm_bytes,
    1799
    0
                });
    1800
    1801
                // QKV weights (quantized)
    1802
    0
                match &layer.qkv_weight {
    1803
    0
                    OwnedQKVWeights::Separate { q, k, v } => {
    1804
    0
                        tensors.push(TensorInfo {
    1805
    0
                            name: format!("blk.{layer_idx}.attn_q.weight"),
    1806
    0
                            dtype: qtype_to_dtype(q.qtype).to_string(),
    1807
    0
                            shape: vec![self.config.hidden_dim, self.config.hidden_dim],
    1808
    0
                            data: q.data.clone(),
    1809
    0
                        });
    1810
    0
                        tensors.push(TensorInfo {
    1811
    0
                            name: format!("blk.{layer_idx}.attn_k.weight"),
    1812
    0
                            dtype: qtype_to_dtype(k.qtype).to_string(),
    1813
    0
                            shape: vec![kv_dim, self.config.hidden_dim],
    1814
    0
                            data: k.data.clone(),
    1815
    0
                        });
    1816
    0
                        tensors.push(TensorInfo {
    1817
    0
                            name: format!("blk.{layer_idx}.attn_v.weight"),
    1818
    0
                            dtype: qtype_to_dtype(v.qtype).to_string(),
    1819
    0
                            shape: vec![kv_dim, self.config.hidden_dim],
    1820
    0
                            data: v.data.clone(),
    1821
    0
                        });
    1822
    0
                    },
    1823
    0
                    OwnedQKVWeights::Fused(t) => {
    1824
    0
                        // Store as fused QKV tensor
    1825
    0
                        tensors.push(TensorInfo {
    1826
    0
                            name: format!("blk.{layer_idx}.attn_qkv.weight"),
    1827
    0
                            dtype: qtype_to_dtype(t.qtype).to_string(),
    1828
    0
                            shape: vec![t.out_dim, t.in_dim],
    1829
    0
                            data: t.data.clone(),
    1830
    0
                        });
    1831
    0
                    },
    1832
                }
    1833
    1834
                // Output projection (quantized)
    1835
    0
                tensors.push(TensorInfo {
    1836
    0
                    name: format!("blk.{layer_idx}.attn_output.weight"),
    1837
    0
                    dtype: qtype_to_dtype(layer.attn_output_weight.qtype).to_string(),
    1838
    0
                    shape: vec![self.config.hidden_dim, self.config.hidden_dim],
    1839
    0
                    data: layer.attn_output_weight.data.clone(),
    1840
    0
                });
    1841
    1842
                // FFN norm (F32)
    1843
    0
                if let Some(ref ffn_norm) = layer.ffn_norm_weight {
    1844
    0
                    let norm_bytes: Vec<u8> = ffn_norm.iter().flat_map(|f| f.to_le_bytes()).collect();
    1845
    0
                    tensors.push(TensorInfo {
    1846
    0
                        name: format!("blk.{layer_idx}.ffn_norm.weight"),
    1847
    0
                        dtype: "F32".to_string(),
    1848
    0
                        shape: vec![self.config.hidden_dim],
    1849
    0
                        data: norm_bytes,
    1850
    0
                    });
    1851
    0
                }
    1852
    1853
                // FFN weights (quantized)
    1854
    0
                if let Some(ref gate) = layer.ffn_gate_weight {
    1855
    0
                    tensors.push(TensorInfo {
    1856
    0
                        name: format!("blk.{layer_idx}.ffn_gate.weight"),
    1857
    0
                        dtype: qtype_to_dtype(gate.qtype).to_string(),
    1858
    0
                        shape: vec![self.config.intermediate_dim, self.config.hidden_dim],
    1859
    0
                        data: gate.data.clone(),
    1860
    0
                    });
    1861
    0
                }
    1862
    1863
    0
                tensors.push(TensorInfo {
    1864
    0
                    name: format!("blk.{layer_idx}.ffn_up.weight"),
    1865
    0
                    dtype: qtype_to_dtype(layer.ffn_up_weight.qtype).to_string(),
    1866
    0
                    shape: vec![self.config.intermediate_dim, self.config.hidden_dim],
    1867
    0
                    data: layer.ffn_up_weight.data.clone(),
    1868
    0
                });
    1869
    1870
    0
                tensors.push(TensorInfo {
    1871
    0
                    name: format!("blk.{layer_idx}.ffn_down.weight"),
    1872
    0
                    dtype: qtype_to_dtype(layer.ffn_down_weight.qtype).to_string(),
    1873
    0
                    shape: vec![self.config.hidden_dim, self.config.intermediate_dim],
    1874
    0
                    data: layer.ffn_down_weight.data.clone(),
    1875
    0
                });
    1876
            }
    1877
    1878
            // Output norm (F32)
    1879
    0
            let output_norm_bytes: Vec<u8> = self
    1880
    0
                .output_norm_weight
    1881
    0
                .iter()
    1882
    0
                .flat_map(|f| f.to_le_bytes())
    1883
    0
                .collect();
    1884
    0
            tensors.push(TensorInfo {
    1885
    0
                name: "output_norm.weight".to_string(),
    1886
    0
                dtype: "F32".to_string(),
    1887
    0
                shape: vec![self.config.hidden_dim],
    1888
    0
                data: output_norm_bytes,
    1889
    0
            });
    1890
    1891
            // LM head (quantized)
    1892
    0
            tensors.push(TensorInfo {
    1893
    0
                name: "output.weight".to_string(),
    1894
    0
                dtype: qtype_to_dtype(self.lm_head_weight.qtype).to_string(),
    1895
    0
                shape: vec![self.config.vocab_size, self.config.hidden_dim],
    1896
    0
                data: self.lm_head_weight.data.clone(),
    1897
    0
            });
    1898
    1899
            // Build metadata JSON
    1900
    0
            let metadata = serde_json::json!({
    1901
    0
                "model_type": "transformer_lm",
    1902
    0
                "architecture": self.config.architecture,
    1903
    0
                "vocab_size": self.config.vocab_size,
    1904
    0
                "hidden_size": self.config.hidden_dim,
    1905
    0
                "num_layers": self.config.num_layers,
    1906
    0
                "num_heads": self.config.num_heads,
    1907
    0
                "num_kv_heads": self.config.num_kv_heads,
    1908
    0
                "intermediate_size": self.config.intermediate_dim,
    1909
    0
                "rms_norm_eps": self.config.eps,
    1910
    0
                "rope_theta": self.config.rope_theta,
    1911
    0
                "context_length": self.config.context_length,
    1912
            });
    1913
    0
            let metadata_bytes =
    1914
    0
                serde_json::to_vec(&metadata).map_err(|e| RealizarError::FormatError {
    1915
    0
                    reason: format!("Failed to serialize metadata: {e}"),
    1916
    0
                })?;
    1917
    0
            let metadata_padded_len = metadata_bytes.len().div_ceil(ALIGNMENT) * ALIGNMENT;
    1918
    1919
            // Build tensor index and data
    1920
    0
            let mut tensor_index_bytes: Vec<u8> = Vec::new();
    1921
    0
            let mut tensor_data_bytes: Vec<u8> = Vec::new();
    1922
    1923
    0
            for tensor in &tensors {
    1924
    0
                // Align tensor data to 64 bytes
    1925
    0
                let padding = (ALIGNMENT - (tensor_data_bytes.len() % ALIGNMENT)) % ALIGNMENT;
    1926
    0
                tensor_data_bytes.extend(std::iter::repeat_n(0u8, padding));
    1927
    0
    1928
    0
                let offset = tensor_data_bytes.len() as u64;
    1929
    0
                let size = tensor.data.len() as u64;
    1930
    0
    1931
    0
                tensor_index_bytes.extend(write_tensor_entry(
    1932
    0
                    &tensor.name,
    1933
    0
                    &tensor.dtype,
    1934
    0
                    &tensor.shape,
    1935
    0
                    offset,
    1936
    0
                    size,
    1937
    0
                ));
    1938
    0
    1939
    0
                tensor_data_bytes.extend_from_slice(&tensor.data);
    1940
    0
            }
    1941
    1942
            // Calculate offsets
    1943
    0
            let metadata_offset = HEADER_SIZE as u64;
    1944
    0
            let tensor_index_offset = metadata_offset + metadata_padded_len as u64;
    1945
    0
            let data_offset = tensor_index_offset + tensor_index_bytes.len() as u64;
    1946
    1947
            // Build header
    1948
    0
            let mut header = vec![0u8; HEADER_SIZE];
    1949
    0
            header[0..4].copy_from_slice(&MAGIC);
    1950
    0
            header[4] = 2; // version major
    1951
    0
            header[5] = 0; // version minor
    1952
    0
            header[6..8].copy_from_slice(&0u16.to_le_bytes()); // flags (quantized = bit 0)
    1953
    0
            header[8..12].copy_from_slice(&(tensors.len() as u32).to_le_bytes());
    1954
    0
            header[12..20].copy_from_slice(&metadata_offset.to_le_bytes());
    1955
    0
            header[20..24].copy_from_slice(&(metadata_bytes.len() as u32).to_le_bytes());
    1956
    0
            header[24..32].copy_from_slice(&tensor_index_offset.to_le_bytes());
    1957
    0
            header[32..40].copy_from_slice(&data_offset.to_le_bytes());
    1958
            // checksum at 40-43 (leave as 0 for now)
    1959
    1960
            // Combine all parts
    1961
    0
            let total_size =
    1962
    0
                HEADER_SIZE + metadata_padded_len + tensor_index_bytes.len() + tensor_data_bytes.len();
    1963
    0
            let mut result = Vec::with_capacity(total_size);
    1964
    0
            result.extend_from_slice(&header);
    1965
    0
            result.extend_from_slice(&metadata_bytes);
    1966
    0
            result.resize(HEADER_SIZE + metadata_padded_len, 0); // pad metadata
    1967
    0
            result.extend_from_slice(&tensor_index_bytes);
    1968
    0
            result.extend_from_slice(&tensor_data_bytes);
    1969
    1970
    0
            Ok(result)
    1971
    0
        }
    1972
    }