/home/noah/src/realizar/src/quantize/dequant.rs
Line | Count | Source |
1 | | //! Dequantization functions for GGUF quantization formats (PMAT-802) |
2 | | //! |
3 | | //! Extracted from quantize/mod.rs - Basic dequantization for Q4_0, Q8_0, F16, |
4 | | //! Q4_1, Q5_0, Q5_1, Q4_K, Q5_K, Q6_K, Q2_K formats. |
5 | | //! |
6 | | //! ## Contents |
7 | | //! - `dequantize_q4_0` - 4-bit quantization (block size 32) |
8 | | //! - `dequantize_q8_0` - 8-bit quantization (block size 32) |
9 | | //! - `dequantize_f16` - Half-precision to f32 |
10 | | //! - `dequantize_q4_1` - 4-bit with scale and min |
11 | | //! - `dequantize_q5_0` - 5-bit quantization |
12 | | //! - `dequantize_q5_1` - 5-bit with scale and min |
13 | | //! - `dequantize_q4_k` - K-quantization 4-bit (super-block 256) |
14 | | //! - `dequantize_q5_k` - K-quantization 5-bit (super-block 256) |
15 | | //! - `dequantize_q6_k` - K-quantization 6-bit (super-block 256) |
16 | | //! - `dequantize_q2_k` - K-quantization 2-bit (super-block 256) |
17 | | |
18 | | use crate::error::{RealizarError, Result}; |
19 | | use crate::quantize::{BLOCK_SIZE, QK_K}; |
20 | | |
21 | | // Re-use helpers from simd module |
22 | | use super::simd::extract_scale_min; |
23 | | |
24 | | /// Dequantize `Q4_0` format weights |
25 | | /// |
26 | | /// # Arguments |
27 | | /// |
28 | | /// * `data` - Raw `Q4_0` quantized data (blocks of scale + 16 bytes) |
29 | | /// |
30 | | /// # Returns |
31 | | /// |
32 | | /// Dequantized float32 values |
33 | | /// |
34 | | /// # Errors |
35 | | /// |
36 | | /// Returns error if data length is not a multiple of block size |
37 | | /// |
38 | | /// # Examples |
39 | | /// |
40 | | /// ```rust,ignore |
41 | | /// let quantized = load_q4_0_weights(); |
42 | | /// let weights = dequantize_q4_0(&quantized)?; |
43 | | /// ``` |
44 | 16 | pub fn dequantize_q4_0(data: &[u8]) -> Result<Vec<f32>> { |
45 | | // Q4_0 block: 2 bytes (f16 scale) + 16 bytes (quants) = 18 bytes |
46 | | // GGML spec: typedef struct { ggml_half d; uint8_t qs[QK4_0/2]; } block_q4_0; |
47 | | const BLOCK_BYTES: usize = 2 + 16; |
48 | | |
49 | 16 | if !data.len().is_multiple_of(BLOCK_BYTES) { |
50 | 2 | return Err(RealizarError::InvalidShape { |
51 | 2 | reason: format!( |
52 | 2 | "Q4_0 data length {} is not a multiple of block size {}", |
53 | 2 | data.len(), |
54 | 2 | BLOCK_BYTES |
55 | 2 | ), |
56 | 2 | }); |
57 | 14 | } |
58 | | |
59 | 14 | let num_blocks = data.len() / BLOCK_BYTES; |
60 | 14 | let mut result = vec![0.0f32; num_blocks * BLOCK_SIZE]; |
61 | | |
62 | 3.59k | for block_idx in 0..num_blocks14 { |
63 | 3.59k | let block_start = block_idx * BLOCK_BYTES; |
64 | 3.59k | let out_start = block_idx * BLOCK_SIZE; |
65 | | |
66 | | // Read scale (f16, per GGML spec) |
67 | 3.59k | let scale_bytes = &data[block_start..block_start + 2]; |
68 | 3.59k | let scale = half::f16::from_le_bytes([scale_bytes[0], scale_bytes[1]]).to_f32(); |
69 | | |
70 | | // Read quantized values (16 bytes) |
71 | 3.59k | let quants_start = block_start + 2; |
72 | 3.59k | let quants = &data[quants_start..quants_start + 16]; |
73 | | |
74 | | // Dequantize following candle's layout: |
75 | | // - Positions 0-15: low nibbles of bytes 0-15 |
76 | | // - Positions 16-31: high nibbles of bytes 0-15 |
77 | 57.5k | for (j, &byte) in quants3.59k .iter3.59k ().enumerate3.59k () { |
78 | 57.5k | // Low 4 bits go to position j |
79 | 57.5k | #[allow(clippy::cast_possible_wrap)] |
80 | 57.5k | let low = (byte & 0x0F) as i16 - 8; |
81 | 57.5k | result[out_start + j] = scale * (low as f32); |
82 | 57.5k | |
83 | 57.5k | // High 4 bits go to position j + 16 |
84 | 57.5k | #[allow(clippy::cast_possible_wrap)] |
85 | 57.5k | let high = (byte >> 4) as i16 - 8; |
86 | 57.5k | result[out_start + j + 16] = scale * (high as f32); |
87 | 57.5k | } |
88 | | } |
89 | | |
90 | 14 | Ok(result) |
91 | 16 | } |
92 | | |
93 | | /// Dequantize `Q8_0` format weights |
94 | | /// |
95 | | /// # Arguments |
96 | | /// |
97 | | /// * `data` - Raw `Q8_0` quantized data (blocks of scale + 32 int8 values) |
98 | | /// |
99 | | /// # Returns |
100 | | /// |
101 | | /// Dequantized float32 values |
102 | | /// |
103 | | /// # Errors |
104 | | /// |
105 | | /// Returns error if data length is not a multiple of block size |
106 | 17 | pub fn dequantize_q8_0(data: &[u8]) -> Result<Vec<f32>> { |
107 | | // Q8_0 block: 2 bytes (f16 scale) + 32 bytes (int8 quants) = 34 bytes |
108 | | // Note: GGML spec uses f16 for scale, not f32! |
109 | | const BLOCK_BYTES: usize = 2 + 32; |
110 | | |
111 | 17 | if !data.len().is_multiple_of(BLOCK_BYTES) { |
112 | 2 | return Err(RealizarError::InvalidShape { |
113 | 2 | reason: format!( |
114 | 2 | "Q8_0 data length {} is not a multiple of block size {}", |
115 | 2 | data.len(), |
116 | 2 | BLOCK_BYTES |
117 | 2 | ), |
118 | 2 | }); |
119 | 15 | } |
120 | | |
121 | 15 | let num_blocks = data.len() / BLOCK_BYTES; |
122 | 15 | let mut result = Vec::with_capacity(num_blocks * BLOCK_SIZE); |
123 | | |
124 | 15 | for block_idx in 0..num_blocks { |
125 | 15 | let block_start = block_idx * BLOCK_BYTES; |
126 | | |
127 | | // Read scale (f16 -> f32) |
128 | 15 | let scale_bits = u16::from_le_bytes([data[block_start], data[block_start + 1]]); |
129 | 15 | let scale = f16_to_f32(scale_bits); |
130 | | |
131 | | // Read quantized values (32 int8 values) |
132 | 15 | let quants_start = block_start + 2; |
133 | 15 | let quants = &data[quants_start..quants_start + 32]; |
134 | | |
135 | | // Dequantize |
136 | 495 | for &byte480 in quants { |
137 | 480 | let value = i8::from_le_bytes([byte]); |
138 | 480 | result.push(scale * f32::from(value)); |
139 | 480 | } |
140 | | } |
141 | | |
142 | 15 | Ok(result) |
143 | 17 | } |
144 | | |
145 | | /// Convert IEEE 754 half-precision (f16) to single-precision (f32) |
146 | | /// |
147 | | /// Handles normal values, subnormals, infinities, and NaN. |
148 | | #[inline] |
149 | 65.1k | pub fn f16_to_f32(h: u16) -> f32 { |
150 | 65.1k | let sign = (h >> 15) & 1; |
151 | 65.1k | let exp = (h >> 10) & 0x1F; |
152 | 65.1k | let mantissa = h & 0x3FF; |
153 | | |
154 | 65.1k | if exp == 0 { |
155 | | // Subnormal or zero |
156 | 54.8k | if mantissa == 0 { |
157 | | // Zero (preserve sign) |
158 | 54.8k | if sign == 1 { |
159 | 4 | -0.0 |
160 | | } else { |
161 | 54.8k | 0.0 |
162 | | } |
163 | | } else { |
164 | | // Subnormal: (mantissa / 1024) * 2^-14 |
165 | 6 | let value = (mantissa as f32 / 1024.0) * (2.0_f32).powi(-14); |
166 | 6 | if sign == 1 { |
167 | 2 | -value |
168 | | } else { |
169 | 4 | value |
170 | | } |
171 | | } |
172 | 10.3k | } else if exp == 31 { |
173 | | // Infinity or NaN |
174 | 13 | if mantissa == 0 { |
175 | 10 | if sign == 1 { |
176 | 5 | f32::NEG_INFINITY |
177 | | } else { |
178 | 5 | f32::INFINITY |
179 | | } |
180 | | } else { |
181 | 3 | f32::NAN |
182 | | } |
183 | | } else { |
184 | | // Normal value: (1 + mantissa/1024) * 2^(exp-15) |
185 | 10.3k | let value = (1.0 + mantissa as f32 / 1024.0) * (2.0_f32).powi(exp as i32 - 15); |
186 | 10.3k | if sign == 1 { |
187 | 8 | -value |
188 | | } else { |
189 | 10.3k | value |
190 | | } |
191 | | } |
192 | 65.1k | } |
193 | | |
194 | | /// Dequantize `F16` format weights to `F32` |
195 | | /// |
196 | | /// # Arguments |
197 | | /// |
198 | | /// * `data` - Raw F16 data (2 bytes per value) |
199 | | /// |
200 | | /// # Returns |
201 | | /// |
202 | | /// Dequantized float32 values |
203 | | /// |
204 | | /// # Errors |
205 | | /// |
206 | | /// Returns error if data length is not a multiple of 2 bytes |
207 | 23 | pub fn dequantize_f16(data: &[u8]) -> Result<Vec<f32>> { |
208 | 23 | if !data.len().is_multiple_of(2) { |
209 | 8 | return Err(RealizarError::InvalidShape { |
210 | 8 | reason: format!( |
211 | 8 | "F16 data length {} is not a multiple of 2 bytes", |
212 | 8 | data.len() |
213 | 8 | ), |
214 | 8 | }); |
215 | 15 | } |
216 | | |
217 | 15 | let num_values = data.len() / 2; |
218 | 15 | let mut result = Vec::with_capacity(num_values); |
219 | | |
220 | 22 | for chunk in data15 .chunks_exact15 (2) { |
221 | 22 | let h = u16::from_le_bytes([chunk[0], chunk[1]]); |
222 | 22 | result.push(f16_to_f32(h)); |
223 | 22 | } |
224 | | |
225 | 15 | Ok(result) |
226 | 23 | } |
227 | | |
228 | | /// Dequantize `Q4_1` format weights |
229 | | /// |
230 | | /// Q4_1 format: 2 bytes (f16 scale) + 2 bytes (f16 min) + 16 bytes (quants) = 20 bytes |
231 | | /// GGUF/candle layout: |
232 | | /// - Positions 0-15: low nibbles of bytes 0-15 |
233 | | /// - Positions 16-31: high nibbles of bytes 0-15 |
234 | 29 | pub fn dequantize_q4_1(data: &[u8]) -> Result<Vec<f32>> { |
235 | | const BLOCK_BYTES: usize = 20; |
236 | | |
237 | 29 | if !data.len().is_multiple_of(BLOCK_BYTES) { |
238 | 6 | return Err(RealizarError::InvalidShape { |
239 | 6 | reason: format!( |
240 | 6 | "Q4_1 data length {} is not a multiple of block size {}", |
241 | 6 | data.len(), |
242 | 6 | BLOCK_BYTES |
243 | 6 | ), |
244 | 6 | }); |
245 | 23 | } |
246 | | |
247 | 23 | let num_blocks = data.len() / BLOCK_BYTES; |
248 | | // Pre-allocate with correct size for candle layout |
249 | 23 | let mut result = vec![0.0f32; num_blocks * BLOCK_SIZE]; |
250 | | |
251 | 29.7k | for block_idx in 0..num_blocks23 { |
252 | 29.7k | let block_start = block_idx * BLOCK_BYTES; |
253 | 29.7k | let out_start = block_idx * BLOCK_SIZE; |
254 | | |
255 | 29.7k | let d_bytes = &data[block_start..block_start + 2]; |
256 | 29.7k | let d = f16_to_f32(u16::from_le_bytes([d_bytes[0], d_bytes[1]])); |
257 | | |
258 | 29.7k | let min_bytes = &data[block_start + 2..block_start + 4]; |
259 | 29.7k | let min = f16_to_f32(u16::from_le_bytes([min_bytes[0], min_bytes[1]])); |
260 | | |
261 | 29.7k | let quants = &data[block_start + 4..block_start + 20]; |
262 | | |
263 | | // Use candle layout (same as Q4_0): |
264 | | // - Low nibbles (byte & 0xF) at positions 0-15 |
265 | | // - High nibbles (byte >> 4) at positions 16-31 |
266 | 475k | for (j, &byte) in quants29.7k .iter29.7k ().enumerate29.7k () { |
267 | 475k | // Low 4 bits go to position j (0-15) |
268 | 475k | let low = byte & 0x0F; |
269 | 475k | result[out_start + j] = d * f32::from(low) + min; |
270 | 475k | |
271 | 475k | // High 4 bits go to position j + 16 (16-31) |
272 | 475k | let high = (byte >> 4) & 0x0F; |
273 | 475k | result[out_start + j + 16] = d * f32::from(high) + min; |
274 | 475k | } |
275 | | } |
276 | | |
277 | 23 | Ok(result) |
278 | 29 | } |
279 | | |
280 | | /// Dequantize `Q5_0` format weights |
281 | | /// |
282 | | /// Q5_0 format: 2 bytes (f16 scale) + 4 bytes (high bits) + 16 bytes (quants) = 22 bytes |
283 | | /// GGUF/candle layout: |
284 | | /// - Positions 0-15: low nibbles + high bits from qh |
285 | | /// - Positions 16-31: high nibbles + high bits from qh |
286 | 26 | pub fn dequantize_q5_0(data: &[u8]) -> Result<Vec<f32>> { |
287 | | const BLOCK_BYTES: usize = 22; |
288 | | |
289 | 26 | if !data.len().is_multiple_of(BLOCK_BYTES) { |
290 | 6 | return Err(RealizarError::InvalidShape { |
291 | 6 | reason: format!( |
292 | 6 | "Q5_0 data length {} is not a multiple of block size {}", |
293 | 6 | data.len(), |
294 | 6 | BLOCK_BYTES |
295 | 6 | ), |
296 | 6 | }); |
297 | 20 | } |
298 | | |
299 | 20 | let num_blocks = data.len() / BLOCK_BYTES; |
300 | | // Pre-allocate with correct size for candle layout |
301 | 20 | let mut result = vec![0.0f32; num_blocks * BLOCK_SIZE]; |
302 | | |
303 | 4.62k | for block_idx in 0..num_blocks20 { |
304 | 4.62k | let block_start = block_idx * BLOCK_BYTES; |
305 | 4.62k | let out_start = block_idx * BLOCK_SIZE; |
306 | | |
307 | 4.62k | let d_bytes = &data[block_start..block_start + 2]; |
308 | 4.62k | let d = f16_to_f32(u16::from_le_bytes([d_bytes[0], d_bytes[1]])); |
309 | | |
310 | 4.62k | let qh = u32::from_le_bytes([ |
311 | 4.62k | data[block_start + 2], |
312 | 4.62k | data[block_start + 3], |
313 | 4.62k | data[block_start + 4], |
314 | 4.62k | data[block_start + 5], |
315 | 4.62k | ]); |
316 | | |
317 | 4.62k | let qs = &data[block_start + 6..block_start + 22]; |
318 | | |
319 | | // Use candle layout: |
320 | | // - Low nibbles (byte & 0xF) at positions 0-15 |
321 | | // - High nibbles (byte >> 4) at positions 16-31 |
322 | 73.9k | for (i, &byte) in qs4.62k .iter4.62k ().enumerate4.62k () { |
323 | 73.9k | // Low 4 bits + 5th bit go to position i (0-15) |
324 | 73.9k | let low_q = byte & 0x0F; |
325 | 73.9k | let high_bit_low = ((qh >> i) & 1) as u8; |
326 | 73.9k | let q_low = low_q | (high_bit_low << 4); |
327 | 73.9k | #[allow(clippy::cast_possible_wrap)] |
328 | 73.9k | let value_low = q_low as i8 - 16; |
329 | 73.9k | result[out_start + i] = d * f32::from(value_low); |
330 | 73.9k | |
331 | 73.9k | // High 4 bits + 5th bit go to position i + 16 (16-31) |
332 | 73.9k | let high_q = (byte >> 4) & 0x0F; |
333 | 73.9k | let high_bit_high = ((qh >> (i + 16)) & 1) as u8; |
334 | 73.9k | let q_high = high_q | (high_bit_high << 4); |
335 | 73.9k | #[allow(clippy::cast_possible_wrap)] |
336 | 73.9k | let value_high = q_high as i8 - 16; |
337 | 73.9k | result[out_start + i + 16] = d * f32::from(value_high); |
338 | 73.9k | } |
339 | | } |
340 | | |
341 | 20 | Ok(result) |
342 | 26 | } |
343 | | |
344 | | /// Dequantize `Q5_1` format weights |
345 | | /// |
346 | | /// Q5_1 format: 2 bytes (f16 scale) + 2 bytes (f16 min) + 4 bytes (high bits) + 16 bytes (quants) = 24 bytes |
347 | | /// GGUF/candle layout: |
348 | | /// - Positions 0-15: low nibbles + high bits from qh |
349 | | /// - Positions 16-31: high nibbles + high bits from qh |
350 | 24 | pub fn dequantize_q5_1(data: &[u8]) -> Result<Vec<f32>> { |
351 | | const BLOCK_BYTES: usize = 24; |
352 | | |
353 | 24 | if !data.len().is_multiple_of(BLOCK_BYTES) { |
354 | 6 | return Err(RealizarError::InvalidShape { |
355 | 6 | reason: format!( |
356 | 6 | "Q5_1 data length {} is not a multiple of block size {}", |
357 | 6 | data.len(), |
358 | 6 | BLOCK_BYTES |
359 | 6 | ), |
360 | 6 | }); |
361 | 18 | } |
362 | | |
363 | 18 | let num_blocks = data.len() / BLOCK_BYTES; |
364 | | // Pre-allocate with correct size for candle layout |
365 | 18 | let mut result = vec![0.0f32; num_blocks * BLOCK_SIZE]; |
366 | | |
367 | 18 | for block_idx17 in 0..num_blocks { |
368 | 17 | let block_start = block_idx * BLOCK_BYTES; |
369 | 17 | let out_start = block_idx * BLOCK_SIZE; |
370 | | |
371 | 17 | let d_bytes = &data[block_start..block_start + 2]; |
372 | 17 | let d = f16_to_f32(u16::from_le_bytes([d_bytes[0], d_bytes[1]])); |
373 | | |
374 | 17 | let min_bytes = &data[block_start + 2..block_start + 4]; |
375 | 17 | let min = f16_to_f32(u16::from_le_bytes([min_bytes[0], min_bytes[1]])); |
376 | | |
377 | 17 | let qh = u32::from_le_bytes([ |
378 | 17 | data[block_start + 4], |
379 | 17 | data[block_start + 5], |
380 | 17 | data[block_start + 6], |
381 | 17 | data[block_start + 7], |
382 | 17 | ]); |
383 | | |
384 | 17 | let qs = &data[block_start + 8..block_start + 24]; |
385 | | |
386 | | // Use candle layout: |
387 | | // - Low nibbles (byte & 0xF) at positions 0-15 |
388 | | // - High nibbles (byte >> 4) at positions 16-31 |
389 | 272 | for (i, &byte) in qs17 .iter17 ().enumerate17 () { |
390 | 272 | // Low 4 bits + 5th bit go to position i (0-15) |
391 | 272 | let low_q = byte & 0x0F; |
392 | 272 | let high_bit_low = ((qh >> i) & 1) as u8; |
393 | 272 | let q_low = low_q | (high_bit_low << 4); |
394 | 272 | result[out_start + i] = d * f32::from(q_low) + min; |
395 | 272 | |
396 | 272 | // High 4 bits + 5th bit go to position i + 16 (16-31) |
397 | 272 | let high_q = (byte >> 4) & 0x0F; |
398 | 272 | let high_bit_high = ((qh >> (i + 16)) & 1) as u8; |
399 | 272 | let q_high = high_q | (high_bit_high << 4); |
400 | 272 | result[out_start + i + 16] = d * f32::from(q_high) + min; |
401 | 272 | } |
402 | | } |
403 | | |
404 | 18 | Ok(result) |
405 | 24 | } |
406 | | |
407 | | /// Dequantize `Q4_K` format weights |
408 | 259 | pub fn dequantize_q4_k(data: &[u8]) -> Result<Vec<f32>> { |
409 | | const SUPER_BLOCK_BYTES: usize = 144; |
410 | | |
411 | 259 | if !data.len().is_multiple_of(SUPER_BLOCK_BYTES) { |
412 | 2 | return Err(RealizarError::InvalidShape { |
413 | 2 | reason: format!( |
414 | 2 | "Q4_K data length {} is not a multiple of super-block size {}", |
415 | 2 | data.len(), |
416 | 2 | SUPER_BLOCK_BYTES |
417 | 2 | ), |
418 | 2 | }); |
419 | 257 | } |
420 | | |
421 | 257 | let num_super_blocks = data.len() / SUPER_BLOCK_BYTES; |
422 | 257 | let mut result = vec![0.0f32; num_super_blocks * QK_K]; |
423 | | |
424 | 736 | for sb_idx in 0..num_super_blocks257 { |
425 | 736 | let sb_start = sb_idx * SUPER_BLOCK_BYTES; |
426 | 736 | let out_start = sb_idx * QK_K; |
427 | | |
428 | 736 | let d = read_f16(&data[sb_start..sb_start + 2]); |
429 | 736 | let dmin = read_f16(&data[sb_start + 2..sb_start + 4]); |
430 | | |
431 | 736 | let mut scales = [0u8; 12]; |
432 | 736 | scales.copy_from_slice(&data[sb_start + 4..sb_start + 16]); |
433 | | |
434 | 736 | let qs_start = sb_start + 16; |
435 | 736 | let qs = &data[qs_start..qs_start + 128]; |
436 | | |
437 | 736 | let mut ys_index = out_start; |
438 | | |
439 | 2.94k | for j in (0..QK_K)736 .step_by736 (64) { |
440 | 2.94k | let q = &qs[j / 2..j / 2 + 32]; |
441 | | |
442 | 2.94k | let is = j / 32; |
443 | 2.94k | let (sc1, m1) = extract_scale_min(&scales, is); |
444 | 2.94k | let d1 = d * sc1; |
445 | 2.94k | let dm1 = dmin * m1; |
446 | | |
447 | 2.94k | let (sc2, m2) = extract_scale_min(&scales, is + 1); |
448 | 2.94k | let d2 = d * sc2; |
449 | 2.94k | let dm2 = dmin * m2; |
450 | | |
451 | 97.1k | for &byte94.2k in q { |
452 | 94.2k | result[ys_index] = d1 * (byte & 0xF) as f32 - dm1; |
453 | 94.2k | ys_index += 1; |
454 | 94.2k | } |
455 | | |
456 | 97.1k | for &byte94.2k in q { |
457 | 94.2k | result[ys_index] = d2 * (byte >> 4) as f32 - dm2; |
458 | 94.2k | ys_index += 1; |
459 | 94.2k | } |
460 | | } |
461 | | } |
462 | | |
463 | 257 | Ok(result) |
464 | 259 | } |
465 | | |
466 | | /// Dequantize `Q5_K` format weights |
467 | 17 | pub fn dequantize_q5_k(data: &[u8]) -> Result<Vec<f32>> { |
468 | | const SUPER_BLOCK_BYTES: usize = 176; |
469 | | |
470 | 17 | if !data.len().is_multiple_of(SUPER_BLOCK_BYTES) { |
471 | 3 | return Err(RealizarError::InvalidShape { |
472 | 3 | reason: format!( |
473 | 3 | "Q5_K data length {} is not a multiple of super-block size {}", |
474 | 3 | data.len(), |
475 | 3 | SUPER_BLOCK_BYTES |
476 | 3 | ), |
477 | 3 | }); |
478 | 14 | } |
479 | | |
480 | 14 | let num_super_blocks = data.len() / SUPER_BLOCK_BYTES; |
481 | 14 | let mut result = Vec::with_capacity(num_super_blocks * QK_K); |
482 | | |
483 | 15 | for sb_idx in 0..num_super_blocks14 { |
484 | 15 | let sb_start = sb_idx * SUPER_BLOCK_BYTES; |
485 | | |
486 | 15 | let d = read_f16(&data[sb_start..sb_start + 2]); |
487 | 15 | let dmin = read_f16(&data[sb_start + 2..sb_start + 4]); |
488 | | |
489 | 15 | let mut scales = [0u8; 12]; |
490 | 15 | scales.copy_from_slice(&data[sb_start + 4..sb_start + 16]); |
491 | | |
492 | 15 | let qh_start = sb_start + 16; |
493 | 15 | let qh = &data[qh_start..qh_start + 32]; |
494 | | |
495 | 15 | let qs_low_start = sb_start + 48; |
496 | 15 | let qs = &data[qs_low_start..qs_low_start + 128]; |
497 | | |
498 | 135 | for block_idx120 in 0..8 { |
499 | 120 | let (scale, min) = extract_scale_min(&scales, block_idx); |
500 | | |
501 | 120 | let block_start = block_idx * 16; |
502 | 120 | let qh_block_start = block_idx * 4; |
503 | | |
504 | 2.04k | for byte_idx1.92k in 0..16 { |
505 | 1.92k | let qs_byte = qs[block_start + byte_idx]; |
506 | 1.92k | |
507 | 1.92k | let high_bits_byte = qh[qh_block_start + byte_idx / 4]; |
508 | 1.92k | let bit_offset = (byte_idx % 4) * 2; |
509 | 1.92k | |
510 | 1.92k | let q_low_4bit = qs_byte & 0x0F; |
511 | 1.92k | let q_low_high_bit = (high_bits_byte >> bit_offset) & 0x01; |
512 | 1.92k | #[allow(clippy::cast_possible_wrap)] |
513 | 1.92k | let q_low = ((q_low_high_bit << 4) | q_low_4bit) as i8; |
514 | 1.92k | let value_low = d * scale * f32::from(q_low) - dmin * min; |
515 | 1.92k | result.push(value_low); |
516 | 1.92k | |
517 | 1.92k | let q_high_4bit = (qs_byte >> 4) & 0x0F; |
518 | 1.92k | let q_high_high_bit = (high_bits_byte >> (bit_offset + 1)) & 0x01; |
519 | 1.92k | #[allow(clippy::cast_possible_wrap)] |
520 | 1.92k | let q_high = ((q_high_high_bit << 4) | q_high_4bit) as i8; |
521 | 1.92k | let value_high = d * scale * f32::from(q_high) - dmin * min; |
522 | 1.92k | result.push(value_high); |
523 | 1.92k | } |
524 | | } |
525 | | } |
526 | | |
527 | 14 | Ok(result) |
528 | 17 | } |
529 | | |
530 | | /// Dequantize `Q6_K` format weights |
531 | 20 | pub fn dequantize_q6_k(data: &[u8]) -> Result<Vec<f32>> { |
532 | | const SUPER_BLOCK_BYTES: usize = 210; |
533 | | |
534 | 20 | if !data.len().is_multiple_of(SUPER_BLOCK_BYTES) { |
535 | 3 | return Err(RealizarError::InvalidShape { |
536 | 3 | reason: format!( |
537 | 3 | "Q6_K data length {} is not a multiple of super-block size {}", |
538 | 3 | data.len(), |
539 | 3 | SUPER_BLOCK_BYTES |
540 | 3 | ), |
541 | 3 | }); |
542 | 17 | } |
543 | | |
544 | 17 | let num_super_blocks = data.len() / SUPER_BLOCK_BYTES; |
545 | 17 | let mut result = vec![0.0f32; num_super_blocks * QK_K]; |
546 | | |
547 | 22 | for sb_idx in 0..num_super_blocks17 { |
548 | 22 | let sb_start = sb_idx * SUPER_BLOCK_BYTES; |
549 | 22 | let out_start = sb_idx * QK_K; |
550 | | |
551 | 22 | let ql = &data[sb_start..sb_start + 128]; |
552 | 22 | let qh = &data[sb_start + 128..sb_start + 192]; |
553 | | |
554 | 22 | let mut scales = [0i8; 16]; |
555 | 352 | for (i, scale) in scales22 .iter_mut22 ().enumerate22 () { |
556 | | #[allow(clippy::cast_possible_wrap)] |
557 | 352 | { |
558 | 352 | *scale = data[sb_start + 192 + i] as i8; |
559 | 352 | } |
560 | | } |
561 | | |
562 | 22 | let d = read_f16(&data[sb_start + 208..sb_start + 210]); |
563 | | |
564 | 44 | for n in (0..QK_K)22 .step_by22 (128) { |
565 | 44 | let idx = n / 128; |
566 | 44 | let sc = &scales[8 * idx..]; |
567 | 44 | let ql_slice = &ql[64 * idx..]; |
568 | 44 | let qh_slice = &qh[32 * idx..]; |
569 | | |
570 | 1.45k | for l1.40k in 0..32 { |
571 | 1.40k | let is = l / 16; |
572 | 1.40k | |
573 | 1.40k | let q1 = ((ql_slice[l] & 0xF) | ((qh_slice[l] & 3) << 4)) as i32 - 32; |
574 | 1.40k | let q2 = ((ql_slice[l + 32] & 0xF) | (((qh_slice[l] >> 2) & 3) << 4)) as i32 - 32; |
575 | 1.40k | let q3 = ((ql_slice[l] >> 4) | (((qh_slice[l] >> 4) & 3) << 4)) as i32 - 32; |
576 | 1.40k | let q4 = ((ql_slice[l + 32] >> 4) | (((qh_slice[l] >> 6) & 3) << 4)) as i32 - 32; |
577 | 1.40k | |
578 | 1.40k | result[out_start + n + l] = d * (sc[is] as f32) * (q1 as f32); |
579 | 1.40k | result[out_start + n + l + 32] = d * (sc[is + 2] as f32) * (q2 as f32); |
580 | 1.40k | result[out_start + n + l + 64] = d * (sc[is + 4] as f32) * (q3 as f32); |
581 | 1.40k | result[out_start + n + l + 96] = d * (sc[is + 6] as f32) * (q4 as f32); |
582 | 1.40k | } |
583 | | } |
584 | | } |
585 | | |
586 | 17 | Ok(result) |
587 | 20 | } |
588 | | |
589 | | /// Dequantize `Q2_K` format weights |
590 | 2 | pub fn dequantize_q2_k(data: &[u8]) -> Result<Vec<f32>> { |
591 | | const SUPER_BLOCK_BYTES: usize = 84; |
592 | | |
593 | 2 | if !data.len().is_multiple_of(SUPER_BLOCK_BYTES) { |
594 | 0 | return Err(RealizarError::InvalidShape { |
595 | 0 | reason: format!( |
596 | 0 | "Q2_K data length {} is not a multiple of super-block size {}", |
597 | 0 | data.len(), |
598 | 0 | SUPER_BLOCK_BYTES |
599 | 0 | ), |
600 | 0 | }); |
601 | 2 | } |
602 | | |
603 | 2 | let num_super_blocks = data.len() / SUPER_BLOCK_BYTES; |
604 | 2 | let mut result = vec![0.0f32; num_super_blocks * QK_K]; |
605 | | |
606 | 2 | for sb_idx in 0..num_super_blocks { |
607 | 2 | let sb_start = sb_idx * SUPER_BLOCK_BYTES; |
608 | 2 | let out_start = sb_idx * QK_K; |
609 | | |
610 | 2 | let scales_data = &data[sb_start..sb_start + 16]; |
611 | 2 | let qs = &data[sb_start + 16..sb_start + 80]; |
612 | | |
613 | 2 | let d = read_f16(&data[sb_start + 80..sb_start + 82]); |
614 | 2 | let dmin = read_f16(&data[sb_start + 82..sb_start + 84]); |
615 | | |
616 | 34 | for j32 in 0..16 { |
617 | 32 | let sc = (scales_data[j] & 0x0F) as f32; |
618 | 32 | let m = (scales_data[j] >> 4) as f32; |
619 | | |
620 | 32 | let d_sc = d * sc; |
621 | 32 | let dm = dmin * m; |
622 | | |
623 | 32 | let qs_offset = j * 4; |
624 | | |
625 | 160 | for k128 in 0..4 { |
626 | 128 | let q_byte = qs[qs_offset + k]; |
627 | 128 | let q0 = (q_byte & 0x03) as f32; |
628 | 128 | let q1 = ((q_byte >> 2) & 0x03) as f32; |
629 | 128 | let q2 = ((q_byte >> 4) & 0x03) as f32; |
630 | 128 | let q3 = ((q_byte >> 6) & 0x03) as f32; |
631 | 128 | |
632 | 128 | let base_idx = out_start + j * 16 + k * 4; |
633 | 128 | result[base_idx] = d_sc * q0 - dm; |
634 | 128 | result[base_idx + 1] = d_sc * q1 - dm; |
635 | 128 | result[base_idx + 2] = d_sc * q2 - dm; |
636 | 128 | result[base_idx + 3] = d_sc * q3 - dm; |
637 | 128 | } |
638 | | } |
639 | | } |
640 | | |
641 | 2 | Ok(result) |
642 | 2 | } |
643 | | |
644 | | /// Helper: Read f16 from bytes and convert to f32 |
645 | | #[inline] |
646 | 2.52M | pub(crate) fn read_f16(bytes: &[u8]) -> f32 { |
647 | 2.52M | let bits = u16::from_le_bytes([bytes[0], bytes[1]]); |
648 | 2.52M | half::f16::from_bits(bits).to_f32() |
649 | 2.52M | } |