/home/noah/src/realizar/src/apr_transformer/dequant.rs
Line | Count | Source |
1 | | //! GGUF K-quant Dequantization Helpers (PMAT-802) |
2 | | //! |
3 | | //! Dequantization routines for APR Q4_K/Q6_K support. |
4 | | |
5 | | // ============================================================================ |
6 | | // GGUF K-quant Dequantization Helpers (for APR Q4_K/Q6_K support) |
7 | | // ============================================================================= |
8 | | |
9 | | /// Convert IEEE 754 half-precision (f16) bits to f32 |
10 | 16 | pub(crate) fn f16_to_f32(bits: u16) -> f32 { |
11 | 16 | let sign = u32::from((bits >> 15) & 1); |
12 | 16 | let exp = u32::from((bits >> 10) & 0x1F); |
13 | 16 | let mant = u32::from(bits & 0x3FF); |
14 | | |
15 | 16 | if exp == 0 { |
16 | 7 | if mant == 0 { |
17 | | // Zero |
18 | 6 | f32::from_bits(sign << 31) |
19 | | } else { |
20 | | // Subnormal - convert to normalized f32 |
21 | 1 | let mut m = mant; |
22 | 1 | let mut e = 0i32; |
23 | 11 | while (m & 0x400) == 0 { |
24 | 10 | m <<= 1; |
25 | 10 | e -= 1; |
26 | 10 | } |
27 | 1 | m &= 0x3FF; |
28 | 1 | let f32_exp = (127 - 15 + 1 + e) as u32; |
29 | 1 | f32::from_bits((sign << 31) | (f32_exp << 23) | (m << 13)) |
30 | | } |
31 | 9 | } else if exp == 31 { |
32 | | // Inf or NaN |
33 | 3 | if mant == 0 { |
34 | 2 | f32::from_bits((sign << 31) | (0xFF << 23)) |
35 | | } else { |
36 | 1 | f32::from_bits((sign << 31) | (0xFF << 23) | (mant << 13)) |
37 | | } |
38 | | } else { |
39 | | // Normal number |
40 | 6 | let f32_exp = (exp as i32 - 15 + 127) as u32; |
41 | 6 | f32::from_bits((sign << 31) | (f32_exp << 23) | (mant << 13)) |
42 | | } |
43 | 16 | } |
44 | | |
45 | | /// Extract scale and min from Q4_K 12-byte packed scales |
46 | | /// |
47 | | /// PAR-001 FIX: Matches llama.cpp's get_scale_min_k4 packing scheme: |
48 | | /// - Blocks 0-3: scale = q[j] & 63, min = q[j+4] & 63 |
49 | | /// - Blocks 4-7: scale = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4) |
50 | | /// min = (q[j+4] >> 4) | ((q[j] >> 6) << 4) |
51 | | #[inline] |
52 | 19 | pub(crate) fn extract_scale_min_apr(scales: &[u8], block_idx: usize) -> (f32, f32) { |
53 | 19 | let j = block_idx; |
54 | 19 | let (scale_bits, min_bits) = if j < 4 { |
55 | | // First 4 blocks: simple layout |
56 | 10 | let d = scales[j] & 63; |
57 | 10 | let m = scales[j + 4] & 63; |
58 | 10 | (d, m) |
59 | | } else { |
60 | | // Last 4 blocks: packed layout using high bits from first 4 bytes |
61 | 9 | let d = (scales[j + 4] & 0x0F) | ((scales[j - 4] >> 6) << 4); |
62 | 9 | let m = (scales[j + 4] >> 4) | ((scales[j] >> 6) << 4); |
63 | 9 | (d, m) |
64 | | }; |
65 | | |
66 | 19 | (f32::from(scale_bits), f32::from(min_bits)) |
67 | 19 | } |
68 | | |
69 | | /// Dequantize Q4_K format (K-quants) for APR tensors |
70 | | /// Q4_K: super blocks of 256 elements |
71 | | /// Each super block: d (f16) + dmin (f16) + scales (12 bytes) + qs (128 bytes) = 144 bytes |
72 | | /// |
73 | | /// PMAT-086 FIX: Correct implementation matching llama.cpp/candle layout: |
74 | | /// - For each 64-value chunk, output 32 low nibbles THEN 32 high nibbles |
75 | | /// - Use sc1/dm1 for low nibbles, sc2/dm2 for high nibbles (different scales per half) |
76 | 4 | pub(crate) fn dequantize_q4_k_apr(data: &[u8], num_elements: usize) -> Vec<f32> { |
77 | | const QK_K: usize = 256; // Super-block size |
78 | | const SUPER_BLOCK_BYTES: usize = 2 + 2 + 12 + 128; // 144 bytes |
79 | | |
80 | 4 | let num_blocks = num_elements.div_ceil(QK_K); |
81 | 4 | let total_bytes = num_blocks * SUPER_BLOCK_BYTES; |
82 | | |
83 | 4 | if total_bytes > data.len() { |
84 | | // Return zeros if data is insufficient |
85 | 1 | return vec![0.0; num_elements]; |
86 | 3 | } |
87 | | |
88 | 3 | let mut result = vec![0.0f32; num_blocks * QK_K]; |
89 | | |
90 | 3 | for sb_idx2 in 0..num_blocks { |
91 | 2 | let sb_start = sb_idx * SUPER_BLOCK_BYTES; |
92 | 2 | let out_start = sb_idx * QK_K; |
93 | | |
94 | | // Read d (f16 scale) and dmin (f16 min) |
95 | 2 | let d = f16_to_f32(u16::from_le_bytes([data[sb_start], data[sb_start + 1]])); |
96 | 2 | let dmin = f16_to_f32(u16::from_le_bytes([data[sb_start + 2], data[sb_start + 3]])); |
97 | | |
98 | | // Read scales (12 bytes) |
99 | 2 | let scales = &data[sb_start + 4..sb_start + 16]; |
100 | | |
101 | | // Read qs (128 bytes) |
102 | 2 | let qs = &data[sb_start + 16..sb_start + 144]; |
103 | | |
104 | | // Dequantize following candle's layout: |
105 | | // For each 64-value chunk, output 32 low nibbles then 32 high nibbles |
106 | 2 | let mut ys_index = out_start; |
107 | | |
108 | 8 | for j in (0..QK_K)2 .step_by2 (64) { |
109 | 8 | let q = &qs[j / 2..j / 2 + 32]; |
110 | | |
111 | | // Get scales for the two 32-value halves |
112 | 8 | let is = j / 32; |
113 | 8 | let (sc1, m1) = extract_scale_min_apr(scales, is); |
114 | 8 | let d1 = d * sc1; |
115 | 8 | let dm1 = dmin * m1; |
116 | | |
117 | 8 | let (sc2, m2) = extract_scale_min_apr(scales, is + 1); |
118 | 8 | let d2 = d * sc2; |
119 | 8 | let dm2 = dmin * m2; |
120 | | |
121 | | // First pass: 32 low nibbles |
122 | 264 | for &byte256 in q { |
123 | 256 | result[ys_index] = d1 * (byte & 0xF) as f32 - dm1; |
124 | 256 | ys_index += 1; |
125 | 256 | } |
126 | | |
127 | | // Second pass: 32 high nibbles |
128 | 264 | for &byte256 in q { |
129 | 256 | result[ys_index] = d2 * (byte >> 4) as f32 - dm2; |
130 | 256 | ys_index += 1; |
131 | 256 | } |
132 | | } |
133 | | } |
134 | | |
135 | 3 | result.truncate(num_elements); |
136 | 3 | result |
137 | 4 | } |
138 | | |
139 | | /// Dequantize Q6_K format (K-quants) for APR tensors |
140 | | /// Q6_K super-block layout (per llama.cpp block_q6_K and candle): |
141 | | /// - ql: 128 bytes (low 4 bits, 256 values, 2 per byte) |
142 | | /// - qh: 64 bytes (high 2 bits, 256 values, 4 per byte) |
143 | | /// - scales: 16 bytes (i8 signed scales for 16 blocks) |
144 | | /// - d: 2 bytes (f16) |
145 | | /// |
146 | | /// Total: 128 + 64 + 16 + 2 = 210 bytes |
147 | 4 | pub(crate) fn dequantize_q6_k_apr(data: &[u8], num_elements: usize) -> Vec<f32> { |
148 | | const QK_K: usize = 256; |
149 | | const SUPER_BLOCK_BYTES: usize = 210; |
150 | | |
151 | 4 | let num_blocks = num_elements.div_ceil(QK_K); |
152 | 4 | let total_bytes = num_blocks * SUPER_BLOCK_BYTES; |
153 | | |
154 | 4 | if total_bytes > data.len() { |
155 | 1 | return vec![0.0; num_elements]; |
156 | 3 | } |
157 | | |
158 | 3 | let mut result = vec![0.0f32; num_blocks * QK_K]; |
159 | | |
160 | 3 | for sb_idx2 in 0..num_blocks { |
161 | 2 | let sb_start = sb_idx * SUPER_BLOCK_BYTES; |
162 | 2 | let out_start = sb_idx * QK_K; |
163 | | |
164 | | // Read ql - low 4 bits (128 bytes) at offset 0 |
165 | 2 | let ql = &data[sb_start..sb_start + 128]; |
166 | | |
167 | | // Read qh - high 2 bits (64 bytes) at offset 128 |
168 | 2 | let qh = &data[sb_start + 128..sb_start + 192]; |
169 | | |
170 | | // Read scales (16 bytes, i8) at offset 192 |
171 | 2 | let mut scales = [0i8; 16]; |
172 | | #[allow(clippy::cast_possible_wrap)] |
173 | 32 | for (i, scale) in scales2 .iter_mut2 ().enumerate2 () { |
174 | 32 | *scale = data[sb_start + 192 + i] as i8; |
175 | 32 | } |
176 | | |
177 | | // Read d (f16 -> f32) at offset 208 (last 2 bytes) |
178 | 2 | let d = f16_to_f32(u16::from_le_bytes([ |
179 | 2 | data[sb_start + 208], |
180 | 2 | data[sb_start + 209], |
181 | 2 | ])); |
182 | | |
183 | | // Dequantize 256 values following candle's exact layout |
184 | | // Process 128 values at a time (n=0, n=128) |
185 | 4 | for n in (0..QK_K)2 .step_by2 (128) { |
186 | 4 | let idx = n / 128; |
187 | 4 | let sc = &scales[8 * idx..]; |
188 | 4 | let ql_slice = &ql[64 * idx..]; |
189 | 4 | let qh_slice = &qh[32 * idx..]; |
190 | | |
191 | 132 | for l128 in 0..32 { |
192 | 128 | let is = l / 16; // Scale index selector (0 or 1 within this 128-block) |
193 | 128 | |
194 | 128 | // Extract 4 values per iteration (at positions l, l+32, l+64, l+96) |
195 | 128 | // q1: low 4 bits of ql[l] + bits 0-1 of qh[l] |
196 | 128 | let q1 = ((ql_slice[l] & 0xF) | ((qh_slice[l] & 3) << 4)) as i32 - 32; |
197 | 128 | // q2: low 4 bits of ql[l+32] + bits 2-3 of qh[l] |
198 | 128 | let q2 = ((ql_slice[l + 32] & 0xF) | (((qh_slice[l] >> 2) & 3) << 4)) as i32 - 32; |
199 | 128 | // q3: high 4 bits of ql[l] + bits 4-5 of qh[l] |
200 | 128 | let q3 = ((ql_slice[l] >> 4) | (((qh_slice[l] >> 4) & 3) << 4)) as i32 - 32; |
201 | 128 | // q4: high 4 bits of ql[l+32] + bits 6-7 of qh[l] |
202 | 128 | let q4 = ((ql_slice[l + 32] >> 4) | (((qh_slice[l] >> 6) & 3) << 4)) as i32 - 32; |
203 | 128 | |
204 | 128 | // Write to output with correct scale indexing |
205 | 128 | result[out_start + n + l] = d * (sc[is] as f32) * (q1 as f32); |
206 | 128 | result[out_start + n + l + 32] = d * (sc[is + 2] as f32) * (q2 as f32); |
207 | 128 | result[out_start + n + l + 64] = d * (sc[is + 4] as f32) * (q3 as f32); |
208 | 128 | result[out_start + n + l + 96] = d * (sc[is + 6] as f32) * (q4 as f32); |
209 | 128 | } |
210 | | } |
211 | | } |
212 | | |
213 | 3 | result.truncate(num_elements); |
214 | 3 | result |
215 | 4 | } |
216 | | |
217 | | // Tests moved to tests/ directory (PMAT-803) |