/home/noah/src/ruchy/src/backend/transpiler/dataframe.rs
Line | Count | Source |
1 | | //! `DataFrame` transpilation for Polars integration |
2 | | |
3 | | #![allow(clippy::missing_errors_doc)] |
4 | | #![allow(clippy::wildcard_imports)] |
5 | | #![allow(clippy::doc_markdown)] |
6 | | |
7 | | use super::*; |
8 | | use crate::frontend::ast::{AggregateOp, DataFrameColumn, DataFrameOp, JoinType}; |
9 | | use anyhow::Result; |
10 | | use proc_macro2::TokenStream; |
11 | | use quote::{format_ident, quote}; |
12 | | |
13 | | impl Transpiler { |
14 | | /// Transpiles DataFrame literals (df![] syntax) |
15 | 3 | pub fn transpile_dataframe(&self, columns: &[DataFrameColumn]) -> Result<TokenStream> { |
16 | 3 | if columns.is_empty() { |
17 | | // Empty DataFrame |
18 | 1 | return Ok(quote! { |
19 | 1 | polars::prelude::DataFrame::empty() |
20 | 1 | }); |
21 | 2 | } |
22 | | |
23 | 2 | let mut series_tokens = Vec::new(); |
24 | | |
25 | 5 | for column3 in columns { |
26 | 3 | let col_name = &column.name; |
27 | | |
28 | | // Transpile the column values |
29 | 3 | let values_tokens = if column.values.is_empty() { |
30 | 1 | quote! { vec![] } |
31 | | } else { |
32 | | // Collect all values into a vector |
33 | 2 | let value_tokens: Result<Vec<_>> = column |
34 | 2 | .values |
35 | 2 | .iter() |
36 | 4 | .map2 (|v| self.transpile_expr(v)) |
37 | 2 | .collect(); |
38 | 2 | let value_tokens = value_tokens?0 ; |
39 | 2 | quote! { vec![#(#value_tokens),*] } |
40 | | }; |
41 | | |
42 | | // Create a Series from the values |
43 | 3 | series_tokens.push(quote! { |
44 | | polars::prelude::Series::new(#col_name, #values_tokens) |
45 | | }); |
46 | | } |
47 | | |
48 | | // Create DataFrame from series |
49 | 2 | Ok(quote! { |
50 | | polars::prelude::DataFrame::new(vec![ |
51 | | #(#series_tokens),* |
52 | | ]).unwrap() |
53 | | }) |
54 | 3 | } |
55 | | |
56 | | /// Transpiles DataFrame operations |
57 | 11 | pub fn transpile_dataframe_operation( |
58 | 11 | &self, |
59 | 11 | df: &Expr, |
60 | 11 | op: &DataFrameOp, |
61 | 11 | ) -> Result<TokenStream> { |
62 | 11 | let df_tokens = self.transpile_expr(df)?0 ; |
63 | | |
64 | 11 | match op { |
65 | 1 | DataFrameOp::Select(columns) => { |
66 | 1 | let col_tokens: Vec<TokenStream> = |
67 | 2 | columns.iter()1 .map1 (|col| quote! { #col }).collect1 (); |
68 | 1 | Ok(quote! { |
69 | | #df_tokens.select(&[#(#col_tokens),*]).unwrap() |
70 | | }) |
71 | | } |
72 | 1 | DataFrameOp::Filter(condition) => { |
73 | 1 | let cond_tokens = self.transpile_expr(condition)?0 ; |
74 | 1 | Ok(quote! { |
75 | 1 | #df_tokens.filter(&#cond_tokens).unwrap() |
76 | 1 | }) |
77 | | } |
78 | 1 | DataFrameOp::GroupBy(columns) => { |
79 | 1 | let col_tokens: Vec<TokenStream> = |
80 | 1 | columns.iter().map(|col| quote! { #col }).collect(); |
81 | 1 | Ok(quote! { |
82 | | #df_tokens.groupby(&[#(#col_tokens),*]).unwrap() |
83 | | }) |
84 | | } |
85 | 1 | DataFrameOp::Sort(columns) => { |
86 | | // Sort by multiple columns |
87 | 1 | let col_tokens: Vec<TokenStream> = |
88 | 1 | columns.iter().map(|col| quote! { #col }).collect(); |
89 | 1 | Ok(quote! { |
90 | | #df_tokens.sort(&[#(#col_tokens),*], false).unwrap() |
91 | | }) |
92 | | } |
93 | 3 | DataFrameOp::Join { other, on, how } => { |
94 | 3 | let other_tokens = self.transpile_expr(other)?0 ; |
95 | 3 | let on_tokens: Vec<TokenStream> = on.iter().map(|col| quote! { #col }).collect(); |
96 | | |
97 | 3 | let join_type = match how { |
98 | 1 | JoinType::Left => quote! { polars::prelude::JoinType::Left }, |
99 | 1 | JoinType::Right => quote! { polars::prelude::JoinType::Right }, |
100 | 1 | JoinType::Inner => quote! { polars::prelude::JoinType::Inner }, |
101 | 0 | JoinType::Outer => quote! { polars::prelude::JoinType::Outer }, |
102 | | }; |
103 | | |
104 | 3 | Ok(quote! { |
105 | | #df_tokens.join( |
106 | | &#other_tokens, |
107 | | &[#(#on_tokens),*], |
108 | | &[#(#on_tokens),*], |
109 | | #join_type |
110 | | ).unwrap() |
111 | | }) |
112 | | } |
113 | 1 | DataFrameOp::Aggregate(agg_ops) => { |
114 | | // Convert AggregateOp to expressions |
115 | 1 | let agg_exprs: Vec<TokenStream> = agg_ops |
116 | 1 | .iter() |
117 | 6 | .map1 (|op| match op { |
118 | 1 | AggregateOp::Sum(col) => quote! { col(#col).sum() }, |
119 | 1 | AggregateOp::Mean(col) => quote! { col(#col).mean() }, |
120 | 1 | AggregateOp::Min(col) => quote! { col(#col).min() }, |
121 | 1 | AggregateOp::Max(col) => quote! { col(#col).max() }, |
122 | 1 | AggregateOp::Count(col) => quote! { col(#col).count() }, |
123 | 1 | AggregateOp::Std(col) => quote! { col(#col).std() }, |
124 | 0 | AggregateOp::Var(col) => quote! { col(#col).var() }, |
125 | 6 | }) |
126 | 1 | .collect(); |
127 | | |
128 | 1 | Ok(quote! { |
129 | | #df_tokens.agg(&[#(#agg_exprs),*]).unwrap() |
130 | | }) |
131 | | } |
132 | 1 | DataFrameOp::Limit(n) => Ok(quote! { |
133 | 1 | #df_tokens.limit(#n) |
134 | 1 | }), |
135 | 1 | DataFrameOp::Head(n) => Ok(quote! { |
136 | 1 | #df_tokens.head(Some(#n)) |
137 | 1 | }), |
138 | 1 | DataFrameOp::Tail(n) => Ok(quote! { |
139 | 1 | #df_tokens.tail(Some(#n)) |
140 | 1 | }), |
141 | | } |
142 | 11 | } |
143 | | |
144 | | /// Transpiles DataFrame method calls (alternative to operation enum) |
145 | 0 | pub fn transpile_dataframe_method( |
146 | 0 | &self, |
147 | 0 | df_expr: &Expr, |
148 | 0 | method: &str, |
149 | 0 | args: &[Expr], |
150 | 0 | ) -> Result<TokenStream> { |
151 | 0 | let df_tokens = self.transpile_expr(df_expr)?; |
152 | 0 | let method_ident = format_ident!("{}", method); |
153 | | |
154 | 0 | let arg_tokens: Result<Vec<_>> = args.iter().map(|a| self.transpile_expr(a)).collect(); |
155 | 0 | let arg_tokens = arg_tokens?; |
156 | | |
157 | | // Map Ruchy DataFrame methods to Polars methods |
158 | 0 | match method { |
159 | 0 | "select" | "filter" | "groupby" | "agg" | "sort" | "join" => Ok(quote! { |
160 | | #df_tokens.#method_ident(#(#arg_tokens),*).unwrap() |
161 | | }), |
162 | 0 | "mean" | "std" | "min" | "max" | "sum" | "count" => { |
163 | | // These are aggregate functions |
164 | 0 | Ok(quote! { |
165 | 0 | #df_tokens.#method_ident() |
166 | 0 | }) |
167 | | } |
168 | 0 | "head" | "tail" => { |
169 | 0 | if args.is_empty() { |
170 | 0 | Ok(quote! { #df_tokens.#method_ident(Some(5)) }) |
171 | | } else { |
172 | 0 | Ok(quote! { #df_tokens.#method_ident(Some(#(#arg_tokens),*)) }) |
173 | | } |
174 | | } |
175 | | _ => { |
176 | | // Default method call |
177 | 0 | Ok(quote! { |
178 | | #df_tokens.#method_ident(#(#arg_tokens),*) |
179 | | }) |
180 | | } |
181 | | } |
182 | 0 | } |
183 | | } |
184 | | |
185 | | #[cfg(test)] |
186 | | mod tests { |
187 | | use super::*; |
188 | | use crate::frontend::ast::{Expr, ExprKind, Literal, Span}; |
189 | | |
190 | 10 | fn make_test_transpiler() -> Transpiler { |
191 | 10 | Transpiler::new() |
192 | 10 | } |
193 | | |
194 | 13 | fn make_literal_expr(val: i64) -> Expr { |
195 | 13 | Expr { |
196 | 13 | kind: ExprKind::Literal(Literal::Integer(val)), |
197 | 13 | span: Span::new(0, 10), |
198 | 13 | attributes: vec![], |
199 | 13 | } |
200 | 13 | } |
201 | | |
202 | | #[test] |
203 | 1 | fn test_empty_dataframe() { |
204 | 1 | let transpiler = make_test_transpiler(); |
205 | 1 | let result = transpiler.transpile_dataframe(&[]).unwrap(); |
206 | 1 | let output = result.to_string(); |
207 | 1 | assert!(output.contains("DataFrame")); |
208 | 1 | assert!(output.contains("empty")); |
209 | 1 | } |
210 | | |
211 | | #[test] |
212 | 1 | fn test_dataframe_with_columns() { |
213 | 1 | let transpiler = make_test_transpiler(); |
214 | 1 | let columns = vec![ |
215 | 1 | DataFrameColumn { |
216 | 1 | name: "col1".to_string(), |
217 | 1 | values: vec![make_literal_expr(1), make_literal_expr(2)], |
218 | 1 | }, |
219 | 1 | DataFrameColumn { |
220 | 1 | name: "col2".to_string(), |
221 | 1 | values: vec![make_literal_expr(3), make_literal_expr(4)], |
222 | 1 | }, |
223 | | ]; |
224 | | |
225 | 1 | let result = transpiler.transpile_dataframe(&columns).unwrap(); |
226 | 1 | let output = result.to_string(); |
227 | 1 | assert!(output.contains("DataFrame")); |
228 | 1 | assert!(output.contains("Series")); |
229 | 1 | assert!(output.contains("col1")); |
230 | 1 | assert!(output.contains("col2")); |
231 | 1 | } |
232 | | |
233 | | #[test] |
234 | 1 | fn test_dataframe_select_operation() { |
235 | 1 | let transpiler = make_test_transpiler(); |
236 | 1 | let df_expr = make_literal_expr(0); // Placeholder |
237 | 1 | let op = DataFrameOp::Select(vec!["col1".to_string(), "col2".to_string()]); |
238 | | |
239 | 1 | let result = transpiler.transpile_dataframe_operation(&df_expr, &op).unwrap(); |
240 | 1 | let output = result.to_string(); |
241 | 1 | assert!(output.contains("select")); |
242 | 1 | assert!(output.contains("col1")); |
243 | 1 | assert!(output.contains("col2")); |
244 | 1 | } |
245 | | |
246 | | #[test] |
247 | 1 | fn test_dataframe_filter_operation() { |
248 | 1 | let transpiler = make_test_transpiler(); |
249 | 1 | let df_expr = make_literal_expr(0); |
250 | 1 | let condition = make_literal_expr(1); |
251 | 1 | let op = DataFrameOp::Filter(Box::new(condition)); |
252 | | |
253 | 1 | let result = transpiler.transpile_dataframe_operation(&df_expr, &op).unwrap(); |
254 | 1 | let output = result.to_string(); |
255 | 1 | assert!(output.contains("filter")); |
256 | 1 | } |
257 | | |
258 | | #[test] |
259 | 1 | fn test_dataframe_groupby_operation() { |
260 | 1 | let transpiler = make_test_transpiler(); |
261 | 1 | let df_expr = make_literal_expr(0); |
262 | 1 | let op = DataFrameOp::GroupBy(vec!["group_col".to_string()]); |
263 | | |
264 | 1 | let result = transpiler.transpile_dataframe_operation(&df_expr, &op).unwrap(); |
265 | 1 | let output = result.to_string(); |
266 | 1 | assert!(output.contains("groupby")); |
267 | 1 | assert!(output.contains("group_col")); |
268 | 1 | } |
269 | | |
270 | | #[test] |
271 | 1 | fn test_dataframe_sort_operation() { |
272 | 1 | let transpiler = make_test_transpiler(); |
273 | 1 | let df_expr = make_literal_expr(0); |
274 | 1 | let op = DataFrameOp::Sort(vec!["sort_col".to_string()]); |
275 | | |
276 | 1 | let result = transpiler.transpile_dataframe_operation(&df_expr, &op).unwrap(); |
277 | 1 | let output = result.to_string(); |
278 | 1 | assert!(output.contains("sort")); |
279 | 1 | assert!(output.contains("sort_col")); |
280 | 1 | } |
281 | | |
282 | | #[test] |
283 | 1 | fn test_dataframe_join_operations() { |
284 | 1 | let transpiler = make_test_transpiler(); |
285 | 1 | let df_expr = make_literal_expr(0); |
286 | 1 | let other_expr = make_literal_expr(1); |
287 | | |
288 | 1 | let join_types = vec![ |
289 | 1 | (JoinType::Inner, "Inner"), |
290 | 1 | (JoinType::Left, "Left"), |
291 | 1 | (JoinType::Right, "Right"), |
292 | | ]; |
293 | | |
294 | 4 | for (join_type3 , expected3 ) in join_types { |
295 | 3 | let op = DataFrameOp::Join { |
296 | 3 | other: Box::new(other_expr.clone()), |
297 | 3 | on: vec!["id".to_string()], |
298 | 3 | how: join_type, |
299 | 3 | }; |
300 | | |
301 | 3 | let result = transpiler.transpile_dataframe_operation(&df_expr, &op).unwrap(); |
302 | 3 | let output = result.to_string(); |
303 | 3 | assert!(output.contains("join")); |
304 | 3 | assert!(output.contains(expected)); |
305 | | } |
306 | 1 | } |
307 | | |
308 | | #[test] |
309 | 1 | fn test_dataframe_aggregate_operations() { |
310 | 1 | let transpiler = make_test_transpiler(); |
311 | 1 | let df_expr = make_literal_expr(0); |
312 | | |
313 | 1 | let agg_ops = vec![ |
314 | 1 | AggregateOp::Mean("col1".to_string()), |
315 | 1 | AggregateOp::Sum("col2".to_string()), |
316 | 1 | AggregateOp::Min("col3".to_string()), |
317 | 1 | AggregateOp::Max("col4".to_string()), |
318 | 1 | AggregateOp::Count("col5".to_string()), |
319 | 1 | AggregateOp::Std("col6".to_string()), |
320 | | ]; |
321 | | |
322 | 1 | let op = DataFrameOp::Aggregate(agg_ops); |
323 | 1 | let result = transpiler.transpile_dataframe_operation(&df_expr, &op).unwrap(); |
324 | 1 | let output = result.to_string(); |
325 | | // Check that it produces some output |
326 | 1 | assert!(!output.is_empty()); |
327 | 1 | } |
328 | | |
329 | | #[test] |
330 | 1 | fn test_dataframe_limit_operations() { |
331 | 1 | let transpiler = make_test_transpiler(); |
332 | 1 | let df_expr = make_literal_expr(0); |
333 | | |
334 | | // Test Limit |
335 | 1 | let op = DataFrameOp::Limit(10); |
336 | 1 | let result = transpiler.transpile_dataframe_operation(&df_expr, &op).unwrap(); |
337 | 1 | let output = result.to_string(); |
338 | 1 | assert!(output.contains("limit")); |
339 | | |
340 | | // Test Head |
341 | 1 | let op = DataFrameOp::Head(5); |
342 | 1 | let result = transpiler.transpile_dataframe_operation(&df_expr, &op).unwrap(); |
343 | 1 | let output = result.to_string(); |
344 | 1 | assert!(output.contains("head")); |
345 | | |
346 | | // Test Tail |
347 | 1 | let op = DataFrameOp::Tail(5); |
348 | 1 | let result = transpiler.transpile_dataframe_operation(&df_expr, &op).unwrap(); |
349 | 1 | let output = result.to_string(); |
350 | 1 | assert!(output.contains("tail")); |
351 | 1 | } |
352 | | |
353 | | #[test] |
354 | 1 | fn test_dataframe_with_empty_column_values() { |
355 | 1 | let transpiler = make_test_transpiler(); |
356 | 1 | let columns = vec![ |
357 | 1 | DataFrameColumn { |
358 | 1 | name: "empty_col".to_string(), |
359 | 1 | values: vec![], |
360 | 1 | }, |
361 | | ]; |
362 | | |
363 | 1 | let result = transpiler.transpile_dataframe(&columns).unwrap(); |
364 | 1 | let output = result.to_string(); |
365 | 1 | assert!(output.contains("Series")); |
366 | 1 | assert!(output.contains("empty_col")); |
367 | 1 | assert!(output.contains("vec")); |
368 | 1 | } |
369 | | } |