/home/noah/src/ruchy/src/runtime/grammar_coverage.rs
Line | Count | Source |
1 | | //! Grammar coverage matrix for REPL testing |
2 | | //! |
3 | | //! Tracks coverage of all grammar productions to ensure complete testing |
4 | | |
5 | | use crate::frontend::ast::Expr; |
6 | | use anyhow::Result; |
7 | | use std::collections::{HashMap, HashSet}; |
8 | | use std::time::Duration; |
9 | | |
10 | | /// Statistics for a single grammar production |
11 | | #[derive(Default, Debug)] |
12 | | pub struct ProductionStats { |
13 | | pub hit_count: usize, |
14 | | pub success_count: usize, |
15 | | pub avg_latency_ns: u64, |
16 | | pub error_patterns: Vec<String>, |
17 | | } |
18 | | |
19 | | /// Grammar coverage tracking matrix |
20 | | #[derive(Default)] |
21 | | pub struct GrammarCoverageMatrix { |
22 | | pub productions: HashMap<&'static str, ProductionStats>, |
23 | | pub ast_variants: HashSet<String>, |
24 | | pub uncovered: Vec<&'static str>, |
25 | | } |
26 | | |
27 | | impl GrammarCoverageMatrix { |
28 | | /// Create a new coverage matrix |
29 | 0 | pub fn new() -> Self { |
30 | 0 | Self::default() |
31 | 0 | } |
32 | | |
33 | | /// Record a parse attempt |
34 | 0 | pub fn record( |
35 | 0 | &mut self, |
36 | 0 | production: &'static str, |
37 | 0 | input: &str, |
38 | 0 | result: Result<Expr>, |
39 | 0 | elapsed: Duration, |
40 | 0 | ) { |
41 | 0 | let stats = self.productions.entry(production).or_default(); |
42 | | |
43 | 0 | stats.hit_count += 1; |
44 | 0 | let elapsed_ns = |
45 | 0 | u64::try_from(elapsed.as_nanos().min(u128::from(u64::MAX))).unwrap_or(u64::MAX); |
46 | 0 | stats.avg_latency_ns = if stats.hit_count == 1 { |
47 | 0 | elapsed_ns |
48 | | } else { |
49 | 0 | (stats.avg_latency_ns * (stats.hit_count as u64 - 1) + elapsed_ns) |
50 | 0 | / stats.hit_count as u64 |
51 | | }; |
52 | | |
53 | 0 | match result { |
54 | 0 | Ok(ast) => { |
55 | 0 | stats.success_count += 1; |
56 | 0 | self.record_ast_variants(&ast); |
57 | 0 | } |
58 | 0 | Err(e) => { |
59 | 0 | let error_msg = format!("{input}: {e}"); |
60 | 0 | if !stats.error_patterns.contains(&error_msg) { |
61 | 0 | stats.error_patterns.push(error_msg); |
62 | 0 | } |
63 | | } |
64 | | } |
65 | 0 | } |
66 | | |
67 | | /// Record AST variant usage |
68 | 0 | fn record_ast_variants(&mut self, expr: &Expr) { |
69 | | use crate::frontend::ast::ExprKind; |
70 | | |
71 | 0 | let variant_name = match &expr.kind { |
72 | 0 | ExprKind::Literal(_) => "Literal", |
73 | 0 | ExprKind::Identifier(_) => "Identifier", |
74 | 0 | ExprKind::Binary { .. } => "Binary", |
75 | 0 | ExprKind::Unary { .. } => "Unary", |
76 | 0 | ExprKind::Call { .. } => "Call", |
77 | 0 | ExprKind::MethodCall { .. } => "MethodCall", |
78 | 0 | ExprKind::If { .. } => "If", |
79 | 0 | ExprKind::Match { .. } => "Match", |
80 | 0 | ExprKind::Block(_) => "Block", |
81 | 0 | ExprKind::Let { .. } => "Let", |
82 | 0 | ExprKind::Function { .. } => "Function", |
83 | 0 | ExprKind::Lambda { .. } => "Lambda", |
84 | 0 | ExprKind::Throw { .. } => "Throw", |
85 | 0 | ExprKind::Ok { .. } => "Ok", |
86 | 0 | ExprKind::Err { .. } => "Err", |
87 | 0 | ExprKind::While { .. } => "While", |
88 | 0 | ExprKind::For { .. } => "For", |
89 | 0 | ExprKind::Struct { .. } => "Struct", |
90 | 0 | ExprKind::StructLiteral { .. } => "StructLiteral", |
91 | 0 | ExprKind::ObjectLiteral { .. } => "ObjectLiteral", |
92 | 0 | ExprKind::FieldAccess { .. } => "FieldAccess", |
93 | 0 | ExprKind::Trait { .. } => "Trait", |
94 | 0 | ExprKind::Impl { .. } => "Impl", |
95 | 0 | ExprKind::Extension { .. } => "Extension", |
96 | 0 | ExprKind::Await { .. } => "Await", |
97 | 0 | ExprKind::List(_) => "List", |
98 | 0 | ExprKind::ListComprehension { .. } => "ListComprehension", |
99 | 0 | ExprKind::StringInterpolation { .. } => "StringInterpolation", |
100 | 0 | ExprKind::QualifiedName { .. } => "QualifiedName", |
101 | 0 | ExprKind::Send { .. } => "Send", |
102 | 0 | ExprKind::Ask { .. } => "Ask", |
103 | 0 | ExprKind::Command { .. } => "Command", |
104 | 0 | ExprKind::Macro { .. } => "Macro", |
105 | 0 | ExprKind::Actor { .. } => "Actor", |
106 | 0 | ExprKind::DataFrame { .. } => "DataFrame", |
107 | 0 | ExprKind::DataFrameOperation { .. } => "DataFrameOperation", |
108 | 0 | ExprKind::Pipeline { .. } => "Pipeline", |
109 | 0 | ExprKind::Import { .. } => "Import", |
110 | 0 | ExprKind::Export { .. } => "Export", |
111 | 0 | ExprKind::Module { .. } => "Module", |
112 | 0 | ExprKind::Range { .. } => "Range", |
113 | 0 | ExprKind::Break { .. } => "Break", |
114 | 0 | ExprKind::Continue { .. } => "Continue", |
115 | 0 | ExprKind::Assign { .. } => "Assign", |
116 | 0 | ExprKind::CompoundAssign { .. } => "CompoundAssign", |
117 | 0 | _ => "Other", |
118 | | }; |
119 | | |
120 | 0 | self.ast_variants.insert(variant_name.to_string()); |
121 | 0 | } |
122 | | |
123 | | /// Check if coverage is complete |
124 | 0 | pub fn is_complete(&self, required_productions: usize) -> bool { |
125 | 0 | self.productions.len() >= required_productions && self.uncovered.is_empty() |
126 | 0 | } |
127 | | |
128 | | /// Assert that coverage is complete |
129 | | /// |
130 | | /// # Panics |
131 | | /// |
132 | | /// Panics if there are uncovered productions or if the number of covered |
133 | | /// productions is less than the required amount. |
134 | 0 | pub fn assert_complete(&self, required_productions: usize) { |
135 | 0 | assert!( |
136 | 0 | self.uncovered.is_empty(), |
137 | 0 | "Uncovered productions: {:?}", |
138 | | self.uncovered |
139 | | ); |
140 | | |
141 | 0 | assert!( |
142 | 0 | self.productions.len() >= required_productions, |
143 | 0 | "Only {} of {} productions covered", |
144 | 0 | self.productions.len(), |
145 | | required_productions |
146 | | ); |
147 | 0 | } |
148 | | |
149 | | /// Generate a coverage report |
150 | 0 | pub fn report(&self) -> String { |
151 | | use std::fmt::Write; |
152 | | |
153 | 0 | let mut report = String::new(); |
154 | 0 | report.push_str("Grammar Coverage Report\n"); |
155 | 0 | report.push_str("=======================\n\n"); |
156 | | |
157 | 0 | let _ = writeln!( |
158 | 0 | &mut report, |
159 | 0 | "Productions covered: {}", |
160 | 0 | self.productions.len() |
161 | | ); |
162 | 0 | let _ = writeln!( |
163 | 0 | &mut report, |
164 | 0 | "AST variants seen: {}", |
165 | 0 | self.ast_variants.len() |
166 | | ); |
167 | | |
168 | 0 | let total_hits: usize = self.productions.values().map(|s| s.hit_count).sum(); |
169 | 0 | let total_success: usize = self.productions.values().map(|s| s.success_count).sum(); |
170 | 0 | let _ = writeln!(&mut report, "Total attempts: {total_hits}"); |
171 | | |
172 | 0 | let success_rate = if total_hits > 0 { |
173 | | #[allow(clippy::cast_precision_loss)] |
174 | 0 | let rate = (total_success as f64 / total_hits as f64) * 100.0; |
175 | 0 | rate |
176 | | } else { |
177 | 0 | 0.0 |
178 | | }; |
179 | 0 | let _ = writeln!(&mut report, "Success rate: {success_rate:.2}%"); |
180 | | |
181 | | // Find slowest productions |
182 | 0 | let mut slowest: Vec<_> = self.productions.iter().collect(); |
183 | 0 | slowest.sort_by_key(|(_, stats)| stats.avg_latency_ns); |
184 | 0 | slowest.reverse(); |
185 | | |
186 | 0 | if !slowest.is_empty() { |
187 | 0 | report.push_str("\nSlowest productions:\n"); |
188 | 0 | for (name, stats) in slowest.iter().take(5) { |
189 | 0 | #[allow(clippy::cast_precision_loss)] |
190 | 0 | let ms = stats.avg_latency_ns as f64 / 1_000_000.0; |
191 | 0 | let _ = writeln!(&mut report, " {name}: {ms:.2}ms"); |
192 | 0 | } |
193 | 0 | } |
194 | | |
195 | 0 | if !self.uncovered.is_empty() { |
196 | 0 | report.push_str("\nUncovered productions:\n"); |
197 | 0 | for prod in &self.uncovered { |
198 | 0 | let _ = writeln!(&mut report, " - {prod}"); |
199 | 0 | } |
200 | 0 | } |
201 | | |
202 | 0 | report |
203 | 0 | } |
204 | | } |
205 | | |
206 | | /// All grammar productions that need coverage |
207 | | pub const GRAMMAR_PRODUCTIONS: &[(&str, &str)] = &[ |
208 | | // Core literals (5) |
209 | | ("literal_int", "42"), |
210 | | ("literal_float", "3.14"), |
211 | | ("literal_string", r#""hello""#), |
212 | | ("literal_bool", "true"), |
213 | | ("literal_unit", "()"), |
214 | | // Binary operators (12) |
215 | | ("op_assign", "x = 5"), |
216 | | ("op_logical_or", "a || b"), |
217 | | ("op_logical_and", "a && b"), |
218 | | ("op_equality", "x == y"), |
219 | | ("op_comparison", "x < y"), |
220 | | ("op_bitwise_or", "a | b"), |
221 | | ("op_bitwise_xor", "a ^ b"), |
222 | | ("op_bitwise_and", "a & b"), |
223 | | ("op_shift", "x << 2"), |
224 | | ("op_range", "0..10"), |
225 | | ("op_add", "x + y"), |
226 | | ("op_mul", "x * y"), |
227 | | // Unary operators (3) |
228 | | ("op_neg", "-x"), |
229 | | ("op_not", "!x"), |
230 | | ("op_ref", "&value"), |
231 | | // Control flow (5) |
232 | | ("if_expr", "if x > 0 { 1 } else { -1 }"), |
233 | | ("match_expr", "match x { Some(y) => y, None => 0 }"), |
234 | | ("for_loop", "for x in 0..10 { print(x) }"), |
235 | | ("while_loop", "while x > 0 { x = x - 1 }"), |
236 | | ("loop_expr", "loop { break 42 }"), |
237 | | // Function calls (5) - CRITICAL: These were missing! |
238 | | ("call_simple", "println(42)"), |
239 | | ("call_args", "println(\"Hello\", \"World\")"), |
240 | | ("call_expr", "add(2 + 3, 4 * 5)"), |
241 | | ("call_nested", "println(add(1, 2))"), |
242 | | ("call_builtin", "print(\"test\")"), |
243 | | // Functions (4) |
244 | | ("fun_decl", "fun add(a: Int, b: Int) -> Int { a + b }"), |
245 | | ("fun_generic", "fun id<T>(x: T) -> T { x }"), |
246 | | ("lambda", "|x| x * 2"), |
247 | | ("lambda_typed", "|x: Int| -> Int { x * 2 }"), |
248 | | // Pattern matching (6) |
249 | | ("pattern_bind", "let x = 5"), |
250 | | ("pattern_tuple", "let (x, y) = (1, 2)"), |
251 | | ("pattern_struct", "let Point { x, y } = p"), |
252 | | ("pattern_enum", "let Some(x) = opt"), |
253 | | ("pattern_slice", "let [head, ..tail] = list"), |
254 | | ("pattern_guard", "match x { n if n > 0 => n }"), |
255 | | // Type system (5) |
256 | | ("type_simple", "let x: Int = 5"), |
257 | | ("type_generic", "let v: Vec<Int> = vec![1,2,3]"), |
258 | | ("type_function", "let f: Fn(Int) -> Int = |x| x"), |
259 | | ("type_tuple", "let t: (Int, String) = (1, \"hi\")"), |
260 | | ("type_option", "let opt: Option<Int> = Some(5)"), |
261 | | // Structs/Traits/Impls (3) |
262 | | ("struct_decl", "struct Point { x: Float, y: Float }"), |
263 | | ("trait_decl", "trait Show { fun show(self) -> String }"), |
264 | | ( |
265 | | "impl_block", |
266 | | "impl Show for Point { fun show(self) -> String { \"...\" } }", |
267 | | ), |
268 | | // Actor system (4) |
269 | | ("actor_decl", "actor Counter { state count: Int = 0 }"), |
270 | | ("actor_handler", "on Increment { self.count += 1 }"), |
271 | | ("send_op", "counter <- Increment"), |
272 | | ("ask_op", "let n = counter <? GetCount"), |
273 | | // DataFrame operations (6) |
274 | | ("df_literal", "df![a = [1,2,3], b = [4,5,6]]"), |
275 | | ("df_filter", "df >> filter(col(\"age\") > 18)"), |
276 | | ("df_select", "df >> select([\"name\", \"age\"])"), |
277 | | ("df_groupby", "df >> groupby(\"dept\")"), |
278 | | ("df_agg", "df >> agg([mean(\"salary\"), count()])"), |
279 | | ("df_join", "df1 >> join(df2, on: \"id\")"), |
280 | | // Pipeline operators (3) |
281 | | ("pipe_simple", "data >> filter(|x| x > 0)"), |
282 | | ("pipe_method", "text >> trim() >> uppercase()"), |
283 | | ("pipe_nested", "x >> (|y| y >> double() >> square())"), |
284 | | // String interpolation (2) |
285 | | ("string_interp", r#"f"Hello {name}""#), |
286 | | ("string_complex", r#"f"Result: {compute(x):.2f}""#), |
287 | | // Import/Export (3) |
288 | | ("import_simple", "import std::fs"), |
289 | | ( |
290 | | "import_multi", |
291 | | "import std::collections::{HashMap, HashSet}", |
292 | | ), |
293 | | ("export", "export { Point, distance }"), |
294 | | // Macros (2) |
295 | | ("macro_println", "println!(\"Hello\", \"World\")"), |
296 | | ("macro_vec", "vec![1, 2, 3]"), |
297 | | ]; |