/home/noah/src/ruchy/src/runtime/grammar_coverage.rs
Line | Count | Source |
1 | | //! Grammar coverage matrix for REPL testing |
2 | | //! |
3 | | //! Tracks coverage of all grammar productions to ensure complete testing |
4 | | |
5 | | use crate::frontend::ast::Expr; |
6 | | use anyhow::Result; |
7 | | use std::collections::{HashMap, HashSet}; |
8 | | use std::time::Duration; |
9 | | |
10 | | /// Statistics for a single grammar production |
11 | | #[derive(Default, Debug)] |
12 | | pub struct ProductionStats { |
13 | | pub hit_count: usize, |
14 | | pub success_count: usize, |
15 | | pub avg_latency_ns: u64, |
16 | | pub error_patterns: Vec<String>, |
17 | | } |
18 | | |
19 | | /// Grammar coverage tracking matrix |
20 | | #[derive(Default)] |
21 | | pub struct GrammarCoverageMatrix { |
22 | | pub productions: HashMap<&'static str, ProductionStats>, |
23 | | pub ast_variants: HashSet<String>, |
24 | | pub uncovered: Vec<&'static str>, |
25 | | } |
26 | | |
27 | | impl GrammarCoverageMatrix { |
28 | | /// Create a new coverage matrix |
29 | 0 | pub fn new() -> Self { |
30 | 0 | Self::default() |
31 | 0 | } |
32 | | |
33 | | /// Record a parse attempt |
34 | 0 | pub fn record( |
35 | 0 | &mut self, |
36 | 0 | production: &'static str, |
37 | 0 | _input: &str, |
38 | 0 | result: Result<Expr>, |
39 | 0 | elapsed: Duration, |
40 | 0 | ) { |
41 | 0 | let stats = self.productions.entry(production).or_default(); |
42 | | |
43 | 0 | stats.hit_count += 1; |
44 | 0 | let elapsed_ns = |
45 | 0 | u64::try_from(elapsed.as_nanos().min(u128::from(u64::MAX))).unwrap_or(u64::MAX); |
46 | 0 | stats.avg_latency_ns = if stats.hit_count == 1 { |
47 | 0 | elapsed_ns |
48 | | } else { |
49 | 0 | (stats.avg_latency_ns * (stats.hit_count as u64 - 1) + elapsed_ns) |
50 | 0 | / stats.hit_count as u64 |
51 | | }; |
52 | | |
53 | 0 | match result { |
54 | 0 | Ok(ast) => { |
55 | 0 | stats.success_count += 1; |
56 | 0 | self.record_ast_variants(&ast); |
57 | 0 | } |
58 | 0 | Err(e) => { |
59 | 0 | let error_msg = e.to_string(); |
60 | 0 | if !stats.error_patterns.contains(&error_msg) { |
61 | 0 | stats.error_patterns.push(error_msg); |
62 | 0 | } |
63 | | } |
64 | | } |
65 | 0 | } |
66 | | |
67 | | /// Record AST variant usage |
68 | 0 | fn record_ast_variants(&mut self, expr: &Expr) { |
69 | | use crate::frontend::ast::ExprKind; |
70 | | |
71 | 0 | let variant_name = match &expr.kind { |
72 | 0 | ExprKind::Literal(_) => "Literal", |
73 | 0 | ExprKind::Identifier(_) => "Identifier", |
74 | 0 | ExprKind::Binary { .. } => "Binary", |
75 | 0 | ExprKind::Unary { .. } => "Unary", |
76 | 0 | ExprKind::Call { .. } => "Call", |
77 | 0 | ExprKind::MethodCall { .. } => "MethodCall", |
78 | 0 | ExprKind::If { .. } => "If", |
79 | 0 | ExprKind::Match { .. } => "Match", |
80 | 0 | ExprKind::Block(_) => "Block", |
81 | 0 | ExprKind::Let { .. } => "Let", |
82 | 0 | ExprKind::Function { .. } => "Function", |
83 | 0 | ExprKind::Lambda { .. } => "Lambda", |
84 | 0 | ExprKind::Throw { .. } => "Throw", |
85 | 0 | ExprKind::Ok { .. } => "Ok", |
86 | 0 | ExprKind::Err { .. } => "Err", |
87 | 0 | ExprKind::While { .. } => "While", |
88 | 0 | ExprKind::For { .. } => "For", |
89 | 0 | ExprKind::Struct { .. } => "Struct", |
90 | 0 | ExprKind::StructLiteral { .. } => "StructLiteral", |
91 | 0 | ExprKind::ObjectLiteral { .. } => "ObjectLiteral", |
92 | 0 | ExprKind::FieldAccess { .. } => "FieldAccess", |
93 | 0 | ExprKind::Trait { .. } => "Trait", |
94 | 0 | ExprKind::Impl { .. } => "Impl", |
95 | 0 | ExprKind::Extension { .. } => "Extension", |
96 | 0 | ExprKind::Await { .. } => "Await", |
97 | 0 | ExprKind::List(_) => "List", |
98 | 0 | ExprKind::ListComprehension { .. } => "ListComprehension", |
99 | 0 | ExprKind::StringInterpolation { .. } => "StringInterpolation", |
100 | 0 | ExprKind::QualifiedName { .. } => "QualifiedName", |
101 | 0 | ExprKind::Send { .. } => "Send", |
102 | 0 | ExprKind::Ask { .. } => "Ask", |
103 | 0 | ExprKind::Command { .. } => "Command", |
104 | 0 | ExprKind::Macro { .. } => "Macro", |
105 | 0 | ExprKind::Actor { .. } => "Actor", |
106 | 0 | ExprKind::DataFrame { .. } => "DataFrame", |
107 | 0 | ExprKind::DataFrameOperation { .. } => "DataFrameOperation", |
108 | 0 | ExprKind::Pipeline { .. } => "Pipeline", |
109 | 0 | ExprKind::Import { .. } => "Import", |
110 | 0 | ExprKind::Export { .. } => "Export", |
111 | 0 | ExprKind::Module { .. } => "Module", |
112 | 0 | ExprKind::Range { .. } => "Range", |
113 | 0 | ExprKind::Break { .. } => "Break", |
114 | 0 | ExprKind::Continue { .. } => "Continue", |
115 | 0 | ExprKind::Assign { .. } => "Assign", |
116 | 0 | ExprKind::CompoundAssign { .. } => "CompoundAssign", |
117 | 0 | _ => "Other", |
118 | | }; |
119 | | |
120 | 0 | self.ast_variants.insert(variant_name.to_string()); |
121 | 0 | } |
122 | | |
123 | | /// Check if coverage is complete |
124 | 0 | pub fn is_complete(&self, required_productions: usize) -> bool { |
125 | 0 | self.productions.len() >= required_productions && self.uncovered.is_empty() |
126 | 0 | } |
127 | | |
128 | | /// Assert that coverage is complete |
129 | | /// |
130 | | /// # Panics |
131 | | /// |
132 | | /// Panics if there are uncovered productions or if the number of covered |
133 | | /// productions is less than the required amount. |
134 | 0 | pub fn assert_complete(&self, required_productions: usize) { |
135 | 0 | assert!( |
136 | 0 | self.uncovered.is_empty(), |
137 | 0 | "Uncovered productions: {:?}", |
138 | | self.uncovered |
139 | | ); |
140 | | |
141 | 0 | assert!( |
142 | 0 | self.productions.len() >= required_productions, |
143 | 0 | "Only {} of {} productions covered", |
144 | 0 | self.productions.len(), |
145 | | required_productions |
146 | | ); |
147 | 0 | } |
148 | | |
149 | | /// Get coverage percentage |
150 | 0 | pub fn get_coverage_percentage(&self) -> f64 { |
151 | 0 | if self.uncovered.is_empty() && self.productions.is_empty() { |
152 | 0 | return 0.0; |
153 | 0 | } |
154 | | |
155 | | // Count uncovered productions that haven't been covered |
156 | 0 | let uncovered_count = self.uncovered.iter() |
157 | 0 | .filter(|prod| !self.productions.contains_key(**prod)) |
158 | 0 | .count(); |
159 | | |
160 | 0 | let total = self.productions.len() + uncovered_count; |
161 | 0 | if total == 0 { |
162 | 0 | return 0.0; |
163 | 0 | } |
164 | | |
165 | | #[allow(clippy::cast_precision_loss)] |
166 | 0 | let percentage = (self.productions.len() as f64 / total as f64) * 100.0; |
167 | 0 | percentage |
168 | 0 | } |
169 | | |
170 | | /// Generate a coverage report (alias for report()) |
171 | 0 | pub fn generate_report(&self) -> String { |
172 | 0 | self.report() |
173 | 0 | } |
174 | | |
175 | | /// Generate a coverage report |
176 | 0 | pub fn report(&self) -> String { |
177 | | use std::fmt::Write; |
178 | | |
179 | 0 | let mut report = String::new(); |
180 | 0 | report.push_str("Grammar Coverage Report\n"); |
181 | 0 | report.push_str("=======================\n\n"); |
182 | | |
183 | 0 | let coverage_percentage = self.get_coverage_percentage(); |
184 | 0 | let _ = writeln!(&mut report, "Coverage: {:.1}%", coverage_percentage); |
185 | | |
186 | 0 | let _ = writeln!( |
187 | 0 | &mut report, |
188 | 0 | "Productions covered: {}", |
189 | 0 | self.productions.len() |
190 | | ); |
191 | 0 | let _ = writeln!( |
192 | 0 | &mut report, |
193 | 0 | "AST variants seen: {}", |
194 | 0 | self.ast_variants.len() |
195 | | ); |
196 | | |
197 | 0 | let total_hits: usize = self.productions.values().map(|s| s.hit_count).sum(); |
198 | 0 | let total_success: usize = self.productions.values().map(|s| s.success_count).sum(); |
199 | 0 | let _ = writeln!(&mut report, "Total attempts: {total_hits}"); |
200 | | |
201 | 0 | let success_rate = if total_hits > 0 { |
202 | | #[allow(clippy::cast_precision_loss)] |
203 | 0 | let rate = (total_success as f64 / total_hits as f64) * 100.0; |
204 | 0 | rate |
205 | | } else { |
206 | 0 | 0.0 |
207 | | }; |
208 | 0 | let _ = writeln!(&mut report, "Success rate: {success_rate:.2}%"); |
209 | | |
210 | | // Find slowest productions |
211 | 0 | let mut slowest: Vec<_> = self.productions.iter().collect(); |
212 | 0 | slowest.sort_by_key(|(_, stats)| stats.avg_latency_ns); |
213 | 0 | slowest.reverse(); |
214 | | |
215 | 0 | if !slowest.is_empty() { |
216 | 0 | report.push_str("\nSlowest productions:\n"); |
217 | 0 | for (name, stats) in slowest.iter().take(5) { |
218 | 0 | #[allow(clippy::cast_precision_loss)] |
219 | 0 | let ms = stats.avg_latency_ns as f64 / 1_000_000.0; |
220 | 0 | let _ = writeln!(&mut report, " {name}: {ms:.2}ms"); |
221 | 0 | } |
222 | 0 | } |
223 | | |
224 | 0 | if !self.uncovered.is_empty() { |
225 | 0 | report.push_str("\nUncovered productions:\n"); |
226 | 0 | for prod in &self.uncovered { |
227 | 0 | let _ = writeln!(&mut report, " - {prod}"); |
228 | 0 | } |
229 | 0 | } |
230 | | |
231 | 0 | report |
232 | 0 | } |
233 | | } |
234 | | |
235 | | /// All grammar productions that need coverage |
236 | | pub const GRAMMAR_PRODUCTIONS: &[(&str, &str)] = &[ |
237 | | // Core literals (5) |
238 | | ("literal_int", "42"), |
239 | | ("literal_float", "3.14"), |
240 | | ("literal_string", r#""hello""#), |
241 | | ("literal_bool", "true"), |
242 | | ("literal_unit", "()"), |
243 | | // Binary operators (12) |
244 | | ("op_assign", "x = 5"), |
245 | | ("op_logical_or", "a || b"), |
246 | | ("op_logical_and", "a && b"), |
247 | | ("op_equality", "x == y"), |
248 | | ("op_comparison", "x < y"), |
249 | | ("op_bitwise_or", "a | b"), |
250 | | ("op_bitwise_xor", "a ^ b"), |
251 | | ("op_bitwise_and", "a & b"), |
252 | | ("op_shift", "x << 2"), |
253 | | ("op_range", "0..10"), |
254 | | ("op_add", "x + y"), |
255 | | ("op_mul", "x * y"), |
256 | | // Unary operators (3) |
257 | | ("op_neg", "-x"), |
258 | | ("op_not", "!x"), |
259 | | ("op_ref", "&value"), |
260 | | // Control flow (5) |
261 | | ("if_expr", "if x > 0 { 1 } else { -1 }"), |
262 | | ("match_expr", "match x { Some(y) => y, None => 0 }"), |
263 | | ("for_loop", "for x in 0..10 { print(x) }"), |
264 | | ("while_loop", "while x > 0 { x = x - 1 }"), |
265 | | ("loop_expr", "loop { break 42 }"), |
266 | | // Function calls (5) - CRITICAL: These were missing! |
267 | | ("call_simple", "println(42)"), |
268 | | ("call_args", "println(\"Hello\", \"World\")"), |
269 | | ("call_expr", "add(2 + 3, 4 * 5)"), |
270 | | ("call_nested", "println(add(1, 2))"), |
271 | | ("call_builtin", "print(\"test\")"), |
272 | | // Functions (4) |
273 | | ("fun_decl", "fun add(a: Int, b: Int) -> Int { a + b }"), |
274 | | ("fun_generic", "fun id<T>(x: T) -> T { x }"), |
275 | | ("lambda", "|x| x * 2"), |
276 | | ("lambda_typed", "|x: Int| -> Int { x * 2 }"), |
277 | | // Pattern matching (6) |
278 | | ("pattern_bind", "let x = 5"), |
279 | | ("pattern_tuple", "let (x, y) = (1, 2)"), |
280 | | ("pattern_struct", "let Point { x, y } = p"), |
281 | | ("pattern_enum", "let Some(x) = opt"), |
282 | | ("pattern_slice", "let [head, ..tail] = list"), |
283 | | ("pattern_guard", "match x { n if n > 0 => n }"), |
284 | | // Type system (5) |
285 | | ("type_simple", "let x: Int = 5"), |
286 | | ("type_generic", "let v: Vec<Int> = vec![1,2,3]"), |
287 | | ("type_function", "let f: Fn(Int) -> Int = |x| x"), |
288 | | ("type_tuple", "let t: (Int, String) = (1, \"hi\")"), |
289 | | ("type_option", "let opt: Option<Int> = Some(5)"), |
290 | | // Structs/Traits/Impls (3) |
291 | | ("struct_decl", "struct Point { x: Float, y: Float }"), |
292 | | ("trait_decl", "trait Show { fun show(self) -> String }"), |
293 | | ( |
294 | | "impl_block", |
295 | | "impl Show for Point { fun show(self) -> String { \"...\" } }", |
296 | | ), |
297 | | // Actor system (4) |
298 | | ("actor_decl", "actor Counter { state count: Int = 0 }"), |
299 | | ("actor_handler", "on Increment { self.count += 1 }"), |
300 | | ("send_op", "counter <- Increment"), |
301 | | ("ask_op", "let n = counter <? GetCount"), |
302 | | // DataFrame operations (6) |
303 | | ("df_literal", "df![a = [1,2,3], b = [4,5,6]]"), |
304 | | ("df_filter", "df >> filter(col(\"age\") > 18)"), |
305 | | ("df_select", "df >> select([\"name\", \"age\"])"), |
306 | | ("df_groupby", "df >> groupby(\"dept\")"), |
307 | | ("df_agg", "df >> agg([mean(\"salary\"), count()])"), |
308 | | ("df_join", "df1 >> join(df2, on: \"id\")"), |
309 | | // Pipeline operators (3) |
310 | | ("pipe_simple", "data >> filter(|x| x > 0)"), |
311 | | ("pipe_method", "text >> trim() >> uppercase()"), |
312 | | ("pipe_nested", "x >> (|y| y >> double() >> square())"), |
313 | | // String interpolation (2) |
314 | | ("string_interp", r#"f"Hello {name}""#), |
315 | | ("string_complex", r#"f"Result: {compute(x):.2f}""#), |
316 | | // Import/Export (3) |
317 | | ("import_simple", "import std::fs"), |
318 | | ( |
319 | | "import_multi", |
320 | | "import std::collections::{HashMap, HashSet}", |
321 | | ), |
322 | | ("export", "export { Point, distance }"), |
323 | | // Macros (2) |
324 | | ("macro_println", "println!(\"Hello\", \"World\")"), |
325 | | ("macro_vec", "vec![1, 2, 3]"), |
326 | | ]; |