Coverage Report

Created: 2025-09-08 21:26

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/home/noah/src/ruchy/src/runtime/grammar_coverage.rs
Line
Count
Source
1
//! Grammar coverage matrix for REPL testing
2
//!
3
//! Tracks coverage of all grammar productions to ensure complete testing
4
5
use crate::frontend::ast::Expr;
6
use anyhow::Result;
7
use std::collections::{HashMap, HashSet};
8
use std::time::Duration;
9
10
/// Statistics for a single grammar production
11
#[derive(Default, Debug)]
12
pub struct ProductionStats {
13
    pub hit_count: usize,
14
    pub success_count: usize,
15
    pub avg_latency_ns: u64,
16
    pub error_patterns: Vec<String>,
17
}
18
19
/// Grammar coverage tracking matrix
20
#[derive(Default)]
21
pub struct GrammarCoverageMatrix {
22
    pub productions: HashMap<&'static str, ProductionStats>,
23
    pub ast_variants: HashSet<String>,
24
    pub uncovered: Vec<&'static str>,
25
}
26
27
impl GrammarCoverageMatrix {
28
    /// Create a new coverage matrix
29
0
    pub fn new() -> Self {
30
0
        Self::default()
31
0
    }
32
33
    /// Record a parse attempt
34
0
    pub fn record(
35
0
        &mut self,
36
0
        production: &'static str,
37
0
        _input: &str,
38
0
        result: Result<Expr>,
39
0
        elapsed: Duration,
40
0
    ) {
41
0
        let stats = self.productions.entry(production).or_default();
42
43
0
        stats.hit_count += 1;
44
0
        let elapsed_ns =
45
0
            u64::try_from(elapsed.as_nanos().min(u128::from(u64::MAX))).unwrap_or(u64::MAX);
46
0
        stats.avg_latency_ns = if stats.hit_count == 1 {
47
0
            elapsed_ns
48
        } else {
49
0
            (stats.avg_latency_ns * (stats.hit_count as u64 - 1) + elapsed_ns)
50
0
                / stats.hit_count as u64
51
        };
52
53
0
        match result {
54
0
            Ok(ast) => {
55
0
                stats.success_count += 1;
56
0
                self.record_ast_variants(&ast);
57
0
            }
58
0
            Err(e) => {
59
0
                let error_msg = e.to_string();
60
0
                if !stats.error_patterns.contains(&error_msg) {
61
0
                    stats.error_patterns.push(error_msg);
62
0
                }
63
            }
64
        }
65
0
    }
66
67
    /// Record AST variant usage
68
0
    fn record_ast_variants(&mut self, expr: &Expr) {
69
        use crate::frontend::ast::ExprKind;
70
71
0
        let variant_name = match &expr.kind {
72
0
            ExprKind::Literal(_) => "Literal",
73
0
            ExprKind::Identifier(_) => "Identifier",
74
0
            ExprKind::Binary { .. } => "Binary",
75
0
            ExprKind::Unary { .. } => "Unary",
76
0
            ExprKind::Call { .. } => "Call",
77
0
            ExprKind::MethodCall { .. } => "MethodCall",
78
0
            ExprKind::If { .. } => "If",
79
0
            ExprKind::Match { .. } => "Match",
80
0
            ExprKind::Block(_) => "Block",
81
0
            ExprKind::Let { .. } => "Let",
82
0
            ExprKind::Function { .. } => "Function",
83
0
            ExprKind::Lambda { .. } => "Lambda",
84
0
            ExprKind::Throw { .. } => "Throw",
85
0
            ExprKind::Ok { .. } => "Ok",
86
0
            ExprKind::Err { .. } => "Err",
87
0
            ExprKind::While { .. } => "While",
88
0
            ExprKind::For { .. } => "For",
89
0
            ExprKind::Struct { .. } => "Struct",
90
0
            ExprKind::StructLiteral { .. } => "StructLiteral",
91
0
            ExprKind::ObjectLiteral { .. } => "ObjectLiteral",
92
0
            ExprKind::FieldAccess { .. } => "FieldAccess",
93
0
            ExprKind::Trait { .. } => "Trait",
94
0
            ExprKind::Impl { .. } => "Impl",
95
0
            ExprKind::Extension { .. } => "Extension",
96
0
            ExprKind::Await { .. } => "Await",
97
0
            ExprKind::List(_) => "List",
98
0
            ExprKind::ListComprehension { .. } => "ListComprehension",
99
0
            ExprKind::StringInterpolation { .. } => "StringInterpolation",
100
0
            ExprKind::QualifiedName { .. } => "QualifiedName",
101
0
            ExprKind::Send { .. } => "Send",
102
0
            ExprKind::Ask { .. } => "Ask",
103
0
            ExprKind::Command { .. } => "Command",
104
0
            ExprKind::Macro { .. } => "Macro",
105
0
            ExprKind::Actor { .. } => "Actor",
106
0
            ExprKind::DataFrame { .. } => "DataFrame",
107
0
            ExprKind::DataFrameOperation { .. } => "DataFrameOperation",
108
0
            ExprKind::Pipeline { .. } => "Pipeline",
109
0
            ExprKind::Import { .. } => "Import",
110
0
            ExprKind::Export { .. } => "Export",
111
0
            ExprKind::Module { .. } => "Module",
112
0
            ExprKind::Range { .. } => "Range",
113
0
            ExprKind::Break { .. } => "Break",
114
0
            ExprKind::Continue { .. } => "Continue",
115
0
            ExprKind::Assign { .. } => "Assign",
116
0
            ExprKind::CompoundAssign { .. } => "CompoundAssign",
117
0
            _ => "Other",
118
        };
119
120
0
        self.ast_variants.insert(variant_name.to_string());
121
0
    }
122
123
    /// Check if coverage is complete
124
0
    pub fn is_complete(&self, required_productions: usize) -> bool {
125
0
        self.productions.len() >= required_productions && self.uncovered.is_empty()
126
0
    }
127
128
    /// Assert that coverage is complete
129
    ///
130
    /// # Panics
131
    ///
132
    /// Panics if there are uncovered productions or if the number of covered
133
    /// productions is less than the required amount.
134
0
    pub fn assert_complete(&self, required_productions: usize) {
135
0
        assert!(
136
0
            self.uncovered.is_empty(),
137
0
            "Uncovered productions: {:?}",
138
            self.uncovered
139
        );
140
141
0
        assert!(
142
0
            self.productions.len() >= required_productions,
143
0
            "Only {} of {} productions covered",
144
0
            self.productions.len(),
145
            required_productions
146
        );
147
0
    }
148
149
    /// Get coverage percentage
150
0
    pub fn get_coverage_percentage(&self) -> f64 {
151
0
        if self.uncovered.is_empty() && self.productions.is_empty() {
152
0
            return 0.0;
153
0
        }
154
        
155
        // Count uncovered productions that haven't been covered
156
0
        let uncovered_count = self.uncovered.iter()
157
0
            .filter(|prod| !self.productions.contains_key(**prod))
158
0
            .count();
159
        
160
0
        let total = self.productions.len() + uncovered_count;
161
0
        if total == 0 {
162
0
            return 0.0;
163
0
        }
164
        
165
        #[allow(clippy::cast_precision_loss)]
166
0
        let percentage = (self.productions.len() as f64 / total as f64) * 100.0;
167
0
        percentage
168
0
    }
169
    
170
    /// Generate a coverage report (alias for report())
171
0
    pub fn generate_report(&self) -> String {
172
0
        self.report()
173
0
    }
174
175
    /// Generate a coverage report
176
0
    pub fn report(&self) -> String {
177
        use std::fmt::Write;
178
179
0
        let mut report = String::new();
180
0
        report.push_str("Grammar Coverage Report\n");
181
0
        report.push_str("=======================\n\n");
182
        
183
0
        let coverage_percentage = self.get_coverage_percentage();
184
0
        let _ = writeln!(&mut report, "Coverage: {:.1}%", coverage_percentage);
185
186
0
        let _ = writeln!(
187
0
            &mut report,
188
0
            "Productions covered: {}",
189
0
            self.productions.len()
190
        );
191
0
        let _ = writeln!(
192
0
            &mut report,
193
0
            "AST variants seen: {}",
194
0
            self.ast_variants.len()
195
        );
196
197
0
        let total_hits: usize = self.productions.values().map(|s| s.hit_count).sum();
198
0
        let total_success: usize = self.productions.values().map(|s| s.success_count).sum();
199
0
        let _ = writeln!(&mut report, "Total attempts: {total_hits}");
200
201
0
        let success_rate = if total_hits > 0 {
202
            #[allow(clippy::cast_precision_loss)]
203
0
            let rate = (total_success as f64 / total_hits as f64) * 100.0;
204
0
            rate
205
        } else {
206
0
            0.0
207
        };
208
0
        let _ = writeln!(&mut report, "Success rate: {success_rate:.2}%");
209
210
        // Find slowest productions
211
0
        let mut slowest: Vec<_> = self.productions.iter().collect();
212
0
        slowest.sort_by_key(|(_, stats)| stats.avg_latency_ns);
213
0
        slowest.reverse();
214
215
0
        if !slowest.is_empty() {
216
0
            report.push_str("\nSlowest productions:\n");
217
0
            for (name, stats) in slowest.iter().take(5) {
218
0
                #[allow(clippy::cast_precision_loss)]
219
0
                let ms = stats.avg_latency_ns as f64 / 1_000_000.0;
220
0
                let _ = writeln!(&mut report, "  {name}: {ms:.2}ms");
221
0
            }
222
0
        }
223
224
0
        if !self.uncovered.is_empty() {
225
0
            report.push_str("\nUncovered productions:\n");
226
0
            for prod in &self.uncovered {
227
0
                let _ = writeln!(&mut report, "  - {prod}");
228
0
            }
229
0
        }
230
231
0
        report
232
0
    }
233
}
234
235
/// All grammar productions that need coverage
236
pub const GRAMMAR_PRODUCTIONS: &[(&str, &str)] = &[
237
    // Core literals (5)
238
    ("literal_int", "42"),
239
    ("literal_float", "3.14"),
240
    ("literal_string", r#""hello""#),
241
    ("literal_bool", "true"),
242
    ("literal_unit", "()"),
243
    // Binary operators (12)
244
    ("op_assign", "x = 5"),
245
    ("op_logical_or", "a || b"),
246
    ("op_logical_and", "a && b"),
247
    ("op_equality", "x == y"),
248
    ("op_comparison", "x < y"),
249
    ("op_bitwise_or", "a | b"),
250
    ("op_bitwise_xor", "a ^ b"),
251
    ("op_bitwise_and", "a & b"),
252
    ("op_shift", "x << 2"),
253
    ("op_range", "0..10"),
254
    ("op_add", "x + y"),
255
    ("op_mul", "x * y"),
256
    // Unary operators (3)
257
    ("op_neg", "-x"),
258
    ("op_not", "!x"),
259
    ("op_ref", "&value"),
260
    // Control flow (5)
261
    ("if_expr", "if x > 0 { 1 } else { -1 }"),
262
    ("match_expr", "match x { Some(y) => y, None => 0 }"),
263
    ("for_loop", "for x in 0..10 { print(x) }"),
264
    ("while_loop", "while x > 0 { x = x - 1 }"),
265
    ("loop_expr", "loop { break 42 }"),
266
    // Function calls (5) - CRITICAL: These were missing!
267
    ("call_simple", "println(42)"),
268
    ("call_args", "println(\"Hello\", \"World\")"),
269
    ("call_expr", "add(2 + 3, 4 * 5)"),
270
    ("call_nested", "println(add(1, 2))"),
271
    ("call_builtin", "print(\"test\")"),
272
    // Functions (4)
273
    ("fun_decl", "fun add(a: Int, b: Int) -> Int { a + b }"),
274
    ("fun_generic", "fun id<T>(x: T) -> T { x }"),
275
    ("lambda", "|x| x * 2"),
276
    ("lambda_typed", "|x: Int| -> Int { x * 2 }"),
277
    // Pattern matching (6)
278
    ("pattern_bind", "let x = 5"),
279
    ("pattern_tuple", "let (x, y) = (1, 2)"),
280
    ("pattern_struct", "let Point { x, y } = p"),
281
    ("pattern_enum", "let Some(x) = opt"),
282
    ("pattern_slice", "let [head, ..tail] = list"),
283
    ("pattern_guard", "match x { n if n > 0 => n }"),
284
    // Type system (5)
285
    ("type_simple", "let x: Int = 5"),
286
    ("type_generic", "let v: Vec<Int> = vec![1,2,3]"),
287
    ("type_function", "let f: Fn(Int) -> Int = |x| x"),
288
    ("type_tuple", "let t: (Int, String) = (1, \"hi\")"),
289
    ("type_option", "let opt: Option<Int> = Some(5)"),
290
    // Structs/Traits/Impls (3)
291
    ("struct_decl", "struct Point { x: Float, y: Float }"),
292
    ("trait_decl", "trait Show { fun show(self) -> String }"),
293
    (
294
        "impl_block",
295
        "impl Show for Point { fun show(self) -> String { \"...\" } }",
296
    ),
297
    // Actor system (4)
298
    ("actor_decl", "actor Counter { state count: Int = 0 }"),
299
    ("actor_handler", "on Increment { self.count += 1 }"),
300
    ("send_op", "counter <- Increment"),
301
    ("ask_op", "let n = counter <? GetCount"),
302
    // DataFrame operations (6)
303
    ("df_literal", "df![a = [1,2,3], b = [4,5,6]]"),
304
    ("df_filter", "df >> filter(col(\"age\") > 18)"),
305
    ("df_select", "df >> select([\"name\", \"age\"])"),
306
    ("df_groupby", "df >> groupby(\"dept\")"),
307
    ("df_agg", "df >> agg([mean(\"salary\"), count()])"),
308
    ("df_join", "df1 >> join(df2, on: \"id\")"),
309
    // Pipeline operators (3)
310
    ("pipe_simple", "data >> filter(|x| x > 0)"),
311
    ("pipe_method", "text >> trim() >> uppercase()"),
312
    ("pipe_nested", "x >> (|y| y >> double() >> square())"),
313
    // String interpolation (2)
314
    ("string_interp", r#"f"Hello {name}""#),
315
    ("string_complex", r#"f"Result: {compute(x):.2f}""#),
316
    // Import/Export (3)
317
    ("import_simple", "import std::fs"),
318
    (
319
        "import_multi",
320
        "import std::collections::{HashMap, HashSet}",
321
    ),
322
    ("export", "export { Point, distance }"),
323
    // Macros (2)
324
    ("macro_println", "println!(\"Hello\", \"World\")"),
325
    ("macro_vec", "vec![1, 2, 3]"),
326
];