Coverage Report

Created: 2025-09-08 21:26

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/home/noah/src/ruchy/src/runtime/replay_converter.rs
Line
Count
Source
1
//! Replay-to-Test Conversion Pipeline
2
//!
3
//! Converts .replay files into comprehensive Rust test cases for regression testing.
4
//! This enables automatic generation of test coverage from real usage patterns.
5
6
use crate::runtime::replay::{ReplSession, Event, EvalResult, InputMode};
7
use anyhow::{Context, Result};
8
use std::fs;
9
use std::path::Path;
10
11
/// Configuration for replay-to-test conversion
12
#[derive(Debug, Clone)]
13
pub struct ConversionConfig {
14
    /// Test module name prefix
15
    pub test_module_prefix: String,
16
    /// Include property tests for state consistency
17
    pub include_property_tests: bool,
18
    /// Include performance benchmarks
19
    pub include_benchmarks: bool,
20
    /// Maximum test timeout in milliseconds
21
    pub timeout_ms: u64,
22
}
23
24
impl Default for ConversionConfig {
25
0
    fn default() -> Self {
26
0
        Self {
27
0
            test_module_prefix: "replay_generated".to_string(),
28
0
            include_property_tests: true,
29
0
            include_benchmarks: false,
30
0
            timeout_ms: 5000,
31
0
        }
32
0
    }
33
}
34
35
/// Test case generated from replay session
36
#[derive(Debug, Clone)]
37
pub struct GeneratedTest {
38
    /// Test function name
39
    pub name: String,
40
    /// Test function code
41
    pub code: String,
42
    /// Test category (unit, integration, property, etc.)
43
    pub category: TestCategory,
44
    /// Expected coverage impact
45
    pub coverage_areas: Vec<String>,
46
}
47
48
/// Category of generated test
49
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
50
pub enum TestCategory {
51
    /// Basic input/output verification
52
    Unit,
53
    /// Multi-step interaction testing
54
    Integration,
55
    /// Property-based testing for invariants
56
    Property,
57
    /// Performance and resource testing
58
    Benchmark,
59
    /// Error handling and recovery
60
    ErrorHandling,
61
}
62
63
/// Replay-to-test converter
64
pub struct ReplayConverter {
65
    config: ConversionConfig,
66
}
67
68
impl ReplayConverter {
69
    /// Create a new converter with default configuration
70
0
    pub fn new() -> Self {
71
0
        Self {
72
0
            config: ConversionConfig::default(),
73
0
        }
74
0
    }
75
76
    /// Create a new converter with custom configuration
77
0
    pub fn with_config(config: ConversionConfig) -> Self {
78
0
        Self { config }
79
0
    }
80
81
    /// Convert a replay file to test cases
82
0
    pub fn convert_file(&self, replay_path: &Path) -> Result<Vec<GeneratedTest>> {
83
0
        let replay_content = fs::read_to_string(replay_path)
84
0
            .context("Failed to read replay file")?;
85
        
86
0
        let session: ReplSession = serde_json::from_str(&replay_content)
87
0
            .context("Failed to parse replay session")?;
88
        
89
0
        self.convert_session(&session, replay_path.file_stem()
90
0
            .and_then(|s| s.to_str())
91
0
            .unwrap_or("unnamed"))
92
0
    }
93
94
    /// Convert a replay session to test cases
95
0
    pub fn convert_session(&self, session: &ReplSession, name_prefix: &str) -> Result<Vec<GeneratedTest>> {
96
0
        let mut tests = Vec::new();
97
        
98
        // Generate unit tests for each input/output pair
99
0
        tests.extend(self.generate_unit_tests(session, name_prefix)?);
100
        
101
        // Generate integration test for full session
102
0
        tests.push(self.generate_integration_test(session, name_prefix)?);
103
        
104
        // Generate property tests if enabled
105
0
        if self.config.include_property_tests {
106
0
            tests.extend(self.generate_property_tests(session, name_prefix)?);
107
0
        }
108
        
109
        // Generate error handling tests
110
0
        tests.extend(self.generate_error_tests(session, name_prefix)?);
111
        
112
0
        Ok(tests)
113
0
    }
114
115
    /// Generate unit tests for individual input/output pairs
116
0
    fn generate_unit_tests(&self, session: &ReplSession, name_prefix: &str) -> Result<Vec<GeneratedTest>> {
117
0
        let mut tests = Vec::new();
118
0
        let mut test_counter = 1;
119
        
120
        // Find input/output pairs
121
0
        let timeline = &session.timeline;
122
0
        for i in 0..timeline.len() {
123
0
            if let Event::Input { text, mode } = &timeline[i].event {
124
                // Find the corresponding output
125
0
                if let Some(output_event) = timeline.get(i + 1) {
126
0
                    if let Event::Output { result, .. } = &output_event.event {
127
0
                        let sanitized_prefix = name_prefix.replace('-', "_");
128
0
        let test_name = format!("{sanitized_prefix}_{test_counter:03}");
129
0
                        let test = self.generate_single_unit_test(&test_name, text, mode, result)?;
130
0
                        tests.push(test);
131
0
                        test_counter += 1;
132
0
                    }
133
0
                }
134
0
            }
135
        }
136
        
137
0
        Ok(tests)
138
0
    }
139
140
    /// Generate a single unit test
141
0
    fn generate_single_unit_test(
142
0
        &self, 
143
0
        test_name: &str, 
144
0
        input: &str, 
145
0
        mode: &InputMode,
146
0
        expected_result: &EvalResult
147
0
    ) -> Result<GeneratedTest> {
148
0
        let sanitized_input = input.replace('\"', "\\\"").replace('\n', "\\n");
149
0
        let timeout = self.config.timeout_ms;
150
        
151
0
        let (expected_output, test_assertion) = match expected_result {
152
0
            EvalResult::Success { value } => {
153
                // Extract actual value from String("...") format if present
154
0
                let actual_value = if value.starts_with("String(\"") && value.ends_with("\")") {
155
                    // Extract content from String("value")
156
0
                    &value[8..value.len()-2]
157
                } else {
158
0
                    value.as_str()
159
                };
160
0
                let sanitized_value = actual_value.replace('\"', "\\\"").replace('\\', "\\\\");
161
0
                (format!("Ok(\"{sanitized_value}\")"), format!("assert!(result.is_ok() && result.unwrap() == r#\"{actual_value}\"#);"))
162
            }
163
0
            EvalResult::Error { message } => {
164
0
                (format!("Err(r#\"{message}\"#)"), format!("assert!(result.is_err() && result.unwrap_err().to_string().contains(r#\"{message}\"#));"))
165
            }
166
            EvalResult::Unit => {
167
0
                ("Ok(\"\")".to_string(), "assert!(result.is_ok() && result.unwrap().is_empty());".to_string())
168
            }
169
        };
170
171
0
        let mode_comment = match mode {
172
0
            InputMode::Interactive => "// Interactive REPL input",
173
0
            InputMode::Paste => "// Pasted/multiline input", 
174
0
            InputMode::File => "// File-loaded input",
175
0
            InputMode::Script => "// Script execution",
176
        };
177
178
0
        let coverage_areas = self.identify_coverage_areas(input);
179
180
0
        let code = format!(r#"
181
0
#[test]
182
0
fn test_{test_name}() -> Result<()> {{
183
0
    {mode_comment}
184
0
    let mut repl = Repl::new()?;
185
0
    
186
0
    let deadline = Some(std::time::Instant::now() + std::time::Duration::from_millis({timeout}));
187
0
    let result = repl.eval("{sanitized_input}");
188
0
    
189
0
    // Expected: {expected_output}
190
0
    {test_assertion}
191
0
    
192
0
    Ok(())
193
0
}}"#);
194
195
0
        Ok(GeneratedTest {
196
0
            name: format!("test_{test_name}"),
197
0
            code,
198
0
            category: TestCategory::Unit,
199
0
            coverage_areas,
200
0
        })
201
0
    }
202
203
    /// Generate integration test for complete session
204
0
    fn generate_integration_test(&self, session: &ReplSession, name_prefix: &str) -> Result<GeneratedTest> {
205
0
        let sanitized_prefix = name_prefix.replace('-', "_");
206
0
        let mut session_code = String::new();
207
0
        let mut assertions = Vec::new();
208
0
        let timeout = self.config.timeout_ms;
209
210
        // Collect all inputs and expected outputs
211
0
        let timeline = &session.timeline;
212
0
        for i in 0..timeline.len() {
213
0
            if let Event::Input { text, .. } = &timeline[i].event {
214
0
                let sanitized_input = text.replace('\"', "\\\"").replace('\n', "\\n");
215
                
216
                // Find corresponding output
217
0
                if let Some(output_event) = timeline.get(i + 1) {
218
0
                    if let Event::Output { result, .. } = &output_event.event {
219
0
                        session_code.push_str(&format!("    let result_{i} = repl.eval(\"{sanitized_input}\");\n"));
220
                        
221
0
                        let assertion = match result {
222
0
                            EvalResult::Success { value } => {
223
                                // Extract actual value from String("...") format if present
224
0
                                let actual_value = if value.starts_with("String(\"") && value.ends_with("\")") {
225
0
                                    &value[8..value.len()-2]
226
                                } else {
227
0
                                    value.as_str()
228
                                };
229
0
                                format!("    assert!(result_{i}.is_ok() && result_{i}.unwrap() == r#\"{actual_value}\"#);\n")
230
                            }
231
0
                            EvalResult::Error { message } => {
232
0
                                format!("    assert!(result_{i}.is_err() && result_{i}.unwrap_err().to_string().contains(r#\"{message}\"#));\n")
233
                            }
234
                            EvalResult::Unit => {
235
0
                                format!("    assert!(result_{i}.is_ok() && result_{i}.unwrap().is_empty());\n")
236
                            }
237
                        };
238
0
                        assertions.push(assertion);
239
0
                    }
240
0
                }
241
0
            }
242
        }
243
244
0
        let code = format!(r"
245
0
#[test]
246
0
fn test_{sanitized_prefix}_session_integration() -> Result<()> {{
247
0
    // Integration test for complete REPL session
248
0
    // Tests state persistence and interaction patterns
249
0
    let mut repl = Repl::new()?;
250
0
    
251
0
    // Session timeout
252
0
    let _deadline = Some(std::time::Instant::now() + std::time::Duration::from_millis({timeout}));
253
0
    
254
0
    // Execute complete session
255
0
{session_code}
256
0
    
257
0
    // Verify all expected outputs
258
0
{assertions}
259
0
    
260
0
    Ok(())
261
0
}}", assertions = assertions.join(""));
262
263
        // Identify comprehensive coverage areas
264
0
        let mut coverage_areas = vec![
265
0
            "session_state".to_string(),
266
0
            "multi_step_interaction".to_string(),
267
0
            "state_persistence".to_string(),
268
        ];
269
        
270
        // Add specific areas based on session content
271
0
        for event in &session.timeline {
272
0
            if let Event::Input { text, .. } = &event.event {
273
0
                coverage_areas.extend(self.identify_coverage_areas(text));
274
0
            }
275
        }
276
0
        coverage_areas.sort();
277
0
        coverage_areas.dedup();
278
279
0
        let sanitized_prefix = name_prefix.replace('-', "_");
280
0
        Ok(GeneratedTest {
281
0
            name: format!("test_{sanitized_prefix}_session_integration"),
282
0
            code,
283
0
            category: TestCategory::Integration,
284
0
            coverage_areas,
285
0
        })
286
0
    }
287
288
    /// Generate property tests for invariants
289
0
    fn generate_property_tests(&self, _session: &ReplSession, name_prefix: &str) -> Result<Vec<GeneratedTest>> {
290
0
        let mut tests = Vec::new();
291
        
292
        // Property: REPL state should be deterministic
293
0
        let sanitized_prefix = name_prefix.replace('-', "_");
294
0
        let determinism_test = GeneratedTest {
295
0
            name: format!("test_{sanitized_prefix}_determinism_property"),
296
0
            code: format!(r#"
297
0
#[test]
298
0
fn test_{sanitized_prefix}_determinism_property() -> Result<()> {{
299
0
    // Property: Session should produce identical results on replay
300
0
    use crate::runtime::replay::*;
301
0
    
302
0
    let mut repl1 = Repl::new()?;
303
0
    let mut repl2 = Repl::new()?;
304
0
    
305
0
    // Execute same sequence on both REPLs
306
0
    let inputs = [
307
0
        // Insert representative inputs from session
308
0
    ];
309
0
    
310
0
    for input in inputs {{
311
0
        let result1 = repl1.eval(input);
312
0
        let result2 = repl2.eval(input);
313
0
        
314
0
        match (result1, result2) {{
315
0
            (Ok(out1), Ok(out2)) => assert_eq!(out1, out2),
316
0
            (Err(_), Err(_)) => {{}}, // Both failed consistently  
317
0
            _ => panic!("Inconsistent REPL behavior: {{}} vs {{}}", input, input),
318
0
        }}
319
0
    }}
320
0
    
321
0
    Ok(())
322
0
}}"#),
323
0
            category: TestCategory::Property,
324
0
            coverage_areas: vec!["determinism".to_string(), "state_consistency".to_string()],
325
0
        };
326
0
        tests.push(determinism_test);
327
328
        // Property: Memory usage should be bounded
329
0
        let memory_test = GeneratedTest {
330
0
            name: format!("test_{sanitized_prefix}_memory_bounds"),
331
0
            code: format!(r#"
332
0
#[test] 
333
0
fn test_{sanitized_prefix}_memory_bounds() -> Result<()> {{
334
0
    // Property: REPL should respect memory limits
335
0
    let mut repl = Repl::new()?;
336
0
    
337
0
    let initial_memory = repl.get_memory_usage();
338
0
    
339
0
    // Execute session operations
340
0
    // ... (session-specific operations)
341
0
    
342
0
    let final_memory = repl.get_memory_usage();
343
0
    
344
0
    // Memory should not exceed reasonable bounds (100MB default)
345
0
    assert!(final_memory < 100 * 1024 * 1024, "Memory usage exceeded bounds: {{}} bytes", final_memory);
346
0
    
347
0
    Ok(())
348
0
}}"#),
349
0
            category: TestCategory::Property,
350
0
            coverage_areas: vec!["memory_management".to_string(), "resource_bounds".to_string()],
351
0
        };
352
0
        tests.push(memory_test);
353
        
354
0
        Ok(tests)
355
0
    }
356
357
    /// Generate error handling tests
358
0
    fn generate_error_tests(&self, session: &ReplSession, name_prefix: &str) -> Result<Vec<GeneratedTest>> {
359
0
        let mut tests = Vec::new();
360
        
361
        // Find error cases in the session
362
0
        for (i, event) in session.timeline.iter().enumerate() {
363
0
            if let Event::Output { result: EvalResult::Error { message }, .. } = &event.event {
364
0
                let sanitized_prefix = name_prefix.replace('-', "_");
365
0
                let test_name = format!("{sanitized_prefix}_{i:03}_error_handling");
366
                
367
                // Find the input that caused this error
368
0
                if i > 0 {
369
0
                    if let Event::Input { text, .. } = &session.timeline[i - 1].event {
370
0
                        let sanitized_input = text.replace('\"', "\\\"");
371
0
                        let sanitized_message = message.replace('\"', "\\\"");
372
0
                        
373
0
                        let test = GeneratedTest {
374
0
                            name: format!("test_{test_name}"),
375
0
                            code: format!(r#"
376
0
#[test]
377
0
fn test_{test_name}() -> Result<()> {{
378
0
    // Error handling test: should gracefully handle invalid input
379
0
    let mut repl = Repl::new()?;
380
0
    
381
0
    let result = repl.eval("{sanitized_input}");
382
0
    
383
0
    // Should fail gracefully with descriptive error
384
0
    assert!(result.is_err());
385
0
    assert!(result.unwrap_err().to_string().contains("{sanitized_message}"));
386
0
    
387
0
    // REPL should remain functional after error
388
0
    let recovery = repl.eval("2 + 2");
389
0
    assert_eq!(recovery, Ok("4".to_string()));
390
0
    
391
0
    Ok(())
392
0
}}"#),
393
0
                            category: TestCategory::ErrorHandling,
394
0
                            coverage_areas: vec![
395
0
                                "error_handling".to_string(),
396
0
                                "error_recovery".to_string(),
397
0
                                "graceful_degradation".to_string(),
398
0
                            ],
399
0
                        };
400
0
                        tests.push(test);
401
0
                    }
402
0
                }
403
0
            }
404
        }
405
        
406
0
        Ok(tests)
407
0
    }
408
409
    /// Identify code coverage areas based on input content
410
0
    fn identify_coverage_areas(&self, input: &str) -> Vec<String> {
411
0
        let mut areas = Vec::new();
412
        
413
        // Language constructs
414
0
        if input.contains("let ") || input.contains("var ") {
415
0
            areas.push("variable_binding".to_string());
416
0
        }
417
0
        if input.contains("fn ") {
418
0
            areas.push("function_definition".to_string());
419
0
        }
420
0
        if input.contains("=>") {
421
0
            areas.push("lambda_expressions".to_string());
422
0
        }
423
0
        if input.contains("match ") {
424
0
            areas.push("pattern_matching".to_string());
425
0
        }
426
0
        if input.contains("if ") {
427
0
            areas.push("conditional_expressions".to_string());
428
0
        }
429
0
        if input.contains("for ") || input.contains("while ") {
430
0
            areas.push("iteration".to_string());
431
0
        }
432
        
433
        // Data structures  
434
0
        if input.contains('[') && input.contains(']') {
435
0
            areas.push("array_operations".to_string());
436
0
        }
437
0
        if input.contains('(') && input.contains(',') {
438
0
            areas.push("tuple_operations".to_string());
439
0
        }
440
0
        if input.contains('{') && input.contains(':') {
441
0
            areas.push("object_operations".to_string());
442
0
        }
443
        
444
        // Operators
445
0
        if input.contains("?.") {
446
0
            areas.push("optional_chaining".to_string());
447
0
        }
448
0
        if input.contains("??") {
449
0
            areas.push("null_coalescing".to_string());
450
0
        }
451
0
        if input.contains("|>") {
452
0
            areas.push("pipeline_operator".to_string());
453
0
        }
454
0
        if input.contains("...") {
455
0
            areas.push("spread_operator".to_string());
456
0
        }
457
        
458
        // String operations
459
0
        if input.contains("f\"") || input.contains("f'") {
460
0
            areas.push("string_interpolation".to_string());
461
0
        }
462
0
        if input.contains(".map(") || input.contains(".filter(") || input.contains(".reduce(") {
463
0
            areas.push("higher_order_functions".to_string());
464
0
        }
465
        
466
        // REPL features
467
0
        if input.starts_with(':') {
468
0
            areas.push("repl_commands".to_string());
469
0
        }
470
0
        if input.contains('?') && !input.contains("??") {
471
0
            areas.push("repl_introspection".to_string());
472
0
        }
473
        
474
        // Error scenarios
475
0
        if input.contains("try ") || input.contains("catch ") {
476
0
            areas.push("error_handling".to_string());
477
0
        }
478
        
479
0
        areas
480
0
    }
481
482
    /// Write generated tests to a file
483
0
    pub fn write_tests(&self, tests: &[GeneratedTest], output_path: &Path) -> Result<()> {
484
0
        let mut content = String::new();
485
        
486
        // File header
487
0
        content.push_str(&format!(r"
488
0
//! Generated regression tests from REPL replay sessions
489
0
//! 
490
0
//! This file is auto-generated by the replay-to-test conversion pipeline.
491
0
//! DO NOT EDIT MANUALLY - regenerate from .replay files instead.
492
0
//!
493
0
//! Generated tests: {}
494
0
//! Coverage areas: {}
495
0
496
0
use anyhow::Result;
497
0
use crate::runtime::Repl;
498
0
499
0
", tests.len(), 
500
0
            tests.iter()
501
0
                .flat_map(|t| &t.coverage_areas)
502
0
                .collect::<std::collections::HashSet<_>>()
503
0
                .len()
504
        ));
505
        
506
        // Group tests by category
507
0
        let categories = [
508
0
            TestCategory::Unit,
509
0
            TestCategory::Integration,
510
0
            TestCategory::Property,
511
0
            TestCategory::ErrorHandling,
512
0
            TestCategory::Benchmark,
513
0
        ];
514
        
515
0
        for category in categories {
516
0
            let category_tests: Vec<_> = tests.iter()
517
0
                .filter(|t| t.category == category)
518
0
                .collect();
519
            
520
0
            if !category_tests.is_empty() {
521
0
                content.push_str(&format!("\n// {:?} Tests ({})\n", 
522
0
                    category, category_tests.len()));
523
0
                content.push_str("// ============================================================================\n\n");
524
                
525
0
                for test in category_tests {
526
0
                    content.push_str(&test.code);
527
0
                    content.push('\n');
528
0
                }
529
0
            }
530
        }
531
        
532
0
        fs::write(output_path, content)
533
0
            .context("Failed to write test file")?;
534
        
535
0
        Ok(())
536
0
    }
537
}
538
539
impl Default for ReplayConverter {
540
0
    fn default() -> Self {
541
0
        Self::new()
542
0
    }
543
}
544
545
#[cfg(test)]
546
mod tests {
547
    use super::*;
548
    use crate::runtime::replay::*;
549
    
550
    #[test]
551
    fn test_replay_converter_basic() {
552
        let converter = ReplayConverter::new();
553
        
554
        // Create a simple mock session
555
        let session = ReplSession {
556
            version: SemVer::new(1, 0, 0),
557
            metadata: SessionMetadata {
558
                session_id: "test".to_string(),
559
                created_at: "2025-01-01T00:00:00Z".to_string(),
560
                ruchy_version: "1.0.0".to_string(),
561
                student_id: None,
562
                assignment_id: None,
563
                tags: vec![],
564
            },
565
            environment: Environment {
566
                seed: 0,
567
                feature_flags: vec![],
568
                resource_limits: ResourceLimits {
569
                    heap_mb: 100,
570
                    stack_kb: 8192,
571
                    cpu_ms: 5000,
572
                },
573
            },
574
            timeline: vec![
575
                TimestampedEvent {
576
                    id: EventId(1),
577
                    timestamp_ns: 1000,
578
                    event: Event::Input {
579
                        text: "2 + 2".to_string(),
580
                        mode: InputMode::Interactive,
581
                    },
582
                    causality: vec![],
583
                },
584
                TimestampedEvent {
585
                    id: EventId(2),
586
                    timestamp_ns: 2000,
587
                    event: Event::Output {
588
                        result: EvalResult::Success { value: "4".to_string() },
589
                        stdout: vec![],
590
                        stderr: vec![],
591
                    },
592
                    causality: vec![],
593
                },
594
            ],
595
            checkpoints: std::collections::BTreeMap::new(),
596
        };
597
        
598
        let tests = converter.convert_session(&session, "basic").unwrap();
599
        
600
        // Should generate at least unit and integration tests
601
        assert!(tests.len() >= 2);
602
        assert!(tests.iter().any(|t| t.category == TestCategory::Unit));
603
        assert!(tests.iter().any(|t| t.category == TestCategory::Integration));
604
    }
605
    
606
    #[test]
607
    fn test_coverage_area_identification() {
608
        let converter = ReplayConverter::new();
609
        
610
        // Test various language constructs
611
        let test_cases = [
612
            ("let x = 42", vec!["variable_binding"]),
613
            ("x.map(y => y * 2)", vec!["lambda_expressions", "higher_order_functions"]),
614
            ("[1, 2, 3]", vec!["array_operations"]),
615
            ("user?.name", vec!["optional_chaining"]),
616
            ("match x { 1 => \"one\" }", vec!["pattern_matching"]),
617
        ];
618
        
619
        for (input, expected_areas) in test_cases {
620
            let areas = converter.identify_coverage_areas(input);
621
            for expected in expected_areas {
622
                assert!(areas.contains(&expected.to_string()), 
623
                    "Expected coverage area '{expected}' not found for input: '{input}'");
624
            }
625
        }
626
    }
627
}