Coverage Report

Created: 2025-09-05 15:26

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/home/noah/src/ruchy/src/testing/harness.rs
Line
Count
Source
1
/// Testing harness for validating Ruchy code
2
/// This module provides a public API for external projects (like ruchy-book)
3
/// to validate that Ruchy code compiles and executes correctly via LLVM
4
use crate::Parser;
5
use crate::Transpiler;
6
use std::fs;
7
use std::io::Write;
8
use std::path::Path;
9
use std::process::Command;
10
use tempfile::NamedTempFile;
11
use thiserror::Error;
12
13
#[derive(Debug, Error)]
14
pub enum TestError {
15
    #[error("Failed to read file: {0}")]
16
    FileRead(String),
17
18
    #[error("Parse error: {0}")]
19
    Parse(String),
20
21
    #[error("Transpile error: {0}")]
22
    Transpile(String),
23
24
    #[error("Compilation error: {0}")]
25
    Compile(String),
26
27
    #[error("Execution error: {0}")]
28
    Execute(String),
29
30
    #[error("Output mismatch: expected {expected}, got {actual}")]
31
    OutputMismatch { expected: String, actual: String },
32
}
33
34
/// Result type for testing operations
35
pub type TestResult<T> = Result<T, TestError>;
36
37
/// Test harness for validating Ruchy code
38
#[derive(Debug, Clone)]
39
pub struct RuchyTestHarness {
40
    /// Whether to keep intermediate files for debugging
41
    pub keep_intermediates: bool,
42
43
    /// Optimization level for LLVM compilation
44
    pub optimization_level: OptLevel,
45
46
    /// Timeout for execution in seconds
47
    pub timeout_secs: u64,
48
}
49
50
#[derive(Debug, Clone, Copy)]
51
pub enum OptLevel {
52
    None,
53
    Basic,
54
    Full,
55
}
56
57
impl Default for RuchyTestHarness {
58
0
    fn default() -> Self {
59
0
        Self {
60
0
            keep_intermediates: false,
61
0
            optimization_level: OptLevel::Basic,
62
0
            timeout_secs: 30,
63
0
        }
64
0
    }
65
}
66
67
impl RuchyTestHarness {
68
    /// Create a new test harness with default settings
69
0
    pub fn new() -> Self {
70
0
        Self::default()
71
0
    }
72
73
    /// Validate a Ruchy file through the full compilation pipeline
74
    ///
75
    /// # Errors
76
    ///
77
    /// Returns an error if the file cannot be read, parsed, transpiled, compiled, or executed.
78
0
    pub fn validate_file(&self, path: &Path) -> TestResult<ValidationResult> {
79
0
        let content = fs::read_to_string(path).map_err(|e| TestError::FileRead(e.to_string()))?;
80
81
0
        self.validate_source(&content, path.to_string_lossy().as_ref())
82
0
    }
83
84
    /// Validate Ruchy source code
85
    ///
86
    /// # Errors
87
    ///
88
    /// Returns an error if the source cannot be parsed, transpiled, compiled, or executed.
89
0
    pub fn validate_source(&self, source: &str, name: &str) -> TestResult<ValidationResult> {
90
        // Parse
91
0
        let mut parser = Parser::new(source);
92
0
        let ast = parser
93
0
            .parse()
94
0
            .map_err(|e| TestError::Parse(format!("{name}: {e:?}")))?;
95
96
        // Transpile to Rust
97
0
        let transpiler = Transpiler::new();
98
0
        let rust_code = transpiler
99
0
            .transpile(&ast)
100
0
            .map_err(|e| TestError::Transpile(format!("{name}: {e:?}")))?;
101
0
        let rust_code = rust_code.to_string();
102
103
        // Compile and execute
104
0
        let execution_result = self.compile_and_run(&rust_code, name)?;
105
106
        Ok(ValidationResult {
107
0
            name: name.to_string(),
108
            parse_success: true,
109
            transpile_success: true,
110
0
            compile_success: execution_result.compiled,
111
0
            execution_output: execution_result.output,
112
0
            rust_code: if self.keep_intermediates {
113
0
                Some(rust_code)
114
            } else {
115
0
                None
116
            },
117
        })
118
0
    }
119
120
    /// Compile Rust code to binary via LLVM and run it
121
0
    fn compile_and_run(&self, rust_code: &str, _name: &str) -> TestResult<ExecutionResult> {
122
        // Write Rust code to working file
123
0
        let mut temp_file = NamedTempFile::new().map_err(|e| TestError::Compile(e.to_string()))?;
124
125
0
        temp_file
126
0
            .write_all(rust_code.as_bytes())
127
0
            .map_err(|e| TestError::Compile(e.to_string()))?;
128
129
0
        temp_file
130
0
            .flush()
131
0
            .map_err(|e| TestError::Compile(e.to_string()))?;
132
133
        // Compile with rustc (LLVM backend)
134
0
        let output_binary = temp_file.path().with_extension("exe");
135
136
0
        let opt_level = match self.optimization_level {
137
0
            OptLevel::None => "opt-level=0",
138
0
            OptLevel::Basic => "opt-level=2",
139
0
            OptLevel::Full => "opt-level=3",
140
        };
141
142
0
        let compile_result = Command::new("rustc")
143
0
            .arg("--edition=2021")
144
0
            .arg("-C")
145
0
            .arg(opt_level)
146
0
            .arg("-o")
147
0
            .arg(&output_binary)
148
0
            .arg(temp_file.path())
149
0
            .output()
150
0
            .map_err(|e| TestError::Compile(e.to_string()))?;
151
152
0
        if !compile_result.status.success() {
153
0
            return Ok(ExecutionResult {
154
0
                compiled: false,
155
0
                output: None,
156
0
                stderr: Some(String::from_utf8_lossy(&compile_result.stderr).to_string()),
157
0
            });
158
0
        }
159
160
        // Run the binary
161
0
        let run_result = Command::new(&output_binary)
162
0
            .output()
163
0
            .map_err(|e| TestError::Execute(e.to_string()))?;
164
165
        // Clean up unless keeping intermediates
166
0
        if !self.keep_intermediates && output_binary.exists() {
167
0
            fs::remove_file(output_binary).ok();
168
0
        }
169
170
        Ok(ExecutionResult {
171
            compiled: true,
172
0
            output: Some(String::from_utf8_lossy(&run_result.stdout).to_string()),
173
0
            stderr: if run_result.stderr.is_empty() {
174
0
                None
175
            } else {
176
0
                Some(String::from_utf8_lossy(&run_result.stderr).to_string())
177
            },
178
        })
179
0
    }
180
181
    /// Validate that source produces expected output
182
    ///
183
    /// # Errors
184
    ///
185
    /// Returns an error if parsing, transpilation, compilation, or execution fails,
186
    /// or if the actual output doesn't match the expected output.
187
0
    pub fn assert_output(&self, source: &str, expected: &str, name: &str) -> TestResult<()> {
188
0
        let result = self.validate_source(source, name)?;
189
190
0
        if let Some(actual) = result.execution_output {
191
0
            if actual.trim() != expected.trim() {
192
0
                return Err(TestError::OutputMismatch {
193
0
                    expected: expected.to_string(),
194
0
                    actual,
195
0
                });
196
0
            }
197
        } else {
198
0
            return Err(TestError::Execute("No output produced".to_string()));
199
        }
200
201
0
        Ok(())
202
0
    }
203
204
    /// Batch validate multiple files
205
    ///
206
    /// # Errors
207
    ///
208
    /// Returns an error if the directory cannot be read or any of the .ruchy files fail to validate.
209
0
    pub fn validate_directory(&self, dir: &Path) -> TestResult<Vec<ValidationResult>> {
210
0
        let mut results = Vec::new();
211
212
0
        for entry in fs::read_dir(dir).map_err(|e| TestError::FileRead(e.to_string()))? {
213
0
            let entry = entry.map_err(|e| TestError::FileRead(e.to_string()))?;
214
0
            let path = entry.path();
215
216
0
            if path.extension().and_then(|s| s.to_str()) == Some("ruchy") {
217
0
                results.push(self.validate_file(&path)?);
218
0
            }
219
        }
220
221
0
        Ok(results)
222
0
    }
223
}
224
225
/// Result of validating a Ruchy source file
226
#[derive(Debug)]
227
pub struct ValidationResult {
228
    pub name: String,
229
    pub parse_success: bool,
230
    pub transpile_success: bool,
231
    pub compile_success: bool,
232
    pub execution_output: Option<String>,
233
    pub rust_code: Option<String>,
234
}
235
236
/// Result of compiling and executing code
237
#[derive(Debug)]
238
struct ExecutionResult {
239
    compiled: bool,
240
    output: Option<String>,
241
    #[allow(dead_code)]
242
    stderr: Option<String>,
243
}