/home/noah/src/ruchy/src/testing/harness.rs
Line | Count | Source |
1 | | /// Testing harness for validating Ruchy code |
2 | | /// This module provides a public API for external projects (like ruchy-book) |
3 | | /// to validate that Ruchy code compiles and executes correctly via LLVM |
4 | | use crate::Parser; |
5 | | use crate::Transpiler; |
6 | | use std::fs; |
7 | | use std::io::Write; |
8 | | use std::path::Path; |
9 | | use std::process::Command; |
10 | | use tempfile::NamedTempFile; |
11 | | use thiserror::Error; |
12 | | |
13 | | #[derive(Debug, Error)] |
14 | | pub enum TestError { |
15 | | #[error("Failed to read file: {0}")] |
16 | | FileRead(String), |
17 | | |
18 | | #[error("Parse error: {0}")] |
19 | | Parse(String), |
20 | | |
21 | | #[error("Transpile error: {0}")] |
22 | | Transpile(String), |
23 | | |
24 | | #[error("Compilation error: {0}")] |
25 | | Compile(String), |
26 | | |
27 | | #[error("Execution error: {0}")] |
28 | | Execute(String), |
29 | | |
30 | | #[error("Output mismatch: expected {expected}, got {actual}")] |
31 | | OutputMismatch { expected: String, actual: String }, |
32 | | } |
33 | | |
34 | | /// Result type for testing operations |
35 | | pub type TestResult<T> = Result<T, TestError>; |
36 | | |
37 | | /// Test harness for validating Ruchy code |
38 | | #[derive(Debug, Clone)] |
39 | | pub struct RuchyTestHarness { |
40 | | /// Whether to keep intermediate files for debugging |
41 | | pub keep_intermediates: bool, |
42 | | |
43 | | /// Optimization level for LLVM compilation |
44 | | pub optimization_level: OptLevel, |
45 | | |
46 | | /// Timeout for execution in seconds |
47 | | pub timeout_secs: u64, |
48 | | } |
49 | | |
50 | | #[derive(Debug, Clone, Copy)] |
51 | | pub enum OptLevel { |
52 | | None, |
53 | | Basic, |
54 | | Full, |
55 | | } |
56 | | |
57 | | impl Default for RuchyTestHarness { |
58 | 0 | fn default() -> Self { |
59 | 0 | Self { |
60 | 0 | keep_intermediates: false, |
61 | 0 | optimization_level: OptLevel::Basic, |
62 | 0 | timeout_secs: 30, |
63 | 0 | } |
64 | 0 | } |
65 | | } |
66 | | |
67 | | impl RuchyTestHarness { |
68 | | /// Create a new test harness with default settings |
69 | 0 | pub fn new() -> Self { |
70 | 0 | Self::default() |
71 | 0 | } |
72 | | |
73 | | /// Validate a Ruchy file through the full compilation pipeline |
74 | | /// |
75 | | /// # Errors |
76 | | /// |
77 | | /// Returns an error if the file cannot be read, parsed, transpiled, compiled, or executed. |
78 | 0 | pub fn validate_file(&self, path: &Path) -> TestResult<ValidationResult> { |
79 | 0 | let content = fs::read_to_string(path).map_err(|e| TestError::FileRead(e.to_string()))?; |
80 | | |
81 | 0 | self.validate_source(&content, path.to_string_lossy().as_ref()) |
82 | 0 | } |
83 | | |
84 | | /// Validate Ruchy source code |
85 | | /// |
86 | | /// # Errors |
87 | | /// |
88 | | /// Returns an error if the source cannot be parsed, transpiled, compiled, or executed. |
89 | 0 | pub fn validate_source(&self, source: &str, name: &str) -> TestResult<ValidationResult> { |
90 | | // Parse |
91 | 0 | let mut parser = Parser::new(source); |
92 | 0 | let ast = parser |
93 | 0 | .parse() |
94 | 0 | .map_err(|e| TestError::Parse(format!("{name}: {e:?}")))?; |
95 | | |
96 | | // Transpile to Rust |
97 | 0 | let transpiler = Transpiler::new(); |
98 | 0 | let rust_code = transpiler |
99 | 0 | .transpile(&ast) |
100 | 0 | .map_err(|e| TestError::Transpile(format!("{name}: {e:?}")))?; |
101 | 0 | let rust_code = rust_code.to_string(); |
102 | | |
103 | | // Compile and execute |
104 | 0 | let execution_result = self.compile_and_run(&rust_code, name)?; |
105 | | |
106 | | Ok(ValidationResult { |
107 | 0 | name: name.to_string(), |
108 | | parse_success: true, |
109 | | transpile_success: true, |
110 | 0 | compile_success: execution_result.compiled, |
111 | 0 | execution_output: execution_result.output, |
112 | 0 | rust_code: if self.keep_intermediates { |
113 | 0 | Some(rust_code) |
114 | | } else { |
115 | 0 | None |
116 | | }, |
117 | | }) |
118 | 0 | } |
119 | | |
120 | | /// Compile Rust code to binary via LLVM and run it |
121 | 0 | fn compile_and_run(&self, rust_code: &str, _name: &str) -> TestResult<ExecutionResult> { |
122 | | // Write Rust code to working file |
123 | 0 | let mut temp_file = NamedTempFile::new().map_err(|e| TestError::Compile(e.to_string()))?; |
124 | | |
125 | 0 | temp_file |
126 | 0 | .write_all(rust_code.as_bytes()) |
127 | 0 | .map_err(|e| TestError::Compile(e.to_string()))?; |
128 | | |
129 | 0 | temp_file |
130 | 0 | .flush() |
131 | 0 | .map_err(|e| TestError::Compile(e.to_string()))?; |
132 | | |
133 | | // Compile with rustc (LLVM backend) |
134 | 0 | let output_binary = temp_file.path().with_extension("exe"); |
135 | | |
136 | 0 | let opt_level = match self.optimization_level { |
137 | 0 | OptLevel::None => "opt-level=0", |
138 | 0 | OptLevel::Basic => "opt-level=2", |
139 | 0 | OptLevel::Full => "opt-level=3", |
140 | | }; |
141 | | |
142 | 0 | let compile_result = Command::new("rustc") |
143 | 0 | .arg("--edition=2021") |
144 | 0 | .arg("-C") |
145 | 0 | .arg(opt_level) |
146 | 0 | .arg("-o") |
147 | 0 | .arg(&output_binary) |
148 | 0 | .arg(temp_file.path()) |
149 | 0 | .output() |
150 | 0 | .map_err(|e| TestError::Compile(e.to_string()))?; |
151 | | |
152 | 0 | if !compile_result.status.success() { |
153 | 0 | return Ok(ExecutionResult { |
154 | 0 | compiled: false, |
155 | 0 | output: None, |
156 | 0 | stderr: Some(String::from_utf8_lossy(&compile_result.stderr).to_string()), |
157 | 0 | }); |
158 | 0 | } |
159 | | |
160 | | // Run the binary |
161 | 0 | let run_result = Command::new(&output_binary) |
162 | 0 | .output() |
163 | 0 | .map_err(|e| TestError::Execute(e.to_string()))?; |
164 | | |
165 | | // Clean up unless keeping intermediates |
166 | 0 | if !self.keep_intermediates && output_binary.exists() { |
167 | 0 | fs::remove_file(output_binary).ok(); |
168 | 0 | } |
169 | | |
170 | | Ok(ExecutionResult { |
171 | | compiled: true, |
172 | 0 | output: Some(String::from_utf8_lossy(&run_result.stdout).to_string()), |
173 | 0 | stderr: if run_result.stderr.is_empty() { |
174 | 0 | None |
175 | | } else { |
176 | 0 | Some(String::from_utf8_lossy(&run_result.stderr).to_string()) |
177 | | }, |
178 | | }) |
179 | 0 | } |
180 | | |
181 | | /// Validate that source produces expected output |
182 | | /// |
183 | | /// # Errors |
184 | | /// |
185 | | /// Returns an error if parsing, transpilation, compilation, or execution fails, |
186 | | /// or if the actual output doesn't match the expected output. |
187 | 0 | pub fn assert_output(&self, source: &str, expected: &str, name: &str) -> TestResult<()> { |
188 | 0 | let result = self.validate_source(source, name)?; |
189 | | |
190 | 0 | if let Some(actual) = result.execution_output { |
191 | 0 | if actual.trim() != expected.trim() { |
192 | 0 | return Err(TestError::OutputMismatch { |
193 | 0 | expected: expected.to_string(), |
194 | 0 | actual, |
195 | 0 | }); |
196 | 0 | } |
197 | | } else { |
198 | 0 | return Err(TestError::Execute("No output produced".to_string())); |
199 | | } |
200 | | |
201 | 0 | Ok(()) |
202 | 0 | } |
203 | | |
204 | | /// Batch validate multiple files |
205 | | /// |
206 | | /// # Errors |
207 | | /// |
208 | | /// Returns an error if the directory cannot be read or any of the .ruchy files fail to validate. |
209 | 0 | pub fn validate_directory(&self, dir: &Path) -> TestResult<Vec<ValidationResult>> { |
210 | 0 | let mut results = Vec::new(); |
211 | | |
212 | 0 | for entry in fs::read_dir(dir).map_err(|e| TestError::FileRead(e.to_string()))? { |
213 | 0 | let entry = entry.map_err(|e| TestError::FileRead(e.to_string()))?; |
214 | 0 | let path = entry.path(); |
215 | | |
216 | 0 | if path.extension().and_then(|s| s.to_str()) == Some("ruchy") { |
217 | 0 | results.push(self.validate_file(&path)?); |
218 | 0 | } |
219 | | } |
220 | | |
221 | 0 | Ok(results) |
222 | 0 | } |
223 | | } |
224 | | |
225 | | /// Result of validating a Ruchy source file |
226 | | #[derive(Debug)] |
227 | | pub struct ValidationResult { |
228 | | pub name: String, |
229 | | pub parse_success: bool, |
230 | | pub transpile_success: bool, |
231 | | pub compile_success: bool, |
232 | | pub execution_output: Option<String>, |
233 | | pub rust_code: Option<String>, |
234 | | } |
235 | | |
236 | | /// Result of compiling and executing code |
237 | | #[derive(Debug)] |
238 | | struct ExecutionResult { |
239 | | compiled: bool, |
240 | | output: Option<String>, |
241 | | #[allow(dead_code)] |
242 | | stderr: Option<String>, |
243 | | } |