/home/noah/src/ruchy/src/testing/snapshot.rs

Source
//! Snapshot Testing Infrastructure
//!
//! Based on docs/ruchy-transpiler-docs.md Section 3: Snapshot Testing
//! Detects any output changes immediately through content-addressed storage

#![allow(clippy::print_stdout)] // Testing infrastructure needs stdout for feedback
#![allow(clippy::print_stderr)] // Testing infrastructure needs stderr for errors

use anyhow::{bail, Result};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::fs;
use std::path::PathBuf;

/// A single snapshot test case
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SnapshotTest {
    /// Unique name for this test
    pub name: String,
    /// Input Ruchy code
    pub input: String,
    /// SHA256 hash of expected output
    pub output_hash: String,
    /// The actual Rust output (for reference)
    pub rust_output: String,
    /// Metadata about when this snapshot was created/updated
    pub metadata: SnapshotMetadata,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SnapshotMetadata {
    pub created_at: String,
    pub updated_at: String,
    pub ruchy_version: String,
    pub rustc_version: String,
}

/// Snapshot test suite
#[derive(Debug, Serialize, Deserialize)]
pub struct SnapshotSuite {
    pub tests: Vec<SnapshotTest>,
    pub config: SnapshotConfig,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SnapshotConfig {
    /// Whether to automatically update snapshots on mismatch
    pub auto_update: bool,
    /// Directory to store snapshot files
    pub snapshot_dir: PathBuf,
    /// Whether to fail on missing snapshots
    pub fail_on_missing: bool,
}

impl Default for SnapshotConfig {
    fn default() -> Self {
        Self {
            auto_update: false,
            snapshot_dir: PathBuf::from("tests/snapshots"),
            fail_on_missing: true,
        }
    }
}

/// Snapshot test runner
pub struct SnapshotRunner {
    config: SnapshotConfig,
    suite: SnapshotSuite,
}

impl SnapshotRunner {
    /// Load snapshot suite from disk
    /// # Errors
    ///
    /// Returns an error if the operation fails
    /// # Errors
    ///
    /// Returns an error if the operation fails
    pub fn load(config: SnapshotConfig) -> Result<Self> {
        let snapshot_file = config.snapshot_dir.join("snapshots.toml");

        let suite = if snapshot_file.exists() {
            let contents = fs::read_to_string(&snapshot_file)?0;
            toml::from_str(&contents)?0
        } else {
            SnapshotSuite {
                tests: Vec::new(),
                config: config.clone(),
            }
        };

        Ok(Self { config, suite })
    }

    /// Run a snapshot test
    /// # Errors
    ///
    /// Returns an error if the operation fails
    /// # Errors
    ///
    /// Returns an error if the operation fails
    pub fn test<F>(&mut self, name: &str, input: &str, transform: F) -> Result<()>
    where
        F: FnOnce(&str) -> Result<String>,
    {
        // Generate output
        let output = transform(input)?0;
        let output_hash = Self::hash(&output);

        // Find existing snapshot
        if let Some(existing4) = self.suite.tests.iter()4.find4(|t| t.name == name) {
            if existing.output_hash == output_hash {
                // Test passed
                println!("✓ Snapshot matched: {name}");
            } else if self.config.auto_update0 {
                // Update the snapshot
                self.update_snapshot(name, input, &output, &output_hash)?;
                println!("✓ Updated snapshot: {name}");
            } else {
                // Fail the test
                bail!(
                    "Snapshot mismatch for '{}':\n  Expected hash: {}\n  Actual hash: {}\n  Output:\n{}",
                    name, existing.output_hash, output_hash, output
                );
            }
        } else {
            // No existing snapshot
            if self.config.fail_on_missing {
                bail!("Missing snapshot for test: {}", name);
            }
            // Create new snapshot
            self.create_snapshot(name, input, &output, &output_hash)?;
            println!("✓ Created snapshot: {name}");
        }

        Ok(())
    }

    /// Update an existing snapshot
    fn update_snapshot(&mut self, name: &str, input: &str, output: &str, hash: &str) -> Result<()> {
        for test in &mut self.suite.tests {
            if test.name == name {
                test.input = input.to_string();
                test.output_hash = hash.to_string();
                test.rust_output = output.to_string();
                test.metadata.updated_at = chrono::Utc::now().to_rfc3339();
                break;
            }
        }

        self.save()?;
        Ok(())
    }

    /// Create a new snapshot
    fn create_snapshot(&mut self, name: &str, input: &str, output: &str, hash: &str) -> Result<()> {
        let test = SnapshotTest {
            name: name.to_string(),
            input: input.to_string(),
            output_hash: hash.to_string(),
            rust_output: output.to_string(),
            metadata: SnapshotMetadata {
                created_at: chrono::Utc::now().to_rfc3339(),
                updated_at: chrono::Utc::now().to_rfc3339(),
                ruchy_version: env!("CARGO_PKG_VERSION").to_string(),
                rustc_version: "1.75.0".to_string(), // Would get from rustc --version
            },
        };

        self.suite.tests.push(test);
        self.save()?;
        Ok(())
    }

    /// Save the snapshot suite to disk
    fn save(&self) -> Result<()> {
        fs::create_dir_all(&self.config.snapshot_dir)?;
        let snapshot_file = self.config.snapshot_dir.join("snapshots.toml");
        let contents = toml::to_string_pretty(&self.suite)?;
        fs::write(snapshot_file, contents)?;
        Ok(())
    }

    /// Calculate SHA256 hash of a string
    fn hash(s: &str) -> String {
        let mut hasher = Sha256::new();
        hasher.update(s.as_bytes());
        format!("{:x}", hasher.finalize())
    }

    /// Run all snapshots and report results
    /// # Errors
    ///
    /// Returns an error if the operation fails
    /// # Errors
    ///
    /// Returns an error if the operation fails
    pub fn run_all<F>(&mut self, transform: F) -> Result<()>
    where
        F: Fn(&str) -> Result<String>,
    {
        let mut passed = 0;
        let mut failed = 0;
        let updated = 0;

        for test in self.suite.tests.clone() {
            match self.test(&test.name, &test.input, |input| transform(input)) {
                Ok(()) => passed += 1,
                Err(e) => {
                    eprintln!("✗ {}: {}", test.name, e);
                    failed += 1;
                }
            }
        }

        println!("\nSnapshot Test Results:");
        println!("  Passed: {passed}");
        println!("  Failed: {failed}");
        if updated > 0 {
            println!("  Updated: {updated}");
        }

        if failed > 0 {
            bail!("{} snapshot tests failed", failed);
        }

        Ok(())
    }
}

/// Automatic bisection to identify regression source
#[allow(clippy::module_name_repetitions)]
pub struct SnapshotBisector {
    #[allow(dead_code)]
    snapshots: Vec<SnapshotTest>,
}

impl SnapshotBisector {
    #[must_use]
    pub fn new(snapshots: Vec<SnapshotTest>) -> Self {
        Self { snapshots }
    }

    /// Find the commit that introduced a regression
    pub fn bisect<F>(&self, test_name: &str, _is_good: F) -> Option<String>
    where
        F: Fn(&str) -> bool,
    {
        // This would integrate with git bisect
        // For now, just a placeholder
        println!("Would bisect to find regression in test: {test_name}");
        None
    }
}

/// Snapshot test definitions for core Ruchy features
#[must_use]
pub fn core_snapshot_tests() -> Vec<(&'static str, &'static str)> {
    vec![
        ("literal_int", "42"),
        ("literal_float", "3.14"),
        ("literal_string", r#""hello""#),
        ("literal_bool_true", "true"),
        ("literal_bool_false", "false"),
        ("binary_add", "1 + 2"),
        ("binary_mul", "3 * 4"),
        ("binary_complex", "1 + 2 * 3"),
        ("binary_parens", "(1 + 2) * 3"),
        ("let_simple", "let x = 10"),
        ("let_nested", "let x = 10 in x + 1"),
        ("function_simple", "fun f(x) { x + 1 }"),
        ("function_multi_param", "fun add(x, y) { x + y }"),
        ("if_simple", "if true { 1 } else { 2 }"),
        ("if_no_else", "if x > 0 { x }"),
        ("list_empty", "[]"),
        ("list_numbers", "[1, 2, 3]"),
        ("pipeline_simple", "data >> filter >> map"),
        ("match_simple", "match x { 1 => \"one\", _ => \"other\" }"),
    ]
}

#[cfg(test)]
mod tests {
    #![allow(clippy::unwrap_used)]
    use super::*;
    use crate::{Parser, Transpiler};

    #[test]
    fn test_snapshot_basic() {
        let config = SnapshotConfig {
            auto_update: true,
            snapshot_dir: PathBuf::from("target/test-snapshots"),
            fail_on_missing: false,
        };

        let mut runner = SnapshotRunner::load(config).unwrap();

        // Test a simple expression
        runner
            .test("simple_addition", "1 + 2", |input| {
                let mut parser = Parser::new(input);
                let ast = parser.parse()?0;
                let transpiler = Transpiler::new();
                let tokens = transpiler.transpile(&ast)?0;
                Ok(tokens.to_string())
            })
            .unwrap();
    }

    #[test]
    fn test_snapshot_determinism() {
        let config = SnapshotConfig {
            auto_update: false,
            snapshot_dir: PathBuf::from("target/test-snapshots-determinism"),
            fail_on_missing: false,
        };

        let mut runner = SnapshotRunner::load(config).unwrap();

        // Run the same test multiple times - should produce identical hashes
        for i3 in 0..3 {
            runner
                .test(&format!("determinism_test_{i}"), "x * 2 + 1", |input| {
                    let mut parser = Parser::new(input);
                    let ast = parser.parse()?0;
                    let transpiler = Transpiler::new();
                    let tokens = transpiler.transpile(&ast)?0;
                    Ok(tokens.to_string())
                })
                .unwrap();
        }
    }
}

Coverage Report

Created: 2025-09-05 15:26

Line	Count	Source
1		//! Snapshot Testing Infrastructure
2		//!
3		//! Based on docs/ruchy-transpiler-docs.md Section 3: Snapshot Testing
4		//! Detects any output changes immediately through content-addressed storage
5
6		#![allow(clippy::print_stdout)] // Testing infrastructure needs stdout for feedback
7		#![allow(clippy::print_stderr)] // Testing infrastructure needs stderr for errors
8
9		use anyhow::{bail, Result};
10		use serde::{Deserialize, Serialize};
11		use sha2::{Digest, Sha256};
12		use std::fs;
13		use std::path::PathBuf;
14
15		/// A single snapshot test case
16		#[derive(Debug, Clone, Serialize, Deserialize)]
17		pub struct SnapshotTest {
18		/// Unique name for this test
19		pub name: String,
20		/// Input Ruchy code
21		pub input: String,
22		/// SHA256 hash of expected output
23		pub output_hash: String,
24		/// The actual Rust output (for reference)
25		pub rust_output: String,
26		/// Metadata about when this snapshot was created/updated
27		pub metadata: SnapshotMetadata,
28		}
29
30		#[derive(Debug, Clone, Serialize, Deserialize)]
31		pub struct SnapshotMetadata {
32		pub created_at: String,
33		pub updated_at: String,
34		pub ruchy_version: String,
35		pub rustc_version: String,
36		}
37
38		/// Snapshot test suite
39		#[derive(Debug, Serialize, Deserialize)]
40		pub struct SnapshotSuite {
41		pub tests: Vec<SnapshotTest>,
42		pub config: SnapshotConfig,
43		}
44
45		#[derive(Debug, Clone, Serialize, Deserialize)]
46		pub struct SnapshotConfig {
47		/// Whether to automatically update snapshots on mismatch
48		pub auto_update: bool,
49		/// Directory to store snapshot files
50		pub snapshot_dir: PathBuf,
51		/// Whether to fail on missing snapshots
52		pub fail_on_missing: bool,
53		}
54
55		impl Default for SnapshotConfig {
56	0	fn default() -> Self {
57	0	Self {
58	0	auto_update: false,
59	0	snapshot_dir: PathBuf::from("tests/snapshots"),
60	0	fail_on_missing: true,
61	0	}
62	0	}
63		}
64
65		/// Snapshot test runner
66		pub struct SnapshotRunner {
67		config: SnapshotConfig,
68		suite: SnapshotSuite,
69		}
70
71		impl SnapshotRunner {
72		/// Load snapshot suite from disk
73		/// # Errors
74		///
75		/// Returns an error if the operation fails
76		/// # Errors
77		///
78		/// Returns an error if the operation fails
79	2	pub fn load(config: SnapshotConfig) -> Result<Self> {
80	2	let snapshot_file = config.snapshot_dir.join("snapshots.toml");
81
82	2	let suite = if snapshot_file.exists() {
83	2	let contents = fs::read_to_string(&snapshot_file) ?0 ;
84	2	toml::from_str(&contents) ?0
85		} else {
86	0	SnapshotSuite {
87	0	tests: Vec::new(),
88	0	config: config.clone(),
89	0	}
90		};
91
92	2	Ok(Self { config, suite })
93	2	}
94
95		/// Run a snapshot test
96		/// # Errors
97		///
98		/// Returns an error if the operation fails
99		/// # Errors
100		///
101		/// Returns an error if the operation fails
102	4	pub fn test<F>(&mut self, name: &str, input: &str, transform: F) -> Result<()>
103	4	where
104	4	F: FnOnce(&str) -> Result<String>,
105		{
106		// Generate output
107	4	let output = transform(input) ?0 ;
108	4	let output_hash = Self::hash(&output);
109
110		// Find existing snapshot
111	7	if let Some( existing4 ) = self.suite.tests.iter()4 . find4 (\|t\| t.name == name) {
112	4	if existing.output_hash == output_hash {
113	4	// Test passed
114	4	println!("✓ Snapshot matched: {name}");
115	4	} else if self.config.auto_update0 {
116		// Update the snapshot
117	0	self.update_snapshot(name, input, &output, &output_hash)?;
118	0	println!("✓ Updated snapshot: {name}");
119		} else {
120		// Fail the test
121	0	bail!(
122	0	"Snapshot mismatch for '{}':\n Expected hash: {}\n Actual hash: {}\n Output:\n{}",
123		name, existing.output_hash, output_hash, output
124		);
125		}
126		} else {
127		// No existing snapshot
128	0	if self.config.fail_on_missing {
129	0	bail!("Missing snapshot for test: {}", name);
130	0	}
131		// Create new snapshot
132	0	self.create_snapshot(name, input, &output, &output_hash)?;
133	0	println!("✓ Created snapshot: {name}");
134		}
135
136	4	Ok(())
137	4	}
138
139		/// Update an existing snapshot
140	0	fn update_snapshot(&mut self, name: &str, input: &str, output: &str, hash: &str) -> Result<()> {
141	0	for test in &mut self.suite.tests {
142	0	if test.name == name {
143	0	test.input = input.to_string();
144	0	test.output_hash = hash.to_string();
145	0	test.rust_output = output.to_string();
146	0	test.metadata.updated_at = chrono::Utc::now().to_rfc3339();
147	0	break;
148	0	}
149		}
150
151	0	self.save()?;
152	0	Ok(())
153	0	}
154
155		/// Create a new snapshot
156	0	fn create_snapshot(&mut self, name: &str, input: &str, output: &str, hash: &str) -> Result<()> {
157	0	let test = SnapshotTest {
158	0	name: name.to_string(),
159	0	input: input.to_string(),
160	0	output_hash: hash.to_string(),
161	0	rust_output: output.to_string(),
162	0	metadata: SnapshotMetadata {
163	0	created_at: chrono::Utc::now().to_rfc3339(),
164	0	updated_at: chrono::Utc::now().to_rfc3339(),
165	0	ruchy_version: env!("CARGO_PKG_VERSION").to_string(),
166	0	rustc_version: "1.75.0".to_string(), // Would get from rustc --version
167	0	},
168	0	};
169
170	0	self.suite.tests.push(test);
171	0	self.save()?;
172	0	Ok(())
173	0	}
174
175		/// Save the snapshot suite to disk
176	0	fn save(&self) -> Result<()> {
177	0	fs::create_dir_all(&self.config.snapshot_dir)?;
178	0	let snapshot_file = self.config.snapshot_dir.join("snapshots.toml");
179	0	let contents = toml::to_string_pretty(&self.suite)?;
180	0	fs::write(snapshot_file, contents)?;
181	0	Ok(())
182	0	}
183
184		/// Calculate SHA256 hash of a string
185	4	fn hash(s: &str) -> String {
186	4	let mut hasher = Sha256::new();
187	4	hasher.update(s.as_bytes());
188	4	format!("{:x}", hasher.finalize())
189	4	}
190
191		/// Run all snapshots and report results
192		/// # Errors
193		///
194		/// Returns an error if the operation fails
195		/// # Errors
196		///
197		/// Returns an error if the operation fails
198	0	pub fn run_all<F>(&mut self, transform: F) -> Result<()>
199	0	where
200	0	F: Fn(&str) -> Result<String>,
201		{
202	0	let mut passed = 0;
203	0	let mut failed = 0;
204	0	let updated = 0;
205
206	0	for test in self.suite.tests.clone() {
207	0	match self.test(&test.name, &test.input, \|input\| transform(input)) {
208	0	Ok(()) => passed += 1,
209	0	Err(e) => {
210	0	eprintln!("✗ {}: {}", test.name, e);
211	0	failed += 1;
212	0	}
213		}
214		}
215
216	0	println!("\nSnapshot Test Results:");
217	0	println!(" Passed: {passed}");
218	0	println!(" Failed: {failed}");
219	0	if updated > 0 {
220	0	println!(" Updated: {updated}");
221	0	}
222
223	0	if failed > 0 {
224	0	bail!("{} snapshot tests failed", failed);
225	0	}
226
227	0	Ok(())
228	0	}
229		}
230
231		/// Automatic bisection to identify regression source
232		#[allow(clippy::module_name_repetitions)]
233		pub struct SnapshotBisector {
234		#[allow(dead_code)]
235		snapshots: Vec<SnapshotTest>,
236		}
237
238		impl SnapshotBisector {
239		#[must_use]
240	0	pub fn new(snapshots: Vec<SnapshotTest>) -> Self {
241	0	Self { snapshots }
242	0	}
243
244		/// Find the commit that introduced a regression
245	0	pub fn bisect<F>(&self, test_name: &str, _is_good: F) -> Option<String>
246	0	where
247	0	F: Fn(&str) -> bool,
248		{
249		// This would integrate with git bisect
250		// For now, just a placeholder
251	0	println!("Would bisect to find regression in test: {test_name}");
252	0	None
253	0	}
254		}
255
256		/// Snapshot test definitions for core Ruchy features
257		#[must_use]
258	0	pub fn core_snapshot_tests() -> Vec<(&'static str, &'static str)> {
259	0	vec![
260	0	("literal_int", "42"),
261	0	("literal_float", "3.14"),
262	0	("literal_string", r#""hello""#),
263	0	("literal_bool_true", "true"),
264	0	("literal_bool_false", "false"),
265	0	("binary_add", "1 + 2"),
266	0	("binary_mul", "3 * 4"),
267	0	("binary_complex", "1 + 2 * 3"),
268	0	("binary_parens", "(1 + 2) * 3"),
269	0	("let_simple", "let x = 10"),
270	0	("let_nested", "let x = 10 in x + 1"),
271	0	("function_simple", "fun f(x) { x + 1 }"),
272	0	("function_multi_param", "fun add(x, y) { x + y }"),
273	0	("if_simple", "if true { 1 } else { 2 }"),
274	0	("if_no_else", "if x > 0 { x }"),
275	0	("list_empty", "[]"),
276	0	("list_numbers", "[1, 2, 3]"),
277	0	("pipeline_simple", "data >> filter >> map"),
278	0	("match_simple", "match x { 1 => \"one\", _ => \"other\" }"),
279		]
280	0	}
281
282		#[cfg(test)]
283		mod tests {
284		#![allow(clippy::unwrap_used)]
285		use super::*;
286		use crate::{Parser, Transpiler};
287
288		#[test]
289	1	fn test_snapshot_basic() {
290	1	let config = SnapshotConfig {
291	1	auto_update: true,
292	1	snapshot_dir: PathBuf::from("target/test-snapshots"),
293	1	fail_on_missing: false,
294	1	};
295
296	1	let mut runner = SnapshotRunner::load(config).unwrap();
297
298		// Test a simple expression
299	1	runner
300	1	.test("simple_addition", "1 + 2", \|input\| {
301	1	let mut parser = Parser::new(input);
302	1	let ast = parser.parse() ?0 ;
303	1	let transpiler = Transpiler::new();
304	1	let tokens = transpiler.transpile(&ast) ?0 ;
305	1	Ok(tokens.to_string())
306	1	})
307	1	.unwrap();
308	1	}
309
310		#[test]
311	1	fn test_snapshot_determinism() {
312	1	let config = SnapshotConfig {
313	1	auto_update: false,
314	1	snapshot_dir: PathBuf::from("target/test-snapshots-determinism"),
315	1	fail_on_missing: false,
316	1	};
317
318	1	let mut runner = SnapshotRunner::load(config).unwrap();
319
320		// Run the same test multiple times - should produce identical hashes
321	4	for i3 in 0..3 {
322	3	runner
323	3	.test(&format!("determinism_test_{i}"), "x * 2 + 1", \|input\| {
324	3	let mut parser = Parser::new(input);
325	3	let ast = parser.parse() ?0 ;
326	3	let transpiler = Transpiler::new();
327	3	let tokens = transpiler.transpile(&ast) ?0 ;
328	3	Ok(tokens.to_string())
329	3	})
330	3	.unwrap();
331		}
332	1	}
333		}