/home/noah/src/ruchy/src/runtime/replay.rs
Line | Count | Source |
1 | | //! REPL Replay Testing System |
2 | | //! |
3 | | //! Provides deterministic replay capabilities for testing and educational assessment. |
4 | | //! Based on docs/specifications/repl-replay-testing-spec.md |
5 | | |
6 | | use anyhow::Result; |
7 | | use serde::{Serialize, Deserialize}; |
8 | | use std::collections::{HashMap, BTreeMap}; |
9 | | use std::time::Instant; |
10 | | use crate::runtime::repl::Value; |
11 | | |
12 | | // ============================================================================ |
13 | | // Core Data Structures |
14 | | // ============================================================================ |
15 | | |
16 | | /// Unique identifier for events in a session |
17 | | #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] |
18 | | pub struct EventId(pub u64); |
19 | | |
20 | | /// Semantic version for compatibility checking |
21 | | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
22 | | pub struct SemVer { |
23 | | major: u32, |
24 | | minor: u32, |
25 | | patch: u32, |
26 | | } |
27 | | |
28 | | impl SemVer { |
29 | 0 | pub fn new(major: u32, minor: u32, patch: u32) -> Self { |
30 | 0 | Self { major, minor, patch } |
31 | 0 | } |
32 | | } |
33 | | |
34 | | /// Complete REPL session recording |
35 | | #[derive(Debug, Clone, Serialize, Deserialize)] |
36 | | pub struct ReplSession { |
37 | | pub version: SemVer, |
38 | | pub metadata: SessionMetadata, |
39 | | pub environment: Environment, |
40 | | pub timeline: Vec<TimestampedEvent>, |
41 | | pub checkpoints: BTreeMap<EventId, StateCheckpoint>, |
42 | | } |
43 | | |
44 | | /// Session metadata for tracking and analysis |
45 | | #[derive(Debug, Clone, Serialize, Deserialize)] |
46 | | pub struct SessionMetadata { |
47 | | pub session_id: String, |
48 | | pub created_at: String, |
49 | | pub ruchy_version: String, |
50 | | pub student_id: Option<String>, |
51 | | pub assignment_id: Option<String>, |
52 | | pub tags: Vec<String>, |
53 | | } |
54 | | |
55 | | /// Environment configuration for deterministic replay |
56 | | #[derive(Debug, Clone, Serialize, Deserialize)] |
57 | | pub struct Environment { |
58 | | pub seed: u64, |
59 | | pub feature_flags: Vec<String>, |
60 | | pub resource_limits: ResourceLimits, |
61 | | } |
62 | | |
63 | | /// Resource limits for bounded execution |
64 | | #[derive(Debug, Clone, Serialize, Deserialize)] |
65 | | pub struct ResourceLimits { |
66 | | pub heap_mb: usize, |
67 | | pub stack_kb: usize, |
68 | | pub cpu_ms: u64, |
69 | | } |
70 | | |
71 | | /// Timestamped event with causality tracking |
72 | | #[derive(Debug, Clone, Serialize, Deserialize)] |
73 | | pub struct TimestampedEvent { |
74 | | pub id: EventId, |
75 | | pub timestamp_ns: u64, |
76 | | pub event: Event, |
77 | | pub causality: Vec<EventId>, // Lamport clock for distributed replay |
78 | | } |
79 | | |
80 | | /// Event types that can occur in a REPL session |
81 | | #[derive(Debug, Clone, Serialize, Deserialize)] |
82 | | pub enum Event { |
83 | | Input { |
84 | | text: String, |
85 | | mode: InputMode, |
86 | | }, |
87 | | Output { |
88 | | result: EvalResult, |
89 | | stdout: Vec<u8>, |
90 | | stderr: Vec<u8>, |
91 | | }, |
92 | | StateChange { |
93 | | bindings_delta: HashMap<String, String>, // Simplified for now |
94 | | state_hash: String, |
95 | | }, |
96 | | ResourceUsage { |
97 | | heap_bytes: usize, |
98 | | stack_depth: usize, |
99 | | cpu_ns: u64, |
100 | | }, |
101 | | } |
102 | | |
103 | | /// Input modes for different interaction patterns |
104 | | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
105 | | pub enum InputMode { |
106 | | Interactive, |
107 | | Paste, |
108 | | File, |
109 | | Script, |
110 | | } |
111 | | |
112 | | /// Result of evaluating an expression |
113 | | #[derive(Debug, Clone, Serialize, Deserialize)] |
114 | | pub enum EvalResult { |
115 | | Success { value: String }, |
116 | | Error { message: String }, |
117 | | Unit, |
118 | | } |
119 | | |
120 | | /// Complete state checkpoint for rollback |
121 | | #[derive(Debug, Clone, Serialize, Deserialize)] |
122 | | pub struct StateCheckpoint { |
123 | | pub bindings: HashMap<String, String>, |
124 | | pub type_environment: HashMap<String, String>, |
125 | | pub state_hash: String, |
126 | | pub resource_usage: ResourceUsage, |
127 | | } |
128 | | |
129 | | /// Resource usage snapshot |
130 | | #[derive(Debug, Clone, Serialize, Deserialize)] |
131 | | pub struct ResourceUsage { |
132 | | pub heap_bytes: usize, |
133 | | pub stack_depth: usize, |
134 | | pub cpu_ns: u64, |
135 | | } |
136 | | |
137 | | // ============================================================================ |
138 | | // Deterministic Execution |
139 | | // ============================================================================ |
140 | | |
141 | | /// Result of deterministic replay execution |
142 | | #[derive(Debug)] |
143 | | pub struct ReplayResult { |
144 | | pub output: Result<Value>, |
145 | | pub state_hash: String, |
146 | | pub resource_usage: ResourceUsage, |
147 | | } |
148 | | |
149 | | /// Trait for deterministic REPL execution |
150 | | pub trait DeterministicRepl { |
151 | | /// Execute with a fixed seed for deterministic behavior |
152 | | fn execute_with_seed(&mut self, input: &str, seed: u64) -> ReplayResult; |
153 | | |
154 | | /// Create a state checkpoint |
155 | | fn checkpoint(&self) -> StateCheckpoint; |
156 | | |
157 | | /// Restore from a checkpoint |
158 | | fn restore(&mut self, checkpoint: &StateCheckpoint) -> Result<()>; |
159 | | |
160 | | /// Validate determinism against another instance |
161 | | fn validate_determinism(&self, other: &Self) -> ValidationResult; |
162 | | } |
163 | | |
164 | | /// Result of determinism validation |
165 | | #[derive(Debug)] |
166 | | pub struct ValidationResult { |
167 | | pub is_deterministic: bool, |
168 | | pub divergences: Vec<Divergence>, |
169 | | } |
170 | | |
171 | | /// Types of divergence in replay |
172 | | #[derive(Debug, Clone)] |
173 | | pub enum Divergence { |
174 | | Output { expected: String, actual: String }, |
175 | | State { expected_hash: String, actual_hash: String }, |
176 | | Resources { expected: ResourceUsage, actual: ResourceUsage }, |
177 | | } |
178 | | |
179 | | // ============================================================================ |
180 | | // Replay Validation Engine |
181 | | // ============================================================================ |
182 | | |
183 | | /// Validates replay sessions for correctness |
184 | | pub struct ReplayValidator { |
185 | | pub strict_mode: bool, |
186 | | pub tolerance: ResourceTolerance, |
187 | | } |
188 | | |
189 | | /// Tolerance for resource usage variations |
190 | | #[derive(Debug, Clone)] |
191 | | pub struct ResourceTolerance { |
192 | | pub heap_bytes_percent: f64, |
193 | | pub cpu_ns_percent: f64, |
194 | | } |
195 | | |
196 | | impl Default for ResourceTolerance { |
197 | 0 | fn default() -> Self { |
198 | 0 | Self { |
199 | 0 | heap_bytes_percent: 10.0, // Allow 10% variation |
200 | 0 | cpu_ns_percent: 20.0, // Allow 20% CPU variation |
201 | 0 | } |
202 | 0 | } |
203 | | } |
204 | | |
205 | | /// Report from validation |
206 | | #[derive(Debug, Default)] |
207 | | pub struct ValidationReport { |
208 | | pub passed: bool, |
209 | | pub total_events: usize, |
210 | | pub successful_events: usize, |
211 | | pub divergences: Vec<(EventId, Divergence)>, |
212 | | } |
213 | | |
214 | | impl ValidationReport { |
215 | 0 | pub fn new() -> Self { |
216 | 0 | Self::default() |
217 | 0 | } |
218 | | |
219 | 0 | pub fn add_divergence(&mut self, event_id: EventId, divergence: Divergence) { |
220 | 0 | self.divergences.push((event_id, divergence)); |
221 | 0 | self.passed = false; |
222 | 0 | } |
223 | | } |
224 | | |
225 | | impl ReplayValidator { |
226 | 0 | pub fn new(strict_mode: bool) -> Self { |
227 | 0 | Self { |
228 | 0 | strict_mode, |
229 | 0 | tolerance: ResourceTolerance::default(), |
230 | 0 | } |
231 | 0 | } |
232 | | |
233 | 0 | pub fn validate_session( |
234 | 0 | &self, |
235 | 0 | recorded: &ReplSession, |
236 | 0 | implementation: &mut impl DeterministicRepl, |
237 | 0 | ) -> ValidationReport { |
238 | 0 | let mut report = ValidationReport::new(); |
239 | 0 | report.total_events = recorded.timeline.len(); |
240 | | |
241 | 0 | for event in &recorded.timeline { |
242 | 0 | if let Event::Input { text, .. } = &event.event { |
243 | 0 | let result = implementation.execute_with_seed(text, recorded.environment.seed); |
244 | | |
245 | | // Find corresponding output event |
246 | 0 | if let Some(expected_output) = self.find_next_output(recorded, event.id) { |
247 | 0 | if self.outputs_equivalent(&result, expected_output) { |
248 | 0 | report.successful_events += 1; |
249 | 0 | } else { |
250 | 0 | report.add_divergence( |
251 | 0 | event.id, |
252 | 0 | Divergence::Output { |
253 | 0 | expected: format!("{expected_output:?}"), |
254 | 0 | actual: format!("{:?}", result.output), |
255 | 0 | } |
256 | 0 | ); |
257 | 0 | } |
258 | 0 | } |
259 | | |
260 | | // Validate resource bounds |
261 | 0 | if !self.tolerance_accepts(&result.resource_usage) { |
262 | 0 | report.add_divergence( |
263 | 0 | event.id, |
264 | 0 | Divergence::Resources { |
265 | 0 | expected: ResourceUsage { |
266 | 0 | heap_bytes: 0, |
267 | 0 | stack_depth: 0, |
268 | 0 | cpu_ns: 0, |
269 | 0 | }, |
270 | 0 | actual: result.resource_usage.clone(), |
271 | 0 | } |
272 | 0 | ); |
273 | 0 | } |
274 | 0 | } |
275 | | } |
276 | | |
277 | 0 | if report.divergences.is_empty() { |
278 | 0 | report.passed = true; |
279 | 0 | } |
280 | | |
281 | 0 | report |
282 | 0 | } |
283 | | |
284 | 0 | fn find_next_output<'a>(&self, session: &'a ReplSession, after_id: EventId) -> Option<&'a Event> { |
285 | 0 | session.timeline |
286 | 0 | .iter() |
287 | 0 | .find(|e| e.id > after_id && matches!(e.event, Event::Output { .. })) |
288 | 0 | .map(|e| &e.event) |
289 | 0 | } |
290 | | |
291 | 0 | fn outputs_equivalent(&self, result: &ReplayResult, expected: &Event) -> bool { |
292 | 0 | match expected { |
293 | 0 | Event::Output { result: expected_result, .. } => { |
294 | 0 | match (&result.output, expected_result) { |
295 | 0 | (Ok(value), EvalResult::Success { value: expected }) => { |
296 | 0 | format!("{value:?}") == *expected |
297 | | } |
298 | 0 | (Err(e), EvalResult::Error { message }) => { |
299 | 0 | e.to_string().contains(message) |
300 | | } |
301 | 0 | _ => false |
302 | | } |
303 | | } |
304 | 0 | _ => false |
305 | | } |
306 | 0 | } |
307 | | |
308 | 0 | fn tolerance_accepts(&self, _usage: &ResourceUsage) -> bool { |
309 | | // For now, always accept - will implement proper bounds checking later |
310 | 0 | true |
311 | 0 | } |
312 | | } |
313 | | |
314 | | // ============================================================================ |
315 | | // Session Recording |
316 | | // ============================================================================ |
317 | | |
318 | | /// Records REPL sessions for later replay |
319 | | pub struct SessionRecorder { |
320 | | session: ReplSession, |
321 | | next_event_id: u64, |
322 | | start_time: Instant, |
323 | | } |
324 | | |
325 | | impl SessionRecorder { |
326 | 0 | pub fn new(metadata: SessionMetadata) -> Self { |
327 | | use std::time::SystemTime; |
328 | | |
329 | | // Generate unique seed based on current time + session_id hash |
330 | 0 | let now = SystemTime::now() |
331 | 0 | .duration_since(SystemTime::UNIX_EPOCH) |
332 | 0 | .unwrap_or_default() |
333 | 0 | .as_nanos() as u64; |
334 | | |
335 | 0 | let session_hash = { |
336 | 0 | let mut hasher = std::collections::hash_map::DefaultHasher::new(); |
337 | 0 | std::hash::Hasher::write(&mut hasher, metadata.session_id.as_bytes()); |
338 | 0 | std::hash::Hasher::finish(&hasher) |
339 | | }; |
340 | | |
341 | 0 | let unique_seed = now.wrapping_add(session_hash); |
342 | | |
343 | 0 | Self { |
344 | 0 | session: ReplSession { |
345 | 0 | version: SemVer::new(1, 0, 0), |
346 | 0 | metadata, |
347 | 0 | environment: Environment { |
348 | 0 | seed: unique_seed, |
349 | 0 | feature_flags: vec![], |
350 | 0 | resource_limits: ResourceLimits { |
351 | 0 | heap_mb: 100, |
352 | 0 | stack_kb: 8192, |
353 | 0 | cpu_ms: 5000, |
354 | 0 | }, |
355 | 0 | }, |
356 | 0 | timeline: vec![], |
357 | 0 | checkpoints: BTreeMap::new(), |
358 | 0 | }, |
359 | 0 | next_event_id: 1, |
360 | 0 | start_time: Instant::now(), |
361 | 0 | } |
362 | 0 | } |
363 | | |
364 | 0 | pub fn record_input(&mut self, text: String, mode: InputMode) -> EventId { |
365 | 0 | let id = EventId(self.next_event_id); |
366 | 0 | self.next_event_id += 1; |
367 | | |
368 | 0 | let event = TimestampedEvent { |
369 | 0 | id, |
370 | 0 | timestamp_ns: self.elapsed_ns(), |
371 | 0 | event: Event::Input { text, mode }, |
372 | 0 | causality: vec![], |
373 | 0 | }; |
374 | | |
375 | 0 | self.session.timeline.push(event); |
376 | 0 | id |
377 | 0 | } |
378 | | |
379 | 0 | pub fn record_output(&mut self, result: Result<Value>) -> EventId { |
380 | 0 | let id = EventId(self.next_event_id); |
381 | 0 | self.next_event_id += 1; |
382 | | |
383 | 0 | let eval_result = match result { |
384 | 0 | Ok(Value::Unit) => EvalResult::Unit, |
385 | 0 | Ok(value) => EvalResult::Success { |
386 | 0 | value: format!("{value:?}") |
387 | 0 | }, |
388 | 0 | Err(e) => EvalResult::Error { |
389 | 0 | message: e.to_string() |
390 | 0 | }, |
391 | | }; |
392 | | |
393 | 0 | let event = TimestampedEvent { |
394 | 0 | id, |
395 | 0 | timestamp_ns: self.elapsed_ns(), |
396 | 0 | event: Event::Output { |
397 | 0 | result: eval_result, |
398 | 0 | stdout: vec![], |
399 | 0 | stderr: vec![], |
400 | 0 | }, |
401 | 0 | causality: vec![], |
402 | 0 | }; |
403 | | |
404 | 0 | self.session.timeline.push(event); |
405 | 0 | id |
406 | 0 | } |
407 | | |
408 | 0 | pub fn add_checkpoint(&mut self, event_id: EventId, checkpoint: StateCheckpoint) { |
409 | 0 | self.session.checkpoints.insert(event_id, checkpoint); |
410 | 0 | } |
411 | | |
412 | 0 | pub fn get_session(&self) -> &ReplSession { |
413 | 0 | &self.session |
414 | 0 | } |
415 | | |
416 | 0 | pub fn into_session(self) -> ReplSession { |
417 | 0 | self.session |
418 | 0 | } |
419 | | |
420 | 0 | fn elapsed_ns(&self) -> u64 { |
421 | 0 | self.start_time.elapsed().as_nanos() as u64 |
422 | 0 | } |
423 | | } |
424 | | |
425 | | // ============================================================================ |
426 | | // Testing Utilities |
427 | | // ============================================================================ |
428 | | |
429 | | #[cfg(test)] |
430 | | mod tests { |
431 | | use super::*; |
432 | | |
433 | | #[test] |
434 | | fn test_session_recording() { |
435 | | let metadata = SessionMetadata { |
436 | | session_id: "test-001".to_string(), |
437 | | created_at: "2025-08-28T10:00:00Z".to_string(), |
438 | | ruchy_version: "1.23.0".to_string(), |
439 | | student_id: None, |
440 | | assignment_id: None, |
441 | | tags: vec!["test".to_string()], |
442 | | }; |
443 | | |
444 | | let mut recorder = SessionRecorder::new(metadata); |
445 | | |
446 | | let input_id = recorder.record_input("let x = 42".to_string(), InputMode::Interactive); |
447 | | assert_eq!(input_id, EventId(1)); |
448 | | |
449 | | let output_id = recorder.record_output(Ok(Value::Unit)); |
450 | | assert_eq!(output_id, EventId(2)); |
451 | | |
452 | | let session = recorder.get_session(); |
453 | | assert_eq!(session.timeline.len(), 2); |
454 | | } |
455 | | |
456 | | #[test] |
457 | | fn test_replay_validation() { |
458 | | // This will be implemented once we have the DeterministicRepl implementation |
459 | | } |
460 | | } |