/home/noah/src/ruchy/src/backend/module_loader.rs
Line | Count | Source |
1 | | //! Multi-file module system implementation |
2 | | //! |
3 | | //! Enables `use external_file;` imports for larger Ruchy programs while preserving |
4 | | //! 100% compatibility with existing inline modules. |
5 | | //! |
6 | | //! # Architecture |
7 | | //! |
8 | | //! - **`ModuleLoader`**: Core component handling file discovery, parsing, and caching |
9 | | //! - **`ParsedModule`**: Represents a loaded module with metadata and dependencies |
10 | | //! - **Search Path Resolution**: Multiple directory support with fallback patterns |
11 | | //! - **Circular Dependency Detection**: Prevents infinite loading loops |
12 | | //! - **Caching**: Avoids re-parsing unchanged files for performance |
13 | | //! |
14 | | //! # Usage |
15 | | //! |
16 | | //! ```rust |
17 | | //! use ruchy::backend::module_loader::ModuleLoader; |
18 | | //! |
19 | | //! let mut loader = ModuleLoader::new(); |
20 | | //! loader.add_search_path("./src"); |
21 | | //! loader.add_search_path("./modules"); |
22 | | //! |
23 | | //! let module = loader.load_module("math")?; // Loads math.ruchy |
24 | | //! ``` |
25 | | |
26 | | use std::collections::HashMap; |
27 | | use std::path::{Path, PathBuf}; |
28 | | use std::fs; |
29 | | use std::time::SystemTime; |
30 | | use anyhow::{Result, bail, Context}; |
31 | | use crate::frontend::parser::Parser; |
32 | | use crate::frontend::ast::{Expr, ExprKind}; |
33 | | |
34 | | /// Core module loading and caching system |
35 | | /// |
36 | | /// Handles file discovery, parsing, dependency resolution, and caching |
37 | | /// for multi-file Ruchy programs. |
38 | | pub struct ModuleLoader { |
39 | | /// Cache of parsed modules to avoid re-parsing unchanged files |
40 | | cache: HashMap<String, ParsedModule>, |
41 | | /// Directories to search for module files |
42 | | search_paths: Vec<PathBuf>, |
43 | | /// Stack of currently loading modules for circular dependency detection |
44 | | loading_stack: Vec<String>, |
45 | | /// Total number of files loaded (for metrics) |
46 | | files_loaded: usize, |
47 | | /// Total cache hits (for performance monitoring) |
48 | | cache_hits: usize, |
49 | | } |
50 | | |
51 | | impl ModuleLoader { |
52 | | /// Create a new `ModuleLoader` with default search paths |
53 | | /// |
54 | | /// Default search paths: |
55 | | /// - `.` (current directory) |
56 | | /// - `./src` (source directory) |
57 | | /// - `./modules` (modules directory) |
58 | | #[must_use] |
59 | 111 | pub fn new() -> Self { |
60 | 111 | Self { |
61 | 111 | cache: HashMap::new(), |
62 | 111 | search_paths: vec![ |
63 | 111 | PathBuf::from("."), // Current directory |
64 | 111 | PathBuf::from("./src"), // Source directory |
65 | 111 | PathBuf::from("./modules"), // Modules directory |
66 | 111 | ], |
67 | 111 | loading_stack: Vec::new(), |
68 | 111 | files_loaded: 0, |
69 | 111 | cache_hits: 0, |
70 | 111 | } |
71 | 111 | } |
72 | | |
73 | | /// Add a directory to the module search path |
74 | | /// |
75 | | /// Modules will be searched in the order paths were added. |
76 | | /// |
77 | | /// # Arguments |
78 | | /// |
79 | | /// * `path` - Directory to search for modules |
80 | 9 | pub fn add_search_path<P: AsRef<Path>>(&mut self, path: P) { |
81 | 9 | self.search_paths.push(path.as_ref().to_path_buf()); |
82 | 9 | } |
83 | | |
84 | | /// Load a module from the file system |
85 | | /// |
86 | | /// Supports these patterns: |
87 | | /// - `module_name.ruchy` - Direct file |
88 | | /// - `module_name/mod.ruchy` - Directory module |
89 | | /// - `module_name.rchy` - Short extension |
90 | | /// |
91 | | /// # Arguments |
92 | | /// |
93 | | /// * `module_name` - Name of the module to load |
94 | | /// |
95 | | /// # Returns |
96 | | /// |
97 | | /// Clone of the parsed module with AST and metadata |
98 | | /// |
99 | | /// # Errors |
100 | | /// |
101 | | /// Returns an error if: |
102 | | /// - Module file not found in any search path |
103 | | /// - Circular dependency detected |
104 | | /// - File parsing fails |
105 | | /// - I/O errors reading the file |
106 | 8 | pub fn load_module(&mut self, module_name: &str) -> Result<ParsedModule> { |
107 | | // Check circular dependencies first |
108 | 8 | if self.loading_stack.contains(&module_name.to_string()) { |
109 | 0 | let stack = self.loading_stack.join(" -> "); |
110 | 0 | let cycle_path = format!("{stack} -> {module_name}"); |
111 | 0 | bail!("Circular dependency detected: {}", cycle_path); |
112 | 8 | } |
113 | | |
114 | | // Check cache for already loaded modules |
115 | 8 | if let Some(cached1 ) = self.cache.get(module_name) { |
116 | 1 | if self.is_cache_valid(cached)?0 { |
117 | 1 | self.cache_hits += 1; |
118 | 1 | return Ok(cached.clone()); |
119 | 0 | } |
120 | 7 | } |
121 | | |
122 | | // Find the module file in search paths |
123 | 7 | let file_path = self.resolve_module_path(module_name) |
124 | 7 | .with_context(|| format!("Failed to find module '{module_name}'"0 ))?0 ; |
125 | | |
126 | | // Read and parse the module file |
127 | 7 | let content = fs::read_to_string(&file_path) |
128 | 7 | .with_context(|| {0 |
129 | 0 | let path = file_path.display(); |
130 | 0 | format!("Failed to read module file: {path}") |
131 | 0 | })?; |
132 | | |
133 | | // Track loading for circular dependency detection |
134 | 7 | self.loading_stack.push(module_name.to_string()); |
135 | | |
136 | | // Parse the module content |
137 | 7 | let mut parser = Parser::new(&content); |
138 | 7 | let ast = parser.parse() |
139 | 7 | .with_context(|| format!("Failed to parse module '{module_name}'"0 ))?0 ; |
140 | | |
141 | | // Extract dependencies from the parsed AST |
142 | 7 | let dependencies = self.extract_dependencies(&ast)?0 ; |
143 | | |
144 | | // Create parsed module metadata |
145 | 7 | let parsed_module = ParsedModule { |
146 | 7 | ast, |
147 | 7 | file_path: file_path.clone(), |
148 | 7 | dependencies: dependencies.clone(), |
149 | 7 | last_modified: fs::metadata(&file_path)?0 .modified()?0 , |
150 | | }; |
151 | | |
152 | | // Load dependencies recursively - check for circular dependencies first |
153 | 7 | for dep2 in &dependencies { |
154 | 2 | if self.loading_stack.contains(&dep.to_string()) { |
155 | 1 | let stack = self.loading_stack.join(" -> "); |
156 | 1 | let cycle_path = format!("{stack} -> {module_name} -> {dep}"); |
157 | 1 | bail!("Circular dependency detected: {}", cycle_path); |
158 | 1 | } |
159 | 1 | self.load_module(dep) |
160 | 1 | .with_context(|| format!("Failed to load dependency '{dep}' for module '{module_name}'"))?; |
161 | | } |
162 | | |
163 | | // Remove from loading stack and cache the result |
164 | 5 | self.loading_stack.pop(); |
165 | | |
166 | | // Cache invalid entry removal and insertion |
167 | 5 | self.cache.remove(module_name); |
168 | 5 | self.cache.insert(module_name.to_string(), parsed_module.clone()); |
169 | 5 | self.files_loaded += 1; |
170 | | |
171 | 5 | Ok(parsed_module) |
172 | 8 | } |
173 | | |
174 | | /// Resolve module name to file system path |
175 | | /// |
176 | | /// Tries these patterns in each search path: |
177 | | /// 1. `{module_name}.ruchy` |
178 | | /// 2. `{module_name}/mod.ruchy` |
179 | | /// 3. `{module_name}.rchy` |
180 | 9 | fn resolve_module_path(&self, module_name: &str) -> Result<PathBuf> { |
181 | 9 | let possible_names = [ |
182 | 9 | format!("{module_name}.ruchy"), |
183 | 9 | format!("{module_name}/mod.ruchy"), |
184 | 9 | format!("{module_name}.rchy"), |
185 | 9 | ]; |
186 | | |
187 | 36 | for search_path35 in &self.search_paths { |
188 | 116 | for name89 in &possible_names { |
189 | 89 | let candidate = search_path.join(name); |
190 | 89 | if candidate.exists() && candidate.is_file()8 { |
191 | 8 | return Ok(candidate); |
192 | 81 | } |
193 | | } |
194 | | } |
195 | | |
196 | 1 | bail!( |
197 | 1 | "Module '{}' not found. Searched in: {}\nLooked for: {}", |
198 | | module_name, |
199 | 1 | self.search_paths.iter() |
200 | 3 | .map1 (|p| p.display().to_string()) |
201 | 1 | .collect::<Vec<_>>() |
202 | 1 | .join(", "), |
203 | 1 | possible_names.join(", ") |
204 | | ); |
205 | 9 | } |
206 | | |
207 | | /// Check if a cached module is still valid (file not modified since parsing) |
208 | 1 | fn is_cache_valid(&self, module: &ParsedModule) -> Result<bool> { |
209 | 1 | let current_modified = fs::metadata(&module.file_path)?0 .modified()?0 ; |
210 | 1 | Ok(current_modified <= module.last_modified) |
211 | 1 | } |
212 | | |
213 | | /// Extract module dependencies from AST |
214 | | /// |
215 | | /// Traverses the AST looking for Import nodes that reference other files |
216 | | /// (not inline modules or standard library imports). |
217 | 7 | fn extract_dependencies(&self, ast: &Expr) -> Result<Vec<String>> { |
218 | 7 | let mut dependencies = Vec::new(); |
219 | 7 | self.collect_dependencies(ast, &mut dependencies); |
220 | 7 | Ok(dependencies) |
221 | 7 | } |
222 | | |
223 | | /// Recursive helper to collect dependencies from AST nodes |
224 | 13 | fn collect_dependencies(&self, expr: &Expr, dependencies: &mut Vec<String>) { |
225 | 13 | match &expr.kind { |
226 | 2 | ExprKind::Import { path, .. } => { |
227 | | // Only treat simple names (no ::) as potential file imports |
228 | 2 | if !path.contains("::") && !path.starts_with("std::") && !path.starts_with("http") { |
229 | 2 | dependencies.push(path.clone()); |
230 | 2 | }0 |
231 | | } |
232 | 1 | ExprKind::Block(exprs) => { |
233 | 2 | for expr1 in exprs { |
234 | 1 | self.collect_dependencies(expr, dependencies); |
235 | 1 | } |
236 | | } |
237 | 0 | ExprKind::Module { body, .. } => { |
238 | 0 | self.collect_dependencies(body, dependencies); |
239 | 0 | } |
240 | 5 | ExprKind::Function { body, .. } => { |
241 | 5 | self.collect_dependencies(body, dependencies); |
242 | 5 | } |
243 | | // Add other expression types that can contain imports |
244 | 5 | _ => { |
245 | 5 | // For now, basic dependency extraction |
246 | 5 | // Future: Add comprehensive AST traversal for all expression types if needed |
247 | 5 | } |
248 | | } |
249 | 13 | } |
250 | | |
251 | | /// Get module loading statistics for performance monitoring |
252 | | #[must_use] |
253 | 7 | pub fn stats(&self) -> ModuleLoaderStats { |
254 | 7 | ModuleLoaderStats { |
255 | 7 | cached_modules: self.cache.len(), |
256 | 7 | files_loaded: self.files_loaded, |
257 | 7 | cache_hits: self.cache_hits, |
258 | 7 | search_paths: self.search_paths.len(), |
259 | 7 | } |
260 | 7 | } |
261 | | |
262 | | /// Clear the module cache |
263 | | /// |
264 | | /// Forces all modules to be reloaded from disk on next access. |
265 | | /// Useful for development when module files are frequently changing. |
266 | 2 | pub fn clear_cache(&mut self) { |
267 | 2 | self.cache.clear(); |
268 | 2 | self.files_loaded = 0; |
269 | 2 | self.cache_hits = 0; |
270 | 2 | } |
271 | | |
272 | | /// Check if a module is currently being loaded (for debugging) |
273 | | #[must_use] |
274 | 0 | pub fn is_loading(&self, module_name: &str) -> bool { |
275 | 0 | self.loading_stack.contains(&module_name.to_string()) |
276 | 0 | } |
277 | | } |
278 | | |
279 | | impl Default for ModuleLoader { |
280 | 0 | fn default() -> Self { |
281 | 0 | Self::new() |
282 | 0 | } |
283 | | } |
284 | | |
285 | | /// Represents a parsed module with metadata and dependencies |
286 | | #[derive(Debug, Clone)] |
287 | | pub struct ParsedModule { |
288 | | /// Parsed AST of the module |
289 | | pub ast: Expr, |
290 | | /// File system path where the module was loaded from |
291 | | pub file_path: PathBuf, |
292 | | /// List of other modules this module depends on |
293 | | pub dependencies: Vec<String>, |
294 | | /// Last modification time of the source file |
295 | | pub last_modified: SystemTime, |
296 | | } |
297 | | |
298 | | impl ParsedModule { |
299 | | /// Get the module name from the file path |
300 | | #[must_use] |
301 | 1 | pub fn name(&self) -> Option<String> { |
302 | 1 | self.file_path |
303 | 1 | .file_stem() |
304 | 1 | .and_then(|stem| stem.to_str()) |
305 | 1 | .map(std::string::ToString::to_string) |
306 | 1 | } |
307 | | |
308 | | /// Check if this module has any dependencies |
309 | | #[must_use] |
310 | 1 | pub fn has_dependencies(&self) -> bool { |
311 | 1 | !self.dependencies.is_empty() |
312 | 1 | } |
313 | | } |
314 | | |
315 | | /// Statistics about module loader performance |
316 | | #[derive(Debug, Clone, Copy)] |
317 | | pub struct ModuleLoaderStats { |
318 | | /// Number of modules currently cached in memory |
319 | | pub cached_modules: usize, |
320 | | /// Total number of files loaded from disk |
321 | | pub files_loaded: usize, |
322 | | /// Number of cache hits (avoided file I/O) |
323 | | pub cache_hits: usize, |
324 | | /// Number of search paths configured |
325 | | pub search_paths: usize, |
326 | | } |
327 | | |
328 | | impl ModuleLoaderStats { |
329 | | /// Calculate cache hit ratio as a percentage |
330 | | #[must_use] |
331 | 1 | pub fn cache_hit_ratio(&self) -> f64 { |
332 | 1 | if self.files_loaded + self.cache_hits == 0 { |
333 | 0 | 0.0 |
334 | | } else { |
335 | 1 | f64::from(self.cache_hits as u32) / f64::from((self.files_loaded + self.cache_hits) as u32) * 100.0 |
336 | | } |
337 | 1 | } |
338 | | } |
339 | | |
340 | | #[cfg(test)] |
341 | | mod tests { |
342 | | use super::*; |
343 | | use tempfile::TempDir; |
344 | | use std::fs; |
345 | | |
346 | 5 | fn create_test_module(temp_dir: &TempDir, name: &str, content: &str) -> Result<()> { |
347 | 5 | let file_path = temp_dir.path().join(format!("{name}.ruchy")); |
348 | 5 | fs::write(file_path, content)?0 ; |
349 | 5 | Ok(()) |
350 | 5 | } |
351 | | |
352 | | #[test] |
353 | 1 | fn test_module_loader_creation() { |
354 | 1 | let loader = ModuleLoader::new(); |
355 | 1 | assert_eq!(loader.cache.len(), 0); |
356 | 1 | assert_eq!(loader.search_paths.len(), 3); |
357 | 1 | assert!(loader.search_paths.contains(&PathBuf::from("."))); |
358 | 1 | assert!(loader.search_paths.contains(&PathBuf::from("./src"))); |
359 | 1 | assert!(loader.search_paths.contains(&PathBuf::from("./modules"))); |
360 | 1 | } |
361 | | |
362 | | #[test] |
363 | 1 | fn test_add_search_path() { |
364 | 1 | let mut loader = ModuleLoader::new(); |
365 | 1 | loader.add_search_path("/custom/path"); |
366 | | |
367 | 1 | assert_eq!(loader.search_paths.len(), 4); |
368 | 1 | assert!(loader.search_paths.contains(&PathBuf::from("/custom/path"))); |
369 | 1 | } |
370 | | |
371 | | #[test] |
372 | 1 | fn test_resolve_module_path_success() -> Result<()> { |
373 | 1 | let temp_dir = TempDir::new()?0 ; |
374 | 1 | let mut loader = ModuleLoader::new(); |
375 | 1 | loader.add_search_path(temp_dir.path()); |
376 | | |
377 | | // Create a test module file |
378 | 1 | create_test_module(&temp_dir, "math", "pub fun add(a, b) { a + b }")?0 ; |
379 | | |
380 | 1 | let resolved = loader.resolve_module_path("math")?0 ; |
381 | 1 | assert_eq!(resolved, temp_dir.path().join("math.ruchy")); |
382 | 1 | assert!(resolved.exists()); |
383 | | |
384 | 1 | Ok(()) |
385 | 1 | } |
386 | | |
387 | | #[test] |
388 | 1 | fn test_resolve_module_path_not_found() { |
389 | 1 | let loader = ModuleLoader::new(); |
390 | 1 | let result = loader.resolve_module_path("nonexistent"); |
391 | | |
392 | 1 | assert!(result.is_err()); |
393 | 1 | let error_msg = result.unwrap_err().to_string(); |
394 | 1 | assert!(error_msg.contains("Module 'nonexistent' not found")); |
395 | 1 | } |
396 | | |
397 | | #[test] |
398 | 1 | fn test_circular_dependency_detection() -> Result<()> { |
399 | 1 | let temp_dir = TempDir::new()?0 ; |
400 | 1 | let mut loader = ModuleLoader::new(); |
401 | 1 | loader.add_search_path(temp_dir.path()); |
402 | | |
403 | | // Create circular dependencies: a imports b, b imports a |
404 | 1 | create_test_module(&temp_dir, "a", "use b;")?0 ; |
405 | 1 | create_test_module(&temp_dir, "b", "use a;")?0 ; |
406 | | |
407 | 1 | let result = loader.load_module("a"); |
408 | 1 | assert!(result.is_err()); |
409 | | |
410 | 1 | let error = result.unwrap_err(); |
411 | 1 | let error_msg = format!("{error:?}"); // Use Debug formatting to get full error chain |
412 | | |
413 | | // Check if the full error chain contains circular dependency |
414 | 1 | let found_circular_dep = error_msg.contains("Circular dependency detected") |
415 | 0 | || error_msg.contains("circular dependency"); |
416 | | |
417 | 1 | assert!(found_circular_dep, "Expected circular dependency error, got: {error_msg}"0 ); |
418 | | |
419 | 1 | Ok(()) |
420 | 1 | } |
421 | | |
422 | | #[test] |
423 | 1 | fn test_stats_tracking() -> Result<()> { |
424 | 1 | let temp_dir = TempDir::new()?0 ; |
425 | 1 | let mut loader = ModuleLoader::new(); |
426 | 1 | loader.add_search_path(temp_dir.path()); |
427 | | |
428 | 1 | create_test_module(&temp_dir, "test", "pub fun test() {}")?0 ; |
429 | | |
430 | 1 | let initial_stats = loader.stats(); |
431 | 1 | assert_eq!(initial_stats.files_loaded, 0); |
432 | 1 | assert_eq!(initial_stats.cache_hits, 0); |
433 | 1 | assert_eq!(initial_stats.cached_modules, 0); |
434 | | |
435 | | // Load module first time |
436 | 1 | loader.load_module("test")?0 ; |
437 | 1 | let after_load = loader.stats(); |
438 | 1 | assert_eq!(after_load.files_loaded, 1); |
439 | 1 | assert_eq!(after_load.cached_modules, 1); |
440 | | |
441 | | // Load same module again (should hit cache) |
442 | 1 | loader.load_module("test")?0 ; |
443 | 1 | let after_cache = loader.stats(); |
444 | 1 | assert_eq!(after_cache.files_loaded, 1); // Same as before |
445 | 1 | assert_eq!(after_cache.cache_hits, 1); // Incremented |
446 | | |
447 | 1 | Ok(()) |
448 | 1 | } |
449 | | |
450 | | #[test] |
451 | 1 | fn test_cache_hit_ratio_calculation() { |
452 | 1 | let stats = ModuleLoaderStats { |
453 | 1 | cached_modules: 5, |
454 | 1 | files_loaded: 10, |
455 | 1 | cache_hits: 20, |
456 | 1 | search_paths: 3, |
457 | 1 | }; |
458 | | |
459 | 1 | let ratio = stats.cache_hit_ratio(); |
460 | 1 | assert!((ratio - 66.67).abs() < 0.01); // 20/(10+20) * 100 = 66.67% |
461 | 1 | } |
462 | | |
463 | | #[test] |
464 | 1 | fn test_clear_cache() -> Result<()> { |
465 | 1 | let temp_dir = TempDir::new()?0 ; |
466 | 1 | let mut loader = ModuleLoader::new(); |
467 | 1 | loader.add_search_path(temp_dir.path()); |
468 | | |
469 | 1 | create_test_module(&temp_dir, "test", "pub fun test() {}")?0 ; |
470 | | |
471 | | // Load module to populate cache |
472 | 1 | loader.load_module("test")?0 ; |
473 | 1 | assert_eq!(loader.cache.len(), 1); |
474 | | |
475 | | // Clear cache |
476 | 1 | loader.clear_cache(); |
477 | 1 | assert_eq!(loader.cache.len(), 0); |
478 | 1 | assert_eq!(loader.files_loaded, 0); |
479 | 1 | assert_eq!(loader.cache_hits, 0); |
480 | | |
481 | 1 | Ok(()) |
482 | 1 | } |
483 | | |
484 | | #[test] |
485 | 1 | fn test_parsed_module_name() -> Result<()> { |
486 | 1 | let temp_dir = TempDir::new()?0 ; |
487 | 1 | let path = temp_dir.path().join("math.ruchy"); |
488 | | |
489 | 1 | let module = ParsedModule { |
490 | 1 | ast: Expr::new(crate::frontend::ast::ExprKind::Literal(crate::frontend::ast::Literal::Unit), |
491 | 1 | crate::frontend::ast::Span { start: 0, end: 0 }), |
492 | 1 | file_path: path, |
493 | 1 | dependencies: Vec::new(), |
494 | 1 | last_modified: SystemTime::now(), |
495 | 1 | }; |
496 | | |
497 | 1 | assert_eq!(module.name(), Some("math".to_string())); |
498 | 1 | assert!(!module.has_dependencies()); |
499 | | |
500 | 1 | Ok(()) |
501 | 1 | } |
502 | | } |