{
    "sourceFile": "tests/run_validation.rs",
    "activeCommit": 0,
    "commits": [
        {
            "activePatchIndex": 2,
            "patches": [
                {
                    "date": 1760243825154,
                    "content": "Index: \n===================================================================\n--- \n+++ \n"
                },
                {
                    "date": 1760251604323,
                    "content": "Index: \n===================================================================\n--- \n+++ \n@@ -0,0 +1,437 @@\n+//! Executable validation runner\n+//!\n+//! This module executes the validation framework against real LLM outputs\n+//! and generates comprehensive scoring reports.\n+\n+use std::fs;\n+use std::process::Command;\n+use std::time::Instant;\n+\n+mod validation_framework;\n+use validation_framework::*;\n+\n+/// Run real LLM commands and capture outputs\n+fn execute_real_command(command: &str, args: &[String]) -> (String, u64) {\n+    let start = Instant::now();\n+\n+    let mut cmd = Command::new(\"cargo\");\n+    cmd.arg(\"run\").arg(\"--release\").arg(\"--\").arg(command);\n+\n+    for arg in args {\n+        cmd.arg(arg);\n+    }\n+\n+    let output = cmd.output().expect(\"Failed to execute command\");\n+    let duration = start.elapsed().as_millis() as u64;\n+\n+    let stdout = String::from_utf8_lossy(&output.stdout);\n+    let stderr = String::from_utf8_lossy(&output.stderr);\n+\n+    (format!(\"{}\\n{}\", stdout, stderr), duration)\n+}\n+\n+/// Enhanced validation framework that runs real commands\n+pub struct RealWorldValidator {\n+    #[allow(dead_code)]\n+    framework: ValidationFramework,\n+}\n+\n+impl Default for RealWorldValidator {\n+    fn default() -> Self {\n+        Self::new()\n+    }\n+}\n+\n+impl RealWorldValidator {\n+    pub fn new() -> Self {\n+        Self {\n+            framework: ValidationFramework::new(),\n+        }\n+    }\n+\n+    /// Run all scenarios with real LLM outputs\n+    pub fn run_real_validation(&self) -> Vec<QualityScore> {\n+        let scenarios = vec![\n+            (\n+                \"simple_template\",\n+                \"ai generate\",\n+                vec![\n+                    \"--description\",\n+                    \"Create a simple hello world web page with HTML and CSS\",\n+                    \"--max-tokens\",\n+                    \"500\",\n+                    \"--output\",\n+                    \"/tmp/test_simple_template.txt\",\n+                ],\n+            ),\n+            (\n+                \"complex_sparql\",\n+                \"ai sparql\",\n+                vec![\n+                    \"--description\",\n+                    \"Find all users with age > 18 who have made purchases in the last 30 days\",\n+                    \"--max-tokens\",\n+                    \"300\",\n+                    \"--output\",\n+                    \"/tmp/test_sparql.txt\",\n+                ],\n+            ),\n+            (\n+                \"blog_frontmatter\",\n+                \"ai frontmatter\",\n+                vec![\n+                    \"--description\",\n+                    \"Blog post about AI ethics with SEO optimization\",\n+                    \"--max-tokens\",\n+                    \"200\",\n+                    \"--output\",\n+                    \"/tmp/test_frontmatter.txt\",\n+                ],\n+            ),\n+            (\n+                \"ontology_graph\",\n+                \"ai graph\",\n+                vec![\n+                    \"--description\",\n+                    \"E-commerce product catalog with categories and pricing\",\n+                    \"--include-examples\",\n+                    \"--max-tokens\",\n+                    \"400\",\n+                    \"--output\",\n+                    \"/tmp/test_ontology.ttl\",\n+                ],\n+            ),\n+        ];\n+\n+        let mut scores = Vec::new();\n+\n+        for (name, command, args) in scenarios {\n+            println!(\"🧪 Running validation for: {}\", name);\n+\n+            let args_vec: Vec<String> = args.iter().map(|s| s.to_string()).collect();\n+            let (output, duration) = execute_real_command(command, &args_vec);\n+\n+            // Read the output file if it exists\n+            let output_file = args_vec\n+                .iter()\n+                .position(|a| a == \"--output\")\n+                .and_then(|i| args_vec.get(i + 1))\n+                .and_then(|path| fs::read_to_string(path).ok())\n+                .unwrap_or_else(|| output.clone());\n+\n+            let mut score = self.score_real_output(&output_file, name, command);\n+            score.metadata.execution_time_ms = duration;\n+\n+            println!(\"   Score: {:.2}/10\", score.total);\n+            scores.push(score);\n+        }\n+\n+        scores\n+    }\n+\n+    /// Score real LLM output with comprehensive criteria\n+    fn score_real_output(&self, output: &str, scenario: &str, command: &str) -> QualityScore {\n+        use std::collections::HashMap;\n+\n+        let mut dimensions = HashMap::new();\n+        let mut feedback = Vec::new();\n+\n+        // Dimension 1: Length appropriateness (0-10)\n+        let length_score = self.score_length(output);\n+        dimensions.insert(\n+            \"length\".to_string(),\n+            DimensionScore {\n+                score: length_score.0,\n+                weight: 0.1,\n+                description: \"Output length is appropriate\".to_string(),\n+                issues: length_score.1.clone(),\n+                suggestions: if length_score.0 < 7.0 {\n+                    vec![\"Adjust max_tokens parameter to get better length\".to_string()]\n+                } else {\n+                    vec![]\n+                },\n+            },\n+        );\n+\n+        // Dimension 2: Format correctness (0-10)\n+        let format_score = self.score_format(output, command);\n+        dimensions.insert(\n+            \"format\".to_string(),\n+            DimensionScore {\n+                score: format_score.0,\n+                weight: 0.2,\n+                description: \"Output format is correct\".to_string(),\n+                issues: format_score.1.clone(),\n+                suggestions: if format_score.0 < 7.0 {\n+                    vec![\"Improve prompt to specify exact format requirements\".to_string()]\n+                } else {\n+                    vec![]\n+                },\n+            },\n+        );\n+\n+        // Dimension 3: Content quality (0-10)\n+        let content_score = self.score_content_quality(output);\n+        dimensions.insert(\n+            \"content_quality\".to_string(),\n+            DimensionScore {\n+                score: content_score.0,\n+                weight: 0.25,\n+                description: \"Content is high quality and relevant\".to_string(),\n+                issues: content_score.1.clone(),\n+                suggestions: if content_score.0 < 7.0 {\n+                    vec![\"Add more specific requirements to the prompt\".to_string()]\n+                } else {\n+                    vec![]\n+                },\n+            },\n+        );\n+\n+        // Dimension 4: Usability (0-10)\n+        let usability_score = self.score_usability(output);\n+        dimensions.insert(\n+            \"usability\".to_string(),\n+            DimensionScore {\n+                score: usability_score.0,\n+                weight: 0.2,\n+                description: \"Output is immediately usable\".to_string(),\n+                issues: usability_score.1.clone(),\n+                suggestions: if usability_score.0 < 7.0 {\n+                    vec![\"Request more complete examples in prompt\".to_string()]\n+                } else {\n+                    vec![]\n+                },\n+            },\n+        );\n+\n+        // Dimension 5: Creativity/Insight (0-10)\n+        let creativity_score = self.score_creativity(output);\n+        dimensions.insert(\n+            \"creativity\".to_string(),\n+            DimensionScore {\n+                score: creativity_score.0,\n+                weight: 0.15,\n+                description: \"Output shows creativity and insight\".to_string(),\n+                issues: creativity_score.1.clone(),\n+                suggestions: if creativity_score.0 < 7.0 {\n+                    vec![\"Increase temperature for more creative outputs\".to_string()]\n+                } else {\n+                    vec![]\n+                },\n+            },\n+        );\n+\n+        // Dimension 6: Professional quality (0-10)\n+        let professional_score = self.score_professional_quality(output);\n+        dimensions.insert(\n+            \"professional\".to_string(),\n+            DimensionScore {\n+                score: professional_score.0,\n+                weight: 0.1,\n+                description: \"Output meets professional standards\".to_string(),\n+                issues: professional_score.1.clone(),\n+                suggestions: if professional_score.0 < 7.0 {\n+                    vec![\"Add quality requirements to prompt\".to_string()]\n+                } else {\n+                    vec![]\n+                },\n+            },\n+        );\n+\n+        // Calculate weighted total\n+        let total = dimensions.values().map(|d| d.score * d.weight).sum::<f32>()\n+            / dimensions.values().map(|d| d.weight).sum::<f32>();\n+\n+        // Collect all feedback\n+        for (name, dim) in &dimensions {\n+            if dim.score < 7.0 {\n+                feedback.push(format!(\n+                    \"⚠️  {} scored {:.1}/10 - needs improvement\",\n+                    name, dim.score\n+                ));\n+                feedback.extend(dim.suggestions.clone());\n+            }\n+        }\n+\n+        // Add overall feedback based on total score\n+        if total >= 9.0 {\n+            feedback.insert(0, \"🌟 Excellent! Output quality is exceptional\".to_string());\n+        } else if total >= 7.0 {\n+            feedback.insert(\n+                0,\n+                \"✅ Good! Output quality is solid with minor improvements possible\".to_string(),\n+            );\n+        } else if total >= 5.0 {\n+            feedback.insert(\n+                0,\n+                \"⚠️  Fair - Output needs significant improvement\".to_string(),\n+            );\n+        } else {\n+            feedback.insert(\n+                0,\n+                \"❌ Poor - Output quality is below acceptable standards\".to_string(),\n+            );\n+        }\n+\n+        QualityScore {\n+            total,\n+            dimensions,\n+            feedback,\n+            metadata: TestMetadata {\n+                command: command.to_string(),\n+                scenario: scenario.to_string(),\n+                timestamp: chrono::Utc::now().to_rfc3339(),\n+                model: \"qwen3-coder:30b\".to_string(),\n+                execution_time_ms: 0,\n+            },\n+        }\n+    }\n+\n+    fn score_length(&self, output: &str) -> (f32, Vec<String>) {\n+        let len = output.len();\n+        let mut issues = Vec::new();\n+\n+        let score = match len {\n+            0..=50 => {\n+                issues.push(\"Output too short\".to_string());\n+                2.0\n+            }\n+            51..=100 => {\n+                issues.push(\"Output quite short\".to_string());\n+                5.0\n+            }\n+            101..=500 => 9.0,\n+            501..=2000 => 10.0,\n+            2001..=5000 => 8.0,\n+            _ => {\n+                issues.push(\"Output might be too verbose\".to_string());\n+                6.0\n+            }\n+        };\n+\n+        (score, issues)\n+    }\n+\n+    fn score_format(&self, output: &str, command: &str) -> (f32, Vec<String>) {\n+        let mut score: f32 = 10.0;\n+        let mut issues = Vec::new();\n+\n+        if command.contains(\"sparql\") {\n+            if !output.to_uppercase().contains(\"SELECT\") {\n+                score -= 5.0;\n+                issues.push(\"Missing SELECT clause\".to_string());\n+            }\n+            if !output.to_uppercase().contains(\"WHERE\") {\n+                score -= 3.0;\n+                issues.push(\"Missing WHERE clause\".to_string());\n+            }\n+        } else if command.contains(\"graph\") {\n+            if !output.contains(\"@prefix\") {\n+                score -= 4.0;\n+                issues.push(\"Missing @prefix declarations\".to_string());\n+            }\n+        } else if command.contains(\"frontmatter\")\n+            && !output.contains(\"title\")\n+            && !output.contains(\"Title\")\n+        {\n+            score -= 3.0;\n+            issues.push(\"Missing title field\".to_string());\n+        }\n+\n+        (score.max(0.0f32), issues)\n+    }\n+\n+    fn score_content_quality(&self, output: &str) -> (f32, Vec<String>) {\n+        let mut score: f32 = 8.0; // Base score\n+        let mut issues = Vec::new();\n+\n+        // Check for placeholder text\n+        if output.contains(\"TODO\") || output.contains(\"FIXME\") || output.contains(\"placeholder\") {\n+            score -= 2.0;\n+            issues.push(\"Contains placeholder text\".to_string());\n+        }\n+\n+        // Check for errors\n+        if output.contains(\"error\") || output.contains(\"Error\") {\n+            score -= 3.0;\n+            issues.push(\"Contains error messages\".to_string());\n+        }\n+\n+        // Check for minimal effort\n+        if output.split_whitespace().count() < 20 {\n+            score -= 2.0;\n+            issues.push(\"Content seems minimal\".to_string());\n+        }\n+\n+        (score.max(0.0f32), issues)\n+    }\n+\n+    fn score_usability(&self, output: &str) -> (f32, Vec<String>) {\n+        let mut score: f32 = 9.0;\n+        let mut issues = Vec::new();\n+\n+        // Check for comments/documentation\n+        if !output.contains(\"#\") && !output.contains(\"//\") && !output.contains(\"/*\") {\n+            score -= 1.0;\n+            issues.push(\"Missing comments or documentation\".to_string());\n+        }\n+\n+        // Check for examples\n+        if output.len() > 200 && !output.contains(\"example\") && !output.contains(\"Example\") {\n+            score -= 1.0;\n+            issues.push(\"Could benefit from examples\".to_string());\n+        }\n+\n+        (score.max(0.0f32), issues)\n+    }\n+\n+    fn score_creativity(&self, output: &str) -> (f32, Vec<String>) {\n+        let mut score: f32 = 7.0; // Base score\n+        let mut issues = Vec::new();\n+\n+        // Simple heuristics for creativity\n+        let unique_words: std::collections::HashSet<_> = output.split_whitespace().collect();\n+\n+        if unique_words.len() < 10 {\n+            score -= 2.0;\n+            issues.push(\"Limited vocabulary used\".to_string());\n+        } else if unique_words.len() > 50 {\n+            score += 2.0; // Bonus for rich vocabulary\n+        }\n+\n+        (score.clamp(0.0f32, 10.0f32), issues)\n+    }\n+\n+    fn score_professional_quality(&self, output: &str) -> (f32, Vec<String>) {\n+        let mut score: f32 = 8.0;\n+        let mut issues = Vec::new();\n+\n+        // Check formatting\n+        if !output.contains('\\n') {\n+            score -= 2.0;\n+            issues.push(\"Poor formatting - no line breaks\".to_string());\n+        }\n+\n+        // Check for proper capitalization\n+        let first_char = output.chars().next();\n+        if let Some(c) = first_char {\n+            if c.is_lowercase() && !output.starts_with('#') {\n+                score -= 1.0;\n+                issues.push(\"Should start with capital letter\".to_string());\n+            }\n+        }\n+\n+        (score.max(0.0f32), issues)\n+    }\n+}\n+\n+#[cfg(test)]\n+mod tests {\n+    use super::*;\n+\n+    #[test]\n+    fn test_real_validator_creation() {\n+        let _validator = RealWorldValidator::new();\n+        // Just ensure it creates successfully\n+    }\n+}\n"
                },
                {
                    "date": 1760251836389,
                    "content": "Index: \n===================================================================\n--- \n+++ \n@@ -434,436 +434,4 @@\n         let _validator = RealWorldValidator::new();\n         // Just ensure it creates successfully\n     }\n }\n-//! Executable validation runner\n-//!\n-//! This module executes the validation framework against real LLM outputs\n-//! and generates comprehensive scoring reports.\n-\n-use std::fs;\n-use std::process::Command;\n-use std::time::Instant;\n-\n-mod validation_framework;\n-use validation_framework::*;\n-\n-/// Run real LLM commands and capture outputs\n-fn execute_real_command(command: &str, args: &[String]) -> (String, u64) {\n-    let start = Instant::now();\n-\n-    let mut cmd = Command::new(\"cargo\");\n-    cmd.arg(\"run\").arg(\"--release\").arg(\"--\").arg(command);\n-\n-    for arg in args {\n-        cmd.arg(arg);\n-    }\n-\n-    let output = cmd.output().expect(\"Failed to execute command\");\n-    let duration = start.elapsed().as_millis() as u64;\n-\n-    let stdout = String::from_utf8_lossy(&output.stdout);\n-    let stderr = String::from_utf8_lossy(&output.stderr);\n-\n-    (format!(\"{}\\n{}\", stdout, stderr), duration)\n-}\n-\n-/// Enhanced validation framework that runs real commands\n-pub struct RealWorldValidator {\n-    framework: ValidationFramework,\n-}\n-\n-impl RealWorldValidator {\n-    pub fn new() -> Self {\n-        Self {\n-            framework: ValidationFramework::new(),\n-        }\n-    }\n-\n-    /// Run all scenarios with real LLM outputs\n-    pub fn run_real_validation(&self) -> Vec<QualityScore> {\n-        let scenarios = vec![\n-            (\n-                \"simple_template\",\n-                \"ai generate\",\n-                vec![\n-                    \"--description\",\n-                    \"Create a simple hello world web page with HTML and CSS\",\n-                    \"--max-tokens\",\n-                    \"500\",\n-                    \"--output\",\n-                    \"/tmp/test_simple_template.txt\",\n-                ],\n-            ),\n-            (\n-                \"complex_sparql\",\n-                \"ai sparql\",\n-                vec![\n-                    \"--description\",\n-                    \"Find all users with age > 18 who have made purchases in the last 30 days\",\n-                    \"--max-tokens\",\n-                    \"300\",\n-                    \"--output\",\n-                    \"/tmp/test_sparql.txt\",\n-                ],\n-            ),\n-            (\n-                \"blog_frontmatter\",\n-                \"ai frontmatter\",\n-                vec![\n-                    \"--description\",\n-                    \"Blog post about AI ethics with SEO optimization\",\n-                    \"--max-tokens\",\n-                    \"200\",\n-                    \"--output\",\n-                    \"/tmp/test_frontmatter.txt\",\n-                ],\n-            ),\n-            (\n-                \"ontology_graph\",\n-                \"ai graph\",\n-                vec![\n-                    \"--description\",\n-                    \"E-commerce product catalog with categories and pricing\",\n-                    \"--include-examples\",\n-                    \"--max-tokens\",\n-                    \"400\",\n-                    \"--output\",\n-                    \"/tmp/test_ontology.ttl\",\n-                ],\n-            ),\n-        ];\n-\n-        let mut scores = Vec::new();\n-\n-        for (name, command, args) in scenarios {\n-            println!(\"🧪 Running validation for: {}\", name);\n-\n-            let args_vec: Vec<String> = args.iter().map(|s| s.to_string()).collect();\n-            let (output, duration) = execute_real_command(command, &args_vec);\n-\n-            // Read the output file if it exists\n-            let output_file = args_vec\n-                .iter()\n-                .position(|a| a == \"--output\")\n-                .and_then(|i| args_vec.get(i + 1))\n-                .and_then(|path| fs::read_to_string(path).ok())\n-                .unwrap_or_else(|| output.clone());\n-\n-            let mut score = self.score_real_output(&output_file, name, command);\n-            score.metadata.execution_time_ms = duration;\n-\n-            println!(\"   Score: {:.2}/10\", score.total);\n-            scores.push(score);\n-        }\n-\n-        scores\n-    }\n-\n-    /// Score real LLM output with comprehensive criteria\n-    fn score_real_output(&self, output: &str, scenario: &str, command: &str) -> QualityScore {\n-        use std::collections::HashMap;\n-\n-        let mut dimensions = HashMap::new();\n-        let mut feedback = Vec::new();\n-\n-        // Dimension 1: Length appropriateness (0-10)\n-        let length_score = self.score_length(output);\n-        dimensions.insert(\n-            \"length\".to_string(),\n-            DimensionScore {\n-                score: length_score.0,\n-                weight: 0.1,\n-                description: \"Output length is appropriate\".to_string(),\n-                issues: length_score.1.clone(),\n-                suggestions: if length_score.0 < 7.0 {\n-                    vec![\"Adjust max_tokens parameter to get better length\".to_string()]\n-                } else {\n-                    vec![]\n-                },\n-            },\n-        );\n-\n-        // Dimension 2: Format correctness (0-10)\n-        let format_score = self.score_format(output, command);\n-        dimensions.insert(\n-            \"format\".to_string(),\n-            DimensionScore {\n-                score: format_score.0,\n-                weight: 0.2,\n-                description: \"Output format is correct\".to_string(),\n-                issues: format_score.1.clone(),\n-                suggestions: if format_score.0 < 7.0 {\n-                    vec![\"Improve prompt to specify exact format requirements\".to_string()]\n-                } else {\n-                    vec![]\n-                },\n-            },\n-        );\n-\n-        // Dimension 3: Content quality (0-10)\n-        let content_score = self.score_content_quality(output);\n-        dimensions.insert(\n-            \"content_quality\".to_string(),\n-            DimensionScore {\n-                score: content_score.0,\n-                weight: 0.25,\n-                description: \"Content is high quality and relevant\".to_string(),\n-                issues: content_score.1.clone(),\n-                suggestions: if content_score.0 < 7.0 {\n-                    vec![\"Add more specific requirements to the prompt\".to_string()]\n-                } else {\n-                    vec![]\n-                },\n-            },\n-        );\n-\n-        // Dimension 4: Usability (0-10)\n-        let usability_score = self.score_usability(output);\n-        dimensions.insert(\n-            \"usability\".to_string(),\n-            DimensionScore {\n-                score: usability_score.0,\n-                weight: 0.2,\n-                description: \"Output is immediately usable\".to_string(),\n-                issues: usability_score.1.clone(),\n-                suggestions: if usability_score.0 < 7.0 {\n-                    vec![\"Request more complete examples in prompt\".to_string()]\n-                } else {\n-                    vec![]\n-                },\n-            },\n-        );\n-\n-        // Dimension 5: Creativity/Insight (0-10)\n-        let creativity_score = self.score_creativity(output);\n-        dimensions.insert(\n-            \"creativity\".to_string(),\n-            DimensionScore {\n-                score: creativity_score.0,\n-                weight: 0.15,\n-                description: \"Output shows creativity and insight\".to_string(),\n-                issues: creativity_score.1.clone(),\n-                suggestions: if creativity_score.0 < 7.0 {\n-                    vec![\"Increase temperature for more creative outputs\".to_string()]\n-                } else {\n-                    vec![]\n-                },\n-            },\n-        );\n-\n-        // Dimension 6: Professional quality (0-10)\n-        let professional_score = self.score_professional_quality(output);\n-        dimensions.insert(\n-            \"professional\".to_string(),\n-            DimensionScore {\n-                score: professional_score.0,\n-                weight: 0.1,\n-                description: \"Output meets professional standards\".to_string(),\n-                issues: professional_score.1.clone(),\n-                suggestions: if professional_score.0 < 7.0 {\n-                    vec![\"Add quality requirements to prompt\".to_string()]\n-                } else {\n-                    vec![]\n-                },\n-            },\n-        );\n-\n-        // Calculate weighted total\n-        let total = dimensions\n-            .iter()\n-            .map(|(_, d)| d.score * d.weight)\n-            .sum::<f32>()\n-            / dimensions.values().map(|d| d.weight).sum::<f32>();\n-\n-        // Collect all feedback\n-        for (name, dim) in &dimensions {\n-            if dim.score < 7.0 {\n-                feedback.push(format!(\n-                    \"⚠️  {} scored {:.1}/10 - needs improvement\",\n-                    name, dim.score\n-                ));\n-                feedback.extend(dim.suggestions.clone());\n-            }\n-        }\n-\n-        // Add overall feedback based on total score\n-        if total >= 9.0 {\n-            feedback.insert(0, \"🌟 Excellent! Output quality is exceptional\".to_string());\n-        } else if total >= 7.0 {\n-            feedback.insert(\n-                0,\n-                \"✅ Good! Output quality is solid with minor improvements possible\".to_string(),\n-            );\n-        } else if total >= 5.0 {\n-            feedback.insert(\n-                0,\n-                \"⚠️  Fair - Output needs significant improvement\".to_string(),\n-            );\n-        } else {\n-            feedback.insert(\n-                0,\n-                \"❌ Poor - Output quality is below acceptable standards\".to_string(),\n-            );\n-        }\n-\n-        QualityScore {\n-            total,\n-            dimensions,\n-            feedback,\n-            metadata: TestMetadata {\n-                command: command.to_string(),\n-                scenario: scenario.to_string(),\n-                timestamp: chrono::Utc::now().to_rfc3339(),\n-                model: \"qwen3-coder:30b\".to_string(),\n-                execution_time_ms: 0,\n-            },\n-        }\n-    }\n-\n-    fn score_length(&self, output: &str) -> (f32, Vec<String>) {\n-        let len = output.len();\n-        let mut issues = Vec::new();\n-\n-        let score = match len {\n-            0..=50 => {\n-                issues.push(\"Output too short\".to_string());\n-                2.0\n-            }\n-            51..=100 => {\n-                issues.push(\"Output quite short\".to_string());\n-                5.0\n-            }\n-            101..=500 => 9.0,\n-            501..=2000 => 10.0,\n-            2001..=5000 => 8.0,\n-            _ => {\n-                issues.push(\"Output might be too verbose\".to_string());\n-                6.0\n-            }\n-        };\n-\n-        (score, issues)\n-    }\n-\n-    fn score_format(&self, output: &str, command: &str) -> (f32, Vec<String>) {\n-        let mut score: f32 = 10.0;\n-        let mut issues = Vec::new();\n-\n-        if command.contains(\"sparql\") {\n-            if !output.to_uppercase().contains(\"SELECT\") {\n-                score -= 5.0;\n-                issues.push(\"Missing SELECT clause\".to_string());\n-            }\n-            if !output.to_uppercase().contains(\"WHERE\") {\n-                score -= 3.0;\n-                issues.push(\"Missing WHERE clause\".to_string());\n-            }\n-        } else if command.contains(\"graph\") {\n-            if !output.contains(\"@prefix\") {\n-                score -= 4.0;\n-                issues.push(\"Missing @prefix declarations\".to_string());\n-            }\n-        } else if command.contains(\"frontmatter\") {\n-            if !output.contains(\"title\") && !output.contains(\"Title\") {\n-                score -= 3.0;\n-                issues.push(\"Missing title field\".to_string());\n-            }\n-        }\n-\n-        (score.max(0.0f32), issues)\n-    }\n-\n-    fn score_content_quality(&self, output: &str) -> (f32, Vec<String>) {\n-        let mut score: f32 = 8.0; // Base score\n-        let mut issues = Vec::new();\n-\n-        // Check for placeholder text\n-        if output.contains(\"TODO\") || output.contains(\"FIXME\") || output.contains(\"placeholder\") {\n-            score -= 2.0;\n-            issues.push(\"Contains placeholder text\".to_string());\n-        }\n-\n-        // Check for errors\n-        if output.contains(\"error\") || output.contains(\"Error\") {\n-            score -= 3.0;\n-            issues.push(\"Contains error messages\".to_string());\n-        }\n-\n-        // Check for minimal effort\n-        if output.split_whitespace().count() < 20 {\n-            score -= 2.0;\n-            issues.push(\"Content seems minimal\".to_string());\n-        }\n-\n-        (score.max(0.0f32), issues)\n-    }\n-\n-    fn score_usability(&self, output: &str) -> (f32, Vec<String>) {\n-        let mut score: f32 = 9.0;\n-        let mut issues = Vec::new();\n-\n-        // Check for comments/documentation\n-        if !output.contains(\"#\") && !output.contains(\"//\") && !output.contains(\"/*\") {\n-            score -= 1.0;\n-            issues.push(\"Missing comments or documentation\".to_string());\n-        }\n-\n-        // Check for examples\n-        if output.len() > 200 && !output.contains(\"example\") && !output.contains(\"Example\") {\n-            score -= 1.0;\n-            issues.push(\"Could benefit from examples\".to_string());\n-        }\n-\n-        (score.max(0.0f32), issues)\n-    }\n-\n-    fn score_creativity(&self, output: &str) -> (f32, Vec<String>) {\n-        let mut score: f32 = 7.0; // Base score\n-        let mut issues = Vec::new();\n-\n-        // Simple heuristics for creativity\n-        let unique_words: std::collections::HashSet<_> = output.split_whitespace().collect();\n-\n-        if unique_words.len() < 10 {\n-            score -= 2.0;\n-            issues.push(\"Limited vocabulary used\".to_string());\n-        } else if unique_words.len() > 50 {\n-            score += 2.0; // Bonus for rich vocabulary\n-        }\n-\n-        (score.min(10.0f32).max(0.0f32), issues)\n-    }\n-\n-    fn score_professional_quality(&self, output: &str) -> (f32, Vec<String>) {\n-        let mut score: f32 = 8.0;\n-        let mut issues = Vec::new();\n-\n-        // Check formatting\n-        if !output.contains('\\n') {\n-            score -= 2.0;\n-            issues.push(\"Poor formatting - no line breaks\".to_string());\n-        }\n-\n-        // Check for proper capitalization\n-        let first_char = output.chars().next();\n-        if let Some(c) = first_char {\n-            if c.is_lowercase() && !output.starts_with('#') {\n-                score -= 1.0;\n-                issues.push(\"Should start with capital letter\".to_string());\n-            }\n-        }\n-\n-        (score.max(0.0f32), issues)\n-    }\n-}\n-\n-#[cfg(test)]\n-mod tests {\n-    use super::*;\n-\n-    #[test]\n-    fn test_real_validator_creation() {\n-        let _validator = RealWorldValidator::new();\n-        assert!(true); // Just ensure it creates successfully\n-    }\n-}\n"
                }
            ],
            "date": 1760243825154,
            "name": "Commit-0",
            "content": "//! Executable validation runner\n//!\n//! This module executes the validation framework against real LLM outputs\n//! and generates comprehensive scoring reports.\n\nuse std::fs;\nuse std::process::Command;\nuse std::time::Instant;\n\nmod validation_framework;\nuse validation_framework::*;\n\n/// Run real LLM commands and capture outputs\nfn execute_real_command(command: &str, args: &[String]) -> (String, u64) {\n    let start = Instant::now();\n\n    let mut cmd = Command::new(\"cargo\");\n    cmd.arg(\"run\").arg(\"--release\").arg(\"--\").arg(command);\n\n    for arg in args {\n        cmd.arg(arg);\n    }\n\n    let output = cmd.output().expect(\"Failed to execute command\");\n    let duration = start.elapsed().as_millis() as u64;\n\n    let stdout = String::from_utf8_lossy(&output.stdout);\n    let stderr = String::from_utf8_lossy(&output.stderr);\n\n    (format!(\"{}\\n{}\", stdout, stderr), duration)\n}\n\n/// Enhanced validation framework that runs real commands\npub struct RealWorldValidator {\n    framework: ValidationFramework,\n}\n\nimpl RealWorldValidator {\n    pub fn new() -> Self {\n        Self {\n            framework: ValidationFramework::new(),\n        }\n    }\n\n    /// Run all scenarios with real LLM outputs\n    pub fn run_real_validation(&self) -> Vec<QualityScore> {\n        let scenarios = vec![\n            (\n                \"simple_template\",\n                \"ai generate\",\n                vec![\n                    \"--description\",\n                    \"Create a simple hello world web page with HTML and CSS\",\n                    \"--max-tokens\",\n                    \"500\",\n                    \"--output\",\n                    \"/tmp/test_simple_template.txt\",\n                ],\n            ),\n            (\n                \"complex_sparql\",\n                \"ai sparql\",\n                vec![\n                    \"--description\",\n                    \"Find all users with age > 18 who have made purchases in the last 30 days\",\n                    \"--max-tokens\",\n                    \"300\",\n                    \"--output\",\n                    \"/tmp/test_sparql.txt\",\n                ],\n            ),\n            (\n                \"blog_frontmatter\",\n                \"ai frontmatter\",\n                vec![\n                    \"--description\",\n                    \"Blog post about AI ethics with SEO optimization\",\n                    \"--max-tokens\",\n                    \"200\",\n                    \"--output\",\n                    \"/tmp/test_frontmatter.txt\",\n                ],\n            ),\n            (\n                \"ontology_graph\",\n                \"ai graph\",\n                vec![\n                    \"--description\",\n                    \"E-commerce product catalog with categories and pricing\",\n                    \"--include-examples\",\n                    \"--max-tokens\",\n                    \"400\",\n                    \"--output\",\n                    \"/tmp/test_ontology.ttl\",\n                ],\n            ),\n        ];\n\n        let mut scores = Vec::new();\n\n        for (name, command, args) in scenarios {\n            println!(\"🧪 Running validation for: {}\", name);\n\n            let args_vec: Vec<String> = args.iter().map(|s| s.to_string()).collect();\n            let (output, duration) = execute_real_command(command, &args_vec);\n\n            // Read the output file if it exists\n            let output_file = args_vec\n                .iter()\n                .position(|a| a == \"--output\")\n                .and_then(|i| args_vec.get(i + 1))\n                .and_then(|path| fs::read_to_string(path).ok())\n                .unwrap_or_else(|| output.clone());\n\n            let mut score = self.score_real_output(&output_file, name, command);\n            score.metadata.execution_time_ms = duration;\n\n            println!(\"   Score: {:.2}/10\", score.total);\n            scores.push(score);\n        }\n\n        scores\n    }\n\n    /// Score real LLM output with comprehensive criteria\n    fn score_real_output(&self, output: &str, scenario: &str, command: &str) -> QualityScore {\n        use std::collections::HashMap;\n\n        let mut dimensions = HashMap::new();\n        let mut feedback = Vec::new();\n\n        // Dimension 1: Length appropriateness (0-10)\n        let length_score = self.score_length(output);\n        dimensions.insert(\n            \"length\".to_string(),\n            DimensionScore {\n                score: length_score.0,\n                weight: 0.1,\n                description: \"Output length is appropriate\".to_string(),\n                issues: length_score.1.clone(),\n                suggestions: if length_score.0 < 7.0 {\n                    vec![\"Adjust max_tokens parameter to get better length\".to_string()]\n                } else {\n                    vec![]\n                },\n            },\n        );\n\n        // Dimension 2: Format correctness (0-10)\n        let format_score = self.score_format(output, command);\n        dimensions.insert(\n            \"format\".to_string(),\n            DimensionScore {\n                score: format_score.0,\n                weight: 0.2,\n                description: \"Output format is correct\".to_string(),\n                issues: format_score.1.clone(),\n                suggestions: if format_score.0 < 7.0 {\n                    vec![\"Improve prompt to specify exact format requirements\".to_string()]\n                } else {\n                    vec![]\n                },\n            },\n        );\n\n        // Dimension 3: Content quality (0-10)\n        let content_score = self.score_content_quality(output);\n        dimensions.insert(\n            \"content_quality\".to_string(),\n            DimensionScore {\n                score: content_score.0,\n                weight: 0.25,\n                description: \"Content is high quality and relevant\".to_string(),\n                issues: content_score.1.clone(),\n                suggestions: if content_score.0 < 7.0 {\n                    vec![\"Add more specific requirements to the prompt\".to_string()]\n                } else {\n                    vec![]\n                },\n            },\n        );\n\n        // Dimension 4: Usability (0-10)\n        let usability_score = self.score_usability(output);\n        dimensions.insert(\n            \"usability\".to_string(),\n            DimensionScore {\n                score: usability_score.0,\n                weight: 0.2,\n                description: \"Output is immediately usable\".to_string(),\n                issues: usability_score.1.clone(),\n                suggestions: if usability_score.0 < 7.0 {\n                    vec![\"Request more complete examples in prompt\".to_string()]\n                } else {\n                    vec![]\n                },\n            },\n        );\n\n        // Dimension 5: Creativity/Insight (0-10)\n        let creativity_score = self.score_creativity(output);\n        dimensions.insert(\n            \"creativity\".to_string(),\n            DimensionScore {\n                score: creativity_score.0,\n                weight: 0.15,\n                description: \"Output shows creativity and insight\".to_string(),\n                issues: creativity_score.1.clone(),\n                suggestions: if creativity_score.0 < 7.0 {\n                    vec![\"Increase temperature for more creative outputs\".to_string()]\n                } else {\n                    vec![]\n                },\n            },\n        );\n\n        // Dimension 6: Professional quality (0-10)\n        let professional_score = self.score_professional_quality(output);\n        dimensions.insert(\n            \"professional\".to_string(),\n            DimensionScore {\n                score: professional_score.0,\n                weight: 0.1,\n                description: \"Output meets professional standards\".to_string(),\n                issues: professional_score.1.clone(),\n                suggestions: if professional_score.0 < 7.0 {\n                    vec![\"Add quality requirements to prompt\".to_string()]\n                } else {\n                    vec![]\n                },\n            },\n        );\n\n        // Calculate weighted total\n        let total = dimensions\n            .iter()\n            .map(|(_, d)| d.score * d.weight)\n            .sum::<f32>()\n            / dimensions.values().map(|d| d.weight).sum::<f32>();\n\n        // Collect all feedback\n        for (name, dim) in &dimensions {\n            if dim.score < 7.0 {\n                feedback.push(format!(\n                    \"⚠️  {} scored {:.1}/10 - needs improvement\",\n                    name, dim.score\n                ));\n                feedback.extend(dim.suggestions.clone());\n            }\n        }\n\n        // Add overall feedback based on total score\n        if total >= 9.0 {\n            feedback.insert(0, \"🌟 Excellent! Output quality is exceptional\".to_string());\n        } else if total >= 7.0 {\n            feedback.insert(\n                0,\n                \"✅ Good! Output quality is solid with minor improvements possible\".to_string(),\n            );\n        } else if total >= 5.0 {\n            feedback.insert(\n                0,\n                \"⚠️  Fair - Output needs significant improvement\".to_string(),\n            );\n        } else {\n            feedback.insert(\n                0,\n                \"❌ Poor - Output quality is below acceptable standards\".to_string(),\n            );\n        }\n\n        QualityScore {\n            total,\n            dimensions,\n            feedback,\n            metadata: TestMetadata {\n                command: command.to_string(),\n                scenario: scenario.to_string(),\n                timestamp: chrono::Utc::now().to_rfc3339(),\n                model: \"qwen3-coder:30b\".to_string(),\n                execution_time_ms: 0,\n            },\n        }\n    }\n\n    fn score_length(&self, output: &str) -> (f32, Vec<String>) {\n        let len = output.len();\n        let mut issues = Vec::new();\n\n        let score = match len {\n            0..=50 => {\n                issues.push(\"Output too short\".to_string());\n                2.0\n            }\n            51..=100 => {\n                issues.push(\"Output quite short\".to_string());\n                5.0\n            }\n            101..=500 => 9.0,\n            501..=2000 => 10.0,\n            2001..=5000 => 8.0,\n            _ => {\n                issues.push(\"Output might be too verbose\".to_string());\n                6.0\n            }\n        };\n\n        (score, issues)\n    }\n\n    fn score_format(&self, output: &str, command: &str) -> (f32, Vec<String>) {\n        let mut score: f32 = 10.0;\n        let mut issues = Vec::new();\n\n        if command.contains(\"sparql\") {\n            if !output.to_uppercase().contains(\"SELECT\") {\n                score -= 5.0;\n                issues.push(\"Missing SELECT clause\".to_string());\n            }\n            if !output.to_uppercase().contains(\"WHERE\") {\n                score -= 3.0;\n                issues.push(\"Missing WHERE clause\".to_string());\n            }\n        } else if command.contains(\"graph\") {\n            if !output.contains(\"@prefix\") {\n                score -= 4.0;\n                issues.push(\"Missing @prefix declarations\".to_string());\n            }\n        } else if command.contains(\"frontmatter\") {\n            if !output.contains(\"title\") && !output.contains(\"Title\") {\n                score -= 3.0;\n                issues.push(\"Missing title field\".to_string());\n            }\n        }\n\n        (score.max(0.0f32), issues)\n    }\n\n    fn score_content_quality(&self, output: &str) -> (f32, Vec<String>) {\n        let mut score: f32 = 8.0; // Base score\n        let mut issues = Vec::new();\n\n        // Check for placeholder text\n        if output.contains(\"TODO\") || output.contains(\"FIXME\") || output.contains(\"placeholder\") {\n            score -= 2.0;\n            issues.push(\"Contains placeholder text\".to_string());\n        }\n\n        // Check for errors\n        if output.contains(\"error\") || output.contains(\"Error\") {\n            score -= 3.0;\n            issues.push(\"Contains error messages\".to_string());\n        }\n\n        // Check for minimal effort\n        if output.split_whitespace().count() < 20 {\n            score -= 2.0;\n            issues.push(\"Content seems minimal\".to_string());\n        }\n\n        (score.max(0.0f32), issues)\n    }\n\n    fn score_usability(&self, output: &str) -> (f32, Vec<String>) {\n        let mut score: f32 = 9.0;\n        let mut issues = Vec::new();\n\n        // Check for comments/documentation\n        if !output.contains(\"#\") && !output.contains(\"//\") && !output.contains(\"/*\") {\n            score -= 1.0;\n            issues.push(\"Missing comments or documentation\".to_string());\n        }\n\n        // Check for examples\n        if output.len() > 200 && !output.contains(\"example\") && !output.contains(\"Example\") {\n            score -= 1.0;\n            issues.push(\"Could benefit from examples\".to_string());\n        }\n\n        (score.max(0.0f32), issues)\n    }\n\n    fn score_creativity(&self, output: &str) -> (f32, Vec<String>) {\n        let mut score: f32 = 7.0; // Base score\n        let mut issues = Vec::new();\n\n        // Simple heuristics for creativity\n        let unique_words: std::collections::HashSet<_> = output.split_whitespace().collect();\n\n        if unique_words.len() < 10 {\n            score -= 2.0;\n            issues.push(\"Limited vocabulary used\".to_string());\n        } else if unique_words.len() > 50 {\n            score += 2.0; // Bonus for rich vocabulary\n        }\n\n        (score.min(10.0f32).max(0.0f32), issues)\n    }\n\n    fn score_professional_quality(&self, output: &str) -> (f32, Vec<String>) {\n        let mut score: f32 = 8.0;\n        let mut issues = Vec::new();\n\n        // Check formatting\n        if !output.contains('\\n') {\n            score -= 2.0;\n            issues.push(\"Poor formatting - no line breaks\".to_string());\n        }\n\n        // Check for proper capitalization\n        let first_char = output.chars().next();\n        if let Some(c) = first_char {\n            if c.is_lowercase() && !output.starts_with('#') {\n                score -= 1.0;\n                issues.push(\"Should start with capital letter\".to_string());\n            }\n        }\n\n        (score.max(0.0f32), issues)\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    #[test]\n    fn test_real_validator_creation() {\n        let _validator = RealWorldValidator::new();\n        assert!(true); // Just ensure it creates successfully\n    }\n}\n"
        }
    ]
}