//! Runs configured `evals.scorers[]` against each eval.
//!
//! Why this is separate from [`super::eval`]: the single-judge path is
//! synchronous and predates the scorer framework. Scorers are async (an
//! `exec` scorer spawns a subprocess) and dispatch through the
//! [`mcptest_scorer_core::ScorerRegistry`]. Keeping the scorer pass in its
//! own module avoids dragging an async runtime through the single-judge
//! code and keeps each module focused. The eval handler runs this pass
//! after the single-judge pass when the config declares scorers.

use std::sync::Arc;

use anyhow::{Context, Result};
use mcptest_scorer_core::{CacheOutcome, RecordingSink, ScoreRequest, ScoreResponse, Verdict};
use serde_yaml::Value as YamlValue;

use crate::scorers;

/// One scorer's outcome on one eval, for rendering and tests.
#[derive(Debug, Clone)]
pub struct ScorerOutcome {
    /// Eval test name.
    pub eval_name: String,
    /// Scorer `type` that ran (or was requested).
    pub scorer_type: String,
    /// `Ok` verdict or the structured error string.
    pub result: Result<ScoreResponse, String>,
}

impl ScorerOutcome {
    /// `true` when the scorer returned a passing verdict.
    pub fn passed(&self) -> bool {
        matches!(
            &self.result,
            Ok(resp) if resp.verdict == Verdict::Pass
        )
    }
}

/// Result of running every configured scorer over every eval.
#[derive(Debug, Clone)]
pub struct ScorerRunReport {
    /// One outcome per (eval, scorer) pair.
    pub outcomes: Vec<ScorerOutcome>,
    /// Sum of known `cost_usd` across outcomes. Null-cost scorers do not
    /// contribute.
    pub total_cost_usd: f64,
    /// `true` when at least one scorer reported a null cost, so the
    /// `--max-cost` cap could not account for it.
    pub saw_null_cost: bool,
}

impl ScorerRunReport {
    /// Number of failing or errored scorer outcomes. Drives the exit code.
    pub fn failure_count(&self) -> usize {
        self.outcomes.iter().filter(|o| !o.passed()).count()
    }
}

/// Run every configured scorer over every eval spec.
///
/// Parses `evals.scorers[]`, builds the OSS registry, and scores each eval
/// through each scorer. An unknown `type` (a vendor name in an OSS build)
/// surfaces the "not available in this build" diagnostic as a failed
/// outcome rather than aborting the run, so one missing enterprise scorer
/// does not mask the others' results.
pub async fn run_scorers(
    config: &YamlValue,
    evals: &[(String, ScoreRequest)],
    max_cost: Option<f64>,
) -> Result<ScorerRunReport> {
    let scorer_configs = scorers::config::parse_scorers(config, &|name| std::env::var(name).ok())
        .context("failed to parse evals.scorers")?;
    let telemetry = Arc::new(RecordingSink::default());
    let registry = scorers::build_registry(&scorer_configs, telemetry.clone());

    let mut outcomes = Vec::new();
    let mut spent = 0.0;
    'budget: for cfg in &scorer_configs {
        let scorer_type = cfg.scorer_type().to_string();
        for (eval_name, request) in evals {
            // WOR-1511: stop once cumulative scorer spend reaches --max-cost, so
            // the scorer pass honors the same cap as the judge pass instead of
            // only warning about it.
            if max_cost.is_some_and(|cap| spent >= cap - 1e-9) {
                break 'budget;
            }
            let result = registry.score(&scorer_type, request.clone()).await;
            if let Ok(resp) = &result {
                spent += resp.cost_usd.unwrap_or(0.0);
            }
            outcomes.push(ScorerOutcome {
                eval_name: eval_name.clone(),
                scorer_type: scorer_type.clone(),
                result: result.map_err(|err| err.to_string()),
            });
        }
    }

    let report = summarize(&telemetry, outcomes);
    Ok(report)
}

/// `true` when the config declares any scorers (under `evals.scorers[]` or
/// a top-level `scorers[]`).
///
/// Lets the handler skip the runtime spin-up entirely when there is
/// nothing to score.
pub fn has_scorers(config: &YamlValue) -> bool {
    scorers::config::scorers_node(config).is_some_and(|s| !s.is_empty())
}

fn summarize(telemetry: &RecordingSink, outcomes: Vec<ScorerOutcome>) -> ScorerRunReport {
    let records = telemetry.records();
    let total_cost_usd = records
        .iter()
        .filter(|r| r.cache == CacheOutcome::Miss)
        .filter_map(|r| r.cost_usd)
        .sum();
    let saw_null_cost = outcomes
        .iter()
        .any(|o| matches!(&o.result, Ok(resp) if resp.cost_usd.is_none()));
    ScorerRunReport {
        outcomes,
        total_cost_usd,
        saw_null_cost,
    }
}

/// Pretty-print the scorer report and the null-cost `--max-cost` warning.
///
/// `max_cost` is taken only to drive the warning copy: a scorer that
/// reports `cost_usd: null` cannot be counted against the cap, and the
/// operator should know the cap is not airtight for those scorers.
pub fn render(report: &ScorerRunReport, max_cost: Option<f64>, quiet: bool) {
    if quiet {
        return;
    }
    for outcome in &report.outcomes {
        match &outcome.result {
            Ok(resp) => {
                let status = if resp.verdict == Verdict::Pass {
                    "PASS"
                } else {
                    "FAIL"
                };
                let cost = match resp.cost_usd {
                    Some(c) => format!("${c:.4}"),
                    None => "n/a".to_string(),
                };
                println!(
                    "{} [{}] {}: cost={} {}",
                    status, outcome.scorer_type, outcome.eval_name, cost, resp.reasoning
                );
            }
            Err(msg) => {
                println!(
                    "ERROR [{}] {}: {}",
                    outcome.scorer_type, outcome.eval_name, msg
                );
            }
        }
    }
    println!("Scorer cost: ${:.4}", report.total_cost_usd);
    if report.saw_null_cost && max_cost.is_some() {
        println!(
            "warning: one or more scorers reported a null cost; null-cost scorers do not count toward --max-cost."
        );
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn config(yaml: &str) -> YamlValue {
        serde_yaml::from_str(yaml).unwrap()
    }

    #[test]
    fn has_scorers_detects_the_block() {
        let with = config(
            r#"
evals:
  scorers:
    - name: j
      type: exec
      command: ["./j.sh"]
"#,
        );
        assert!(has_scorers(&with));
        assert!(!has_scorers(&config("evals: {}")));
    }

    #[tokio::test]
    async fn unknown_type_surfaces_not_available_diagnostic() {
        let cfg = config(
            r#"
evals:
  scorers:
    - name: bt
      type: braintrust
"#,
        );
        let evals = vec![("e1".to_string(), ScoreRequest::new("q", "a", "r"))];
        let report = run_scorers(&cfg, &evals, None).await.unwrap();
        assert_eq!(report.outcomes.len(), 1);
        let outcome = &report.outcomes[0];
        assert!(!outcome.passed());
        let Err(msg) = &outcome.result else {
            panic!("expected an error outcome");
        };
        assert!(msg.contains("not available in this build"), "msg={msg}");
    }

    #[tokio::test]
    async fn exec_scorer_runs_and_passes() {
        let cfg = config(
            r#"
evals:
  scorers:
    - name: judge
      type: exec
      command:
        - sh
        - -c
        - "cat >/dev/null; printf '{\"verdict\":\"pass\",\"cost_usd\":0.01}'"
"#,
        );
        let evals = vec![("e1".to_string(), ScoreRequest::new("q", "a", "r"))];
        let report = run_scorers(&cfg, &evals, None).await.unwrap();
        assert_eq!(report.outcomes.len(), 1);
        assert!(report.outcomes[0].passed());
        assert!((report.total_cost_usd - 0.01).abs() < 1e-9);
        assert_eq!(report.failure_count(), 0);
    }

    #[tokio::test]
    async fn null_cost_scorer_sets_warning_flag() {
        let cfg = config(
            r#"
evals:
  scorers:
    - name: judge
      type: exec
      command:
        - sh
        - -c
        - "cat >/dev/null; printf '{\"verdict\":\"pass\",\"cost_usd\":null}'"
"#,
        );
        let evals = vec![("e1".to_string(), ScoreRequest::new("q", "a", "r"))];
        let report = run_scorers(&cfg, &evals, None).await.unwrap();
        assert!(report.saw_null_cost);
    }
}
