//! Handler for `mcptest eval`.

use anyhow::{Context, Result};
use mcptest_scorer_core::ScoreRequest;
use serde_yaml::Value as YamlValue;

use crate::cli::args::EvalArgs;
use crate::cli::exit_codes::{EXIT_FAILURE, EXIT_INVALID_INVOCATION, EXIT_OK};
use crate::cli::GlobalArgs;
use crate::commands;

/// Handle `mcptest eval`. Grades each `evals:` entry against its rubric with a
/// real provider resolved from the environment. With no provider key, or with
/// no deterministic `response:` to grade, each eval defers (passed, with a
/// note) so a key-free CI run stays green. When the config declares
/// `evals.scorers[]`, the external scorers run as a second pass and fold their
/// failures into the exit code (the worse of the two wins).
pub(crate) fn eval_command(global: &GlobalArgs, args: &EvalArgs) -> Result<i32> {
    let cwd = std::env::current_dir()?;
    let path = match global.config.clone() {
        Some(path) => path,
        // WOR-1350: shared default resolution (`mcptest.yaml`, then
        // `mcptest.yml`) so eval finds the same file run would.
        None => match crate::cli::default_config::resolve_default_config(&cwd) {
            Ok(resolved) => resolved.into_path(),
            Err(ambiguous) => {
                eprintln!("mcptest eval: {}", ambiguous.message());
                return Ok(EXIT_INVALID_INVOCATION);
            }
        },
    };

    // WOR-1355: refuse a directory before the raw read so eval matches
    // run/validate, surfacing the loader's actionable IsDirectory message
    // with the invalid-invocation exit code instead of a generic anyhow
    // "failed to read ... Is a directory" at exit 1.
    if path.is_dir() {
        let err = mcptest_config::Error::IsDirectory { path: path.clone() };
        eprintln!("mcptest eval: {err}");
        return Ok(EXIT_INVALID_INVOCATION);
    }

    let evals = commands::eval::load_evals(&path)?;

    // `--explain` is a dry run: print the plan and exit without grading or
    // calling any provider.
    if args.explain {
        commands::eval_grade::explain_evals(&evals, commands::eval::DEFAULT_EVAL_MODEL);
        return Ok(EXIT_OK);
    }

    if evals.is_empty() && !global.quiet {
        println!("mcptest eval: no `evals:` block in {}", path.display());
    }

    // Resolve a judge provider from the environment. A missing key (or an
    // unsupported model family) yields no provider, and grading defers.
    let lookup = mcptest_agent::from_env(
        commands::eval::DEFAULT_EVAL_MODEL,
        &mcptest_core::network::ProxyConfig::default(),
    );
    let provider = match lookup {
        Ok(mcptest_agent::ProviderLookup::Resolved { provider, .. }) => Some(provider),
        _ => None,
    };
    let provider_ref = provider.as_ref().map(|p| p.as_ref());
    let model = provider_ref.map(|_| commands::eval::DEFAULT_EVAL_MODEL);

    // Surface the projected judge-call count up front so a jury (which
    // multiplies calls by its size) does not surprise the cost budget. Goes to
    // stderr so a machine `--reporter` keeps clean stdout.
    if !global.quiet && !evals.is_empty() {
        eprintln!(
            "mcptest eval: {} evals, ~{} projected judge calls",
            evals.len(),
            commands::eval_grade::projected_judge_calls(&evals)
        );
    }

    // WOR-1320: build the LLM-judge verdict cache for this grading run.
    // `--no-verdict-cache` always wins; otherwise the YAML
    // `evals.cache.verdicts` opt-in (default off, per ADR 0008) decides.
    // Single-judge grading inside `grade_evals` consults it through
    // `commands::eval_cache`; jury and panel grading always judge fresh.
    let mut verdict_cache = build_verdict_cache_for(&path, args.no_verdict_cache)?;

    let runtime = tokio::runtime::Runtime::new().context("could not build tokio runtime")?;
    let report = runtime.block_on(grade_with_optional_live(
        &path,
        &evals,
        provider_ref,
        model,
        args.max_cost,
        global.quiet,
        &mut verdict_cache,
    ));
    render_eval(&report, global)?;
    let judge_code = commands::eval::exit_code_for(&report);

    let scorer_code = run_scorer_pass(&path, args, global.quiet, &report.live_candidates)?;
    // The worse of the two exit codes wins: a scorer failure must not be
    // masked by an all-pass grading run, and vice versa.
    Ok(judge_code.max(scorer_code))
}

/// Parse the config once more to read `evals.cache.verdicts` and build the
/// verdict cache with the CLI override applied (WOR-1320). The config was
/// already read and parsed by `load_evals`, so failures here are unexpected
/// and surface as errors rather than silently disabling the cache.
fn build_verdict_cache_for(
    path: &std::path::Path,
    no_verdict_cache: bool,
) -> Result<mcptest_core::eval::VerdictCache> {
    let body = std::fs::read_to_string(path)
        .with_context(|| format!("failed to read {}", path.display()))?;
    let value: YamlValue = serde_yaml::from_str(&body)
        .with_context(|| format!("failed to parse {} as YAML", path.display()))?;
    Ok(commands::eval::build_verdict_cache(
        &value,
        no_verdict_cache,
    ))
}

/// Run the external-scorer pass when the config declares any.
///
/// Returns `0` when no scorers are configured (so the single-judge code
/// stands), `1` when any scorer outcome failed or errored. The scorers run
/// on a fresh tokio runtime because the surrounding `eval` command is sync.
fn run_scorer_pass(
    path: &std::path::Path,
    args: &EvalArgs,
    quiet: bool,
    live_candidates: &std::collections::BTreeMap<String, String>,
) -> Result<i32> {
    let body = match std::fs::read_to_string(path) {
        Ok(body) => body,
        // The single-judge pass already reported a missing/unreadable
        // config; do not double-report here.
        Err(_) => return Ok(EXIT_OK),
    };
    let config: YamlValue = serde_yaml::from_str(&body)
        .with_context(|| format!("failed to parse {} as YAML", path.display()))?;
    if !commands::has_scorers(&config) {
        return Ok(EXIT_OK);
    }

    let evals = eval_requests(&config, live_candidates);
    let runtime = tokio::runtime::Runtime::new().context("could not build tokio runtime")?;
    let report = runtime.block_on(commands::run_scorers(&config, &evals, args.max_cost))?;
    commands::render_scorers(&report, args.max_cost, quiet);
    if report.failure_count() > 0 {
        Ok(EXIT_FAILURE)
    } else {
        Ok(EXIT_OK)
    }
}

/// Build a `(name, ScoreRequest)` per eval from the parsed config.
///
/// The eval's `prompt` is the scorer `input` and its `rubric` is the
/// `criteria`. For a deterministic eval (one that declares a `response:`, the
/// recommended CI tier) the response is the candidate `output`, so the external
/// scorer grades the real candidate rather than an empty string (WOR-1386). For
/// a live eval (no `response:`), the candidate is the one the judge pass already
/// produced, passed in via `live_candidates` keyed by eval name, so the scorer
/// grades the real model output without re-running the model (WOR-1399); a live
/// eval with no captured candidate (a key-free run that deferred) still scores
/// an empty string.
fn eval_requests(
    config: &YamlValue,
    live_candidates: &std::collections::BTreeMap<String, String>,
) -> Vec<(String, ScoreRequest)> {
    // `evals` is a sequence of tests in the legacy shape; in the object
    // shape (which carries `scorers:`) the test list lives under
    // `evals.tests`. Accept both so a scorer-bearing config still names its
    // evals.
    let evals_node = config.get("evals");
    let arr = evals_node.and_then(YamlValue::as_sequence).or_else(|| {
        evals_node
            .and_then(|e| e.get("tests"))
            .and_then(YamlValue::as_sequence)
    });
    let Some(arr) = arr else {
        return Vec::new();
    };
    arr.iter()
        .filter_map(|node| {
            let name = node.get("name").and_then(YamlValue::as_str)?.to_owned();
            let prompt = node
                .get("prompt")
                .and_then(YamlValue::as_str)
                .unwrap_or("")
                .to_owned();
            let rubric = node
                .get("rubric")
                .and_then(YamlValue::as_str)
                .unwrap_or("")
                .to_owned();
            // The deterministic candidate is the eval's `response:`. For a live
            // eval it comes from the judge pass via `live_candidates`; only when
            // neither is present does the scorer fall back to an empty string.
            let output = node
                .get("response")
                .and_then(YamlValue::as_str)
                .map(str::to_owned)
                .or_else(|| live_candidates.get(&name).cloned())
                .unwrap_or_default();
            Some((name, ScoreRequest::new(prompt, output, rubric)))
        })
        .collect()
}

/// Render the eval run: the canonical [`Report`] through the selected
/// `--reporter` (so all nine formats render the same envelope), or the default
/// pretty summary when no reporter is set. The canonical report is redacted so
/// a secret in the judge rationale never reaches a reporter.
fn render_eval(report: &commands::eval::EvalRunReport, global: &GlobalArgs) -> Result<()> {
    let Some(format) = global.reporter.and_then(|r| r.render_format()) else {
        commands::eval::render_report(report, global.quiet);
        return Ok(());
    };
    let canonical =
        commands::eval::to_report(report).redacted(&mcptest_core::redaction::Redactor::new());
    let reporter = crate::cli::reporters::build_reporter(format, false, false, false, None);
    let mut out = std::io::stdout();
    reporter
        .render(&canonical, &mut out)
        .context("rendering the eval report")?;
    Ok(())
}

/// Grade the evals, connecting servers for any live-tier eval first.
///
/// When a provider is resolved and at least one eval has no deterministic
/// `response:`, that eval's `server:` is connected so the rubric grades a live
/// agent run. With no provider, or when every eval carries a `response:`, no
/// server is connected and the live tier is skipped (the key-free path).
async fn grade_with_optional_live(
    path: &std::path::Path,
    evals: &[commands::eval::EvalSpec],
    provider: Option<&dyn mcptest_core::eval::provider::LlmProvider>,
    model: Option<&str>,
    max_cost: Option<f64>,
    quiet: bool,
    verdict_cache: &mut mcptest_core::eval::VerdictCache,
) -> commands::eval::EvalRunReport {
    let needs_live = provider.is_some() && evals.iter().any(|e| e.response.is_none());
    if !needs_live {
        return commands::eval_grade::grade_evals(
            evals,
            provider,
            model,
            None,
            max_cost,
            verdict_cache,
        )
        .await;
    }
    let pool = mcptest_core::connector::ServerPool::new();
    let budget = std::sync::Arc::new(mcptest_agent::Budget::new(None, None));
    connect_eval_servers(path, evals, &pool, quiet).await;
    let live = commands::eval_grade::LiveCandidates {
        pool: &pool,
        budget,
    };
    commands::eval_grade::grade_evals(evals, provider, model, Some(&live), max_cost, verdict_cache)
        .await
}

/// Connect every distinct server a live eval (one with no `response:`)
/// references into `pool`, best effort. An unknown or unreachable server is
/// noted and skipped so the eval defers rather than aborting the command; the
/// deterministic `response:` path stays the recommended CI tier.
async fn connect_eval_servers(
    path: &std::path::Path,
    evals: &[commands::eval::EvalSpec],
    pool: &mcptest_core::connector::ServerPool,
    quiet: bool,
) {
    let Ok(body) = std::fs::read_to_string(path) else {
        return;
    };
    let Ok(yaml) = serde_yaml::from_str::<YamlValue>(&body) else {
        return;
    };
    let Ok(yaml) = crate::cli::handlers::interp::interpolate_run_config_reported(yaml, path, quiet)
    else {
        return;
    };
    let Ok(mut suite) = mcptest_core::suite::parse(&yaml) else {
        return;
    };
    // WOR-1504: resolve relative `cassette:` paths against the suite directory,
    // matching `mcptest run` (run_live.rs), so the same suite replays the same
    // file under `eval` from any working directory.
    if let Some(dir) = path.parent() {
        mcptest_core::suite::rebase_cassette_paths(&mut suite.servers, dir);
    }
    let proxy = mcptest_core::network::ProxyConfig::default();
    let mut seen = std::collections::BTreeSet::new();
    for spec in evals {
        if spec.response.is_some() || spec.server.is_empty() || !seen.insert(spec.server.clone()) {
            continue;
        }
        let Some(server_spec) = suite.servers.get(&spec.server) else {
            if !quiet {
                eprintln!(
                    "mcptest eval: live eval `{}` references unknown server `{}`; it will defer",
                    spec.name, spec.server
                );
            }
            continue;
        };
        match mcptest_core::connector::connect_server(
            &spec.server,
            server_spec,
            "mcptest",
            mcptest_core::version::semver(),
            &proxy,
            &[],
        )
        .await
        {
            Ok(live) => pool.insert(spec.server.clone(), live).await,
            Err(e) => {
                if !quiet {
                    eprintln!(
                        "mcptest eval: connect to `{}` failed ({e}); live evals on it will defer",
                        spec.server
                    );
                }
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn parse(yaml: &str) -> YamlValue {
        serde_yaml::from_str(yaml).expect("test yaml parses")
    }

    fn empty() -> std::collections::BTreeMap<String, String> {
        std::collections::BTreeMap::new()
    }

    #[test]
    fn deterministic_eval_feeds_its_response_to_the_scorer_output() {
        // A deterministic eval declares a `response:`; that response is the
        // candidate the external scorer must grade (WOR-1386).
        let config = parse(
            "evals:\n  - name: greeting\n    prompt: say hi\n    response: \"hello there\"\n    rubric: friendly\n",
        );
        let reqs = eval_requests(&config, &empty());
        assert_eq!(reqs.len(), 1);
        let (name, req) = &reqs[0];
        assert_eq!(name, "greeting");
        assert_eq!(req.input, "say hi");
        assert_eq!(req.output, "hello there");
        assert_eq!(req.criteria, "friendly");
    }

    #[test]
    fn live_eval_without_a_captured_candidate_scores_an_empty_output() {
        // A live eval (no `response:`) whose model run was not captured (a
        // key-free run that deferred) has no entry in the candidate map, so the
        // scorer output stays empty rather than inventing one.
        let config =
            parse("evals:\n  - name: live\n    prompt: ask\n    server: weather\n    rubric: ok\n");
        let reqs = eval_requests(&config, &empty());
        assert_eq!(reqs.len(), 1);
        assert_eq!(reqs[0].1.output, "");
    }

    #[test]
    fn live_eval_scores_the_captured_candidate_without_re_running() {
        // A live eval whose candidate the judge pass captured grades that real
        // model output, not an empty string, and without a second model call
        // (WOR-1399).
        let config =
            parse("evals:\n  - name: live\n    prompt: ask\n    server: weather\n    rubric: ok\n");
        let mut candidates = empty();
        candidates.insert("live".to_string(), "the model said hello".to_string());
        let reqs = eval_requests(&config, &candidates);
        assert_eq!(reqs.len(), 1);
        assert_eq!(reqs[0].1.output, "the model said hello");
        // A deterministic `response:` still wins over a captured candidate.
        let det = parse(
            "evals:\n  - name: live\n    prompt: ask\n    response: declared\n    rubric: ok\n",
        );
        let reqs = eval_requests(&det, &candidates);
        assert_eq!(reqs[0].1.output, "declared");
    }

    #[test]
    fn object_shaped_evals_under_tests_are_still_read() {
        // The scorer-bearing object shape nests the test list under
        // `evals.tests`; the response still reaches the scorer output.
        let config = parse(
            "evals:\n  scorers:\n    - type: exec\n      command: [\"true\"]\n  tests:\n    - name: t\n      prompt: p\n      response: candidate\n      rubric: r\n",
        );
        let reqs = eval_requests(&config, &empty());
        assert_eq!(reqs.len(), 1);
        assert_eq!(reqs[0].1.output, "candidate");
    }
}
