//! `mcptest eval` runs single-judge quality evaluations.
//!
//! The OSS surface here covers the single-juror path: every eval in the
//! suite is graded by one LLM judge, the verdict and rationale are
//! pretty-printed, and a [`CostBudget`] tracks cumulative spend. Multi-juror
//! consensus, bias mitigations, and inter-juror agreement are v1.1 features
//! that wire through this same entry point.

use std::path::Path;

use std::collections::{BTreeMap, HashMap};

use anyhow::{Context, Result};
use mcptest_core::eval::rubric::RubricConfig;
use mcptest_core::eval::{CostBudget, VerdictCache, VerdictCacheConfig, VerdictKey};
use serde::{Deserialize, Serialize};
use serde_yaml::Value as YamlValue;

/// Per-test eval verdict shown to the user.
///
/// Held as a struct (rather than a tuple) so the reporter can grow a new
/// field (latency, juror name) without churning every call site.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EvalVerdict {
    /// Test name from the YAML `evals[*].name` field.
    pub name: String,
    /// `true` when the juror's score met the threshold (default 0.7).
    pub passed: bool,
    /// Score in `[0, 1]` returned by the juror.
    pub score: f64,
    /// Free-form reasoning text returned by the juror.
    pub reasoning: String,
    /// Cost (USD) attributed to this test.
    pub cost_usd: f64,
}

/// Outcome of a full eval run, returned by [`run_eval_with_judge`].
///
/// Tests build a fake [`SingleJudge`] and consume this struct directly so
/// they can assert on individual verdicts without parsing stdout.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EvalRunReport {
    /// One verdict per eval test in the suite.
    pub verdicts: Vec<EvalVerdict>,
    /// Cumulative cost across every juror call.
    pub total_cost_usd: f64,
    /// `true` when the cost cap tripped mid-run.
    pub budget_exceeded: bool,
    /// Candidate text produced for each live eval (one with no deterministic
    /// `response:`), keyed by eval name. The external-scorer pass reuses these
    /// so a live eval grades its real model output instead of an empty string,
    /// without re-running the model (WOR-1399). Empty for a deterministic or
    /// key-free run, and skipped on serialize so those outputs are unchanged.
    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
    pub live_candidates: BTreeMap<String, String>,
}

impl EvalRunReport {
    /// Number of failing verdicts. Drives the process exit code: `0` when
    /// zero, `1` when one or more, `5` when the budget tripped.
    pub fn failure_count(&self) -> usize {
        self.verdicts.iter().filter(|v| !v.passed).count()
    }

    /// Number of passing verdicts (the "completed tasks").
    pub fn passed_count(&self) -> usize {
        self.verdicts.iter().filter(|v| v.passed).count()
    }

    /// Effective cost per completed task (WOR-1393): total dollars divided by
    /// the number of passing verdicts, not by the number attempted. A run that
    /// is cheap per call but fails and retries is correctly scored as expensive
    /// per completed task. `None` when nothing passed (the figure is undefined)
    /// or nothing was spent, so the reporter omits the line rather than printing
    /// a divide-by-zero or a free run's noise.
    pub fn effective_cost_per_task(&self) -> Option<f64> {
        let passed = self.passed_count();
        if passed == 0 || self.total_cost_usd <= 0.0 {
            return None;
        }
        Some(self.total_cost_usd / passed as f64)
    }
}

/// Minimal trait the eval runner needs from a juror.
///
/// Why a focused trait rather than [`mcptest_core::eval::LlmProvider`]:
/// the eval runner only needs "given a prompt and a rubric, give me a
/// score + reasoning + cost". Wiring the full provider trait would drag
/// async-trait through every test in this crate. The CLI handler builds
/// an adapter from a real `LlmProvider` later; the integration test fakes
/// one in-process.
pub trait SingleJudge {
    /// Score the response against the rubric. Returns
    /// `(score in [0,1], reasoning, cost_usd)`.
    fn score(
        &self,
        eval_name: &str,
        prompt: &str,
        rubric: &str,
    ) -> Result<(f64, String, f64), anyhow::Error>;
}

/// Synchronous `SingleJudge` runner over an `evals:` config.
///
/// Loads `evals:` from the config, applies the global cost cap, runs each
/// test through the supplied judge, and prints per-test verdicts plus the
/// total cost. Returns the desired exit code (`0` pass, `1` fail, `5`
/// budget exceeded).
///
/// WOR-1320 note: the binary's `mcptest eval` handler does not route here;
/// it grades through `super::eval_grade::grade_evals`, which consults the
/// same verdict cache via `super::eval_cache`. This entry point stays as
/// the in-process harness for tests that need a synchronous judge without
/// an async provider.
pub fn run_eval<J: SingleJudge>(
    config_path: &Path,
    judge: &J,
    max_cost: Option<f64>,
    no_verdict_cache: bool,
    quiet: bool,
) -> Result<i32> {
    let body = std::fs::read_to_string(config_path)
        .with_context(|| format!("failed to read {}", config_path.display()))?;
    let value: YamlValue = serde_yaml::from_str(&body)
        .with_context(|| format!("failed to parse {} as YAML", config_path.display()))?;
    let evals = extract_evals(&value)?;
    if evals.is_empty() {
        if !quiet {
            println!(
                "mcptest eval: no `evals:` block in {}",
                config_path.display()
            );
        }
        return Ok(0);
    }

    let mut cache = build_verdict_cache(&value, no_verdict_cache);
    let report = run_eval_with_judge(&evals, judge, max_cost, &mut cache);
    render_report(&report, quiet);
    Ok(exit_code_for(&report))
}

/// Pure runner used by [`run_eval`] and integration tests.
///
/// Walks each eval, calls the judge, and tallies cost against the budget.
/// The `cache` argument is consulted only when its `enabled` flag is set;
/// disabled caches always miss (and never store), so a `--no-verdict-cache`
/// run never reads stale verdicts (WOR-1303).
pub fn run_eval_with_judge<J: SingleJudge>(
    evals: &[EvalSpec],
    judge: &J,
    max_cost: Option<f64>,
    cache: &mut VerdictCache,
) -> EvalRunReport {
    let mut budget = match max_cost {
        Some(cap) => CostBudget::with_cap(cap),
        None => CostBudget::unbounded(),
    };
    let mut verdicts: Vec<EvalVerdict> = Vec::with_capacity(evals.len());
    let mut budget_exceeded = false;
    let mut total_cost = 0.0f64;

    for spec in evals {
        // Pre-flight: refuse to dispatch if even a zero-cost call would
        // exceed the cap (because the cap is already exhausted).
        if budget.would_exceed(&mcptest_core::eval::Cost::default()) {
            budget_exceeded = true;
            break;
        }
        let (score, reasoning, cost_usd) = judge_through_cache(spec, judge, cache);
        total_cost += cost_usd;
        let realized = mcptest_core::eval::Cost {
            input_dollars: cost_usd,
            ..mcptest_core::eval::Cost::default()
        };
        budget.record_call(&realized);
        let passed = score >= spec.threshold;
        verdicts.push(EvalVerdict {
            name: spec.name.clone(),
            passed,
            score,
            reasoning,
            cost_usd,
        });
        if max_cost.is_some_and(|cap| total_cost > cap + 1e-9) {
            budget_exceeded = true;
            break;
        }
    }

    EvalRunReport {
        verdicts,
        total_cost_usd: total_cost,
        budget_exceeded,
        // The single-judge path grades a deterministic candidate, so there is
        // no live candidate to hand the scorer pass.
        live_candidates: BTreeMap::new(),
    }
}

/// Score one eval through the verdict cache (WOR-1303).
///
/// The hit/miss semantics live in [`super::eval_cache`] (WOR-1320), shared
/// with the rubric-engine wrapper used by `mcptest eval` grading and the
/// `run --live` agent eval metrics, so the cache contract has exactly one
/// implementation: a fresh hit costs nothing and prefixes its reasoning
/// `cached:`, a juror error is never cached, a disabled cache always misses
/// (the `--no-verdict-cache` behavior), and cache I/O failures degrade to
/// misses.
fn judge_through_cache<J: SingleJudge>(
    spec: &EvalSpec,
    judge: &J,
    cache: &mut VerdictCache,
) -> (f64, String, f64) {
    let key = VerdictKey {
        // The generic single-judge path carries no model identity; the
        // constant keeps its slots separate from any future model-keyed user.
        juror_model: "single-judge",
        juror_prompt_template_version: super::eval_cache::JUROR_PROMPT_TEMPLATE_VERSION,
        criteria: &spec.rubric,
        prompt: &spec.prompt,
        response: spec.response.as_deref().unwrap_or(""),
    };
    if let Some(hit) = super::eval_cache::lookup(cache, &key) {
        return (hit.score, hit.reasoning, 0.0);
    }
    let (score, reasoning, cost_usd) = match judge.score(&spec.name, &spec.prompt, &spec.rubric) {
        Ok(t) => t,
        Err(err) => return (0.0, format!("juror error: {err}"), 0.0),
    };
    super::eval_cache::store(cache, &key, score >= spec.threshold, score, &reasoning);
    (score, reasoning, cost_usd)
}

/// Parsed `evals:` entry. Mirrors the schema EvalTest shape, with a
/// default threshold of `0.7` per the schema docs.
#[derive(Debug, Clone, PartialEq)]
pub struct EvalSpec {
    /// Test name.
    pub name: String,
    /// Server key (informational; the runner-side dispatch lands later).
    pub server: String,
    /// Prompt sent to the model.
    pub prompt: String,
    /// Free-form grading rubric. Empty when a structured `rubric_config` is
    /// supplied instead.
    pub rubric: String,
    /// Structured rubric (weighted `criteria` or a decision `tree`), parsed
    /// from a mapping-valued `rubric:`. `None` when the rubric is a plain
    /// string (or absent). The grading engine prefers this when present.
    pub rubric_config: Option<RubricConfig>,
    /// Deterministic candidate response to grade. When present, the rubric
    /// grades this fixed text: key-free and reproducible, the CI-safe path.
    /// When absent, the live tool-using agent run produces the candidate.
    pub response: Option<String>,
    /// Pass threshold on a `[0, 1]` scale. For a structured rubric the
    /// engine uses the threshold inside `rubric_config`; this field is the
    /// threshold for the free-form string form.
    pub threshold: f64,
    /// Optional per-eval judge configuration (model override, jury). `None`
    /// grades with a single judge on the run's default model.
    pub judge: Option<JudgeConfig>,
    /// Optional matrix: run this eval's rubric across several models and/or
    /// prompts. Expanded into one cell per combination before grading. `None`
    /// is a single eval.
    pub matrix: Option<EvalMatrix>,
}

/// A matrix that fans one eval out across models and/or prompts. Each cell
/// reuses the same rubric, judge config, and threshold, so the per-cell scores
/// are an apples-to-apples comparison for picking a model or prompt.
#[derive(Debug, Clone, PartialEq, Default)]
pub struct EvalMatrix {
    /// Judge models to compare; one cell per model. Empty means the eval's own
    /// model.
    pub models: Vec<String>,
    /// Prompts to compare; one cell per prompt. Empty means the eval's own
    /// prompt.
    pub prompts: Vec<String>,
}

/// Per-eval judge configuration parsed from a `judge:` block.
#[derive(Debug, Clone, PartialEq, Default)]
pub struct JudgeConfig {
    /// Judge model override; `None` falls back to the run's default model.
    pub model: Option<String>,
    /// Optional jury: grade with N independent judges and pass on a consensus
    /// fraction. `None` is a single judge.
    pub jury: Option<JuryPlan>,
    /// Optional multi-model panel (ensemble). Takes precedence over `jury` and
    /// `model` when present.
    pub panel: Option<JudgePanel>,
}

/// A multi-model judge panel: each model grades the rubric once and the
/// per-model verdicts combine by `aggregate`, with `tie_break` resolving an
/// even majority split.
///
/// OSS panels are single-provider: every model id is served by the one
/// resolved provider (so a panel of same-vendor models works with one key).
/// Mixing distinct vendor provider instances in one panel is an enterprise
/// feature, consistent with the scorers boundary.
#[derive(Debug, Clone, PartialEq)]
pub struct JudgePanel {
    /// Model ids on the panel, each judged once by the resolved provider.
    pub models: Vec<String>,
    /// How per-model verdicts combine into the panel verdict.
    pub aggregate: PanelAggregate,
    /// How an even majority split is broken.
    pub tie_break: TieBreak,
}

/// How a [`JudgePanel`]'s per-model scores combine into a pass/fail.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum PanelAggregate {
    /// Pass when the mean of per-model scores meets the threshold (default).
    #[default]
    Mean,
    /// Pass when the median of per-model scores meets the threshold.
    Median,
    /// Pass when a majority of models individually pass.
    Majority,
}

impl PanelAggregate {
    /// Short label for the reasoning line.
    pub(crate) fn label(self) -> &'static str {
        match self {
            PanelAggregate::Mean => "mean",
            PanelAggregate::Median => "median",
            PanelAggregate::Majority => "majority",
        }
    }
}

/// How a [`PanelAggregate::Majority`] tie (equal passes and fails) resolves.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum TieBreak {
    /// A tie fails the eval (conservative default).
    #[default]
    Fail,
    /// A tie passes the eval.
    Pass,
}

/// A jury plan: N judges and the pass-fraction consensus they must reach.
///
/// OSS jury is single-provider: the same model judges `size` times and the
/// eval passes when at least `consensus` of those judgments pass. Cross-vendor
/// panels resolve only in the enterprise build (see WOR-1017).
#[derive(Debug, Clone, PartialEq)]
pub struct JuryPlan {
    /// Number of judges (independent rubric evaluations) to run, at least 1.
    pub size: u32,
    /// Fraction of judges that must pass for the eval to pass, in `[0, 1]`.
    pub consensus: f64,
}

/// Extract the `evals:` array from a parsed YAML config.
///
/// Returns an error (rather than silently skipping) when an entry is missing a
/// required field or carries a malformed structured rubric, so a typo surfaces
/// at load time instead of vanishing from the run.
fn extract_evals(value: &YamlValue) -> Result<Vec<EvalSpec>> {
    let named = extract_named_rubrics(value)?;
    let Some(arr) = value.get("evals").and_then(YamlValue::as_sequence) else {
        return Ok(Vec::new());
    };
    arr.iter()
        .enumerate()
        .map(|(i, node)| eval_spec_from_yaml(node, i, &named))
        .collect()
}

/// Parse the top-level `rubrics:` map into named [`RubricConfig`]s an eval can
/// reference with `rubric: { ref: <name> }`. Empty when the key is absent.
fn extract_named_rubrics(value: &YamlValue) -> Result<HashMap<String, RubricConfig>> {
    let Some(map) = value.get("rubrics").and_then(YamlValue::as_mapping) else {
        return Ok(HashMap::new());
    };
    let mut out = HashMap::with_capacity(map.len());
    for (key, node) in map {
        let name = key
            .as_str()
            .context("`rubrics:` keys must be strings")?
            .to_owned();
        let cfg = mcptest_core::eval::parse_rubric(node, &format!("rubrics/{name}"))
            .with_context(|| format!("named rubric `{name}` is invalid"))?;
        out.insert(name, cfg);
    }
    Ok(out)
}

fn eval_spec_from_yaml(
    node: &YamlValue,
    index: usize,
    named: &HashMap<String, RubricConfig>,
) -> Result<EvalSpec> {
    let name = node
        .get("name")
        .and_then(YamlValue::as_str)
        .with_context(|| format!("evals[{index}] is missing a `name`"))?
        .to_owned();
    let server = node
        .get("server")
        .and_then(YamlValue::as_str)
        .unwrap_or("")
        .to_owned();
    let prompt = node
        .get("prompt")
        .and_then(YamlValue::as_str)
        .with_context(|| format!("eval `{name}` is missing a `prompt`"))?
        .to_owned();
    // `rubric:` is either a free-form string or a structured mapping
    // (weighted `criteria` or a decision `tree`), parsed by the shared
    // rubric parser so this matches the agent-side `eval.rubric`.
    let (rubric, rubric_config) = match node.get("rubric") {
        None => (String::new(), None),
        Some(YamlValue::String(s)) => (s.clone(), None),
        // `rubric: { ref: <name> }` references a rubric in the top-level
        // `rubrics:` map, with optional `threshold`/`strict` overrides.
        Some(structured) if structured.get("ref").is_some() => {
            let ref_name = structured
                .get("ref")
                .and_then(YamlValue::as_str)
                .with_context(|| format!("eval `{name}`: rubric `ref` must be a string"))?;
            let mut cfg = named
                .get(ref_name)
                .cloned()
                .with_context(|| format!("eval `{name}` references unknown rubric `{ref_name}`"))?;
            if let Some(t) = structured.get("threshold").and_then(YamlValue::as_f64) {
                cfg.threshold = t;
            }
            if let Some(s) = structured.get("strict").and_then(YamlValue::as_bool) {
                cfg.strict = s;
            }
            (String::new(), Some(cfg))
        }
        // `rubric: { preset: <name> }` starts from a built-in rubric, with
        // optional `threshold`/`strict` overrides and extra `criteria`
        // appended to extend it.
        Some(structured) if structured.get("preset").is_some() => {
            let preset_name = structured
                .get("preset")
                .and_then(YamlValue::as_str)
                .with_context(|| format!("eval `{name}`: rubric `preset` must be a string"))?;
            let mut cfg = mcptest_core::eval::preset(preset_name).with_context(|| {
                format!(
                    "eval `{name}` uses unknown rubric preset `{preset_name}` (known: {})",
                    mcptest_core::eval::PRESET_NAMES.join(", ")
                )
            })?;
            if let Some(t) = structured.get("threshold").and_then(YamlValue::as_f64) {
                cfg.threshold = t;
            }
            if let Some(s) = structured.get("strict").and_then(YamlValue::as_bool) {
                cfg.strict = s;
            }
            // Appended criteria extend the preset (parsed via the shared
            // parser, which also validates each criterion).
            if structured.get("criteria").is_some() {
                let path = format!("evals/{index}/rubric");
                let extra = mcptest_core::eval::parse_rubric(structured, &path)
                    .with_context(|| format!("eval `{name}` has invalid extra criteria"))?;
                cfg.criteria.extend(extra.criteria);
            }
            (String::new(), Some(cfg))
        }
        Some(structured) => {
            let path = format!("evals/{index}/rubric");
            let cfg = mcptest_core::eval::parse_rubric(structured, &path)
                .with_context(|| format!("eval `{name}` has an invalid rubric"))?;
            (String::new(), Some(cfg))
        }
    };
    let response = node
        .get("response")
        .and_then(YamlValue::as_str)
        .map(str::to_owned);
    let threshold = node
        .get("threshold")
        .and_then(YamlValue::as_f64)
        .unwrap_or(0.7);
    let judge = parse_judge(node);
    let matrix = parse_matrix(node);
    Ok(EvalSpec {
        name,
        server,
        prompt,
        rubric,
        rubric_config,
        response,
        threshold,
        judge,
        matrix,
    })
}

/// Parse the optional `matrix:` block: `models` and/or `prompts` lists. Returns
/// `None` when absent or when both lists are empty (nothing to fan out).
fn parse_matrix(node: &YamlValue) -> Option<EvalMatrix> {
    let m = node.get("matrix")?;
    let list = |key: &str| -> Vec<String> {
        m.get(key)
            .and_then(YamlValue::as_sequence)
            .map(|seq| {
                seq.iter()
                    .filter_map(|x| x.as_str().map(str::to_owned))
                    .collect()
            })
            .unwrap_or_default()
    };
    let matrix = EvalMatrix {
        models: list("models"),
        prompts: list("prompts"),
    };
    (!matrix.models.is_empty() || !matrix.prompts.is_empty()).then_some(matrix)
}

/// Fan each eval that declares a `matrix:` out into one cell per
/// (model, prompt) combination. A cell reuses the base eval's rubric, judge
/// config, and threshold; it overrides the judge model (for a `models` axis)
/// and the prompt (for a `prompts` axis), and its name records the cell so the
/// per-cell scores read as a comparison. Evals without a matrix pass through.
pub(crate) fn expand_matrix(evals: Vec<EvalSpec>) -> Vec<EvalSpec> {
    let mut out = Vec::with_capacity(evals.len());
    for spec in evals {
        let Some(matrix) = spec.matrix.clone() else {
            out.push(spec);
            continue;
        };
        // An empty axis contributes a single "use the base value" slot.
        let models: Vec<Option<&str>> = if matrix.models.is_empty() {
            vec![None]
        } else {
            matrix.models.iter().map(|m| Some(m.as_str())).collect()
        };
        let prompts: Vec<Option<(usize, &str)>> = if matrix.prompts.is_empty() {
            vec![None]
        } else {
            matrix
                .prompts
                .iter()
                .enumerate()
                .map(|(i, p)| Some((i + 1, p.as_str())))
                .collect()
        };
        for model in &models {
            for prompt in &prompts {
                let mut cell = spec.clone();
                cell.matrix = None;
                let mut dims = Vec::new();
                if let Some(model) = model {
                    let mut judge = cell.judge.clone().unwrap_or_default();
                    judge.model = Some((*model).to_string());
                    cell.judge = Some(judge);
                    dims.push(format!("model={model}"));
                }
                if let Some((idx, prompt)) = prompt {
                    cell.prompt = (*prompt).to_string();
                    dims.push(format!("prompt {idx}"));
                }
                cell.name = format!("{} [{}]", spec.name, dims.join(", "));
                out.push(cell);
            }
        }
    }
    out
}

/// Parse the optional per-eval `judge:` block: a `model` override and an
/// optional `jury: { size, consensus }`. Absent fields take sensible defaults
/// (jury size at least 1, consensus 0.5). Returns `None` when there is no
/// `judge:` key, so a single judge on the default model is used.
fn parse_judge(node: &YamlValue) -> Option<JudgeConfig> {
    let j = node.get("judge")?;
    let model = j
        .get("model")
        .and_then(YamlValue::as_str)
        .map(str::to_owned);
    let jury = j.get("jury").map(|jy| JuryPlan {
        size: jy
            .get("size")
            .and_then(YamlValue::as_u64)
            .unwrap_or(1)
            .max(1) as u32,
        consensus: jy
            .get("consensus")
            .and_then(YamlValue::as_f64)
            .unwrap_or(0.5),
    });
    let panel = parse_panel(j);
    Some(JudgeConfig { model, jury, panel })
}

/// Parse the optional `panel:` model list under `judge:` plus the sibling
/// `aggregate` and `tie_break`. Returns `None` when there is no `panel` key or
/// it lists no models, so the single-judge or jury path is used instead.
fn parse_panel(judge: &YamlValue) -> Option<JudgePanel> {
    let models: Vec<String> = judge
        .get("panel")?
        .as_sequence()?
        .iter()
        .filter_map(|m| m.as_str().map(str::to_owned))
        .collect();
    if models.is_empty() {
        return None;
    }
    let aggregate = match judge.get("aggregate").and_then(YamlValue::as_str) {
        Some("median") => PanelAggregate::Median,
        Some("majority") => PanelAggregate::Majority,
        // `mean` and anything else (the schema validates the vocabulary)
        // default to mean.
        _ => PanelAggregate::Mean,
    };
    let tie_break = match judge.get("tie_break").and_then(YamlValue::as_str) {
        Some("pass") => TieBreak::Pass,
        _ => TieBreak::Fail,
    };
    Some(JudgePanel {
        models,
        aggregate,
        tie_break,
    })
}

/// Build the verdict cache from the YAML `evals.cache.verdicts` field,
/// applying the CLI override.
///
/// `--no-verdict-cache` always wins; otherwise the YAML opt-in (default
/// off, per ADR 0008) decides. The function is `pub` so the integration
/// test can verify the override directly.
pub fn build_verdict_cache(value: &YamlValue, no_verdict_cache: bool) -> VerdictCache {
    if no_verdict_cache {
        return VerdictCache::disabled();
    }
    let yaml_enabled = value
        .get("evals")
        .and_then(|e| e.get("cache"))
        .and_then(|c| c.get("verdicts"))
        .and_then(YamlValue::as_bool)
        .unwrap_or(false);
    if !yaml_enabled {
        return VerdictCache::disabled();
    }
    let cfg = VerdictCacheConfig {
        enabled: true,
        ..VerdictCacheConfig::default()
    };
    VerdictCache::new(cfg)
}

/// Pretty-print a report to stdout.
///
/// Format mirrors the existing `run` summary: one line per eval plus a
/// trailing total. JSON output stays on the canonical Report when the
/// reporter wiring catches up.
pub(crate) fn render_report(report: &EvalRunReport, quiet: bool) {
    if quiet {
        return;
    }
    for v in &report.verdicts {
        let status = if v.passed { "PASS" } else { "FAIL" };
        println!(
            "{}: {} score={:.2} cost=${:.4} reasoning={}",
            status, v.name, v.score, v.cost_usd, v.reasoning
        );
    }
    println!(
        "Total cost: ${:.4} across {} tests",
        report.total_cost_usd,
        report.verdicts.len()
    );
    // WOR-1393: cost per completed task, not per attempted test, so a run that
    // fails and retries reads as more expensive per result it actually produced.
    if let Some(per_task) = report.effective_cost_per_task() {
        println!(
            "Effective cost per completed task: ${:.4} ({} of {} passed)",
            per_task,
            report.passed_count(),
            report.verdicts.len()
        );
    }
    if report.budget_exceeded {
        println!("Cost budget exceeded; halting run early.");
    }
}

/// Map a run report onto the standard CLI exit codes.
///
/// `5` for a tripped budget, `1` for any failed verdict, `0` for an
/// all-pass run.
pub(crate) fn exit_code_for(report: &EvalRunReport) -> i32 {
    if report.budget_exceeded {
        return 5;
    }
    if report.failure_count() > 0 {
        return 1;
    }
    0
}

/// Fold an [`EvalRunReport`] into the canonical run [`Report`] so every reporter
/// renders the eval envelope (score, pass/fail, per-criterion or tree-path
/// reasons, cost) from one shared shape, with no per-reporter special-casing.
/// A failing eval carries its reasoning as the failure message; the normalized
/// score rides on every test. Cost folds into the report's aggregate cost.
pub(crate) fn to_report(run: &EvalRunReport) -> mcptest_core::report::Report {
    use mcptest_core::report::{Report, RunCost, RunSummary, TestFailure, TestResult, TestStatus};
    let mut passed = 0usize;
    let mut failed = 0usize;
    let mut tests_with_cost = 0u64;
    let tests = run
        .verdicts
        .iter()
        .map(|v| {
            let status = if v.passed {
                passed += 1;
                TestStatus::Pass
            } else {
                failed += 1;
                TestStatus::Fail
            };
            if v.cost_usd > 0.0 {
                tests_with_cost += 1;
            }
            let failure = (!v.passed).then(|| TestFailure {
                message: v.reasoning.clone(),
                diff: None,
                network_diagnostic: None,
            });
            TestResult {
                name: v.name.clone(),
                status,
                duration_ms: 0,
                cache_hit: false,
                started_at: None,
                ended_at: None,
                failure,
                file: None,
                line: None,
                steps: Vec::new(),
                consensus: None,
                rule_id: None,
                severity: None,
                score: Some(v.score),
                derived_metrics: Vec::new(),
                natural_language: false,
            }
        })
        .collect();
    let cost = (run.total_cost_usd > 0.0).then(|| RunCost {
        total_cost_usd: run.total_cost_usd,
        cost_per_juror_avg: 0.0,
        cost_per_test_avg: if tests_with_cost > 0 {
            run.total_cost_usd / tests_with_cost as f64
        } else {
            0.0
        },
        juror_calls: 0,
        tests_with_cost,
        // WOR-1393: cost per passing verdict, so the json carries the same
        // completed-task figure the pretty report prints.
        cost_per_completed_task: run.effective_cost_per_task(),
    });
    Report {
        schema_version: "0.1".to_string(),
        mcptest_version: mcptest_core::version::full().to_string(),
        // The runner leaves these empty at build time and fills metadata
        // separately; the eval path mirrors that.
        generated_at: String::new(),
        run_id: String::new(),
        summary: RunSummary {
            total: run.verdicts.len(),
            passed,
            failed,
            skipped: 0,
            inconclusive: 0,
            duration_ms: 0,
            cache_hits: 0,
            verdict_cache_hits: 0,
            started_at: None,
            ended_at: None,
        },
        tests,
        cost,
        metadata: None,
        matrix: None,
        protocol_violations: Vec::new(),
    }
}

/// Default judge model when the config does not pin one. Per-eval judge
/// configuration (model, jury) lands in WOR-1007.
pub const DEFAULT_EVAL_MODEL: &str = "claude-sonnet-4-5";

/// Read and parse the `evals:` block from a config file.
pub(crate) fn load_evals(config_path: &Path) -> Result<Vec<EvalSpec>> {
    let body = std::fs::read_to_string(config_path)
        .with_context(|| format!("failed to read {}", config_path.display()))?;
    let value: YamlValue = serde_yaml::from_str(&body)
        .with_context(|| format!("failed to parse {} as YAML", config_path.display()))?;
    extract_evals(&value).map(expand_matrix)
}

#[cfg(test)]
mod tests {
    use super::*;
    // The grading runtime moved to the sibling `eval_grade` module; the tests
    // here exercise it through these entry points.
    use crate::commands::eval_grade::{grade_evals, projected_judge_calls};

    struct FakeJudge {
        score: f64,
        cost: f64,
    }

    impl SingleJudge for FakeJudge {
        fn score(&self, _name: &str, _prompt: &str, _rubric: &str) -> Result<(f64, String, f64)> {
            Ok((self.score, "fake reasoning".to_owned(), self.cost))
        }
    }

    fn spec(name: &str, threshold: f64) -> EvalSpec {
        EvalSpec {
            name: name.into(),
            server: "svr".into(),
            prompt: "prompt".into(),
            rubric: "rubric".into(),
            rubric_config: None,
            response: None,
            threshold,
            judge: None,
            matrix: None,
        }
    }

    #[tokio::test]
    async fn grades_a_response_against_a_string_rubric() {
        let mut s = spec("helpful", 0.5);
        s.rubric = "answer must be helpful".into();
        s.response = Some("here is a helpful, on-topic answer".into());
        let provider = mcptest_agent::stub::StubProvider::text(r#"{"score": 0.9}"#);
        let report = grade_evals(
            std::slice::from_ref(&s),
            Some(&provider),
            Some("stub"),
            None,
            None,
            &mut VerdictCache::disabled(),
        )
        .await;
        assert_eq!(report.verdicts.len(), 1);
        assert!(report.verdicts[0].passed);
        assert!((report.verdicts[0].score - 0.9).abs() < 1e-9);
    }

    #[tokio::test]
    async fn second_grading_run_serves_the_verdict_from_the_cache() {
        // WOR-1320: the `mcptest eval` grading path (grade_evals through the
        // eval_cache wrapper) replays a warm verdict. The stub provider
        // scripts exactly one reply, so a second live judging would error;
        // a passing second run with the `cached:` prefix and zero cost
        // proves the verdict came from the cache, not the judge.
        let dir = tempfile::tempdir().unwrap();
        let mut s = spec("cached eval", 0.5);
        s.rubric = "is it good".into();
        s.response = Some("an answer".into());
        let provider = mcptest_agent::stub::StubProvider::text(r#"{"score": 0.9}"#);
        let mut cache = VerdictCache::enabled_at(dir.path());

        let first = grade_evals(
            std::slice::from_ref(&s),
            Some(&provider),
            Some("stub"),
            None,
            None,
            &mut cache,
        )
        .await;
        assert!(first.verdicts[0].passed);
        assert!(!first.verdicts[0].reasoning.starts_with("cached:"));

        let second = grade_evals(
            std::slice::from_ref(&s),
            Some(&provider),
            Some("stub"),
            None,
            None,
            &mut cache,
        )
        .await;
        assert!(second.verdicts[0].passed);
        assert!(
            second.verdicts[0].reasoning.starts_with("cached:"),
            "a cache hit is visible in the reasoning: {}",
            second.verdicts[0].reasoning
        );
        assert!(
            second.total_cost_usd.abs() < 1e-12,
            "a cache hit spends nothing"
        );
    }

    #[tokio::test]
    async fn grading_defers_without_a_provider() {
        let mut s = spec("q", 0.5);
        s.response = Some("x".into());
        let report = grade_evals(
            std::slice::from_ref(&s),
            None,
            None,
            None,
            None,
            &mut VerdictCache::disabled(),
        )
        .await;
        assert!(report.verdicts[0].passed);
        assert!(report.verdicts[0].reasoning.contains("deferred"));
    }

    #[tokio::test]
    async fn grading_defers_without_a_response() {
        let s = spec("q", 0.5);
        let provider = mcptest_agent::stub::StubProvider::text(r#"{"score": 0.9}"#);
        let report = grade_evals(
            std::slice::from_ref(&s),
            Some(&provider),
            Some("stub"),
            None,
            None,
            &mut VerdictCache::disabled(),
        )
        .await;
        assert!(report.verdicts[0].passed);
        assert!(report.verdicts[0].reasoning.contains("response"));
    }

    #[test]
    fn parses_free_form_string_rubric() {
        let value: YamlValue = serde_yaml::from_str(
            r#"
evals:
  - name: smoke
    server: main
    prompt: "do a thing"
    rubric: "must mention the release tag"
    threshold: 0.6
"#,
        )
        .unwrap();
        let evals = extract_evals(&value).expect("parses");
        assert_eq!(evals.len(), 1);
        assert_eq!(evals[0].rubric, "must mention the release tag");
        assert!(evals[0].rubric_config.is_none());
        assert!((evals[0].threshold - 0.6).abs() < 1e-9);
    }

    #[test]
    fn parses_structured_criteria_rubric() {
        let value: YamlValue = serde_yaml::from_str(
            r#"
evals:
  - name: quality
    server: main
    prompt: "summarize"
    rubric:
      threshold: 0.8
      criteria:
        - name: on topic
          description: "stays on subject"
          weight: 2
        - name: concise
          description: "no filler"
"#,
        )
        .unwrap();
        let evals = extract_evals(&value).expect("parses");
        assert_eq!(evals.len(), 1);
        assert!(evals[0].rubric.is_empty());
        let cfg = evals[0].rubric_config.as_ref().expect("structured rubric");
        assert_eq!(cfg.criteria.len(), 2);
        assert!((cfg.threshold - 0.8).abs() < 1e-9);
    }

    #[test]
    fn malformed_structured_rubric_is_an_error() {
        let value: YamlValue = serde_yaml::from_str(
            r#"
evals:
  - name: broken
    server: main
    prompt: "x"
    rubric:
      criteria: []
"#,
        )
        .unwrap();
        assert!(extract_evals(&value).is_err());
    }

    #[test]
    fn resolves_a_named_rubric_via_ref_with_override() {
        let value: YamlValue = serde_yaml::from_str(
            r#"
rubrics:
  quality:
    threshold: 0.5
    criteria:
      - name: on topic
        description: "stays on subject"
evals:
  - name: uses shared rubric
    server: main
    prompt: "x"
    rubric:
      ref: quality
      threshold: 0.9
"#,
        )
        .unwrap();
        let evals = extract_evals(&value).expect("parses");
        let cfg = evals[0].rubric_config.as_ref().expect("resolved rubric");
        assert_eq!(cfg.criteria.len(), 1);
        // The inline override wins over the named rubric's threshold.
        assert!((cfg.threshold - 0.9).abs() < 1e-9);
    }

    #[test]
    fn resolves_a_preset_and_appends_extra_criteria() {
        let value: YamlValue = serde_yaml::from_str(
            r#"
evals:
  - name: grounded and brief
    server: main
    prompt: "x"
    rubric:
      preset: groundedness
      threshold: 0.8
      criteria:
        - name: brief
          description: "no filler"
"#,
        )
        .unwrap();
        let evals = extract_evals(&value).expect("parses");
        let cfg = evals[0].rubric_config.as_ref().expect("resolved preset");
        // groundedness ships 2 criteria; the appended one makes 3.
        assert_eq!(cfg.criteria.len(), 3);
        assert!((cfg.threshold - 0.8).abs() < 1e-9);
    }

    #[test]
    fn unknown_preset_is_an_error() {
        let value: YamlValue = serde_yaml::from_str(
            r#"
evals:
  - name: bad
    server: main
    prompt: "x"
    rubric:
      preset: does-not-exist
"#,
        )
        .unwrap();
        assert!(extract_evals(&value).is_err());
    }

    #[test]
    fn unknown_rubric_ref_is_an_error() {
        let value: YamlValue = serde_yaml::from_str(
            r#"
evals:
  - name: dangling
    server: main
    prompt: "x"
    rubric:
      ref: nope
"#,
        )
        .unwrap();
        assert!(extract_evals(&value).is_err());
    }

    #[test]
    fn missing_prompt_is_an_error() {
        let value: YamlValue = serde_yaml::from_str(
            r#"
evals:
  - name: noprompt
    server: main
"#,
        )
        .unwrap();
        assert!(extract_evals(&value).is_err());
    }

    #[test]
    fn passing_run_returns_zero_failures() {
        let evals = vec![spec("a", 0.5), spec("b", 0.5)];
        let mut cache = VerdictCache::disabled();
        let report = run_eval_with_judge(
            &evals,
            &FakeJudge {
                score: 0.9,
                cost: 0.01,
            },
            None,
            &mut cache,
        );
        assert_eq!(report.failure_count(), 0);
        assert!((report.total_cost_usd - 0.02).abs() < 1e-9);
        assert_eq!(exit_code_for(&report), 0);
    }

    #[test]
    fn failing_score_below_threshold_marks_test_failed() {
        let evals = vec![spec("a", 0.9)];
        let mut cache = VerdictCache::disabled();
        let report = run_eval_with_judge(
            &evals,
            &FakeJudge {
                score: 0.5,
                cost: 0.0,
            },
            None,
            &mut cache,
        );
        assert_eq!(report.failure_count(), 1);
        assert_eq!(exit_code_for(&report), 1);
    }

    #[test]
    fn budget_exceeded_halts_early_and_reports_code_five() {
        let evals = vec![spec("a", 0.0), spec("b", 0.0), spec("c", 0.0)];
        let mut cache = VerdictCache::disabled();
        let report = run_eval_with_judge(
            &evals,
            &FakeJudge {
                score: 1.0,
                cost: 0.2,
            },
            Some(0.25),
            &mut cache,
        );
        assert!(report.budget_exceeded);
        assert_eq!(exit_code_for(&report), 5);
        assert!(report.verdicts.len() < 3);
    }

    /// Judge that counts how many times it is consulted, so the cache tests
    /// can assert a hit skipped the call entirely.
    struct CountingJudge {
        calls: std::cell::Cell<usize>,
        score: f64,
    }

    impl SingleJudge for CountingJudge {
        fn score(&self, _name: &str, _prompt: &str, _rubric: &str) -> Result<(f64, String, f64)> {
            self.calls.set(self.calls.get() + 1);
            Ok((self.score, "freshly judged".to_owned(), 0.01))
        }
    }

    #[test]
    fn warm_enabled_verdict_cache_skips_the_judge_call() {
        let dir = tempfile::tempdir().unwrap();
        let evals = vec![spec("a", 0.5)];
        let judge = CountingJudge {
            calls: std::cell::Cell::new(0),
            score: 0.9,
        };
        let mut cache = VerdictCache::enabled_at(dir.path());

        let first = run_eval_with_judge(&evals, &judge, None, &mut cache);
        assert_eq!(judge.calls.get(), 1);
        assert!(first.verdicts[0].passed);

        let second = run_eval_with_judge(&evals, &judge, None, &mut cache);
        assert_eq!(judge.calls.get(), 1, "second run served from the cache");
        assert!(second.verdicts[0].passed);
        assert!(
            second.verdicts[0].reasoning.starts_with("cached:"),
            "a cache hit is visible in the reasoning: {}",
            second.verdicts[0].reasoning
        );
        assert!(
            second.total_cost_usd.abs() < 1e-12,
            "a cache hit spends nothing"
        );
    }

    #[test]
    fn no_verdict_cache_flag_forces_a_fresh_judge_call_every_run() {
        // The YAML opts in, the flag overrides: build_verdict_cache returns a
        // disabled cache that always misses (and never stores), so the judge
        // runs every time (WOR-1303).
        let yaml: YamlValue = serde_yaml::from_str(
            r#"
evals:
  cache:
    verdicts: true
"#,
        )
        .unwrap();
        let mut cache = build_verdict_cache(&yaml, true);
        assert!(!cache.is_enabled());

        let evals = vec![spec("a", 0.5)];
        let judge = CountingJudge {
            calls: std::cell::Cell::new(0),
            score: 0.9,
        };
        run_eval_with_judge(&evals, &judge, None, &mut cache);
        run_eval_with_judge(&evals, &judge, None, &mut cache);
        assert_eq!(
            judge.calls.get(),
            2,
            "with the cache disabled, every run judges afresh"
        );
    }

    #[tokio::test]
    async fn likert_scale_shows_the_native_value_in_reasoning() {
        // A normalized 0.75 on a 1-5 Likert renders as 4.0/5 for the human
        // reader while the verdict's `score` stays the normalized 0.75.
        let value: YamlValue = serde_yaml::from_str(
            r#"
evals:
  - name: graded
    server: main
    prompt: "x"
    response: "an answer"
    rubric:
      scale: { likert: { min: 1, max: 5 } }
      criteria:
        - name: quality
          description: "is it good"
"#,
        )
        .unwrap();
        let evals = extract_evals(&value).expect("parses");
        let provider = mcptest_agent::stub::StubProvider::text(r#"{"score": 0.75}"#);
        let report = grade_evals(
            &evals,
            Some(&provider),
            Some("stub"),
            None,
            None,
            &mut VerdictCache::disabled(),
        )
        .await;
        assert!((report.verdicts[0].score - 0.75).abs() < 1e-9);
        assert!(
            report.verdicts[0].reasoning.contains("4.0/5"),
            "native scale in reasoning: {}",
            report.verdicts[0].reasoning
        );
    }

    #[test]
    fn parses_a_judge_block_with_model_and_jury() {
        let value: YamlValue = serde_yaml::from_str(
            r#"
evals:
  - name: juried
    server: main
    prompt: "x"
    rubric: "good?"
    judge:
      model: claude-sonnet-4-5
      jury: { size: 3, consensus: 0.66 }
"#,
        )
        .unwrap();
        let evals = extract_evals(&value).expect("parses");
        let judge = evals[0].judge.as_ref().expect("judge block");
        assert_eq!(judge.model.as_deref(), Some("claude-sonnet-4-5"));
        let jury = judge.jury.as_ref().expect("jury");
        assert_eq!(jury.size, 3);
        assert!((jury.consensus - 0.66).abs() < 1e-9);
    }

    #[tokio::test]
    async fn jury_passes_when_enough_judges_agree() {
        // Three judges score 0.9, 0.2, 0.9 against a 0.5 threshold: 2 of 3
        // pass, which clears the 0.66 consensus.
        let mut s = spec("juried", 0.5);
        s.rubric = "is it good".into();
        s.response = Some("an answer".into());
        s.judge = Some(JudgeConfig {
            model: None,
            jury: Some(JuryPlan {
                size: 3,
                consensus: 0.66,
            }),
            panel: None,
        });
        let provider = mcptest_agent::stub::StubProvider::new(vec![
            mcptest_agent::stub::StubReply::Text(r#"{"score": 0.9}"#.into()),
            mcptest_agent::stub::StubReply::Text(r#"{"score": 0.2}"#.into()),
            mcptest_agent::stub::StubReply::Text(r#"{"score": 0.9}"#.into()),
        ]);
        let report = grade_evals(
            std::slice::from_ref(&s),
            Some(&provider),
            Some("stub"),
            None,
            None,
            &mut VerdictCache::disabled(),
        )
        .await;
        assert!(report.verdicts[0].passed, "2/3 >= 0.66");
        assert!(
            report.verdicts[0].reasoning.contains("2/3"),
            "reasoning names the vote: {}",
            report.verdicts[0].reasoning
        );
    }

    #[tokio::test]
    async fn jury_fails_when_consensus_is_not_reached() {
        let mut s = spec("juried", 0.5);
        s.rubric = "is it good".into();
        s.response = Some("an answer".into());
        s.judge = Some(JudgeConfig {
            model: None,
            jury: Some(JuryPlan {
                size: 3,
                consensus: 0.9,
            }),
            panel: None,
        });
        let provider = mcptest_agent::stub::StubProvider::new(vec![
            mcptest_agent::stub::StubReply::Text(r#"{"score": 0.9}"#.into()),
            mcptest_agent::stub::StubReply::Text(r#"{"score": 0.2}"#.into()),
            mcptest_agent::stub::StubReply::Text(r#"{"score": 0.9}"#.into()),
        ]);
        let report = grade_evals(
            std::slice::from_ref(&s),
            Some(&provider),
            Some("stub"),
            None,
            None,
            &mut VerdictCache::disabled(),
        )
        .await;
        assert!(!report.verdicts[0].passed, "2/3 < 0.9 consensus");
    }

    #[test]
    fn projected_calls_multiply_by_jury_size() {
        // One eval, two criteria, jury size 3 = 6 projected judge calls.
        let value: YamlValue = serde_yaml::from_str(
            r#"
evals:
  - name: q
    server: main
    prompt: "x"
    rubric:
      criteria:
        - name: a
          description: "a"
        - name: b
          description: "b"
    judge:
      jury: { size: 3, consensus: 0.5 }
"#,
        )
        .unwrap();
        let evals = extract_evals(&value).unwrap();
        assert_eq!(projected_judge_calls(&evals), 6);
    }

    #[test]
    fn parses_a_judge_panel_with_aggregate_and_tie_break() {
        let value: YamlValue = serde_yaml::from_str(
            r#"
evals:
  - name: panelled
    server: main
    prompt: "x"
    rubric: "good?"
    judge:
      panel: [claude-sonnet-4-5, claude-haiku-4-5]
      aggregate: median
      tie_break: pass
"#,
        )
        .unwrap();
        let evals = extract_evals(&value).expect("parses");
        let panel = evals[0]
            .judge
            .as_ref()
            .and_then(|j| j.panel.as_ref())
            .expect("panel");
        assert_eq!(panel.models.len(), 2);
        assert_eq!(panel.aggregate, PanelAggregate::Median);
        assert_eq!(panel.tie_break, TieBreak::Pass);
    }

    fn panel_spec(models: &[&str], aggregate: PanelAggregate, tie_break: TieBreak) -> EvalSpec {
        let mut s = spec("panelled", 0.5);
        s.rubric = "is it good".into();
        s.response = Some("an answer".into());
        s.judge = Some(JudgeConfig {
            model: None,
            jury: None,
            panel: Some(JudgePanel {
                models: models.iter().map(|m| m.to_string()).collect(),
                aggregate,
                tie_break,
            }),
        });
        s
    }

    fn score_replies(scores: &[f64]) -> mcptest_agent::stub::StubProvider {
        let replies = scores
            .iter()
            .map(|s| mcptest_agent::stub::StubReply::Text(format!(r#"{{"score": {s}}}"#)))
            .collect();
        mcptest_agent::stub::StubProvider::new(replies)
    }

    #[tokio::test]
    async fn panel_majority_passes_when_most_models_pass() {
        // Three models score 0.9, 0.2, 0.9 against a 0.5 threshold: 2 of 3
        // pass, a majority.
        let s = panel_spec(
            &["m1", "m2", "m3"],
            PanelAggregate::Majority,
            TieBreak::Fail,
        );
        let provider = score_replies(&[0.9, 0.2, 0.9]);
        let report = grade_evals(
            std::slice::from_ref(&s),
            Some(&provider),
            Some("stub"),
            None,
            None,
            &mut VerdictCache::disabled(),
        )
        .await;
        assert!(report.verdicts[0].passed, "2 of 3 is a majority");
        assert!(report.verdicts[0].reasoning.contains("panel"));
    }

    #[tokio::test]
    async fn panel_majority_tie_breaks_by_the_rule() {
        // Two models split 1-1; the tie-break decides.
        let fail = panel_spec(&["m1", "m2"], PanelAggregate::Majority, TieBreak::Fail);
        let report = grade_evals(
            std::slice::from_ref(&fail),
            Some(&score_replies(&[0.9, 0.2])),
            Some("stub"),
            None,
            None,
            &mut VerdictCache::disabled(),
        )
        .await;
        assert!(!report.verdicts[0].passed, "1-1 tie breaks to fail");

        let pass = panel_spec(&["m1", "m2"], PanelAggregate::Majority, TieBreak::Pass);
        let report = grade_evals(
            std::slice::from_ref(&pass),
            Some(&score_replies(&[0.9, 0.2])),
            Some("stub"),
            None,
            None,
            &mut VerdictCache::disabled(),
        )
        .await;
        assert!(report.verdicts[0].passed, "1-1 tie breaks to pass");
    }

    #[tokio::test]
    async fn panel_mean_aggregate_uses_the_average_score() {
        // Mean of 0.9 and 0.2 is 0.55, which clears the 0.5 threshold even
        // though a majority vote would tie.
        let s = panel_spec(&["m1", "m2"], PanelAggregate::Mean, TieBreak::Fail);
        let provider = score_replies(&[0.9, 0.2]);
        let report = grade_evals(
            std::slice::from_ref(&s),
            Some(&provider),
            Some("stub"),
            None,
            None,
            &mut VerdictCache::disabled(),
        )
        .await;
        assert!(report.verdicts[0].passed, "mean 0.55 >= 0.5");
        assert!((report.verdicts[0].score - 0.55).abs() < 1e-9);
    }

    #[test]
    fn the_eval_envelope_renders_in_every_reporter() {
        use crate::cli::reporters::{build_reporter, RenderFormat};
        let run = EvalRunReport {
            verdicts: vec![
                EvalVerdict {
                    name: "stays on topic".into(),
                    passed: true,
                    score: 0.9,
                    reasoning: "on topic=0.90".into(),
                    cost_usd: 0.0,
                },
                EvalVerdict {
                    name: "cites the release tag".into(),
                    passed: false,
                    score: 0.3,
                    reasoning: "missing tag=0.30".into(),
                    cost_usd: 0.0,
                },
            ],
            total_cost_usd: 0.0,
            budget_exceeded: false,
            live_candidates: BTreeMap::new(),
        };
        let report = to_report(&run);
        assert_eq!(report.summary.total, 2);
        assert_eq!(report.summary.passed, 1);
        assert_eq!(report.summary.failed, 1);

        // Every reporter consumes the same canonical report and produces
        // output, with no eval-specific branch in any of them.
        for format in [
            RenderFormat::Pretty,
            RenderFormat::Json,
            RenderFormat::Junit,
            RenderFormat::Md,
            RenderFormat::Html,
            RenderFormat::Sarif,
            RenderFormat::Gitlab,
            RenderFormat::Ndjson,
            RenderFormat::Tap,
        ] {
            let mut buf = Vec::new();
            build_reporter(format, false, false, false, None)
                .render(&report, &mut buf)
                .unwrap_or_else(|e| panic!("{format:?} render failed: {e}"));
            assert!(
                !buf.is_empty(),
                "{format:?} produced no output for the eval envelope"
            );
        }

        // The JSON reporter is the round-trippable canonical shape: it must
        // carry the score and the per-eval verdicts.
        let mut json = Vec::new();
        build_reporter(RenderFormat::Json, false, false, false, None)
            .render(&report, &mut json)
            .unwrap();
        let json = String::from_utf8(json).unwrap();
        assert!(json.contains("stays on topic"));
        assert!(json.contains("cites the release tag"));
        assert!(json.contains("0.9"), "the normalized score renders: {json}");
    }

    #[test]
    fn effective_cost_per_task_divides_by_passing_not_attempted() {
        // Two passing tasks, one failing, $0.30 total: $0.30 / 2 passed = $0.15
        // per completed task, more than the $0.10 per attempted test, because
        // the failed attempt still cost money but completed nothing.
        let verdict = |name: &str, passed: bool, cost: f64| EvalVerdict {
            name: name.into(),
            passed,
            score: if passed { 1.0 } else { 0.0 },
            reasoning: String::new(),
            cost_usd: cost,
        };
        let run = EvalRunReport {
            verdicts: vec![
                verdict("a", true, 0.10),
                verdict("b", true, 0.10),
                verdict("c", false, 0.10),
            ],
            total_cost_usd: 0.30,
            budget_exceeded: false,
            live_candidates: BTreeMap::new(),
        };
        assert_eq!(run.passed_count(), 2);
        assert_eq!(run.effective_cost_per_task(), Some(0.15));
        // It also reaches the json via the canonical report's cost block.
        let report = to_report(&run);
        assert_eq!(report.cost.unwrap().cost_per_completed_task, Some(0.15));
    }

    #[test]
    fn effective_cost_per_task_is_none_when_nothing_passed_or_nothing_spent() {
        let failing = EvalRunReport {
            verdicts: vec![EvalVerdict {
                name: "x".into(),
                passed: false,
                score: 0.0,
                reasoning: String::new(),
                cost_usd: 0.5,
            }],
            total_cost_usd: 0.5,
            budget_exceeded: false,
            live_candidates: BTreeMap::new(),
        };
        assert_eq!(failing.effective_cost_per_task(), None);
        let free = EvalRunReport {
            verdicts: vec![EvalVerdict {
                name: "x".into(),
                passed: true,
                score: 1.0,
                reasoning: String::new(),
                cost_usd: 0.0,
            }],
            total_cost_usd: 0.0,
            budget_exceeded: false,
            live_candidates: BTreeMap::new(),
        };
        assert_eq!(free.effective_cost_per_task(), None);
    }

    #[test]
    fn a_matrix_expands_into_one_cell_per_model_and_prompt() {
        let value: YamlValue = serde_yaml::from_str(
            r#"
evals:
  - name: quality
    server: main
    prompt: "base prompt"
    response: "an answer"
    rubric: "good?"
    matrix:
      models: [claude-sonnet-4-5, claude-haiku-4-5]
      prompts: ["summarize tersely", "summarize in detail"]
"#,
        )
        .unwrap();
        let evals = extract_evals(&value).expect("parses");
        assert!(evals[0].matrix.is_some());
        let cells = expand_matrix(evals);
        // 2 models x 2 prompts = 4 cells, each reusing the same rubric.
        assert_eq!(cells.len(), 4);
        assert!(cells.iter().all(|c| c.matrix.is_none()));
        assert!(cells
            .iter()
            .all(|c| c.judge.as_ref().and_then(|j| j.model.as_deref()).is_some()));
        assert!(cells[0].name.contains("model=claude-sonnet-4-5"));
        assert!(cells[0].name.contains("prompt 1"));
        assert!(cells.iter().any(|c| c.prompt == "summarize in detail"));
    }

    #[test]
    fn a_models_only_matrix_makes_one_cell_per_model() {
        let value: YamlValue = serde_yaml::from_str(
            r#"
evals:
  - name: judge-compare
    server: main
    prompt: "x"
    response: "an answer"
    rubric: "good?"
    matrix:
      models: [m1, m2, m3]
"#,
        )
        .unwrap();
        let cells = expand_matrix(extract_evals(&value).unwrap());
        assert_eq!(cells.len(), 3);
        assert_eq!(
            cells[0].judge.as_ref().unwrap().model.as_deref(),
            Some("m1")
        );
        assert!(cells[2].name.contains("model=m3"));
    }

    #[test]
    fn an_eval_without_a_matrix_passes_through_expansion() {
        let value: YamlValue = serde_yaml::from_str(
            r#"
evals:
  - name: plain
    server: main
    prompt: "x"
    rubric: "good?"
"#,
        )
        .unwrap();
        let cells = expand_matrix(extract_evals(&value).unwrap());
        assert_eq!(cells.len(), 1);
        assert_eq!(cells[0].name, "plain");
    }
}
