//! Grading runtime for the top-level `evals:` test type.
//!
//! Turns a parsed [`EvalSpec`] into a verdict: resolves the candidate
//! (deterministic `response:` or a live agent run), runs the rubric engine with
//! the configured judge (single, jury, or multi-model panel), totals cost, and
//! gates on the threshold. Split out of `commands::eval` so that module stays
//! under the production-module-size guard; the types and reporter conversion
//! stay there.

use mcptest_core::eval::provider::LlmProvider;
use mcptest_core::eval::rubric::{evaluate_rubric, AggregationMode, Criterion, RubricConfig};
use mcptest_core::eval::VerdictCache;

use super::eval::{
    EvalRunReport, EvalSpec, EvalVerdict, JudgePanel, JuryPlan, PanelAggregate, TieBreak,
};
use super::eval_cache::{evaluate_rubric_cached, rubric_reasoning, RubricGrade};

/// Print a dry-run plan for each eval: what it would grade, against which
/// rubric, with which candidate source and judge model, and how many judge
/// calls it would make. Calls no provider and spends nothing. Used by
/// `mcptest eval --explain`.
pub(crate) fn explain_evals(evals: &[EvalSpec], model: &str) {
    if evals.is_empty() {
        println!("mcptest eval --explain: no evals to plan.");
        return;
    }
    println!("Eval plan ({} evals, judge model {model}):\n", evals.len());
    for spec in evals {
        let candidate = match &spec.response {
            Some(_) => "fixed response (deterministic)",
            None => "live agent run (needs a provider and a connected server)",
        };
        println!("- {} [{}]", spec.name, candidate);
        match &spec.rubric_config {
            Some(cfg) if cfg.tree.is_some() => {
                println!("    rubric: decision tree (threshold {:.2})", cfg.threshold);
                println!("    judge calls: 1 per question along the path taken");
            }
            Some(cfg) => {
                let aggregation = match cfg.aggregation {
                    AggregationMode::Min => ", min (worst criterion)",
                    _ => "",
                };
                println!(
                    "    rubric: {} criteria (threshold {:.2}){}{}",
                    cfg.criteria.len(),
                    cfg.threshold,
                    if cfg.strict { ", strict" } else { "" },
                    aggregation
                );
                for c in &cfg.criteria {
                    let tags = match (c.required, c.guard) {
                        (true, _) => " (required)",
                        (_, true) => " (guard)",
                        _ => "",
                    };
                    println!("      - {}{}: {}", c.name, tags, c.description);
                }
                println!("    judge calls: {}", cfg.criteria.len());
            }
            None => {
                println!(
                    "    rubric: free-form string (threshold {:.2})",
                    spec.threshold
                );
                println!("    \"{}\"", spec.rubric);
                println!("    judge calls: 1");
            }
        }
    }
}

/// The live-tier candidate source for grading: the connected server pool plus
/// the agent budget. Present when `mcptest eval` resolved a provider and
/// connected the servers the evals reference; `None` for the key-free or
/// no-server path, where evals without a `response:` defer.
pub(crate) struct LiveCandidates<'a> {
    /// Pool of connected servers, keyed by the `server:` name.
    pub pool: &'a mcptest_core::connector::ServerPool,
    /// Agent run budget (token and cost caps), shared across all live evals.
    pub budget: std::sync::Arc<mcptest_agent::Budget>,
}

/// Grade every eval against its rubric with a real provider.
///
/// The candidate is the deterministic `response` field when present, otherwise
/// a live tool-using agent run against the eval's connected `server` (the whole
/// run envelope is graded, matching the agent-side `eval.rubric` target). An
/// eval with no provider, or no candidate source at all, defers (passed, with a
/// note) so a key-free CI run stays green. Cost is not yet surfaced by the
/// rubric engine, so per-verdict cost is reported as zero until WOR-1007
/// threads it through.
///
/// `cache` is the WOR-1320 verdict cache, consulted only on the single-judge
/// path (see [`grade_candidate`]); a disabled cache always misses, which is
/// what `--no-verdict-cache` builds.
pub(crate) async fn grade_evals(
    evals: &[EvalSpec],
    provider: Option<&dyn LlmProvider>,
    model: Option<&str>,
    live: Option<&LiveCandidates<'_>>,
    max_cost: Option<f64>,
    cache: &mut VerdictCache,
) -> EvalRunReport {
    let mut verdicts = Vec::with_capacity(evals.len());
    let mut live_candidates = std::collections::BTreeMap::new();
    let mut total_cost = 0.0;
    let mut budget_exceeded = false;
    for spec in evals {
        // WOR-1511: the budget left for this eval, so a jury inside grade_one
        // stops before blowing `--max-cost` ~N-fold (the between-eval check
        // below only fires after an eval has already fully run).
        let remaining = max_cost.map(|cap| (cap - total_cost).max(0.0));
        let (verdict, live_candidate) =
            grade_one(spec, provider, model, live, remaining, cache).await;
        if let Some(text) = live_candidate {
            live_candidates.insert(spec.name.clone(), text);
        }
        total_cost += verdict.cost_usd;
        verdicts.push(verdict);
        // Stop once the cumulative judge spend crosses the cap so a long suite
        // does not blow the budget after it is already exhausted.
        if max_cost.is_some_and(|cap| total_cost > cap + 1e-9) {
            budget_exceeded = true;
            break;
        }
    }
    EvalRunReport {
        verdicts,
        total_cost_usd: total_cost,
        budget_exceeded,
        live_candidates,
    }
}

/// Projected number of judge calls a full grading run would make, so the run
/// header can warn before any spend. Each criterion is one call (a free-form
/// string or a tree counts as one), multiplied by the jury size when set.
pub(crate) fn projected_judge_calls(evals: &[EvalSpec]) -> usize {
    evals.iter().map(judge_calls_for).sum()
}

fn judge_calls_for(spec: &EvalSpec) -> usize {
    let per_judge = match &spec.rubric_config {
        Some(cfg) if !cfg.criteria.is_empty() => cfg.criteria.len(),
        // A tree walks at least one ask; a free-form string is one call.
        _ => 1,
    };
    // A panel multiplies by its model count; a jury by its size; otherwise one.
    let judges = match spec.judge.as_ref() {
        Some(j) if j.panel.as_ref().is_some_and(|p| !p.models.is_empty()) => {
            j.panel.as_ref().map(|p| p.models.len()).unwrap_or(1)
        }
        Some(j) => j
            .jury
            .as_ref()
            .map(|jr| jr.size.max(1) as usize)
            .unwrap_or(1),
        None => 1,
    };
    per_judge * judges
}

async fn grade_one(
    spec: &EvalSpec,
    provider: Option<&dyn LlmProvider>,
    model: Option<&str>,
    live: Option<&LiveCandidates<'_>>,
    remaining: Option<f64>,
    cache: &mut VerdictCache,
) -> (EvalVerdict, Option<String>) {
    let (Some(provider), Some(default_model)) = (provider, model) else {
        return (
            deferred(spec, "no LLM provider (set a model API key to grade)"),
            None,
        );
    };
    // The candidate is produced with the run's default model (a live agent
    // run); the judge model can be overridden per eval and grades that
    // candidate.
    let candidate = match resolve_candidate(spec, provider, default_model, live).await {
        Ok(Some(candidate)) => candidate,
        Ok(None) => {
            return (
                deferred(
                    spec,
                    "no `response` to grade and no connected `server` for a live run",
                ),
                None,
            )
        }
        Err(note) => return (error_verdict(spec, note), None),
    };
    // Capture the live candidate text for the external-scorer pass, but only
    // for a live eval (no deterministic `response`): a model run produced this
    // candidate and the scorer pass would otherwise score an empty string.
    let live_candidate = spec.response.is_none().then(|| {
        candidate
            .as_str()
            .map(str::to_owned)
            .unwrap_or_else(|| candidate.to_string())
    });
    let config = rubric_for(spec);
    let judge_model = spec
        .judge
        .as_ref()
        .and_then(|j| j.model.as_deref())
        .unwrap_or(default_model);
    // A panel (ensemble) takes precedence over a jury, which takes precedence
    // over a single judge.
    let verdict = if let Some(panel) = spec.judge.as_ref().and_then(|j| j.panel.as_ref()) {
        grade_panel(spec, &config, &candidate, provider, panel).await
    } else {
        let jury = spec.judge.as_ref().and_then(|j| j.jury.as_ref());
        grade_candidate(
            spec,
            &config,
            &candidate,
            provider,
            judge_model,
            jury,
            remaining,
            cache,
        )
        .await
    };
    (verdict, live_candidate)
}

/// Grade a candidate with a multi-model panel: each model judges once, scores
/// combine by the panel's aggregate rule, and an even majority split is broken
/// by `tie_break`. The reported score is the mean across the panel; the cost is
/// the sum. OSS panels run every model through the one resolved provider.
///
/// Panels never consult the verdict cache (WOR-1320): the reasoning line
/// reports a per-model breakdown that a flat [`mcptest_core::eval::VerdictRecord`]
/// cannot carry, so a cached panel verdict would replay with its evidence
/// missing. Single-judge grading is the cached path.
async fn grade_panel(
    spec: &EvalSpec,
    config: &RubricConfig,
    candidate: &serde_json::Value,
    provider: &dyn LlmProvider,
    panel: &JudgePanel,
) -> EvalVerdict {
    let mut scores = Vec::with_capacity(panel.models.len());
    let mut passes = 0u32;
    let mut fails = 0u32;
    let mut cost = 0.0;
    let mut per_model = Vec::with_capacity(panel.models.len());
    for model in &panel.models {
        match evaluate_rubric(config, candidate, provider, model).await {
            Ok(out) => {
                if out.passed {
                    passes += 1;
                } else {
                    fails += 1;
                }
                scores.push(out.score);
                cost += out.cost_usd;
                per_model.push(format!(
                    "{model}={:.2}{}",
                    out.score,
                    if out.passed { " pass" } else { " fail" }
                ));
            }
            Err(e) => return error_verdict(spec, format!("panel judge `{model}` error: {e}")),
        }
    }
    let mean = scores.iter().sum::<f64>() / scores.len().max(1) as f64;
    let passed = match panel.aggregate {
        PanelAggregate::Mean => mean >= config.threshold,
        PanelAggregate::Median => median(&scores) >= config.threshold,
        PanelAggregate::Majority => match passes.cmp(&fails) {
            std::cmp::Ordering::Greater => true,
            std::cmp::Ordering::Less => false,
            std::cmp::Ordering::Equal => matches!(panel.tie_break, TieBreak::Pass),
        },
    };
    EvalVerdict {
        name: spec.name.clone(),
        passed,
        score: mean,
        reasoning: format!(
            "panel ({} of {} models): {}",
            panel.aggregate.label(),
            panel.models.len(),
            per_model.join(", ")
        ),
        cost_usd: cost,
    }
}

/// Median of a score list. Returns 0.0 for an empty list; for an even count it
/// averages the two middle values.
fn median(scores: &[f64]) -> f64 {
    if scores.is_empty() {
        return 0.0;
    }
    let mut sorted = scores.to_vec();
    sorted.sort_by(f64::total_cmp);
    let mid = sorted.len() / 2;
    if sorted.len() % 2 == 0 {
        (sorted[mid - 1] + sorted[mid]) / 2.0
    } else {
        sorted[mid]
    }
}

/// Grade a resolved candidate against the rubric, optionally as a jury.
///
/// The single-judge path goes through the WOR-1320 verdict cache: a fresh
/// hit replays the stored verdict at zero cost with a `cached:` prefix on
/// its reasoning. A jury never consults the cache: its whole value is `size`
/// independent judgments, and a cache hit would collapse them into one
/// sample (every juror shares the same key). It runs the rubric `size` times
/// and passes when at least `consensus` of the judges pass; the reported
/// score is the mean and the cost is the sum across judges.
#[allow(clippy::too_many_arguments)]
async fn grade_candidate(
    spec: &EvalSpec,
    config: &RubricConfig,
    candidate: &serde_json::Value,
    provider: &dyn LlmProvider,
    judge_model: &str,
    jury: Option<&JuryPlan>,
    remaining: Option<f64>,
    cache: &mut VerdictCache,
) -> EvalVerdict {
    let Some(plan) = jury else {
        return match evaluate_rubric_cached(
            config,
            candidate,
            provider,
            judge_model,
            &spec.prompt,
            cache,
        )
        .await
        {
            Ok(RubricGrade::Fresh(out)) => EvalVerdict {
                name: spec.name.clone(),
                passed: out.passed,
                score: out.score,
                reasoning: rubric_reasoning(&out, &config.scale),
                cost_usd: out.cost_usd,
            },
            Ok(RubricGrade::Cached(hit)) => EvalVerdict {
                name: spec.name.clone(),
                passed: hit.passed,
                score: hit.score,
                reasoning: hit.reasoning,
                cost_usd: 0.0,
            },
            Err(e) => error_verdict(spec, format!("rubric provider error: {e}")),
        };
    };
    let n = plan.size.max(1);
    let mut passes = 0u32;
    let mut score_sum = 0.0;
    let mut cost = 0.0;
    let mut ran = 0u32;
    for _ in 0..n {
        // WOR-1511: stop the jury once this eval's spend reaches its budget so a
        // large `jury.size` cannot blow `--max-cost` ~N-fold.
        if !budget_allows(cost, remaining) {
            break;
        }
        match evaluate_rubric(config, candidate, provider, judge_model).await {
            Ok(out) => {
                if out.passed {
                    passes += 1;
                }
                score_sum += out.score;
                cost += out.cost_usd;
                ran += 1;
            }
            Err(e) => return error_verdict(spec, format!("rubric provider error: {e}")),
        }
    }
    // Consensus and the mean score are over the judges that actually ran, so a
    // budget-truncated jury still reports a coherent fraction.
    let denom = ran.max(1);
    let frac = passes as f64 / denom as f64;
    let truncated = if ran < n { " (budget-truncated)" } else { "" };
    EvalVerdict {
        name: spec.name.clone(),
        passed: frac >= plan.consensus,
        score: score_sum / denom as f64,
        reasoning: format!(
            "jury {passes}/{ran} judges passed (consensus {:.2}){truncated}",
            plan.consensus
        ),
        cost_usd: cost,
    }
}

/// Whether another jury judge may run, given the spend so far and the budget
/// remaining for this eval. A `None` budget always allows; otherwise the jury
/// stops once its spend reaches the cap (WOR-1511), so a large `jury.size`
/// cannot overshoot `--max-cost` before the between-eval check fires.
fn budget_allows(cost_so_far: f64, remaining: Option<f64>) -> bool {
    remaining.is_none_or(|cap| cost_so_far < cap - 1e-9)
}

/// A failed verdict carrying an error or skip note.
fn error_verdict(spec: &EvalSpec, note: String) -> EvalVerdict {
    EvalVerdict {
        name: spec.name.clone(),
        passed: false,
        score: 0.0,
        reasoning: note,
        cost_usd: 0.0,
    }
}

/// Resolve the candidate the rubric grades: the deterministic `response:` when
/// present, otherwise a live tool-using agent run against the eval's connected
/// `server`. Returns `Ok(None)` when neither source is available (the eval
/// defers), or an error note when a live run was attempted and failed.
async fn resolve_candidate(
    spec: &EvalSpec,
    provider: &dyn LlmProvider,
    model: &str,
    live: Option<&LiveCandidates<'_>>,
) -> Result<Option<serde_json::Value>, String> {
    if let Some(response) = spec.response.as_deref() {
        return Ok(Some(serde_json::json!(response)));
    }
    let Some(live) = live else {
        return Ok(None);
    };
    if spec.server.is_empty() {
        return Ok(None);
    }
    // The server must already be in the pool; the handler connects every
    // server a live eval references before grading.
    let Some(server) = live.pool.get(&spec.server).await else {
        return Ok(None);
    };
    let handles = [mcptest_agent::ServerHandle {
        name: spec.server.as_str(),
        client: &server.client,
    }];
    super::eval_live::run_live_candidate(
        spec,
        &handles,
        provider,
        model,
        std::sync::Arc::clone(&live.budget),
    )
    .await
    .map(Some)
    .map_err(|e| e.to_string())
}

/// A passed verdict that explains why grading was skipped, for the key-free
/// and no-candidate cases so a CI gate stays green rather than failing on a
/// missing key.
fn deferred(spec: &EvalSpec, reason: &str) -> EvalVerdict {
    EvalVerdict {
        name: spec.name.clone(),
        passed: true,
        score: 1.0,
        reasoning: format!("deferred: {reason}"),
        cost_usd: 0.0,
    }
}

/// The rubric to grade against: the structured rubric when present, else the
/// free-form string wrapped as a single weighted criterion so both forms run
/// through one engine.
fn rubric_for(spec: &EvalSpec) -> RubricConfig {
    if let Some(cfg) = &spec.rubric_config {
        return cfg.clone();
    }
    RubricConfig {
        criteria: vec![Criterion {
            name: "rubric".to_string(),
            weight: 1.0,
            description: spec.rubric.clone(),
            ..Criterion::default()
        }],
        threshold: spec.threshold,
        ..RubricConfig::default()
    }
}

// The one-line reasoning renderer (`rubric_reasoning`) moved to
// `super::eval_cache` (WOR-1320) so the stored cache rationale and the
// displayed reasoning come from one implementation.

#[cfg(test)]
mod tests {
    use super::budget_allows;

    #[test]
    fn budget_allows_runs_freely_with_no_cap() {
        assert!(budget_allows(0.0, None));
        assert!(budget_allows(1_000.0, None));
    }

    #[test]
    fn budget_allows_stops_once_spend_reaches_the_cap() {
        assert!(budget_allows(0.0, Some(1.0)));
        assert!(budget_allows(0.99, Some(1.0)));
        assert!(!budget_allows(1.0, Some(1.0)));
        assert!(!budget_allows(2.0, Some(1.0)));
    }
}
