//! Leaf scorers for the agent `eval:` block.
//!
//! Holds the two provider-touching scorers that turn one judged metric into a
//! [`mcptest_core::checks::CheckOutcome`] report row: [`eval_rubric_metric`]
//! (a single rubric over the whole run trace, including the  decision
//! tree) and [`eval_multi_turn_mcp_use`] (per-turn MCP-use alignment averaged
//! across a multi-turn conversation). Pulled out of [`super::run_live_agents`]
//! so that module stays under the production-module-size guard; the dispatch
//! that picks which metric to run for each `eval:` entry stays there.
//!
//! Both scorers judge through the WOR-1320 verdict cache
//! (`crate::commands::eval_cache::evaluate_rubric_cached`): a fresh cached
//! verdict replays at zero cost with a `cached:` prefix on its note, and
//! `--no-verdict-cache` (or a missing `evals.cache.verdicts: true` opt-in)
//! builds a disabled cache that always misses. With no provider these
//! metrics defer with an explicit note; they never fall back to the stub
//! provider (WOR-1312).

use mcptest_core::checks::CheckOutcome;
use mcptest_core::eval::provider::LlmProvider;
use mcptest_core::eval::rubric::RubricConfig;
use mcptest_core::eval::VerdictCache;

use crate::commands::eval_cache::{evaluate_rubric_cached, RubricGrade};

/// Score `multi_turn_mcp_use`: judge each `conversation.per_turn`
/// slice for MCP-use alignment and average the per-turn scores. Needs a
/// multi-turn trace (a `turns:` run) and a real provider; with neither it
/// defers (passed, with a note) so a single-turn or key-free run stays green.
/// Each per-turn judging goes through `cache` keyed on that turn's slice, so
/// only an identical turn replays a cached score.
pub(super) async fn eval_multi_turn_mcp_use(
    name: &str,
    candidate: &serde_json::Value,
    threshold: f64,
    strict: bool,
    provider: Option<&dyn LlmProvider>,
    model: Option<&str>,
    cache: &mut VerdictCache,
) -> CheckOutcome {
    use mcptest_core::eval::rubric::mcp_use_rubric;
    let name = name.to_string();
    let turns = candidate
        .get("conversation")
        .and_then(|c| c.get("per_turn"))
        .and_then(|v| v.as_array())
        .cloned()
        .unwrap_or_default();
    if turns.is_empty() {
        return CheckOutcome {
            name,
            passed: true,
            failure: Some(
                "deferred: no multi-turn trace (add a `turns:` block to score per-turn MCP use)"
                    .to_string(),
            ),
        };
    }
    let (Some(provider), Some(model)) = (provider, model) else {
        return CheckOutcome {
            name,
            passed: true,
            failure: Some("deferred: no LLM provider (set a model API key to score)".to_string()),
        };
    };
    let mut total = 0.0;
    // Track the lowest-scoring turn so a failure names the weak spot.
    let mut weakest: Option<(usize, f64, Option<String>)> = None;
    for (i, turn) in turns.iter().enumerate() {
        let intent = turn.get("user").and_then(|v| v.as_str()).unwrap_or("");
        let rubric = mcp_use_rubric(intent, threshold, strict);
        // The turn's user intent is already baked into the rubric and the
        // turn slice is the candidate, so the key's `prompt` slot is empty.
        match evaluate_rubric_cached(&rubric, turn, provider, model, "", cache).await {
            Ok(grade) => {
                let (score, reason) = match grade {
                    RubricGrade::Fresh(out) => (
                        out.score,
                        out.per_criterion.first().and_then(|c| c.reason.clone()),
                    ),
                    RubricGrade::Cached(hit) => (hit.score, Some(hit.reasoning)),
                };
                total += score;
                if weakest.as_ref().map(|(_, s, _)| score < *s).unwrap_or(true) {
                    weakest = Some((i, score, reason));
                }
            }
            Err(e) => {
                return CheckOutcome {
                    name,
                    passed: false,
                    failure: Some(format!("multi-turn-mcp-use provider error: {e}")),
                }
            }
        }
    }
    let mean = total / turns.len() as f64;
    let passed = if strict {
        mean >= 1.0
    } else {
        mean >= threshold
    };
    let failure = if passed {
        None
    } else {
        let base = format!(
            "aggregate {:.2} < threshold {:.2} over {} turns",
            mean,
            threshold,
            turns.len()
        );
        Some(match weakest {
            Some((i, s, Some(r))) => format!("{base}; weakest turn {} ({:.2}): {}", i + 1, s, r),
            Some((i, s, None)) => format!("{base}; weakest turn {} ({:.2})", i + 1, s),
            None => base,
        })
    };
    CheckOutcome {
        name,
        passed,
        failure,
    }
}

/// Score one `rubric` metric over `candidate` and fold it into a
/// [`mcptest_core::checks::CheckOutcome`] report row. Judging goes through
/// the verdict cache: a hit replays the stored verdict at zero cost and its
/// `cached:` note rides on the row so a replay is visible in the report.
pub(super) async fn eval_rubric_metric(
    name: &str,
    config: &RubricConfig,
    candidate: &serde_json::Value,
    provider: Option<&dyn LlmProvider>,
    model: Option<&str>,
    cache: &mut VerdictCache,
) -> CheckOutcome {
    let name = name.to_string();
    let (Some(provider), Some(model)) = (provider, model) else {
        // No real provider ran (stub-only): defer rather than fail so key-free
        // CI stays green, mirroring how the llm-judge matcher defers.
        return CheckOutcome {
            name,
            passed: true,
            failure: Some("deferred: no LLM provider (set a model API key to score)".to_string()),
        };
    };
    // The criteria (which embed the agent prompt for the built-in metrics)
    // plus the run-trace candidate fully determine the judge inputs, so the
    // key's `prompt` slot is empty on this path.
    match evaluate_rubric_cached(config, candidate, provider, model, "", cache).await {
        Ok(RubricGrade::Cached(hit)) => CheckOutcome {
            name,
            passed: hit.passed,
            // The cached rationale doubles as the row note (even on a pass)
            // so a replayed verdict never looks like a fresh judging.
            failure: Some(hit.reasoning),
        },
        Ok(RubricGrade::Fresh(out)) => {
            let failure = if out.passed {
                None
            } else if !out.path.is_empty() {
                // Tree mode: the taken path is the explanation.
                let mut note = format!(
                    "score {:.2} < threshold {:.2}; path: {}",
                    out.score,
                    config.threshold,
                    out.path.join(", ")
                );
                if let Some(reason) = &out.leaf_reason {
                    note.push_str(&format!(" ({reason})"));
                }
                Some(note)
            } else {
                let worst = out
                    .per_criterion
                    .iter()
                    .min_by(|a, b| a.score.total_cmp(&b.score))
                    .map(|c| {
                        format!(
                            "score {:.2} < threshold {:.2}; weakest `{}` ({:.2})",
                            out.score, config.threshold, c.name, c.score
                        )
                    });
                worst.or_else(|| Some(format!("score {:.2} below threshold", out.score)))
            };
            CheckOutcome {
                name,
                passed: out.passed,
                failure,
            }
        }
        Err(e) => CheckOutcome {
            name,
            passed: false,
            failure: Some(format!("rubric provider error: {e}")),
        },
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use mcptest_agent::stub::{StubProvider, StubReply};
    use serde_json::json;

    /// Build a candidate envelope with `conversation.per_turn` from a list of
    /// `(user, tool_name)` turns, mirroring what the multi-turn driver records.
    fn multi_turn_candidate(turns: &[(&str, &str)]) -> serde_json::Value {
        let per_turn: Vec<serde_json::Value> = turns
            .iter()
            .map(|(user, tool)| {
                json!({
                    "user": user,
                    "tool_calls": [{"name": tool, "server": "s", "args": {}}],
                    "tool_results": [{"call_index": 0, "server": "s", "is_error": false, "result": {}}],
                    "final_response": "done"
                })
            })
            .collect();
        json!({ "conversation": { "per_turn": per_turn } })
    }

    #[tokio::test]
    async fn defers_without_a_multi_turn_trace() {
        // A single-prompt run has no `per_turn`; the metric defers (passes with
        // a note) rather than failing.
        let candidate = json!({ "conversation": { "per_turn": [] } });
        let provider = StubProvider::text(r#"{"score": 1.0}"#);
        let out = eval_multi_turn_mcp_use(
            "row",
            &candidate,
            0.6,
            false,
            Some(&provider),
            Some("stub"),
            &mut VerdictCache::disabled(),
        )
        .await;
        assert!(out.passed);
        assert!(out
            .failure
            .as_deref()
            .unwrap()
            .contains("no multi-turn trace"));
    }

    #[tokio::test]
    async fn defers_without_a_provider() {
        let candidate = multi_turn_candidate(&[("q1", "get_weather")]);
        let out = eval_multi_turn_mcp_use(
            "row",
            &candidate,
            0.6,
            false,
            None,
            None,
            &mut VerdictCache::disabled(),
        )
        .await;
        assert!(out.passed);
        assert!(out.failure.as_deref().unwrap().contains("no LLM provider"));
    }

    #[tokio::test]
    async fn good_use_every_turn_passes() {
        // Two turns, each judged 1.0 (one judge call per turn): the mean clears
        // the threshold.
        let candidate = multi_turn_candidate(&[("q1", "get_weather"), ("q2", "get_weather")]);
        let provider = StubProvider::new(vec![text_score(1.0), text_score(1.0)]);
        let out = eval_multi_turn_mcp_use(
            "row",
            &candidate,
            0.6,
            false,
            Some(&provider),
            Some("stub"),
            &mut VerdictCache::disabled(),
        )
        .await;
        assert!(out.passed, "mean 1.0 >= 0.6");
        assert!(out.failure.is_none());
    }

    #[tokio::test]
    async fn one_bad_turn_drags_the_aggregate_below_threshold() {
        // First turn good (1.0), second turn picks the wrong tool (0.2): the
        // mean (0.6) sits just under the 0.7 threshold and the metric fails,
        // naming the weak turn.
        let candidate = multi_turn_candidate(&[("q1", "get_weather"), ("q2", "delete_everything")]);
        let provider = StubProvider::new(vec![text_score(1.0), text_score(0.2)]);
        let out = eval_multi_turn_mcp_use(
            "row",
            &candidate,
            0.7,
            false,
            Some(&provider),
            Some("stub"),
            &mut VerdictCache::disabled(),
        )
        .await;
        assert!(!out.passed, "mean 0.6 < 0.7");
        let note = out.failure.as_deref().unwrap();
        assert!(
            note.contains("weakest turn 2"),
            "names the weak turn: {note}"
        );
    }

    fn text_score(score: f64) -> StubReply {
        StubReply::Text(format!(r#"{{"score": {score}}}"#))
    }
}
