//! Assertion evaluation and combined-score folding for the executor.
//!
//! Walks a test's assertions and assert-sets against the response
//! envelope, folds the results into the combined-score model, applies
//! derived-metric gates, and decides pass/fail. Split out of [`super`]
//! (the executor module) so both halves stay under the AGENTS
//! module-size ceiling; `execute_one` calls in through
//! [`evaluate_expectations`].

use serde_json::Value;

use crate::eval::provider::LlmProvider;
use crate::report::{DerivedMetricReport, TestFailure};
use crate::runner::TestOutcome;
use crate::suite::scoring::{combined_score, named_scores, test_passes, ScoredItem};
use crate::suite::{AssertSet, Assertion};
use crate::transform::TransformContext;

use super::jury_metrics;
use super::jury_metrics::AssertionEval;
use super::{Scoring, SnapshotContext};

/// Evaluate the test's assertions and assert-sets, fold them into a
/// combined score, compute derived metrics, and decide pass/fail.
///
/// When `scoring` carries no threshold and no derived-metric gate, the
/// outcome is the legacy contract: every assertion and assert-set must
/// pass, and the failure message names the first failing item. When a
/// threshold is present, the test passes on a combined weighted score at
/// or above it. The score and derived metrics ride back on the
/// [`TestOutcome`] regardless of pass/fail so the report can surface them.
///
/// WOR-1290: a test that declared expectations but evaluated none of them
/// (every assertion and assert-set skipped) fails instead of passing
/// vacuously; see [`finalize_outcome`]. A test that declared no
/// expectations at all keeps the legacy default pass, because declaring
/// nothing is the author's explicit choice while declaring checks that
/// all silently skip is not.
#[allow(clippy::too_many_arguments)]
pub(super) async fn evaluate_expectations(
    response: &Value,
    expectations: &[Assertion],
    scoring: &Scoring,
    provider: Option<&dyn LlmProvider>,
    default_judge_model: Option<&str>,
    ctx: &TransformContext,
    generator_model: Option<&str>,
    snapshot: &SnapshotContext,
) -> TestOutcome {
    let (mut items, mut first_failure, mut skipped, jury_consensus) = fold_assertions(
        response,
        expectations,
        provider,
        default_judge_model,
        ctx,
        generator_model,
        snapshot,
    )
    .await;

    for set in &scoring.assert_sets {
        match eval_assert_set(
            response,
            set,
            provider,
            default_judge_model,
            ctx,
            snapshot,
            generator_model,
        )
        .await
        {
            AssertionEval::Pass => items.push(ScoredItem {
                name: set.name.clone(),
                weight: set.weight,
                passed: true,
            }),
            AssertionEval::Fail(reason) => {
                if first_failure.is_none() {
                    first_failure = Some(reason);
                }
                items.push(ScoredItem {
                    name: set.name.clone(),
                    weight: set.weight,
                    passed: false,
                });
            }
            // A set whose members all skipped is itself skipped: excluded
            // from the outer items (neither pass nor fail, weight inert)
            // and counted so the declared-but-nothing-evaluated guard in
            // `finalize_outcome` can account for it (WOR-1290).
            AssertionEval::Skipped(note) => {
                tracing::info!(
                    target: "mcptest_core::executor",
                    set = %set.name.as_deref().unwrap_or("(unnamed)"),
                    note = %note,
                    "assert-set skipped"
                );
                skipped += 1;
            }
        }
    }

    let declared = expectations.len() + scoring.assert_sets.len();
    // Attach the jury consensus (WOR-1384) so the runner forwards it onto
    // TestResult.consensus; a test with no jury assertion leaves it absent.
    finalize_outcome(&items, scoring, first_failure, declared, skipped)
        .with_consensus(jury_consensus)
}

/// Walk the test's top-level assertions into scored items, the first
/// failure under the legacy contract, and the count of skipped
/// assertions. Split from [`evaluate_expectations`] so each half stays
/// under the AGENTS function-size cap.
///
/// Runs the test's single `llm-jury` assertion once, up front, so its
/// analytics back the `jury.*` targets and its verdict is reused for the
/// jury assertion itself instead of dispatching the jury twice.
#[allow(clippy::too_many_arguments)]
async fn fold_assertions(
    response: &Value,
    expectations: &[Assertion],
    provider: Option<&dyn LlmProvider>,
    default_judge_model: Option<&str>,
    ctx: &TransformContext,
    generator_model: Option<&str>,
    snapshot: &SnapshotContext,
) -> (
    Vec<ScoredItem>,
    Option<String>,
    usize,
    Option<crate::report::JuryConsensus>,
) {
    let mut items: Vec<ScoredItem> = Vec::new();
    let mut first_failure: Option<String> = None;
    let mut skipped: usize = 0;

    let jury_pre =
        jury_metrics::prepare_jury(response, expectations, provider, ctx, generator_model).await;
    let (jury_index, jury_run, jury_obj) = match jury_pre {
        Some((index, run)) => {
            let obj = jury_metrics::metrics_object(&run.metrics);
            (Some(index), Some(run), Some(obj))
        }
        None => (None, None, None),
    };

    for (i, assertion) in expectations.iter().enumerate() {
        // Fold the cached jury verdict for the jury assertion rather than
        // dispatching the jury a second time.
        let eval = if Some(i) == jury_index {
            let run = jury_run.as_ref().expect("jury_run present when index set");
            match &run.outcome {
                Ok(()) => AssertionEval::Pass,
                Err(reason) => AssertionEval::Fail(reason.clone()),
            }
        } else {
            jury_metrics::eval_assertion(
                response,
                assertion,
                provider,
                default_judge_model,
                ctx,
                jury_obj.as_ref(),
                snapshot,
                generator_model,
            )
            .await
        };
        let passed = match eval {
            AssertionEval::Pass => true,
            AssertionEval::Fail(reason) => {
                if first_failure.is_none() {
                    first_failure = Some(format!(
                        "assertion #{i} (`{}`) failed: {reason}",
                        assertion.target
                    ));
                }
                false
            }
            AssertionEval::Skipped(note) => {
                tracing::info!(
                    target: "mcptest_core::executor",
                    assertion = %assertion.target,
                    note = %note,
                    "assertion skipped"
                );
                // A skipped assertion neither passes nor fails: drop it
                // from the scored items so its weight cannot move the
                // combined score, but count it so a test whose declared
                // checks all skipped fails instead of passing vacuously
                // (WOR-1290).
                skipped += 1;
                continue;
            }
        };
        items.push(ScoredItem {
            name: assertion.name.clone(),
            weight: assertion.weight,
            passed,
        });
    }
    // Surface the jury's per-juror consensus (WOR-1384) so the caller can fold
    // it onto the TestOutcome. `None` when the test ran no `llm-jury` assertion.
    let consensus = jury_run.and_then(|run| run.consensus);
    (items, first_failure, skipped, consensus)
}

/// Evaluate every assertion inside an assert-set and fold the evaluated
/// members against the set's internal threshold.
///
/// Returns the set's own tri-state verdict: `Pass`/`Fail` when at least
/// one member evaluated, `Skipped` when every member skipped, so the
/// caller can exclude a fully skipped set from the outer score exactly
/// like a skipped top-level assertion (WOR-1290).
#[allow(clippy::too_many_arguments)]
async fn eval_assert_set(
    response: &Value,
    set: &AssertSet,
    provider: Option<&dyn LlmProvider>,
    default_judge_model: Option<&str>,
    ctx: &TransformContext,
    snapshot: &SnapshotContext,
    generator_model: Option<&str>,
) -> AssertionEval {
    let mut evals: Vec<(f64, AssertionEval)> = Vec::with_capacity(set.assertions.len());
    for assertion in &set.assertions {
        // Jury targets are a test-level concept (one jury per test), so an
        // assert-set never resolves `jury.*`; pass no metrics object.
        let eval = jury_metrics::eval_assertion(
            response,
            assertion,
            provider,
            default_judge_model,
            ctx,
            None,
            snapshot,
            generator_model,
        )
        .await;
        evals.push((assertion.weight, eval));
    }
    fold_assert_set(set, evals)
}

/// Fold a set's member evaluations into the set's verdict. Pure half of
/// [`eval_assert_set`], split out so the skip accounting is unit
/// testable without a live matcher path that can skip.
///
/// A skipped member is dropped from the set's scored items (neither pass
/// nor fail), mirroring top-level handling, so its weight cannot lift a
/// failing set over its threshold or drag a passing one under it
/// (WOR-1290; skipped members previously counted as passes).
fn fold_assert_set(set: &AssertSet, evals: Vec<(f64, AssertionEval)>) -> AssertionEval {
    let mut inner: Vec<ScoredItem> = Vec::with_capacity(evals.len());
    let mut skipped: usize = 0;
    for (weight, eval) in evals {
        let passed = match eval {
            AssertionEval::Pass => true,
            AssertionEval::Fail(_) => false,
            AssertionEval::Skipped(_) => {
                skipped += 1;
                continue;
            }
        };
        inner.push(ScoredItem {
            name: None,
            weight,
            passed,
        });
    }
    let label = set.name.as_deref().unwrap_or("(unnamed)");
    // Every member skipped: the set produced no evidence either way, so
    // it is skipped as a whole rather than passing on an empty fraction.
    if inner.is_empty() && skipped > 0 {
        return AssertionEval::Skipped(format!(
            "assert-set `{label}`: all {skipped} assertions skipped"
        ));
    }
    let score = combined_score(&inner);
    if score >= set.threshold {
        return AssertionEval::Pass;
    }
    let skip_note = if skipped > 0 {
        format!(", {skipped} member(s) skipped")
    } else {
        String::new()
    };
    AssertionEval::Fail(format!(
        "assert-set `{label}` scored {score:.3} below threshold {:.3}{skip_note}",
        set.threshold
    ))
}

/// Turn the scored items plus the scoring config into a [`TestOutcome`]
///. Decides pass/fail (legacy all-pass when no threshold,
/// score-gated otherwise), computes derived metrics, lets a thresholded
/// derived metric veto a pass, and attaches the score telemetry.
///
/// `declared` and `skipped` back the WOR-1290 vacuous-pass guard: when
/// the test declared at least one assertion or assert-set but every one
/// of them skipped, the test fails, because a pass built from zero
/// evaluated checks is indistinguishable from a healthy green test. A
/// test that declared nothing (`declared == 0`) keeps the default pass.
fn finalize_outcome(
    items: &[ScoredItem],
    scoring: &Scoring,
    first_failure: Option<String>,
    declared: usize,
    skipped: usize,
) -> TestOutcome {
    let scores = named_scores(items);
    // Evaluate in declaration order so a derived metric can reference an
    // earlier derived metric by name (chained derivation, WOR-1198).
    let derived = crate::suite::scoring::derive_all(&scoring.derived_metrics, &scores);
    let derived_failure = derived.iter().find(|d| d.passed == Some(false)).map(|d| {
        format!(
            "derived metric `{}` scored {:.3} below its threshold",
            d.name, d.value
        )
    });

    // The vacuous-pass guard fires before the score model: an empty item
    // list scores 1.0, which would clear any threshold.
    let nothing_evaluated = declared > 0 && items.is_empty();
    let assertions_pass = !nothing_evaluated && test_passes(items, scoring.threshold);
    // Surface the score only under the opt-in score model so a legacy
    // suite's report JSON is byte-for-byte unchanged. A `threshold:` opts
    // in; a matrix-level assertion opts in via `emit_score`
    // so its cells carry a comparable score even when the cell
    // sets no threshold. A test that sets neither still reports `None`.
    let score = if scoring.threshold.is_some() || scoring.emit_score {
        Some(combined_score(items))
    } else {
        None
    };
    let derived_report: Vec<DerivedMetricReport> = derived
        .iter()
        .map(|d| DerivedMetricReport {
            name: d.name.clone(),
            value: d.value,
            passed: d.passed,
        })
        .collect();

    let outcome = if assertions_pass && derived_failure.is_none() {
        if skipped > 0 {
            // TestOutcome carries no note field for a passing test, so the
            // partial-skip signal rides on the log; the failure path
            // carries the count in its message instead.
            tracing::warn!(
                target: "mcptest_core::executor",
                skipped,
                declared,
                "test passed with skipped assertions; only evaluated checks were scored"
            );
        }
        TestOutcome::pass()
    } else {
        let message = pick_failure_message(
            scoring,
            &first_failure,
            derived_failure,
            items,
            declared,
            skipped,
        );
        TestOutcome::fail_with(TestFailure {
            message,
            diff: None,
            network_diagnostic: None,
        })
    };
    outcome.with_score(score, derived_report)
}

/// Choose the failure message for a failing test. The vacuous case
/// (declared checks, none evaluated) names both counts so the operator
/// sees why a test with no failing assertion still failed; a thresholded
/// suite reports the combined score; the legacy contract reports the
/// first failing assertion or set; a derived-metric veto reports the
/// metric. A non-zero skip count rides along on every variant so partial
/// skips stay visible in the report (WOR-1290).
fn pick_failure_message(
    scoring: &Scoring,
    first_failure: &Option<String>,
    derived_failure: Option<String>,
    items: &[ScoredItem],
    declared: usize,
    skipped: usize,
) -> String {
    if declared > 0 && items.is_empty() {
        return format!(
            "no assertions were evaluated: {declared} declared, {skipped} skipped \
             (targets missing or not applicable)"
        );
    }
    let base = if let Some(threshold) = scoring.threshold {
        let score = combined_score(items);
        if score < threshold {
            format!("combined score {score:.3} is below threshold {threshold:.3}")
        } else {
            // The score met the threshold but a derived metric vetoed it.
            derived_failure.unwrap_or_else(|| "test failed its derived-metric gate".to_string())
        }
    } else {
        derived_failure
            .or_else(|| first_failure.clone())
            .unwrap_or_else(|| "test failed".to_string())
    };
    if skipped > 0 {
        format!("{base} ({skipped} assertion(s) skipped)")
    } else {
        base
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::report::TestStatus;
    use serde_json::json;

    fn set_with(threshold: f64) -> AssertSet {
        AssertSet {
            name: Some("checks".into()),
            threshold,
            weight: 1.0,
            assertions: Vec::new(),
        }
    }

    fn skip() -> AssertionEval {
        AssertionEval::Skipped("target not applicable".into())
    }

    #[test]
    fn declared_assertions_all_skipped_fails_with_the_skip_count() {
        let outcome = finalize_outcome(&[], &Scoring::default(), None, 3, 3);
        assert_eq!(outcome.status, TestStatus::Fail);
        let message = outcome.failure.unwrap().message;
        assert_eq!(
            message,
            "no assertions were evaluated: 3 declared, 3 skipped \
             (targets missing or not applicable)"
        );
    }

    #[test]
    fn declared_assertions_all_skipped_fails_even_under_a_threshold() {
        // combined_score over zero items is 1.0, which would clear any
        // threshold; the vacuous-pass guard must fire before the score can.
        let scoring = Scoring {
            threshold: Some(0.1),
            ..Scoring::default()
        };
        let outcome = finalize_outcome(&[], &scoring, None, 2, 2);
        assert_eq!(outcome.status, TestStatus::Fail);
        let message = outcome.failure.unwrap().message;
        assert!(
            message.contains("no assertions were evaluated"),
            "{message}"
        );
    }

    #[tokio::test]
    async fn test_with_no_expectations_still_passes() {
        // Declaring nothing is the author's explicit choice (the WOR-1290
        // scope decision); only declared-but-all-skipped is the bug.
        let outcome = evaluate_expectations(
            &json!({"result": {"ok": true}}),
            &[],
            &Scoring::default(),
            None,
            None,
            &TransformContext::default(),
            None,
            &SnapshotContext::default(),
        )
        .await;
        assert_eq!(outcome.status, TestStatus::Pass);
    }

    #[test]
    fn assert_set_with_all_members_skipped_does_not_pass_the_set() {
        // Even a zero-threshold set (which always passes when evaluated)
        // must report skipped, not pass, when no member actually ran.
        let evals = vec![(1.0, skip()), (2.0, skip())];
        let folded = fold_assert_set(&set_with(0.0), evals);
        assert!(matches!(folded, AssertionEval::Skipped(_)));
    }

    #[test]
    fn assert_set_with_mixed_skip_and_pass_scores_only_evaluated_members() {
        // One pass and one fail among the evaluated members is 0.500. The
        // heavy skipped member would lift the set over its 0.9 threshold
        // if it still counted as a pass (the pre-WOR-1290 behavior).
        let evals = vec![
            (1.0, AssertionEval::Pass),
            (1.0, AssertionEval::Fail("nope".into())),
            (100.0, skip()),
        ];
        match fold_assert_set(&set_with(0.9), evals) {
            AssertionEval::Fail(reason) => {
                assert!(reason.contains("0.500"), "{reason}");
                assert!(reason.contains("1 member(s) skipped"), "{reason}");
            }
            _ => panic!("set must fail on its evaluated members alone"),
        }
        // And the inverse: a skipped member never drags an otherwise
        // passing set below its threshold.
        let evals = vec![(1.0, AssertionEval::Pass), (100.0, skip())];
        assert!(matches!(
            fold_assert_set(&set_with(1.0), evals),
            AssertionEval::Pass
        ));
    }

    #[test]
    fn fully_skipped_assert_set_counts_toward_the_vacuous_guard() {
        // Wiring check for the outer accounting: one declared set that
        // skipped entirely means declared 1, skipped 1, zero items, and
        // the test fails rather than passing on an empty score.
        let outcome = finalize_outcome(&[], &Scoring::default(), None, 1, 1);
        assert_eq!(outcome.status, TestStatus::Fail);
        let message = outcome.failure.unwrap().message;
        assert_eq!(
            message,
            "no assertions were evaluated: 1 declared, 1 skipped \
             (targets missing or not applicable)"
        );
    }
}
