//! Computed-metric check blocks for the live runner.
//!
//! A check block (`calibration:` and `tool_quality:` today) is a test-like
//! top-level entry: it computes a metrics object from its own input and gates
//! it with the existing assertion grammar. This module evaluates the selected
//! check entries after the tool and agent run, folds each into one PASS/FAIL
//! row on the run report, and re-tallies the summary so the exit code and
//! every reporter see them without a new reporter format.
//!
//! `tool_quality:` needs a live `tools/list` catalog, which it fetches up
//! front via [`fetch_tool_quality_catalogs`] (while the server pool is alive)
//! and scores after the run.

use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;
use std::sync::Arc;

use anyhow::{Context, Result};
use serde_json::Value;

use mcptest_core::checks::CheckOutcome;
use mcptest_core::connector::ServerPool;
use mcptest_core::report::{Report, TestFailure, TestResult, TestStatus};
use mcptest_core::suite::ResolvedSuite;

/// Fetch the `tools/list` catalog for every selected `tool_quality:` check.
///
/// Runs while the server pool is still alive (the pool is shut down inside the
/// run-with-hooks drive), keyed by check name so the post-run scoring pass can
/// match each catalog to its entry. An unreachable server or a `tools/list`
/// failure is a run error and bubbles up, matching how a tool test surfaces a
/// dead server.
pub(crate) async fn fetch_tool_quality_catalogs(
    suite: &ResolvedSuite,
    selected: &BTreeSet<String>,
    pool: &Arc<ServerPool>,
) -> Result<BTreeMap<String, Value>> {
    let mut catalogs = BTreeMap::new();
    for check in &suite.tool_quality {
        if !selected.contains(&check.name) {
            continue;
        }
        let server = pool.get(&check.server).await.with_context(|| {
            format!(
                "tool_quality check `{}` references server `{}`, which is not connected",
                check.name, check.server
            )
        })?;
        let catalog = server
            .client
            .request("tools/list", serde_json::json!({}))
            .await
            .with_context(|| {
                format!(
                    "tool_quality check `{}`: tools/list on server `{}` failed",
                    check.name, check.server
                )
            })?;
        catalogs.insert(check.name.clone(), catalog);
    }
    Ok(catalogs)
}

/// Run every selected check block, append its row to `report`, and re-tally
/// the summary.
///
/// Only entries whose name survived selection (`--filter`, `--include`, shard)
/// run, matching how tool tests are gated. A label-load failure on a
/// calibration entry is a suite-load error and bubbles up as an `Err` so the
/// command exits non-zero with a clear message rather than silently skipping
/// the gate. `tool_quality_catalogs` carries the pre-fetched catalogs keyed by
/// check name (see [`fetch_tool_quality_catalogs`]).
pub(crate) fn run_check_blocks(
    report: &mut Report,
    suite: &ResolvedSuite,
    selected: &BTreeSet<String>,
    config_path: &Path,
    tool_quality_catalogs: &BTreeMap<String, Value>,
) -> Result<()> {
    let base_dir = config_path
        .parent()
        .unwrap_or_else(|| Path::new("."))
        .to_path_buf();

    let mut added = false;
    for check in &suite.calibration {
        if !selected.contains(&check.name) {
            continue;
        }
        let outcome = mcptest_core::checks::calibration::evaluate(check, &base_dir)
            .with_context(|| format!("calibration check `{}` failed to load", check.name))?;
        report.tests.push(check_row(outcome));
        added = true;
    }

    for check in &suite.tool_quality {
        if !selected.contains(&check.name) {
            continue;
        }
        // The catalog was fetched up front while the pool was alive. A missing
        // entry means the fetch was skipped (it should not happen for a
        // selected check), so fail loudly rather than silently passing.
        let catalog = tool_quality_catalogs.get(&check.name).with_context(|| {
            format!(
                "tool_quality check `{}` has no fetched catalog (internal error)",
                check.name
            )
        })?;
        let outcome = mcptest_core::checks::tool_quality::evaluate(check, catalog);
        report.tests.push(check_row(outcome));
        added = true;
    }

    if added {
        retally(report);
    }
    Ok(())
}

/// Append already-computed check outcomes as report rows and re-tally the
/// summary.
///
/// The within-session stability gate is computed per agent cell inside the
/// agent dispatch loop (it needs the N run envelopes), so its outcomes arrive
/// as a ready list rather than from a top-level block. This folds each into the
/// same PASS/FAIL row shape the check blocks use so the exit code and every
/// reporter treat them identically. A no-op on an empty list.
pub(crate) fn append_check_outcomes(report: &mut Report, outcomes: Vec<CheckOutcome>) {
    if outcomes.is_empty() {
        return;
    }
    for outcome in outcomes {
        report.tests.push(check_row(outcome));
    }
    retally(report);
}

/// Build the report row for one check outcome. A passing check with
/// a note (the empty-labels warning) still reads as a pass, but the note rides
/// in the failure message slot so the reporter surfaces it.
fn check_row(outcome: CheckOutcome) -> TestResult {
    let (status, failure) = if outcome.passed {
        let failure = outcome.failure.map(|note| TestFailure {
            message: note,
            diff: None,
            network_diagnostic: None,
        });
        (TestStatus::Pass, failure)
    } else {
        (
            TestStatus::Fail,
            Some(TestFailure {
                message: outcome
                    .failure
                    .unwrap_or_else(|| "check failed".to_string()),
                diff: None,
                network_diagnostic: None,
            }),
        )
    };
    TestResult {
        name: outcome.name,
        status,
        duration_ms: 1,
        cache_hit: false,
        started_at: None,
        ended_at: None,
        failure,
        file: None,
        line: None,
        steps: Vec::new(),
        consensus: None,
        rule_id: None,
        severity: None,
        score: None,
        derived_metrics: Vec::new(),
        natural_language: false,
    }
}

/// Recompute the run summary's pass/fail/total counts from the test list after
/// appending check rows, so the one-line summary and the exit code reflect
/// them. Leaves the timing and cache counters alone (check blocks add no model
/// or cache cost).
fn retally(report: &mut Report) {
    let total = report.tests.len();
    let passed = report
        .tests
        .iter()
        .filter(|t| matches!(t.status, TestStatus::Pass))
        .count();
    let failed = report
        .tests
        .iter()
        .filter(|t| matches!(t.status, TestStatus::Fail))
        .count();
    let skipped = report
        .tests
        .iter()
        .filter(|t| matches!(t.status, TestStatus::Skip))
        .count();
    report.summary.total = total;
    report.summary.passed = passed;
    report.summary.failed = failed;
    report.summary.skipped = skipped;
}

/// Evaluate an agent test's `recovery:` gate over its run trace, returning a
/// PASS/FAIL outcome to fold into the report (WOR-1077). The gate scores the
/// injected hang in virtual time (no wall-clock sleep), so a poor-recovery
/// agent fails as a quality failure (exit 1). Returns `None` when the agent
/// test declares no `recovery:` block or produced no run envelope.
pub(super) fn recovery_outcome(
    agent: &mcptest_core::suite::AgentTest,
    case_name: &str,
    quiet: bool,
    injection: &[mcptest_core::suite::FaultSpec],
    envelopes: &[Value],
) -> Option<CheckOutcome> {
    let gate = agent.recovery.as_ref()?;
    // Score the last run's trace, matching how the `eval:` metrics pick the
    // candidate envelope.
    let candidate = envelopes.last()?;
    let outcome = mcptest_core::eval::evaluate_recovery(candidate, injection, gate);
    if !quiet {
        let verdict = if outcome.passed { "PASS" } else { "FAIL" };
        println!("recovery gate [{verdict}] {case_name}: {}", outcome.detail);
    }
    Some(CheckOutcome {
        name: format!("{case_name} (recovery)"),
        passed: outcome.passed,
        failure: (!outcome.passed).then(|| outcome.detail.clone()),
    })
}

/// Per-cell agent outcomes and telemetry produced during plan build, folded
/// onto the report after the run. Bundled so [`attach_post_run_sections`] takes
/// one parameter for the by-value pieces rather than five.
pub(super) struct PostRunSections {
    /// Within-session `stability:` gate outcomes, one per cell that declared one.
    pub stability_outcomes: Vec<CheckOutcome>,
    /// Objective `equal_function_sets:` tool-selection F1 gate outcomes.
    pub f1_outcomes: Vec<CheckOutcome>,
    /// `eval:` metric outcomes (rubric, ...), one per metric per cell.
    pub eval_outcomes: Vec<CheckOutcome>,
    /// `recovery:` gate outcomes (WOR-1077), one per cell that declared one.
    pub recovery_outcomes: Vec<CheckOutcome>,
    /// Per-row agent wall-clock and cost (WOR-1202).
    pub agent_metrics: Vec<super::run_live_agent_metrics::AgentRowMetric>,
}

/// Fold every post-run section onto the report: the per-cell agent gate
/// outcomes (stability, tool-selection F1, `eval:` metrics, recovery), the run
/// metadata block, the comparison-matrix grid, and the agent wall-clock/cost
/// telemetry (WOR-1202). Sequenced here so [`super::run_live`]'s
/// `run_live_async` stays under the function-length lint.
pub(super) fn attach_post_run_sections(
    report: &mut Report,
    suite: &ResolvedSuite,
    args: &crate::cli::args::RunArgs,
    matrix_cells: &[super::run_live_matrix::MatrixCell],
    sections: PostRunSections,
) {
    // Fold each agent gate outcome into the report as a PASS/FAIL row so the
    // exit code and every reporter see it alongside the per-run rows.
    append_check_outcomes(report, sections.stability_outcomes);
    append_check_outcomes(report, sections.f1_outcomes);
    append_check_outcomes(report, sections.eval_outcomes);
    append_check_outcomes(report, sections.recovery_outcomes);
    // Run metadata (profile + per-server posture) for the enterprise policy
    // evaluator; additive, so a legacy report keeps its None metadata.
    super::run_live_metadata::attach_run_metadata(report, args, suite);
    // Carry the matrix grid so `--reporter matrix` and the cloud viewer render
    // the test-by-model grid from the same artifact. No-op for a non-matrix run.
    super::run_live_matrix::attach_matrix_section(report, matrix_cells);
    // WOR-1202: fold each agent run's real wall-clock and cost onto the report.
    super::run_live_agent_metrics::apply_agent_metrics(report, &sections.agent_metrics);
}

/// Inputs for [`attach_run_sections`]. Bundled (like the run path's other
/// context structs) so the caller passes one value and [`run_live_async`]
/// stays under the function-length lint.
///
/// [`run_live_async`]: super::run_live::run_live_async
pub(super) struct RunSectionsCtx<'a> {
    pub suite: &'a ResolvedSuite,
    pub selected: &'a BTreeSet<String>,
    pub config_path: &'a Path,
    pub tool_quality_catalogs: &'a BTreeMap<String, Value>,
    pub args: &'a crate::cli::args::RunArgs,
    pub matrix_cells: &'a [super::run_live_matrix::MatrixCell],
    pub sections: PostRunSections,
}

/// Run the computed-metric check blocks and fold every post-run section onto
/// the report in one call. A thin wrapper over [`run_check_blocks`] and
/// [`attach_post_run_sections`] so the run path's main function keeps a single
/// post-drive step.
pub(super) fn attach_run_sections(report: &mut Report, ctx: RunSectionsCtx<'_>) -> Result<()> {
    run_check_blocks(
        report,
        ctx.suite,
        ctx.selected,
        ctx.config_path,
        ctx.tool_quality_catalogs,
    )?;
    attach_post_run_sections(report, ctx.suite, ctx.args, ctx.matrix_cells, ctx.sections);
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;

    fn empty_report() -> Report {
        Report {
            schema_version: "0.1".into(),
            mcptest_version: "test".into(),
            generated_at: String::new(),
            run_id: String::new(),
            // Derived (WOR-1317) so the fixture cannot drift from its
            // (empty) test vec.
            summary: mcptest_core::report::RunSummary::from_tests(&[]),
            tests: Vec::new(),
            cost: None,
            metadata: None,
            matrix: None,
            protocol_violations: Vec::new(),
        }
    }

    #[test]
    fn passing_check_row_is_a_pass_with_no_failure() {
        let row = check_row(CheckOutcome {
            name: "judge".into(),
            passed: true,
            failure: None,
        });
        assert_eq!(row.status, TestStatus::Pass);
        assert!(row.failure.is_none());
    }

    #[test]
    fn passing_check_row_keeps_a_warning_note() {
        let row = check_row(CheckOutcome {
            name: "judge".into(),
            passed: true,
            failure: Some("warning: labels file is empty".into()),
        });
        assert_eq!(row.status, TestStatus::Pass);
        assert_eq!(
            row.failure.unwrap().message,
            "warning: labels file is empty"
        );
    }

    #[test]
    fn failing_check_row_carries_the_message() {
        let row = check_row(CheckOutcome {
            name: "judge".into(),
            passed: false,
            failure: Some("ece too high".into()),
        });
        assert_eq!(row.status, TestStatus::Fail);
        assert_eq!(row.failure.unwrap().message, "ece too high");
    }

    #[test]
    fn retally_counts_appended_rows() {
        let mut report = empty_report();
        report.tests.push(check_row(CheckOutcome {
            name: "ok".into(),
            passed: true,
            failure: None,
        }));
        report.tests.push(check_row(CheckOutcome {
            name: "bad".into(),
            passed: false,
            failure: Some("nope".into()),
        }));
        retally(&mut report);
        assert_eq!(report.summary.total, 2);
        assert_eq!(report.summary.passed, 1);
        assert_eq!(report.summary.failed, 1);
    }
}
