//! Runner-driving and hook orchestration for the live runner.
//!
//! Split out of [`super::run_live`] so that module stays under the
//! production-module-size guard. Holds the functions that turn a planned
//! set of specs into a run report: the suite-level hook bracket
//! ([`run_with_hooks`]), the executor build plus per-test hook wrapping
//! ([`drive_runner`]), the generic runner drive ([`run_with_runner`]),
//! and the all-failed report a failing `beforeAll` produces
//! ([`all_failed_report`]).

use anyhow::{Context, Result};

use mcptest_core::hooks::{HookContext, HookPatch, HookPhase};

use super::hooks::{run_phase, HookOutcome, HookedExecutor};

/// Run `beforeAll`, drive the runner (with per-test hooks when declared),
/// then `afterAll`, returning the run report.
///
/// A failed `beforeAll`, or a `beforeAll` context hook that asks to skip,
/// short-circuits to an all-failed report without dispatching any test,
/// since no test could run against the setup it needed. A `beforeAll`
/// context hook's `{ env, vars }` patch seeds every per-test hook context.
/// An `afterAll` failure is reported but does not change the report (the
/// run already happened). The pool is shut down inside [`drive_runner`] (or
/// here on the beforeAll short-circuit).
#[allow(clippy::too_many_arguments)]
pub(super) async fn run_with_hooks(
    pool: &std::sync::Arc<mcptest_core::connector::ServerPool>,
    plans: std::collections::HashMap<String, std::sync::Arc<mcptest_core::executor::PlannedTest>>,
    specs: &[mcptest_core::runner::TestSpec],
    options: mcptest_core::runner::RunnerOptions,
    last_real_provider: Option<std::sync::Arc<dyn mcptest_core::eval::provider::LlmProvider>>,
    last_real_model: Option<String>,
    hooks: &mcptest_core::suite::Hooks,
    suite_name: &str,
    snapshot: mcptest_core::executor::SnapshotContext,
    allow_stub_provider: bool,
    restart_ctx: Option<super::run_live_restart::RestartContext>,
    coverage: Option<super::run_live_coverage::CoverageHandle>,
) -> Result<mcptest_core::report::Report> {
    let suite_ctx = HookContext {
        suite_name: suite_name.to_string(),
        ..HookContext::default()
    };
    let seed = match run_phase(HookPhase::BeforeAll, hooks, &suite_ctx).await {
        HookOutcome::Failed(reason) => {
            pool.shutdown_all().await;
            return Ok(all_failed_report(specs, &reason));
        }
        HookOutcome::Skip(reason) => {
            pool.shutdown_all().await;
            return Ok(all_skipped_report(specs, &reason));
        }
        HookOutcome::Ok(patch) => patch,
    };
    let report = drive_runner(
        pool,
        plans,
        specs,
        options,
        last_real_provider,
        last_real_model,
        hooks,
        suite_name,
        seed,
        snapshot,
        allow_stub_provider,
        restart_ctx,
        coverage,
    )
    .await?;
    if let HookOutcome::Failed(reason) = run_phase(HookPhase::AfterAll, hooks, &suite_ctx).await {
        eprintln!("mcptest run: afterAll hook reported a failure: {reason}");
    }
    Ok(report)
}

/// Build a report in which every selected test failed for the same
/// reason. Used when a suite-level `beforeAll` hook fails: the
/// run never dispatched a test, but the operator should see one failed
/// row per test with the hook's reason rather than an opaque run error.
fn all_failed_report(
    specs: &[mcptest_core::runner::TestSpec],
    reason: &str,
) -> mcptest_core::report::Report {
    use mcptest_core::report::{Report, RunSummary, TestFailure, TestResult, TestStatus};
    let tests: Vec<TestResult> = specs
        .iter()
        .map(|s| TestResult {
            name: s.name.clone(),
            status: TestStatus::Fail,
            duration_ms: 0,
            cache_hit: false,
            started_at: None,
            ended_at: None,
            failure: Some(TestFailure {
                message: format!("beforeAll hook failed: {reason}"),
                diff: None,
                network_diagnostic: None,
            }),
            file: s.file.clone(),
            line: s.line,
            steps: Vec::new(),
            consensus: None,
            rule_id: None,
            severity: None,
            score: None,
            derived_metrics: Vec::new(),
            natural_language: false,
        })
        .collect();
    Report {
        schema_version: "0.1".to_string(),
        mcptest_version: mcptest_core::version::full().to_string(),
        generated_at: String::new(),
        run_id: String::new(),
        // Derived (WOR-1317): every spec became a Fail row above, so the
        // counts come from the rows rather than a hand-rolled literal.
        summary: RunSummary::from_tests(&tests),
        tests,
        cost: None,
        metadata: None,
        matrix: None,
        protocol_violations: Vec::new(),
    }
}

/// Build a report in which every selected test was skipped for the same
/// reason. Used when a `beforeAll` context hook asks to skip: no
/// test runs, but the operator sees one skipped row per test with the hook's
/// reason rather than an opaque run error.
fn all_skipped_report(
    specs: &[mcptest_core::runner::TestSpec],
    reason: &str,
) -> mcptest_core::report::Report {
    use mcptest_core::report::{Report, RunSummary, TestResult, TestStatus};
    eprintln!("mcptest run: every test skipped by beforeAll hook: {reason}");
    let tests: Vec<TestResult> = specs
        .iter()
        .map(|s| TestResult {
            name: s.name.clone(),
            status: TestStatus::Skip,
            duration_ms: 0,
            cache_hit: false,
            started_at: None,
            ended_at: None,
            failure: None,
            file: s.file.clone(),
            line: s.line,
            steps: Vec::new(),
            consensus: None,
            rule_id: None,
            severity: None,
            score: None,
            derived_metrics: Vec::new(),
            natural_language: false,
        })
        .collect();
    Report {
        schema_version: "0.1".to_string(),
        mcptest_version: mcptest_core::version::full().to_string(),
        generated_at: String::new(),
        run_id: String::new(),
        // Derived (WOR-1317): every spec became a Skip row above, so the
        // counts come from the rows rather than a hand-rolled literal.
        summary: RunSummary::from_tests(&tests),
        tests,
        cost: None,
        metadata: None,
        matrix: None,
        protocol_violations: Vec::new(),
    }
}

/// Build the executor (reusing the last real provider for `llm-jury`, with
/// the stub fallback gated behind `--allow-stub-provider`, WOR-1319), drive
/// the runner end to end, and shut the server pool down.
///
/// When per-test hooks are declared, the executor is wrapped in a
/// [`HookedExecutor`] and the runner is forced serial so the
/// `beforeEach` / `afterEach` bracket holds for stateful MCP sessions.
/// When a non-default restart policy is in effect, the stack also gets a
/// [`RestartingExecutor`](super::run_live_restart::RestartingExecutor)
/// respawning servers at test boundaries (WOR-1323).
#[allow(clippy::too_many_arguments)]
async fn drive_runner(
    pool: &std::sync::Arc<mcptest_core::connector::ServerPool>,
    plans: std::collections::HashMap<String, std::sync::Arc<mcptest_core::executor::PlannedTest>>,
    specs: &[mcptest_core::runner::TestSpec],
    options: mcptest_core::runner::RunnerOptions,
    last_real_provider: Option<std::sync::Arc<dyn mcptest_core::eval::provider::LlmProvider>>,
    last_real_model: Option<String>,
    hooks: &mcptest_core::suite::Hooks,
    suite_name: &str,
    seed: HookPatch,
    snapshot: mcptest_core::executor::SnapshotContext,
    allow_stub_provider: bool,
    restart_ctx: Option<super::run_live_restart::RestartContext>,
    coverage: Option<super::run_live_coverage::CoverageHandle>,
) -> Result<mcptest_core::report::Report> {
    use mcptest_agent::stub::StubProvider;
    use mcptest_core::eval::provider::LlmProvider;
    use mcptest_core::executor::McpExecutor;
    use std::sync::Arc;

    // WOR-1319: a jury that grades on the canned stub passes fabricated
    // verdicts, so the fallback is a hard error unless the operator opted in.
    let jury_tests = jury_test_names(&plans);
    let decision = decide_jury_provider(
        &jury_tests,
        last_real_provider.is_some(),
        allow_stub_provider,
    )?;
    // Reuse the last real provider for `llm-jury` matchers when available so
    // suites with real-key runs evaluate jurors against the same model family,
    // not against the deterministic stub.
    let jury_provider: Arc<dyn LlmProvider> = last_real_provider.unwrap_or_else(|| {
        Arc::new(StubProvider::text(format!(
            r#"{{"pass": true, "score": 0.9, "reason": "{STUB_JURY_REASON}"}}"#
        )))
    });
    let mut executor = McpExecutor::new(Arc::clone(pool), plans)
        .with_provider(Arc::clone(&jury_provider))
        .with_snapshots(snapshot.dir, snapshot.mode);
    // An `llm-judge` block without `model:` grades with the agent's own model
    // (the one behind the reused provider), so the judge call goes to a model
    // the installed provider can actually serve.
    if let Some(model) = last_real_model {
        executor = executor.with_judge_model(model);
    }
    // WOR-1386: `--coverage` installs a shared recorder so each dispatched
    // tool call is observed; the caller folds the tracker into a report.
    if let Some(tracker) = coverage {
        executor = executor.with_coverage(tracker);
    }
    // WOR-1398: `--cache` installs a per-suite result cache so a cacheable
    // tool result is served from the store instead of dispatched live.
    if options.result_cache.enabled {
        if let Some(root) = &options.result_cache.root {
            let store = Arc::new(mcptest_core::cache::store::FsCacheStore::with_root(
                root.clone(),
            ));
            executor =
                executor.with_cache(store, options.result_cache.read, options.result_cache.write);
        }
    }

    let mut report = run_with_wrappers(
        executor,
        pool,
        restart_ctx,
        specs,
        options,
        hooks,
        suite_name,
        seed,
    )
    .await?;

    // WOR-1319: an allowed stub jury is marked on the report, one passing
    // `(jury provider)` row per jury test whose note leads with `stub:`, so
    // neither a human scan nor a machine diff can read it as a real verdict.
    if matches!(decision, JuryProviderDecision::AllowedStub) {
        super::run_live_checks::append_check_outcomes(
            &mut report,
            jury_tests.iter().map(|t| stub_jury_marker(t)).collect(),
        );
    }

    // WOR-1311: harvest each connection's protocol violations while the
    // clients are still alive, warn on stderr (machine-read stdout
    // reporters stay clean), and attach them to the run report so the
    // JSON artifact carries the evidence.
    let violations = pool.protocol_violations().await;
    let entries = mcptest_core::report::protocol_violation_entries(&violations);
    if !entries.is_empty() {
        for line in protocol_violation_warning_lines(&entries) {
            eprintln!("{line}");
        }
        report.protocol_violations = entries;
    }

    let shutdown_errors = pool.shutdown_all().await;
    for err in &shutdown_errors {
        eprintln!("mcptest run: shutdown warning: {err}");
    }
    Ok(report)
}

/// Stack the optional wrappers around the concrete executor and drive
/// the runner.
///
/// The order is deliberate (WOR-1323): the restart wrapper sits outside
/// the hook bracket so a respawn lands before `beforeEach` runs, which is
/// what `docs/test-isolation.md` commits to. Per-test hooks force serial
/// execution here so their bracket ordering holds; a non-default restart
/// policy was already forced serial in `apply_run_options`.
#[allow(clippy::too_many_arguments)]
async fn run_with_wrappers(
    executor: mcptest_core::executor::McpExecutor,
    pool: &std::sync::Arc<mcptest_core::connector::ServerPool>,
    restart_ctx: Option<super::run_live_restart::RestartContext>,
    specs: &[mcptest_core::runner::TestSpec],
    mut options: mcptest_core::runner::RunnerOptions,
    hooks: &mcptest_core::suite::Hooks,
    suite_name: &str,
    seed: HookPatch,
) -> Result<mcptest_core::report::Report> {
    use super::run_live_restart::RestartingExecutor;
    use mcptest_core::runner::Parallelism;
    use std::sync::Arc;

    let has_each_hooks = !hooks.before_each.is_empty() || !hooks.after_each.is_empty();
    if has_each_hooks {
        options.parallel = Parallelism::Fixed(1);
    }
    match (has_each_hooks, restart_ctx) {
        (true, Some(ctx)) => {
            let hooked = HookedExecutor::new(executor, suite_name, hooks, seed);
            let restarting = RestartingExecutor::new(hooked, Arc::clone(pool), ctx);
            run_with_runner(restarting, specs, options).await
        }
        (true, None) => {
            let hooked = HookedExecutor::new(executor, suite_name, hooks, seed);
            run_with_runner(hooked, specs, options).await
        }
        (false, Some(ctx)) => {
            let restarting = RestartingExecutor::new(executor, Arc::clone(pool), ctx);
            run_with_runner(restarting, specs, options).await
        }
        (false, None) => run_with_runner(executor, specs, options).await,
    }
}

/// Render the stderr warning block for the run's protocol violations
/// (WOR-1311): a header plus one line per (server, kind) with the count
/// and one example. A free function returning lines so the format is
/// unit-testable without capturing stderr.
fn protocol_violation_warning_lines(
    entries: &[mcptest_core::report::ProtocolViolationReport],
) -> Vec<String> {
    let mut lines = vec!["mcptest run: protocol violations observed during this run:".to_string()];
    for v in entries {
        if v.example.is_empty() {
            lines.push(format!("  `{}`: {} x{}", v.server, v.kind, v.count));
        } else {
            lines.push(format!(
                "  `{}`: {} x{} (example: {})",
                v.server, v.kind, v.count, v.example
            ));
        }
    }
    lines
}

/// Canned verdict reason the stub jury provider returns. Leads with `stub:`
/// so the per-juror note in any rendered verdict is unmistakable (WOR-1319).
const STUB_JURY_REASON: &str = "stub: canned jury verdict (--allow-stub-provider)";

/// How the executor's `llm-jury` provider slot was decided (WOR-1319).
#[derive(Debug, PartialEq, Eq)]
enum JuryProviderDecision {
    /// A real provider resolved during the run; jurors reuse it.
    UseReal,
    /// No selected test declares an `llm-jury` matcher, so the stub
    /// placeholder is inert and needs no opt-in.
    NoJuryDeclared,
    /// The operator opted into the canned stub with `--allow-stub-provider`;
    /// the run marks every jury test with a `stub:` report row.
    AllowedStub,
}

/// Gate the jury stub fallback (WOR-1319, mirroring the WOR-1312 agent gate).
///
/// A jury graded by the canned stub passes a fabricated verdict, so when a
/// jury test is in the plan and no real provider resolved, the run errors by
/// default, naming the tests and the escape hatch. `--allow-stub-provider`
/// opts back into the fallback for offline smoke runs. Free function so the
/// decision is unit-testable without a connected pool.
fn decide_jury_provider(
    jury_tests: &[String],
    has_real_provider: bool,
    allow_stub_provider: bool,
) -> Result<JuryProviderDecision> {
    if jury_tests.is_empty() {
        return Ok(JuryProviderDecision::NoJuryDeclared);
    }
    if has_real_provider {
        return Ok(JuryProviderDecision::UseReal);
    }
    if allow_stub_provider {
        return Ok(JuryProviderDecision::AllowedStub);
    }
    anyhow::bail!(
        "test(s) `{}` use `llm-jury` but no real LLM provider resolved for this run, so the \
         jurors would grade on the canned stub and pass a fabricated verdict; set a provider \
         credential so an agent cell in the suite resolves a real provider (the jury reuses \
         it), or pass --allow-stub-provider for an offline smoke run against the canned stub",
        jury_tests.join("`, `")
    )
}

/// Names of planned tests that declare at least one `llm-jury` matcher,
/// in top-level `expect:` entries or inside `assert_sets:`. Sorted so the
/// gate's error message and the report markers are deterministic.
fn jury_test_names(
    plans: &std::collections::HashMap<String, std::sync::Arc<mcptest_core::executor::PlannedTest>>,
) -> Vec<String> {
    use mcptest_core::executor::PlannedTest;
    use mcptest_core::suite::{Assertion, Matcher};
    fn any_jury(assertions: &[Assertion]) -> bool {
        assertions
            .iter()
            .any(|a| matches!(a.matcher, Matcher::LlmJury(_)))
    }
    let mut names: Vec<String> = plans
        .iter()
        .filter(|(_, plan)| {
            let (expectations, scoring) = match plan.as_ref() {
                PlannedTest::Tool(p) => (&p.expectations, &p.scoring),
                PlannedTest::Agent(p) => (&p.expectations, &p.scoring),
                PlannedTest::Compliance(p) => (&p.expectations, &p.scoring),
            };
            any_jury(expectations) || scoring.assert_sets.iter().any(|s| any_jury(&s.assertions))
        })
        .map(|(name, _)| name.clone())
        .collect();
    names.sort();
    names
}

/// Report row marking a jury test whose verdict came from the stub provider
/// (WOR-1319). Same shape as the WOR-1312 agent `(provider)` marker: a
/// passing check row whose note leads with `stub:` so a report scan (human
/// or machine) cannot mistake the verdict for a real jury run.
fn stub_jury_marker(test_name: &str) -> mcptest_core::checks::CheckOutcome {
    mcptest_core::checks::CheckOutcome {
        name: format!("{test_name} (jury provider)"),
        passed: true,
        failure: Some(
            "stub: canned jury verdict (no real provider resolved; --allow-stub-provider)"
                .to_string(),
        ),
    }
}

/// Drive one [`Runner`] over `specs` to a report. Generic over the
/// executor so the hook-wrapped and bare executors share one path.
async fn run_with_runner<E: mcptest_core::runner::TestExecutor + 'static>(
    executor: E,
    specs: &[mcptest_core::runner::TestSpec],
    options: mcptest_core::runner::RunnerOptions,
) -> Result<mcptest_core::report::Report> {
    use mcptest_core::runner::Runner;
    let mut runner = Runner::new(options, executor);
    runner.prepare().await.context("runner prepare failed")?;
    let report = runner
        .run_all(specs)
        .await
        .context("runner run_all failed")?;
    runner.finalize().await.context("runner finalize failed")?;
    Ok(report)
}

#[cfg(test)]
mod tests {
    use super::*;
    use mcptest_core::executor::{AgentPlan, PlannedTest, Scoring};
    use mcptest_core::suite::{AssertSet, Assertion, Matcher};
    use serde_json::json;
    use std::collections::HashMap;
    use std::sync::Arc;

    fn jury_assertion() -> Assertion {
        Assertion {
            target: "final_response".to_string(),
            matcher: Matcher::LlmJury(json!({
                "rubric": "is it helpful",
                "jurors": [{"model": "claude-sonnet-4-5"}],
            })),
            weight: 1.0,
            name: None,
            transform: None,
        }
    }

    fn exact_assertion() -> Assertion {
        Assertion {
            target: "final_response".to_string(),
            matcher: Matcher::Exact(json!("ok")),
            weight: 1.0,
            name: None,
            transform: None,
        }
    }

    fn agent_plan(expectations: Vec<Assertion>, scoring: Scoring) -> Arc<PlannedTest> {
        Arc::new(PlannedTest::Agent(AgentPlan {
            trace_envelope: json!({"final_response": "ok"}),
            expectations,
            scoring,
            generator_model: None,
        }))
    }

    #[test]
    fn jury_without_provider_or_opt_in_errors_naming_the_test_and_flag() {
        let err = decide_jury_provider(&["weather agent".to_string()], false, false)
            .expect_err("hard error by default");
        let msg = err.to_string();
        assert!(msg.contains("weather agent"), "names the test: {msg}");
        assert!(msg.contains("llm-jury"), "names the matcher: {msg}");
        assert!(
            msg.contains("--allow-stub-provider"),
            "names the escape hatch: {msg}"
        );
        assert!(
            msg.contains("fabricated"),
            "says why the stub verdict is dangerous: {msg}"
        );
    }

    #[test]
    fn jury_with_opt_in_is_an_allowed_stub() {
        let decision = decide_jury_provider(&["t".to_string()], false, true)
            .expect("opt-in permits the fallback");
        assert_eq!(decision, JuryProviderDecision::AllowedStub);
    }

    #[test]
    fn jury_with_real_provider_needs_no_opt_in() {
        let decision =
            decide_jury_provider(&["t".to_string()], true, false).expect("real provider is fine");
        assert_eq!(decision, JuryProviderDecision::UseReal);
    }

    #[test]
    fn no_jury_in_the_plan_skips_the_gate_entirely() {
        let decision =
            decide_jury_provider(&[], false, false).expect("nothing to gate without a jury");
        assert_eq!(decision, JuryProviderDecision::NoJuryDeclared);
    }

    #[test]
    fn jury_test_names_finds_top_level_and_assert_set_juries() {
        let mut plans: HashMap<String, Arc<PlannedTest>> = HashMap::new();
        plans.insert(
            "top-level jury".to_string(),
            agent_plan(vec![jury_assertion()], Scoring::default()),
        );
        plans.insert(
            "set jury".to_string(),
            agent_plan(
                vec![exact_assertion()],
                Scoring {
                    assert_sets: vec![AssertSet {
                        name: None,
                        threshold: 1.0,
                        weight: 1.0,
                        assertions: vec![jury_assertion()],
                    }],
                    ..Scoring::default()
                },
            ),
        );
        plans.insert(
            "no jury".to_string(),
            agent_plan(vec![exact_assertion()], Scoring::default()),
        );
        assert_eq!(
            jury_test_names(&plans),
            vec!["set jury".to_string(), "top-level jury".to_string()]
        );
    }

    #[test]
    fn violation_warning_names_server_kind_count_and_example() {
        let entries = vec![mcptest_core::report::ProtocolViolationReport {
            server: "github".into(),
            kind: "string-id echo".into(),
            count: 3,
            example: "server echoed numeric request id 1 as the string \"1\"".into(),
        }];
        let lines = protocol_violation_warning_lines(&entries);
        assert_eq!(lines.len(), 2);
        assert!(lines[0].contains("protocol violations observed"));
        assert_eq!(
            lines[1],
            "  `github`: string-id echo x3 \
             (example: server echoed numeric request id 1 as the string \"1\")"
        );
    }

    #[test]
    fn violation_warning_drops_the_example_clause_when_none_was_retained() {
        let entries = vec![mcptest_core::report::ProtocolViolationReport {
            server: "srv".into(),
            kind: "orphan response".into(),
            count: 40,
            example: String::new(),
        }];
        let lines = protocol_violation_warning_lines(&entries);
        assert_eq!(lines[1], "  `srv`: orphan response x40");
    }

    #[test]
    fn before_all_failure_report_derives_its_summary_from_the_rows() {
        use mcptest_core::runner::TestSpec;
        let specs = vec![TestSpec::new("a"), TestSpec::new("b")];
        let report = all_failed_report(&specs, "setup exploded");
        assert_eq!(report.summary.total, 2);
        assert_eq!(report.summary.failed, 2);
        assert_eq!(report.summary.passed, 0);
        assert!(report.summary.is_consistent(&report.tests));
        let skipped = all_skipped_report(&specs, "hook asked to skip");
        assert_eq!(skipped.summary.skipped, 2);
        assert!(skipped.summary.is_consistent(&skipped.tests));
    }

    #[test]
    fn stub_jury_marker_note_leads_with_the_stub_prefix() {
        let outcome = stub_jury_marker("weather agent");
        assert!(outcome.passed, "the marker never fails the run by itself");
        assert_eq!(outcome.name, "weather agent (jury provider)");
        let note = outcome.failure.expect("marker carries a note");
        assert!(note.starts_with("stub:"), "note is unmistakable: {note}");
    }

    #[test]
    fn stub_jury_reason_leads_with_the_stub_prefix() {
        // The canned verdict the stub provider returns is the per-juror note,
        // so it must lead with `stub:` too.
        assert!(STUB_JURY_REASON.starts_with("stub:"));
    }
}
