//! Concrete MCP [`TestExecutor`].
//!
//! Bridges the runner's transport-free [`TestExecutor`] trait to the
//! protocol [`crate::protocol::Client`]. Each `execute_one` looks up
//! the [`PlannedTest`] by name, dispatches to the right backend (a
//! [`crate::connector::LiveServer`] for tool tests, a pre-recorded
//! [`PlannedTest::Agent`] envelope for agent tests), walks the
//! [`Assertion`] list, and returns the [`TestOutcome`].
//!
//! Matcher routing: `exact`, `contains`, `regex`, and
//! `schema` are evaluated by the shared [`crate::matchers`] crate so the
//! live path and the matchers crate cannot diverge. `subset` stays inline
//! (the matchers crate has no subset module). The provider-backed matchers,
//! `llm-jury`, `similar`, and `llm-judge`, run through the
//! private `check_llm_jury` / `check_similar` / `check_llm_judge` helpers in
//! the async evaluator when the executor was built with a
//! [`crate::eval::provider::LlmProvider`]; `snapshot` runs through
//! `check_snapshot` against the executor's filesystem record/replay. The
//! synchronous `check_matcher` only guards against calling these four
//! directly.

use std::collections::HashMap;
use std::sync::Arc;

use async_trait::async_trait;
use serde_json::Value;

use crate::connector::ServerPool;
use crate::eval::provider::LlmProvider;
use crate::fixtures::NamedError;
use crate::matchers::{
    cel, contains, exact, is_json, is_sql, is_xml, levenshtein, named, regex, schema, tools_call,
};
use crate::report::TestStatus;
use crate::runner::{TestExecutor, TestOutcome, TestSpec};
use crate::suite::{Action, AssertSet, Assertion, DerivedMetric, Matcher};
use crate::transform::TransformContext;

use dispatch::{
    agent_transform_context, apply_response_transform, assertions_expect_error, call_server,
    run_metamorphic, synthesize_error_envelope, transform_context, ElicitSetup,
};

/// Concrete MCP executor consumed by `Runner::run_all`.
///
/// One per run. Builds an `Arc<ServerPool>` of connected servers and
/// a name-indexed `HashMap` of [`PlannedTest`] entries the runner's
/// [`TestSpec`] keys look up. The optional `provider` is reused for
/// every juror when an assertion uses the `llm-jury` matcher; the
/// matcher itself names the per-juror model, but the executor only
/// owns one provider trait object today.
pub struct McpExecutor {
    pool: Arc<ServerPool>,
    plans: HashMap<String, Arc<PlannedTest>>,
    provider: Option<Arc<dyn LlmProvider>>,
    /// Model id the `llm-judge` matcher grades with when its block omits an
    /// explicit `model:`. The run path sets this to the model that built
    /// [`Self::provider`] (the agent's model), so a judge with no model
    /// grades with the same model the agent used.
    default_judge_model: Option<String>,
    snapshot: SnapshotContext,
    /// Optional per-run coverage recorder (WOR-1386). When set, every
    /// dispatched tool call records one [`crate::coverage::Observation`] so
    /// `mcptest run --coverage` reports what the run actually exercised. The
    /// mutex serializes records across the runner's parallel tests; `None`
    /// (the default) keeps the no-recording path with zero overhead.
    coverage: Option<Arc<std::sync::Mutex<crate::coverage::CoverageTracker>>>,
    /// Optional run-path result cache (WOR-1398); `None` never touches it.
    result_cache: Option<result_cache::ResultCache>,
}

/// Pre-resolved view of one test. Tool tests carry the server name
/// and action so the executor can dispatch on the live MCP transport.
/// Agent tests carry a pre-recorded conversation envelope so the
/// executor can run matcher targets like `tool_calls[0].name` and
/// `final_response` without re-running the model loop on every retry.
#[derive(Debug, Clone)]
pub enum PlannedTest {
    /// Tool, resource, or prompt test against a live MCP server.
    Tool(ToolPlan),
    /// Agent test whose conversation trace has already been collected.
    Agent(AgentPlan),
    /// Compliance check issuing a raw protocol method (WOR-1347).
    Compliance(CompliancePlan),
}

/// Compliance-check plan: a raw protocol method against a live server.
///
/// Exists because a `compliance:` test names a wire method
/// (`initialize`, `tools/list`, ...), not a tool/resource/prompt, so it
/// fits no [`Action`] variant. The `initialize` method is special-cased
/// at dispatch: re-issuing it mid-session is not valid MCP, so the
/// executor evaluates the handshake the connector captured instead.
#[derive(Debug, Clone)]
pub struct CompliancePlan {
    /// Server name the check references. Looked up in [`ServerPool`].
    pub server: String,
    /// Protocol method to issue (or `initialize` to replay the captured
    /// handshake). Sent with empty `{}` params; the v1 check surface
    /// carries none.
    pub method: String,
    /// Assertions evaluated against the `{"result": ...}` envelope.
    pub expectations: Vec<Assertion>,
    /// Combined-score inputs. Compliance checks keep the legacy
    /// every-assertion-must-pass default; the field exists so every
    /// [`PlannedTest`] variant carries one source of scoring truth.
    pub scoring: Scoring,
}

/// Tool-test plan: server name, the action to invoke, and the
/// assertions to evaluate against the wrapped `{"result": ...}` envelope.
#[derive(Debug, Clone)]
pub struct ToolPlan {
    /// Server name the test references. Looked up in [`ServerPool`].
    pub server: String,
    /// What to invoke.
    pub action: Action,
    /// Assertions to evaluate against the response envelope.
    pub expectations: Vec<Assertion>,
    /// Step-level budgets and header checks from the long-form `expect:`
    /// mapping (WOR-1280). The empty default keeps the legacy no-gate path.
    pub budgets: crate::suite::ExpectBudgets,
    /// Transport class of the target server, so header checks can fail
    /// actionably on targets that have no HTTP headers.
    pub transport: TargetTransport,
    /// Combined-score inputs (assert-sets, threshold, derived metrics).
    /// Defaults to the legacy all-pass model.
    pub scoring: Scoring,
    /// Optional request/response transform. `None` keeps the
    /// no-transform path: the request is sent as built and the response is
    /// matched as received.
    pub transform: Option<crate::suite::TransformSpec>,
    /// Optional named-error fixture, pre-resolved at plan-build time.
    ///
    /// When `Some`, the executor synthesizes a JSON-RPC error envelope
    /// from the [`NamedError`] and feeds it through the assertion
    /// pipeline as if it had come back from the server. No
    /// `ServerPool` lookup, no transport spawn, no network call. The
    /// CLI's plan builder is responsible for resolving the test's
    /// `inject_error: <name>` against `ResolvedSuite.fixtures` and
    /// populating this field; the cross-reference validator ran at
    /// suite-load time so an unresolved name never reaches the
    /// executor.
    ///
    /// Boxed because a `NamedError` is larger than a usize-shaped
    /// payload and most tool plans never carry one, so the indirection
    /// keeps `PlannedTest` small on the common path.
    pub inject_error: Option<Box<NamedError>>,
    /// Optional metamorphic relations (WOR-1237). When set, after the base
    /// assertions the executor issues one transformed call per relation and
    /// gates the test on whether each relation held.
    pub metamorphic: Option<crate::suite::MetamorphicSpec>,
    /// Optional schema-driven input fuzzing (WOR-1238). When set, after the
    /// base assertions the executor fuzzes the tool's arguments and gates on
    /// clean failure handling.
    pub fuzz: Option<crate::suite::FuzzSpec>,
    /// Optional negative-path JSON-RPC error conformance (WOR-1239). When set,
    /// the executor runs the taxonomy-keyed bad-request probes and gates on
    /// clean rejection.
    pub negative_path: Option<crate::suite::NegativePathSpec>,
    /// Optional answer source for a 2026-07-28 `InputRequiredResult`
    /// elicitation (WOR-1383). Resolved at plan-build time from the test's
    /// `input_responses` / `input_responder` or the suite-level default; the
    /// executor uses it to satisfy an elicitation and retry. `None` when the
    /// test declares no source (an eliciting tool then fails fast).
    pub responder: Option<elicitation::Responder>,
}

/// Agent-test plan: a serialized [`crate::matchers`]-addressable trace
/// envelope and the assertions to evaluate against it.
///
/// The CLI drives the model loop through `mcptest-agent` before the
/// runner starts and stashes the resulting envelope here so the runner
/// can retry the assertion stage without re-running the model.
#[derive(Debug, Clone)]
pub struct AgentPlan {
    /// JSON envelope produced by `mcptest_agent::ConversationTrace::to_envelope`.
    /// Matchers resolve target paths like `tool_calls[0].name` and
    /// `conversation.tokens.total` against this value.
    pub trace_envelope: Value,
    /// Assertions to evaluate against [`AgentPlan::trace_envelope`].
    pub expectations: Vec<Assertion>,
    /// Combined-score inputs (assert-sets, threshold, derived metrics).
    /// Defaults to the legacy all-pass model.
    pub scoring: Scoring,
    /// The model that produced this trace, fed to the `jury.same_family` bias
    /// signal. The generator under test in an agent test is the
    /// agent's own model. `None` when the trace's model is unknown (a replayed
    /// cassette, say), in which case `same_family` reads `false`.
    pub generator_model: Option<String>,
}

/// Score-model inputs for one planned test.
///
/// Bundles the assert-sets, the optional combined-score threshold, and
/// the derived-metric declarations the executor folds into a verdict.
/// The [`Default`] is the legacy contract: no assert-sets, no threshold
/// (so every assertion must pass), no derived metrics. A test that
/// declares none of the  keys carries this default and behaves
/// exactly as it did before.
#[derive(Debug, Clone, Default)]
pub struct Scoring {
    /// Named assert-set bundles, each with its own internal threshold.
    pub assert_sets: Vec<AssertSet>,
    /// Combined-score threshold on a 0..1 scale. `None` keeps the legacy
    /// all-pass rule.
    pub threshold: Option<f64>,
    /// Derived-metric declarations reported (and optionally gating).
    pub derived_metrics: Vec<DerivedMetric>,
    /// Force the combined score onto the report even without a threshold
    ///. A matrix-level assertion (`matrix_assert:`) compares cells
    /// by their `report.tests[].score`, which the score model otherwise
    /// emits only for thresholded tests. The CLI sets this for cells of a
    /// matrix-assert test. Defaults `false`, so a test that sets neither a
    /// threshold nor this flag still reports `score: None` and its legacy
    /// report JSON is byte-for-byte unchanged.
    pub emit_score: bool,
}

impl PlannedTest {
    fn scoring(&self) -> &Scoring {
        match self {
            PlannedTest::Tool(p) => &p.scoring,
            PlannedTest::Agent(p) => &p.scoring,
            PlannedTest::Compliance(p) => &p.scoring,
        }
    }

    fn expectations(&self) -> &[Assertion] {
        match self {
            PlannedTest::Tool(p) => &p.expectations,
            PlannedTest::Agent(p) => &p.expectations,
            PlannedTest::Compliance(p) => &p.expectations,
        }
    }
}

impl McpExecutor {
    /// Build a new executor against the given pool and plan map.
    /// Callers (the CLI) populate both before construction.
    pub fn new(pool: Arc<ServerPool>, plans: HashMap<String, Arc<PlannedTest>>) -> Self {
        Self {
            pool,
            plans,
            provider: None,
            default_judge_model: None,
            snapshot: SnapshotContext::default(),
            coverage: None,
            result_cache: None,
        }
    }

    /// Install a default [`LlmProvider`] used to dispatch jurors when an
    /// assertion uses the `llm-jury` matcher. Without a provider the
    /// executor returns a structured failure for any `llm-jury`
    /// assertion (rather than panicking), so agent suites that use the
    /// jury matcher must call this builder.
    pub fn with_provider(mut self, provider: Arc<dyn LlmProvider>) -> Self {
        self.provider = Some(provider);
        self
    }

    /// Set the default model the `llm-judge` matcher grades with when its
    /// block omits `model:`. Pair with [`Self::with_provider`]: the model
    /// must be one the installed provider can serve. When unset,
    /// an `llm-judge` block without `model:` fails with a clear message.
    pub fn with_judge_model(mut self, model: impl Into<String>) -> Self {
        self.default_judge_model = Some(model.into());
        self
    }

    /// Install the filesystem context the `snapshot` matcher records and
    /// replays against. `dir` is the directory snapshot
    /// files live in (the suite's `snapshots/` folder); `mode` carries the
    /// `--update-snapshots` decision. Without this builder the executor uses
    /// [`SnapshotContext::default`] (a `snapshots/` dir under the working
    /// directory, never auto-updating).
    pub fn with_snapshots(
        mut self,
        dir: std::path::PathBuf,
        mode: crate::matchers::snapshot::UpdateSnapshotsMode,
    ) -> Self {
        self.snapshot = SnapshotContext { dir, mode };
        self
    }

    /// Install a per-run coverage recorder (WOR-1386). Pass a shared tracker;
    /// the executor records one observation per dispatched tool call and the
    /// caller folds the tracker into a report once the run finishes. Without
    /// this builder the executor records nothing.
    pub fn with_coverage(
        mut self,
        tracker: Arc<std::sync::Mutex<crate::coverage::CoverageTracker>>,
    ) -> Self {
        self.coverage = Some(tracker);
        self
    }

    /// Record one coverage observation for a dispatched tool call, when a
    /// recorder is installed. Only `tools/call` actions count toward tool
    /// coverage; resource reads and prompt gets are not tool invocations. A
    /// poisoned mutex is ignored rather than crashing a run over telemetry.
    fn record_coverage(&self, tool: &ToolPlan) {
        let Some(tracker) = &self.coverage else {
            return;
        };
        let Action::ToolCall { tool: name, args } = &tool.action else {
            return;
        };
        let observation = crate::coverage::Observation {
            tool: name.clone(),
            args: args.clone(),
            error_path: tool
                .inject_error
                .as_ref()
                .map(|e| e.name.clone())
                .filter(|n| !n.is_empty()),
            transport: tool.transport.label().to_string(),
        };
        if let Ok(mut guard) = tracker.lock() {
            guard.record(&observation);
        }
    }

    /// Resolve a tool plan to the response envelope the assertions see.
    ///
    /// Returns a [`DispatchedTool`] bundling the wrapped `{"result": ...}`
    /// envelope, the transform context, the wall-clock duration of the
    /// live call, and the captured HTTP response headers (WOR-1287). The
    /// duration and the headers are `None` on the fixture-injection path,
    /// which dispatches nothing, so the corresponding gates have nothing
    /// to measure there. Any dispatch or transform failure comes back as
    /// a ready-made failing [`TestOutcome`] the caller returns unchanged.
    /// Split out of `execute_one` to keep that function under the length
    /// lint.
    async fn dispatch_tool(
        &self,
        test_name: &str,
        tool: &ToolPlan,
    ) -> Result<DispatchedTool, TestOutcome> {
        // Fixture injection short-circuits the live dispatch path. The
        // CLI pre-resolved the `inject_error: <name>` reference against
        // the suite's fixtures table, so the executor synthesizes the
        // error envelope from the typed `NamedError` and feeds it
        // straight into the assertion pipeline. No pool lookup, no
        // transport, no cassette write.
        if let Some(err) = &tool.inject_error {
            let ctx = transform_context(test_name, tool);
            return Ok(DispatchedTool {
                response: synthesize_error_envelope(err),
                ctx,
                measured: None,
                response_headers: None,
            });
        }
        // WOR-1398: serve a cached result before the pool lookup so an
        // opted-in run never pays for the server on a hit.
        if let Some(hit) = self.cached_dispatch(test_name, tool) {
            return Ok(hit);
        }
        let Some(server) = self.pool.get(&tool.server).await else {
            return Err(TestOutcome::fail(format!(
                "test `{test_name}` references unknown server `{}`",
                tool.server
            )));
        };
        // The request transform runs inside `call_server` (after the params
        // are built, before send); the response transform runs here (after
        // the call returns, before assertions). When a tool cassette layer
        // lands, the request transform must run before record and the
        // response transform after replay so the recording stays faithful
        // (deferred follow-up).
        let ctx = transform_context(test_name, tool);
        let started = std::time::Instant::now();
        // When the test asserts on `result.error`, a live JSON-RPC error is
        // routed into an assertable envelope instead of failing the call
        // (WOR-1357); otherwise an unexpected error still fails hard.
        let expects_error = assertions_expect_error(&tool.expectations);
        // Thread the resolved elicitation answer source so a tool that returns
        // an InputRequiredResult is satisfied and retried (WOR-1383).
        let elicit = tool.responder.as_ref().map(|responder| ElicitSetup {
            responder,
            test_name,
            server: &tool.server,
        });
        let (raw, request_id) = call_server(
            &server.client,
            &tool.action,
            tool.transform.as_ref(),
            &ctx,
            expects_error,
            elicit.as_ref(),
        )
        .await
        .map_err(TestOutcome::fail)?;
        // Time only the dispatch, not the response transform: the budget
        // protects against a slow server, and a slow user transform already
        // has its own failure surface.
        let measured = started.elapsed();
        // WOR-1287: claim this call's response headers off the transport's
        // side channel. `None` on stdio and cassette targets, which carry
        // no capture channel at all.
        let response_headers = server
            .header_capture
            .as_ref()
            .and_then(|capture| capture.take(request_id));
        let response = apply_response_transform(raw, tool.transform.as_ref(), &ctx)
            .await
            .map_err(TestOutcome::fail)?;
        // WOR-1398: store the fresh response so the next opted-in run can
        // serve it. A no-op unless caching and writes are both on.
        self.cache_put(tool, &response);
        Ok(DispatchedTool {
            response,
            ctx,
            measured: Some(measured),
            response_headers,
        })
    }

    /// Resolve a compliance plan to the `{"result": ...}` envelope its
    /// assertions see (WOR-1347).
    ///
    /// `initialize` replays the handshake the connector captured at
    /// connect time, because re-issuing it on an established session is
    /// not valid MCP. Every other method is sent as a raw request with
    /// empty params. A dispatch failure comes back as a ready-made
    /// failing [`TestOutcome`] the caller returns unchanged.
    async fn dispatch_compliance(
        &self,
        test_name: &str,
        check: &CompliancePlan,
    ) -> Result<(Value, TransformContext), TestOutcome> {
        let Some(server) = self.pool.get(&check.server).await else {
            return Err(TestOutcome::fail(format!(
                "test `{test_name}` references unknown server `{}`",
                check.server
            )));
        };
        let raw = if check.method == "initialize" {
            server.client.initialize_response().await.ok_or_else(|| {
                TestOutcome::fail(format!(
                    "test `{test_name}`: no captured `initialize` handshake on server `{}`",
                    check.server
                ))
            })?
        } else {
            server
                .client
                .request(check.method.clone(), serde_json::json!({}))
                .await
                .map_err(|e| TestOutcome::fail(format!("{} failed: {e}", check.method)))?
        };
        let ctx = TransformContext {
            test_name: test_name.to_string(),
            server: check.server.clone(),
            tool: check.method.clone(),
            vars: std::collections::BTreeMap::new(),
        };
        Ok((serde_json::json!({ "result": raw }), ctx))
    }
}

/// Everything `execute_one` needs back from one tool dispatch (WOR-1287).
///
/// A struct rather than a four-element tuple so the call site stays
/// readable and future per-call observations have a named place to land.
struct DispatchedTool {
    /// Wrapped `{"result": ...}` envelope the assertions run against.
    response: Value,
    /// Transform context naming the test, server, and tool.
    ctx: TransformContext,
    /// Wall-clock duration of the live call; `None` when nothing was
    /// dispatched (fixture injection).
    measured: Option<std::time::Duration>,
    /// HTTP response headers captured for this call; `None` on transports
    /// without headers or when nothing was dispatched.
    response_headers: Option<HashMap<String, String>>,
}

#[async_trait]
impl TestExecutor for McpExecutor {
    async fn execute_one(&self, test: &TestSpec) -> TestOutcome {
        let Some(plan) = self.plans.get(&test.name).cloned() else {
            return TestOutcome::fail(format!("no plan registered for test `{}`", test.name));
        };
        // The transform context names the test/server/tool for any
        // request, response, or per-assertion transform subprocess.
        // Both branches build it so assertion transforms work on agent tests
        // too, where only the test name is meaningful.
        // `measured` is the wall-clock duration of the live call, feeding the
        // `max_duration_ms` gate (WOR-1280). It stays `None` on every path
        // that makes no live call (fixture injection, agent traces).
        let mut measured: Option<std::time::Duration> = None;
        // Per-call HTTP response headers captured by the transport
        // (WOR-1287); only a live HTTP dispatch populates this.
        let mut response_headers: Option<HashMap<String, String>> = None;
        let (response, ctx) = match plan.as_ref() {
            PlannedTest::Tool(tool) => match self.dispatch_tool(&test.name, tool).await {
                Ok(dispatched) => {
                    measured = dispatched.measured;
                    response_headers = dispatched.response_headers;
                    // WOR-1386: record what this run exercised, before the
                    // assertion verdict, so `--coverage` reflects the calls
                    // made rather than only the ones that passed.
                    self.record_coverage(tool);
                    (dispatched.response, dispatched.ctx)
                }
                Err(outcome) => return outcome,
            },
            PlannedTest::Agent(agent) => (
                agent.trace_envelope.clone(),
                agent_transform_context(&test.name),
            ),
            PlannedTest::Compliance(check) => {
                match self.dispatch_compliance(&test.name, check).await {
                    Ok(dispatched) => dispatched,
                    Err(outcome) => return outcome,
                }
            }
        };
        // The generator model under test backs the `jury.same_family`
        // signal. Only an agent test has one; a tool test's response has no
        // generating model.
        let generator_model = match plan.as_ref() {
            PlannedTest::Agent(agent) => agent.generator_model.as_deref(),
            PlannedTest::Tool(_) | PlannedTest::Compliance(_) => None,
        };
        let base_outcome = evaluate_expectations(
            &response,
            plan.expectations(),
            plan.scoring(),
            self.provider.as_deref(),
            self.default_judge_model.as_deref(),
            &ctx,
            generator_model,
            &self.snapshot,
        )
        .await;
        // WOR-1280: the step-level budgets (duration, response tokens,
        // header checks) gate after the assertions so a primary assertion
        // failure is never masked by a budget breach.
        if let PlannedTest::Tool(tool) = plan.as_ref() {
            if base_outcome.status == TestStatus::Pass {
                if let Err(message) = budget_gate::evaluate_budget_gate(
                    &tool.budgets,
                    &tool.server,
                    tool.transport,
                    measured,
                    &response,
                    response_headers.as_ref(),
                ) {
                    return TestOutcome::fail(message);
                }
            }
        }
        // A tool test with metamorphic relations (WOR-1237) or a fuzz block
        // (WOR-1238) runs extra live calls after the base assertions and gates
        // on the result. Skipped on the inject_error path (no live call) and
        // when the base test already failed, so the primary failure is never
        // masked. Both share one server handle.
        if let PlannedTest::Tool(tool) = plan.as_ref() {
            let extra =
                tool.metamorphic.is_some() || tool.fuzz.is_some() || tool.negative_path.is_some();
            if let Action::ToolCall {
                tool: tool_name,
                args,
            } = &tool.action
            {
                if extra && tool.inject_error.is_none() && base_outcome.status == TestStatus::Pass {
                    if let Some(server) = self.pool.get(&tool.server).await {
                        if let Some(spec) = &tool.metamorphic {
                            if let Err(message) =
                                run_metamorphic(&server.client, tool_name, args, spec, &ctx).await
                            {
                                return TestOutcome::fail(message);
                            }
                        }
                        if let Some(spec) = &tool.fuzz {
                            if let Err(message) =
                                run_fuzz(&server.client, tool_name, args, spec).await
                            {
                                return TestOutcome::fail(message);
                            }
                        }
                        if let Some(spec) = &tool.negative_path {
                            if let Err(message) =
                                run_negative_path(&server.client, tool_name, args, spec).await
                            {
                                return TestOutcome::fail(message);
                            }
                        }
                    }
                }
            }
        }
        base_outcome
    }
}

// Dispatch helpers live in the sibling `dispatch` module so this file
// stays under the production-module-size cap; the call sites use them
// via short paths declared above.

/// Back-compat thin wrapper kept for the `llm-jury` matcher's regular path
/// and the assert-set evaluator. Runs the jury once and folds it to
/// pass/fail; jury analytics on the inline jury path are still available via
/// the test-level pre-pass.
pub(super) async fn check_llm_jury(
    actual: &Value,
    raw: &Value,
    provider: Option<&dyn LlmProvider>,
) -> Result<(), String> {
    // The inline/assert-set jury path does not gate on `jury.*` analytics, so
    // it needs no generator model: the same-family signal is only exposed via
    // the test-level pre-pass (`prepare_jury`).
    jury_metrics::run_llm_jury(actual, raw, provider, None)
        .await?
        .outcome
}

mod assertion_transform;
mod async_matchers;
mod budget_gate;
mod composition;
mod dispatch;
pub mod elicitation;
mod evaluate;
mod fuzz_run;
mod judge_matcher;
mod jury_metrics;
mod matcher_result;
mod metric_assert;
mod negative_path_run;
mod path;
mod result_cache;
mod snapshot;
use async_matchers::{check_citation, check_similar};
pub use budget_gate::TargetTransport;
use evaluate::evaluate_expectations;
pub use fuzz_run::fuzz_tool;
use fuzz_run::run_fuzz;
use matcher_result::{check_subset, from_assertion, from_result};
pub use metric_assert::eval_metric_assertion;
use negative_path_run::run_negative_path;
pub use path::{absent_target_hint, resolve_target};
use snapshot::check_snapshot;
pub use snapshot::SnapshotContext;

use assertion_transform::apply_assertion_transform;

/// Evaluate one non-jury matcher against the resolved target value.
///
/// `exact`, `contains`, `regex`, and `schema` delegate to the shared
/// [`crate::matchers`] crate, so the live executor and the
/// matchers crate share one definition of each matcher rather than two
/// that can drift apart. The matcher value carried in the [`Matcher`]
/// variant is the expectation; `actual` is the value resolved from the
/// response envelope. `subset` stays inline because the matchers crate
/// has no subset module. Returns `Ok(())` on pass and a rendered
/// failure string (with JSON-Pointer paths and notes preserved) on a
/// failing assertion.
pub(crate) fn check_matcher(actual: &Value, matcher: &Matcher) -> Result<(), String> {
    match matcher {
        Matcher::Exact(expected) => from_result(exact::evaluate(expected, actual)),
        Matcher::Contains(expected) => from_result(contains::evaluate(expected, actual)),
        Matcher::Regex(pattern) => from_result(regex::evaluate(pattern, actual)),
        Matcher::Subset(expected) => check_subset(actual, expected),
        Matcher::Schema(document) => from_result(schema::evaluate(document, actual)),
        Matcher::Snapshot(_) => {
            // `snapshot` runs through [`check_snapshot`] in the async
            // evaluator because it needs the suite's filesystem context.
            // Reaching it here means the caller bypassed
            // [`evaluate_expectations`], which is a programmer error.
            Err(
                "matcher `snapshot` must be evaluated through `evaluate_expectations`, \
                 not the synchronous `check_matcher` helper"
                    .to_string(),
            )
        }
        Matcher::LlmJudge(_) => {
            // `llm-judge` runs through [`check_llm_judge`] in the async
            // evaluator (it needs a provider). Reaching it here means the
            // caller bypassed [`evaluate_expectations`], a programmer error.
            Err(
                "matcher `llm-judge` must be evaluated through `evaluate_expectations`, \
                 not the synchronous `check_matcher` helper"
                    .to_string(),
            )
        }
        Matcher::LlmJury(_) => {
            // `llm-jury` runs through [`check_llm_jury`] in the async
            // evaluator. Reaching it here means the caller bypassed
            // [`evaluate_expectations`], which is a programmer error.
            Err(
                "matcher `llm-jury` must be evaluated through `evaluate_expectations`, \
                 not the synchronous `check_matcher` helper"
                    .to_string(),
            )
        }
        Matcher::ContainsAll(needles) => from_result(named::contains_all(needles, actual)),
        Matcher::ContainsAny(needles) => from_result(named::contains_any(needles, actual)),
        Matcher::IContains(needle) => {
            from_result(named::icontains(&Value::String(needle.clone()), actual))
        }
        Matcher::StartsWith(prefix) => {
            from_result(named::starts_with(&Value::String(prefix.clone()), actual))
        }
        Matcher::IsJson(argument) => from_result(is_json::evaluate(argument, actual)),
        Matcher::IsValidToolsCall(argument) => from_result(tools_call::evaluate(argument, actual)),
        Matcher::IsXml(argument) => from_result(is_xml::evaluate(argument, actual)),
        Matcher::IsSql(argument) => from_result(is_sql::evaluate(argument, actual)),
        Matcher::Levenshtein(argument) => from_result(levenshtein::evaluate(argument, actual)),
        Matcher::Cel(program) => from_result(cel::evaluate(program, actual)),
        Matcher::Similar(_) => {
            // `similar` runs through [`check_similar`] in the async evaluator.
            // Reaching it here means the caller bypassed
            // [`evaluate_expectations`], which is a programmer error.
            Err(
                "matcher `similar` must be evaluated through `evaluate_expectations`, \
                 not the synchronous `check_matcher` helper"
                    .to_string(),
            )
        }
        Matcher::Citation(_) => {
            // `citation` runs through [`check_citation`] in the async evaluator
            // (it needs a provider judge call and a network fetch). Reaching it
            // here means the caller bypassed [`evaluate_expectations`].
            Err(
                "matcher `citation` must be evaluated through `evaluate_expectations`, \
                 not the synchronous `check_matcher` helper"
                    .to_string(),
            )
        }
        Matcher::Not(inner) => check_not(actual, inner),
        Matcher::OneOf(children) => composition::check(actual, children, composition::Rule::OneOf),
        Matcher::AnyOf(children) => composition::check(actual, children, composition::Rule::AnyOf),
        Matcher::AllOf(children) => composition::check(actual, children, composition::Rule::AllOf),
    }
}

/// Evaluate a `not:` matcher by inverting the inner matcher's pass/fail.
///
/// `not:` is scoped to deterministic matchers. The deferred/async matchers
/// (`snapshot`, `llm-judge`, `llm-jury`) have no synchronous pass/fail to
/// invert here, so wrapping one returns a clear evaluation error rather than
/// negating a "deferred" failure into a misleading pass. An inner structural
/// error (bad regex, malformed argument) is propagated unchanged: a broken
/// matcher is a broken matcher whether or not it is negated, so we surface the
/// real cause rather than inverting it into a spurious pass.
fn check_not(actual: &Value, inner: &Matcher) -> Result<(), String> {
    match deterministic_pass(actual, inner) {
        // Inner is deferred/async; cannot be negated synchronously.
        DeterministicOutcome::NotNegatable(name) => Err(format!(
            "matcher `not` cannot wrap the deferred/async matcher `{name}`; \
             negation is limited to deterministic matchers"
        )),
        // Inner passed, so the negation fails.
        DeterministicOutcome::Passed => {
            Err("expected NOT to match, but the inner matcher passed".to_string())
        }
        // Inner failed its assertion, so the negation passes.
        DeterministicOutcome::Failed => Ok(()),
        // A structural error is not a failed assertion; propagate it.
        DeterministicOutcome::Error(reason) => Err(reason),
    }
}

/// Three-state outcome of a deterministic matcher, used by [`check_not`] to
/// invert a real assertion result without mistaking a structural error for a
/// failed assertion. `from_result`-style flattening keeps only pass-or-message;
/// negation needs to keep the three states apart.
pub(super) enum DeterministicOutcome {
    /// The matcher's assertion held.
    Passed,
    /// The matcher's assertion did not hold. The message is discarded because
    /// negation only cares that it failed, not why.
    Failed,
    /// The matcher could not run (bad input, bad regex, invalid schema).
    Error(String),
    /// The inner matcher is deferred/async and cannot be negated here.
    NotNegatable(&'static str),
}

/// Run a deterministic matcher and report which of the three states it landed
/// in. Mirrors the dispatch in [`check_matcher`] but keeps pass, fail, and
/// structural error distinct.
pub(super) fn deterministic_pass(actual: &Value, matcher: &Matcher) -> DeterministicOutcome {
    let result = match matcher {
        Matcher::Exact(expected) => exact::evaluate(expected, actual),
        Matcher::Contains(expected) => contains::evaluate(expected, actual),
        Matcher::Regex(pattern) => regex::evaluate(pattern, actual),
        Matcher::Schema(document) => schema::evaluate(document, actual),
        Matcher::ContainsAll(needles) => named::contains_all(needles, actual),
        Matcher::ContainsAny(needles) => named::contains_any(needles, actual),
        Matcher::IContains(needle) => named::icontains(&Value::String(needle.clone()), actual),
        Matcher::StartsWith(prefix) => named::starts_with(&Value::String(prefix.clone()), actual),
        Matcher::IsJson(argument) => is_json::evaluate(argument, actual),
        Matcher::IsValidToolsCall(argument) => tools_call::evaluate(argument, actual),
        Matcher::IsXml(argument) => is_xml::evaluate(argument, actual),
        Matcher::IsSql(argument) => is_sql::evaluate(argument, actual),
        Matcher::Levenshtein(argument) => levenshtein::evaluate(argument, actual),
        Matcher::Cel(program) => cel::evaluate(program, actual),
        // `subset` is inline-only; reuse its checker and treat any message
        // as a failed assertion (it never errors structurally).
        Matcher::Subset(expected) => {
            return match check_subset(actual, expected) {
                Ok(()) => DeterministicOutcome::Passed,
                Err(_) => DeterministicOutcome::Failed,
            }
        }
        Matcher::Snapshot(_) => return DeterministicOutcome::NotNegatable("snapshot"),
        Matcher::LlmJudge(_) => return DeterministicOutcome::NotNegatable("llm-judge"),
        Matcher::LlmJury(_) => return DeterministicOutcome::NotNegatable("llm-jury"),
        Matcher::Similar(_) => return DeterministicOutcome::NotNegatable("similar"),
        Matcher::Citation(_) => return DeterministicOutcome::NotNegatable("citation"),
        // Nested `not:` negates again; recurse through `check_not`, which
        // already inverts the inner result.
        Matcher::Not(nested) => {
            return match check_not(actual, nested) {
                Ok(()) => DeterministicOutcome::Passed,
                Err(_) => DeterministicOutcome::Failed,
            }
        }
        // Nested compositions recurse through `composition::check`.
        Matcher::OneOf(_) | Matcher::AnyOf(_) | Matcher::AllOf(_) => {
            return composition::deterministic_from(actual, matcher)
        }
    };
    match result {
        Err(e) => DeterministicOutcome::Error(e.to_string()),
        Ok(outcome) if outcome.pass => DeterministicOutcome::Passed,
        Ok(_) => DeterministicOutcome::Failed,
    }
}

#[cfg(test)]
mod tests {
    use super::judge_matcher::check_llm_judge;
    use super::*;
    use crate::suite::TransformSpec;
    use serde_json::json;

    #[test]
    fn resolve_simple_dot_path() {
        let v = json!({"result": {"isError": false}});
        assert_eq!(resolve_target(&v, "result.isError"), Value::Bool(false));
    }

    #[test]
    fn resolve_array_index() {
        let v = json!({"result": {"content": [{"type": "text", "text": "hi"}]}});
        assert_eq!(
            resolve_target(&v, "result.content[0].text"),
            Value::String("hi".into())
        );
    }

    #[test]
    fn resolve_quoted_key() {
        let v = json!({"a": {"weird key": 42}});
        assert_eq!(resolve_target(&v, "a[\"weird key\"]"), json!(42));
    }

    #[test]
    fn resolve_missing_path_returns_null() {
        let v = json!({"a": 1});
        assert_eq!(resolve_target(&v, "a.b.c"), Value::Null);
    }

    #[test]
    fn exact_matcher_pass_and_fail() {
        assert!(check_matcher(&json!(1), &Matcher::Exact(json!(1))).is_ok());
        assert!(check_matcher(&json!(1), &Matcher::Exact(json!(2))).is_err());
    }

    // This test previously encoded the executor's old, divergent
    // `contains` behavior on the live path, which delegates to
    // `matchers::contains`: case-sensitive substring for strings, object
    // subset, and array multiset-superset. The assertions below pin that.
    #[test]
    fn contains_uses_substring_and_object_subset() {
        // Object: expected fragment is a subset of actual (extra keys ignored).
        assert!(check_matcher(
            &json!({"a": 1, "b": 2, "id": 7}),
            &Matcher::Contains(json!({"a": 1}))
        )
        .is_ok());
        // Strings: case-sensitive substring. The whole string contains itself,
        // and a substring matches; a missing substring fails.
        assert!(check_matcher(&json!("world"), &Matcher::Contains(json!("world"))).is_ok());
        assert!(check_matcher(&json!("hello world"), &Matcher::Contains(json!("world"))).is_ok());
        assert!(check_matcher(&json!("hello world"), &Matcher::Contains(json!("World"))).is_err());
        // A missing key fails with a subset diff.
        assert!(check_matcher(&json!({"a": 1}), &Matcher::Contains(json!({"b": 2}))).is_err());
    }

    // The divergence bug. With plain array membership the OLD
    // executor judged `[1, 1]` contained in `[1, 2, 3]` (each needle found
    // *some* equal element). The matchers crate uses multiset semantics, so
    // two `1`s require two `1`s in the actual array. This test pins the
    // corrected behavior on the live path and renders the JSON-Pointer diff.
    #[test]
    fn contains_array_uses_multiset_superset_on_live_path() {
        // Multiset-superset succeeds when duplicates are available.
        assert!(check_matcher(&json!([1, 1, 2, 3]), &Matcher::Contains(json!([1, 1, 2]))).is_ok());
        // The case the old array-membership code judged differently: only one
        // `1` is available, so `[1, 1]` is NOT contained.
        let err = check_matcher(&json!([1, 2, 3]), &Matcher::Contains(json!([1, 1])))
            .expect_err("multiset should reject the second 1");
        assert!(
            err.contains("/1"),
            "diff should carry the index path: {err}"
        );
        assert!(
            err.contains("multiset"),
            "note should mention multiset: {err}"
        );
    }

    #[test]
    fn regex_matcher_against_string_only() {
        assert!(
            check_matcher(&json!("abc-123"), &Matcher::Regex(r"^abc-\d+$".to_string())).is_ok()
        );
        assert!(check_matcher(&json!("abc"), &Matcher::Regex(r"^abc-\d+$".to_string())).is_err());
        assert!(check_matcher(&json!(42), &Matcher::Regex(r".*".to_string())).is_err());
    }

    #[test]
    fn subset_passes_when_expected_keys_match() {
        let actual = json!({"a": 1, "b": 2, "c": 3});
        let expected = Matcher::Subset(json!({"a": 1, "b": 2}));
        assert!(check_matcher(&actual, &expected).is_ok());
    }

    #[test]
    fn subset_fails_when_a_key_is_missing() {
        let actual = json!({"a": 1});
        let expected = Matcher::Subset(json!({"a": 1, "b": 2}));
        assert!(check_matcher(&actual, &expected).is_err());
    }

    #[test]
    fn subset_walks_nested_objects() {
        let actual = json!({"outer": {"a": 1, "b": 2}});
        let expected = Matcher::Subset(json!({"outer": {"a": 1}}));
        assert!(check_matcher(&actual, &expected).is_ok());
    }

    // Schema used to be a deferred stub on the live path (a
    // `schema:` assertion was silently never evaluated). It now routes
    // through `matchers::schema`, so a conforming value passes.
    #[test]
    fn schema_matcher_passes_for_conforming_value_on_live_path() {
        let schema = json!({
            "type": "object",
            "required": ["name"],
            "properties": {"name": {"type": "string"}}
        });
        let actual = json!({"name": "alice", "extra": 1});
        assert!(check_matcher(&actual, &Matcher::Schema(schema)).is_ok());
    }

    // The inverse. A non-conforming value now fails on the live
    // path with a JSON-Pointer diff instead of the old "parsed but not yet
    // evaluated" stub.
    #[test]
    fn schema_matcher_fails_with_diff_for_nonconforming_value() {
        let schema = json!({
            "type": "object",
            "properties": {"age": {"type": "integer"}}
        });
        let actual = json!({"age": "thirty"});
        let err = check_matcher(&actual, &Matcher::Schema(schema))
            .expect_err("non-conforming value should fail schema");
        assert!(
            err.contains("/age"),
            "diff should carry the field path: {err}"
        );
        assert!(
            !err.contains("not yet evaluated"),
            "schema must be evaluated, not deferred: {err}"
        );
    }

    // A malformed schema document is a structural error, not an
    // assertion failure, and must surface as `Err` rather than a silent pass.
    #[test]
    fn schema_matcher_surfaces_invalid_schema_as_error() {
        let bad_schema = json!({"type": 42});
        let err = check_matcher(&json!(null), &Matcher::Schema(bad_schema))
            .expect_err("invalid schema document should error");
        assert!(err.to_lowercase().contains("schema"), "got: {err}");
    }

    #[test]
    fn snapshot_matcher_routes_through_async_evaluator_not_check_matcher() {
        // `snapshot` needs the suite's filesystem context, so the synchronous
        // helper rejects it; reaching `check_matcher` is a programmer error.
        let result = check_matcher(&json!(0), &Matcher::Snapshot("tools/echo/baseline".into()));
        let err = result.expect_err("should be err");
        assert!(err.contains("snapshot"));
        assert!(err.contains("evaluate_expectations"));
    }

    #[tokio::test]
    async fn snapshot_assertion_records_then_replays_through_evaluate_expectations() {
        let dir = std::env::temp_dir().join(format!("mcptest-eval-snap-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        let record = SnapshotContext {
            dir: dir.clone(),
            mode: crate::matchers::snapshot::UpdateSnapshotsMode::Always,
        };
        let snap = SnapshotContext {
            dir: dir.clone(),
            mode: crate::matchers::snapshot::UpdateSnapshotsMode::Never,
        };
        let response = json!({"result": {"content": [{"type": "text", "text": "v1"}]}});
        let exps = vec![asrt("result", Matcher::Snapshot("echo/baseline".into()))];
        // Recording requires the explicit update mode (WOR-1292).
        let first = evaluate_expectations(
            &response,
            &exps,
            &Scoring::default(),
            None,
            None,
            &TransformContext::default(),
            None,
            &record,
        )
        .await;
        assert_eq!(first.status, crate::report::TestStatus::Pass);
        assert!(dir.join("echo/baseline.json").exists());
        // Second run with the same response replays and passes.
        let second = evaluate_expectations(
            &response,
            &exps,
            &Scoring::default(),
            None,
            None,
            &TransformContext::default(),
            None,
            &snap,
        )
        .await;
        assert_eq!(second.status, crate::report::TestStatus::Pass);
        // A drifted response fails the snapshot assertion.
        let drifted = json!({"result": {"content": [{"type": "text", "text": "v2"}]}});
        let third = evaluate_expectations(
            &drifted,
            &exps,
            &Scoring::default(),
            None,
            None,
            &TransformContext::default(),
            None,
            &snap,
        )
        .await;
        assert_eq!(third.status, crate::report::TestStatus::Fail);
        let _ = std::fs::remove_dir_all(&dir);
    }

    #[test]
    fn llm_judge_matcher_reports_deferred_failure() {
        let result = check_matcher(&json!(0), &Matcher::LlmJudge(json!({"rubric": "ok"})));
        let err = result.expect_err("should be err");
        assert!(err.contains("llm-judge"));
        assert!(err.contains("must be evaluated through `evaluate_expectations`"));
    }

    // The new deterministic matchers dispatch on the live path.

    #[test]
    fn contains_all_dispatches_on_live_path() {
        assert!(check_matcher(
            &json!("a foo bar"),
            &Matcher::ContainsAll(json!(["foo", "bar"]))
        )
        .is_ok());
        assert!(check_matcher(
            &json!("a foo"),
            &Matcher::ContainsAll(json!(["foo", "bar"]))
        )
        .is_err());
    }

    #[test]
    fn contains_any_dispatches_on_live_path() {
        assert!(check_matcher(
            &json!("only bar"),
            &Matcher::ContainsAny(json!(["x", "bar"]))
        )
        .is_ok());
        assert!(check_matcher(&json!("nope"), &Matcher::ContainsAny(json!(["xx", "yy"]))).is_err());
    }

    #[test]
    fn icontains_and_starts_with_dispatch_on_live_path() {
        assert!(check_matcher(&json!("Hello World"), &Matcher::IContains("world".into())).is_ok());
        assert!(check_matcher(&json!("Hello"), &Matcher::StartsWith("Hel".into())).is_ok());
        assert!(check_matcher(&json!("Hello"), &Matcher::StartsWith("ello".into())).is_err());
    }

    #[test]
    fn is_json_dispatches_on_live_path() {
        assert!(check_matcher(&json!("{\"a\":1}"), &Matcher::IsJson(Value::Null)).is_ok());
        assert!(check_matcher(&json!("not json"), &Matcher::IsJson(Value::Null)).is_err());
    }

    #[test]
    fn is_valid_tools_call_dispatches_on_live_path() {
        let call = json!({"name": "search", "arguments": {"q": "x"}});
        assert!(check_matcher(&call, &Matcher::IsValidToolsCall(Value::Null)).is_ok());
        let bad = json!({"arguments": {}});
        assert!(check_matcher(&bad, &Matcher::IsValidToolsCall(Value::Null)).is_err());
    }

    #[test]
    fn levenshtein_dispatches_on_live_path() {
        let arg = json!({"value": "sitting", "max": 3});
        assert!(check_matcher(&json!("kitten"), &Matcher::Levenshtein(arg.clone())).is_ok());
        let tight = json!({"value": "sitting", "max": 1});
        assert!(check_matcher(&json!("kitten"), &Matcher::Levenshtein(tight)).is_err());
    }

    // `not:` inverts a deterministic inner matcher.

    #[test]
    fn not_wrapping_a_failing_inner_matcher_passes() {
        // `exact: 2` fails against `1`, so `not: { exact: 2 }` passes.
        let m = Matcher::Not(Box::new(Matcher::Exact(json!(2))));
        assert!(check_matcher(&json!(1), &m).is_ok());
    }

    #[test]
    fn not_wrapping_a_passing_inner_matcher_fails() {
        // `exact: 1` passes against `1`, so `not: { exact: 1 }` fails.
        let m = Matcher::Not(Box::new(Matcher::Exact(json!(1))));
        let err = check_matcher(&json!(1), &m).expect_err("negation of a pass must fail");
        assert!(
            err.contains("NOT"),
            "message should explain negation: {err}"
        );
    }

    #[test]
    fn double_negation_is_identity() {
        let m = Matcher::Not(Box::new(Matcher::Not(Box::new(Matcher::Exact(json!(1))))));
        // not(not(exact 1)) against 1 should behave like exact 1: pass.
        assert!(check_matcher(&json!(1), &m).is_ok());
        // And fail against 2.
        assert!(check_matcher(&json!(2), &m).is_err());
    }

    #[test]
    fn not_propagates_structural_error_instead_of_inverting_it() {
        // A bad regex is a structural error; negating it must not turn the
        // error into a spurious pass.
        let m = Matcher::Not(Box::new(Matcher::Regex("(".into())));
        let err = check_matcher(&json!("x"), &m).expect_err("structural error must propagate");
        assert!(err.to_lowercase().contains("regex"), "got: {err}");
    }

    #[test]
    fn not_rejects_wrapping_a_deferred_matcher() {
        let m = Matcher::Not(Box::new(Matcher::Snapshot("baseline".into())));
        let err = check_matcher(&json!(0), &m).expect_err("cannot negate a deferred matcher");
        assert!(err.contains("deterministic"), "got: {err}");
        assert!(err.contains("snapshot"), "got: {err}");
    }

    // === : composition matchers (oneOf / anyOf / allOf) ===

    #[test]
    fn one_of_passes_when_exactly_one_inner_passes() {
        let m = Matcher::OneOf(vec![Matcher::Exact(json!(1)), Matcher::Exact(json!(2))]);
        assert!(check_matcher(&json!(1), &m).is_ok());
        assert!(check_matcher(&json!(2), &m).is_ok());
    }

    #[test]
    fn one_of_fails_when_zero_inner_pass() {
        let m = Matcher::OneOf(vec![Matcher::Exact(json!("a")), Matcher::Exact(json!("b"))]);
        let err = check_matcher(&json!("c"), &m).expect_err("zero passes must fail");
        assert!(err.contains("oneOf"), "{err}");
        assert!(err.contains("0 of 2"), "{err}");
    }

    #[test]
    fn one_of_fails_when_more_than_one_inner_passes() {
        // Two overlapping matchers: both pass for 1, so `oneOf` fails.
        let m = Matcher::OneOf(vec![Matcher::Contains(json!(1)), Matcher::Exact(json!(1))]);
        let err = check_matcher(&json!(1), &m).expect_err("two passes must fail oneOf");
        assert!(err.contains("oneOf"), "{err}");
        assert!(err.contains("2 of 2"), "{err}");
    }

    #[test]
    fn any_of_passes_when_at_least_one_inner_passes() {
        let m = Matcher::AnyOf(vec![
            Matcher::Exact(json!("a")),
            Matcher::Exact(json!("b")),
            Matcher::Exact(json!("c")),
        ]);
        assert!(check_matcher(&json!("a"), &m).is_ok());
        assert!(check_matcher(&json!("c"), &m).is_ok());
    }

    #[test]
    fn any_of_fails_when_no_inner_pass() {
        let m = Matcher::AnyOf(vec![Matcher::Exact(json!(1)), Matcher::Exact(json!(2))]);
        let err = check_matcher(&json!(3), &m).expect_err("zero passes must fail anyOf");
        assert!(err.contains("anyOf"), "{err}");
    }

    #[test]
    fn all_of_passes_when_every_inner_passes() {
        let m = Matcher::AllOf(vec![
            Matcher::Contains(json!({"a": 1})),
            Matcher::Contains(json!({"b": 2})),
        ]);
        assert!(check_matcher(&json!({"a": 1, "b": 2, "c": 3}), &m).is_ok());
    }

    #[test]
    fn all_of_fails_when_any_inner_fails() {
        let m = Matcher::AllOf(vec![
            Matcher::Contains(json!({"a": 1})),
            Matcher::Contains(json!({"b": 999})),
        ]);
        let err = check_matcher(&json!({"a": 1, "b": 2}), &m).expect_err("one fail breaks allOf");
        assert!(err.contains("allOf"), "{err}");
        assert!(err.contains("1 of 2 failed"), "{err}");
    }

    #[test]
    fn composition_propagates_structural_error_from_inner() {
        // A bad regex inside an anyOf is a structural error; the wrapper
        // surfaces it rather than masking it as a failed branch.
        let m = Matcher::AnyOf(vec![Matcher::Regex("(".into()), Matcher::Exact(json!(0))]);
        let err = check_matcher(&json!(0), &m).expect_err("structural inner error propagates");
        assert!(err.to_lowercase().contains("regex"), "{err}");
    }

    #[test]
    fn composition_rejects_a_deferred_inner_matcher() {
        let m = Matcher::AllOf(vec![
            Matcher::Exact(json!(1)),
            Matcher::Snapshot("baseline".into()),
        ]);
        let err = check_matcher(&json!(1), &m).expect_err("deferred inner is a clear error");
        assert!(err.contains("deterministic"), "{err}");
        assert!(err.contains("snapshot"), "{err}");
    }

    #[test]
    fn composition_nests_with_not_and_other_compositions() {
        // not(allOf(exact 1, exact 1)) against 1 should: allOf passes,
        // so not flips it to fail.
        let m = Matcher::Not(Box::new(Matcher::AllOf(vec![
            Matcher::Exact(json!(1)),
            Matcher::Exact(json!(1)),
        ])));
        assert!(check_matcher(&json!(1), &m).is_err());
        // And anyOf(not(exact 1), exact 1) against 1: first branch
        // (not exact 1) fails, second (exact 1) passes -> anyOf passes.
        let m = Matcher::AnyOf(vec![
            Matcher::Not(Box::new(Matcher::Exact(json!(1)))),
            Matcher::Exact(json!(1)),
        ]);
        assert!(check_matcher(&json!(1), &m).is_ok());
    }

    /// Build a default-weight, unnamed assertion for the executor tests.
    fn asrt(target: &str, matcher: Matcher) -> Assertion {
        Assertion {
            target: target.into(),
            matcher,
            weight: 1.0,
            name: None,
            transform: None,
        }
    }

    /// Build an assertion with an explicit weight and name.
    fn weighted_named(target: &str, matcher: Matcher, weight: f64, name: &str) -> Assertion {
        Assertion {
            target: target.into(),
            matcher,
            weight,
            name: Some(name.into()),
            transform: None,
        }
    }

    // Transform wiring around the tool call.

    /// Build a minimal tool plan for the transform-helper tests.
    fn tool_plan_with_transform(transform: Option<TransformSpec>) -> ToolPlan {
        use crate::suite::Action;
        ToolPlan {
            server: "fs".into(),
            action: Action::ToolCall {
                tool: "echo".into(),
                args: json!({"q": "hi"}),
            },
            expectations: Vec::new(),
            budgets: Default::default(),
            transport: TargetTransport::default(),
            scoring: Scoring::default(),
            transform,
            inject_error: None,
            metamorphic: None,
            fuzz: None,
            negative_path: None,
            responder: None,
        }
    }

    #[test]
    fn transform_context_carries_test_server_and_tool() {
        let plan = tool_plan_with_transform(None);
        let ctx = transform_context("my test", &plan);
        assert_eq!(ctx.test_name, "my test");
        assert_eq!(ctx.server, "fs");
        assert_eq!(ctx.tool, "echo");
    }

    #[tokio::test]
    async fn response_transform_is_noop_without_a_command() {
        let plan = tool_plan_with_transform(None);
        let ctx = transform_context("t", &plan);
        let response = json!({"result": {"ok": true}});
        let out = apply_response_transform(response.clone(), plan.transform.as_ref(), &ctx)
            .await
            .expect("no-op transform");
        assert_eq!(out, response);
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn response_transform_mutates_the_matched_value() {
        use crate::suite::{TransformCommand, TransformStep};
        // Strip a volatile id from the result before assertions see it.
        let spec = TransformSpec {
            request: None,
            response: Some(TransformStep::Command(TransformCommand {
                program: "/bin/sh".into(),
                args: vec![
                    "-c".into(),
                    r#"cat >/dev/null; echo '{"result":{"ok":true}}'"#.into(),
                ],
            })),
        };
        let plan = tool_plan_with_transform(Some(spec));
        let ctx = transform_context("t", &plan);
        let response = json!({"result": {"ok": true, "id": "vol-1"}});
        let out = apply_response_transform(response, plan.transform.as_ref(), &ctx)
            .await
            .expect("transform succeeds");
        assert_eq!(out, json!({"result": {"ok": true}}));
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn response_transform_error_renders_a_failure_message() {
        use crate::suite::{TransformCommand, TransformStep};
        let spec = TransformSpec {
            request: None,
            response: Some(TransformStep::Command(TransformCommand {
                program: "/bin/sh".into(),
                args: vec!["-c".into(), "exit 4".into()],
            })),
        };
        let plan = tool_plan_with_transform(Some(spec));
        let ctx = transform_context("t", &plan);
        let err = apply_response_transform(json!({"result": null}), plan.transform.as_ref(), &ctx)
            .await
            .expect_err("non-zero exit fails");
        assert!(err.contains("response transform"), "{err}");
        assert!(err.contains("exited with 4"), "{err}");
    }

    // Inline jq response transform runs in-process (no subprocess),
    // so these tests run on every platform, not just unix.

    #[tokio::test]
    async fn inline_jq_response_transform_reshapes_the_matched_value() {
        use crate::suite::TransformStep;
        // The raw response wraps the text under result.content; the matcher
        // wants the bare string. Without the transform a `result` matcher on
        // the text would fail; the jq unwrap makes it pass.
        let spec = TransformSpec {
            request: None,
            response: Some(TransformStep::Jq(".result.content[0].text".into())),
        };
        let plan = tool_plan_with_transform(Some(spec));
        let ctx = transform_context("t", &plan);
        let response = json!({"result": {"content": [{"text": "ok"}]}});
        let out = apply_response_transform(response, plan.transform.as_ref(), &ctx)
            .await
            .expect("jq transform succeeds");
        assert_eq!(out, json!("ok"));
    }

    #[tokio::test]
    async fn inline_jq_response_transform_error_renders_a_failure_message() {
        use crate::suite::TransformStep;
        // `empty` yields zero outputs, which violates the one-in-one-out
        // contract and must fail the transform with a clear message.
        let spec = TransformSpec {
            request: None,
            response: Some(TransformStep::Jq("empty".into())),
        };
        let plan = tool_plan_with_transform(Some(spec));
        let ctx = transform_context("t", &plan);
        let err = apply_response_transform(json!({"result": null}), plan.transform.as_ref(), &ctx)
            .await
            .expect_err("zero outputs fails");
        assert!(err.contains("response transform"), "{err}");
        assert!(err.contains("expected exactly one"), "{err}");
    }

    // Per-assertion transform runs through the evaluator before the
    // matcher sees the extracted value.

    /// Build an assertion carrying a `/bin/sh -c` transform command.
    #[cfg(unix)]
    fn asrt_with_transform(target: &str, matcher: Matcher, script: &str) -> Assertion {
        use crate::suite::TransformCommand;
        Assertion {
            target: target.into(),
            matcher,
            weight: 1.0,
            name: None,
            transform: Some(TransformCommand {
                program: "/bin/sh".into(),
                args: vec!["-c".into(), script.into()],
            }),
        }
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn assertion_transform_makes_an_otherwise_failing_matcher_pass() {
        // The raw value is "ECHO"; `exact: "echo"` would fail. A transform
        // lowercases it before the matcher runs, so the assertion passes.
        let response = json!({"result": {"name": "ECHO"}});
        let exps = vec![asrt_with_transform(
            "result.name",
            Matcher::Exact(json!("echo")),
            r#"cat >/dev/null; printf '"echo"'"#,
        )];
        let outcome = evaluate_expectations(
            &response,
            &exps,
            &Scoring::default(),
            None,
            None,
            &TransformContext::default(),
            None,
            &SnapshotContext::default(),
        )
        .await;
        assert_eq!(outcome.status, crate::report::TestStatus::Pass);
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn assertion_transform_failure_fails_the_assertion_with_a_clear_message() {
        let response = json!({"result": {"name": "anything"}});
        let exps = vec![asrt_with_transform(
            "result.name",
            Matcher::Exact(json!("echo")),
            "exit 6",
        )];
        let outcome = evaluate_expectations(
            &response,
            &exps,
            &Scoring::default(),
            None,
            None,
            &TransformContext::default(),
            None,
            &SnapshotContext::default(),
        )
        .await;
        assert_eq!(outcome.status, crate::report::TestStatus::Fail);
        let failure = outcome.failure.expect("failure payload");
        assert!(
            failure.message.contains("assertion transform"),
            "{}",
            failure.message
        );
        assert!(
            failure.message.contains("exited with 6"),
            "{}",
            failure.message
        );
    }

    #[tokio::test]
    async fn evaluate_passes_when_every_assertion_holds() {
        let response = json!({"result": {"isError": false, "content": [{"text": "ok"}]}});
        let exps = vec![
            asrt("result.isError", Matcher::Exact(json!(false))),
            asrt("result.content[0].text", Matcher::Contains(json!("ok"))),
        ];
        match evaluate_expectations(
            &response,
            &exps,
            &Scoring::default(),
            None,
            None,
            &TransformContext::default(),
            None,
            &SnapshotContext::default(),
        )
        .await
        .status
        {
            crate::report::TestStatus::Pass => {}
            other => panic!("expected pass, got {other:?}"),
        }
    }

    #[tokio::test]
    async fn evaluate_fails_on_first_failing_assertion_with_index() {
        let response = json!({"result": {"isError": true}});
        let exps = vec![
            asrt("result.isError", Matcher::Exact(json!(false))),
            asrt("result.never", Matcher::Exact(json!("never reached"))),
        ];
        let outcome = evaluate_expectations(
            &response,
            &exps,
            &Scoring::default(),
            None,
            None,
            &TransformContext::default(),
            None,
            &SnapshotContext::default(),
        )
        .await;
        let failure = outcome.failure.expect("failure payload");
        assert!(failure.message.contains("#0"));
        assert!(failure.message.contains("isError"));
    }

    #[tokio::test]
    async fn evaluate_returns_clear_error_when_jury_provider_missing() {
        let response = json!({"final_response": "anything"});
        let exps = vec![asrt(
            "final_response",
            Matcher::LlmJury(json!({
                "rubric": "is the answer plausible",
                "jurors": [{"model": "stub"}],
                "quorum": 0.5,
            })),
        )];
        let outcome = evaluate_expectations(
            &response,
            &exps,
            &Scoring::default(),
            None,
            None,
            &TransformContext::default(),
            None,
            &SnapshotContext::default(),
        )
        .await;
        let failure = outcome.failure.expect("failure payload");
        assert!(failure.message.contains("llm-jury"));
        assert!(failure.message.contains("LlmProvider"));
    }

    // Combined-score model. Back-compat first: a suite with no
    // threshold still requires every assertion to pass and reports no score.

    #[tokio::test]
    async fn no_threshold_keeps_all_pass_contract_and_omits_score() {
        let response = json!({"result": {"a": 1, "b": 2}});
        let exps = vec![
            asrt("result.a", Matcher::Exact(json!(1))),
            asrt("result.b", Matcher::Exact(json!(999))),
        ];
        let outcome = evaluate_expectations(
            &response,
            &exps,
            &Scoring::default(),
            None,
            None,
            &TransformContext::default(),
            None,
            &SnapshotContext::default(),
        )
        .await;
        assert_eq!(outcome.status, crate::report::TestStatus::Fail);
        assert!(
            outcome.score.is_none(),
            "no-threshold suite reports no score"
        );
    }

    // `emit_score` forces the score onto the report without a
    // threshold so a matrix-level assertion can compare cells. It must not
    // change pass/fail: the legacy all-pass rule still decides the verdict.
    #[tokio::test]
    async fn emit_score_reports_score_without_changing_the_verdict() {
        // One of two assertions fails. With no threshold the legacy all-pass
        // rule fails the test, but the score (0.5) still rides back because
        // `emit_score` is set. This is the exact path a matrix-assert cell
        // takes: scored for comparison, judged by the legacy contract.
        let response = json!({"result": {"a": 1, "b": 2}});
        let exps = vec![
            asrt("result.a", Matcher::Exact(json!(1))),
            asrt("result.b", Matcher::Exact(json!(999))),
        ];
        let scoring = Scoring {
            emit_score: true,
            ..Scoring::default()
        };
        let outcome = evaluate_expectations(
            &response,
            &exps,
            &scoring,
            None,
            None,
            &TransformContext::default(),
            None,
            &SnapshotContext::default(),
        )
        .await;
        // Verdict unchanged from the no-threshold legacy contract.
        assert_eq!(outcome.status, crate::report::TestStatus::Fail);
        // The score is present for the matrix comparison even though the
        // test sets no threshold.
        assert_eq!(outcome.score, Some(0.5));
    }

    // The no-regression guard. A plain test (no threshold, no
    // `emit_score`) must still report `score: None`, exactly as before the
    // matrix-assert feature, so its on-disk report JSON is unchanged.
    #[tokio::test]
    async fn default_scoring_still_omits_the_score() {
        let response = json!({"result": {"a": 1}});
        let exps = vec![asrt("result.a", Matcher::Exact(json!(1)))];
        let outcome = evaluate_expectations(
            &response,
            &exps,
            &Scoring::default(),
            None,
            None,
            &TransformContext::default(),
            None,
            &SnapshotContext::default(),
        )
        .await;
        assert_eq!(outcome.status, crate::report::TestStatus::Pass);
        assert!(
            outcome.score.is_none(),
            "a plain test must report no score so its report JSON is unchanged"
        );
    }

    #[tokio::test]
    async fn threshold_passes_when_combined_score_meets_it() {
        // One of two assertions fails: combined score 0.5, threshold 0.5 passes.
        let response = json!({"result": {"a": 1, "b": 2}});
        let exps = vec![
            asrt("result.a", Matcher::Exact(json!(1))),
            asrt("result.b", Matcher::Exact(json!(999))),
        ];
        let scoring = Scoring {
            threshold: Some(0.5),
            ..Scoring::default()
        };
        let outcome = evaluate_expectations(
            &response,
            &exps,
            &scoring,
            None,
            None,
            &TransformContext::default(),
            None,
            &SnapshotContext::default(),
        )
        .await;
        assert_eq!(outcome.status, crate::report::TestStatus::Pass);
        assert_eq!(outcome.score, Some(0.5));
    }

    #[tokio::test]
    async fn threshold_fails_just_above_the_score() {
        let response = json!({"result": {"a": 1, "b": 2}});
        let exps = vec![
            asrt("result.a", Matcher::Exact(json!(1))),
            asrt("result.b", Matcher::Exact(json!(999))),
        ];
        let scoring = Scoring {
            threshold: Some(0.6),
            ..Scoring::default()
        };
        let outcome = evaluate_expectations(
            &response,
            &exps,
            &scoring,
            None,
            None,
            &TransformContext::default(),
            None,
            &SnapshotContext::default(),
        )
        .await;
        assert_eq!(outcome.status, crate::report::TestStatus::Fail);
        let failure = outcome.failure.expect("failure payload");
        assert!(
            failure.message.contains("combined score"),
            "{}",
            failure.message
        );
    }

    #[tokio::test]
    async fn weight_biases_the_combined_score() {
        // The heavy assertion passes; the light one fails. Score = 3/4 = 0.75.
        let response = json!({"result": {"a": 1, "b": 2}});
        let exps = vec![
            weighted_named("result.a", Matcher::Exact(json!(1)), 3.0, "a"),
            weighted_named("result.b", Matcher::Exact(json!(999)), 1.0, "b"),
        ];
        let scoring = Scoring {
            threshold: Some(0.75),
            ..Scoring::default()
        };
        let outcome = evaluate_expectations(
            &response,
            &exps,
            &scoring,
            None,
            None,
            &TransformContext::default(),
            None,
            &SnapshotContext::default(),
        )
        .await;
        assert_eq!(outcome.status, crate::report::TestStatus::Pass);
        assert_eq!(outcome.score, Some(0.75));
    }

    #[tokio::test]
    async fn assert_set_passes_at_its_internal_threshold() {
        // 2 of 3 pass: internal score 0.667 >= set threshold 0.6, so the set
        // passes and the test (no test threshold) passes.
        let response = json!({"result": {"a": 1, "b": 2, "c": 3}});
        let set = AssertSet {
            name: Some("string-checks".into()),
            threshold: 0.6,
            weight: 1.0,
            assertions: vec![
                asrt("result.a", Matcher::Exact(json!(1))),
                asrt("result.b", Matcher::Exact(json!(2))),
                asrt("result.c", Matcher::Exact(json!(999))),
            ],
        };
        let scoring = Scoring {
            assert_sets: vec![set],
            ..Scoring::default()
        };
        let outcome = evaluate_expectations(
            &response,
            &[],
            &scoring,
            None,
            None,
            &TransformContext::default(),
            None,
            &SnapshotContext::default(),
        )
        .await;
        assert_eq!(outcome.status, crate::report::TestStatus::Pass);
    }

    #[tokio::test]
    async fn assert_set_fails_below_its_internal_threshold() {
        // Only 1 of 3 passes: internal score 0.333 < threshold 0.6, set fails;
        // with no test threshold the legacy all-pass rule fails the test.
        let response = json!({"result": {"a": 1, "b": 2, "c": 3}});
        let set = AssertSet {
            name: Some("string-checks".into()),
            threshold: 0.6,
            weight: 1.0,
            assertions: vec![
                asrt("result.a", Matcher::Exact(json!(1))),
                asrt("result.b", Matcher::Exact(json!(888))),
                asrt("result.c", Matcher::Exact(json!(999))),
            ],
        };
        let scoring = Scoring {
            assert_sets: vec![set],
            ..Scoring::default()
        };
        let outcome = evaluate_expectations(
            &response,
            &[],
            &scoring,
            None,
            None,
            &TransformContext::default(),
            None,
            &SnapshotContext::default(),
        )
        .await;
        assert_eq!(outcome.status, crate::report::TestStatus::Fail);
        let failure = outcome.failure.expect("failure payload");
        assert!(
            failure.message.contains("assert-set"),
            "{}",
            failure.message
        );
    }

    #[tokio::test]
    async fn derived_metric_is_reported_and_can_gate() {
        use crate::suite::{DerivedAggregation, DerivedKind, DerivedMetric, DerivedTerm};
        // Two named assertions: a passes, b fails. A weighted-average derived
        // metric over them scores 0.5; with a 0.7 threshold the metric vetoes
        // the test even though the test threshold itself is met.
        let response = json!({"result": {"a": 1, "b": 2}});
        let exps = vec![
            weighted_named("result.a", Matcher::Exact(json!(1)), 1.0, "a"),
            weighted_named("result.b", Matcher::Exact(json!(999)), 1.0, "b"),
        ];
        let metric = DerivedMetric {
            name: "blended".into(),
            value: DerivedAggregation {
                kind: DerivedKind::WeightedAverage,
                terms: vec![
                    DerivedTerm {
                        reference: "a".into(),
                        weight: 1.0,
                    },
                    DerivedTerm {
                        reference: "b".into(),
                        weight: 1.0,
                    },
                ],
            },
            threshold: Some(0.7),
        };
        let scoring = Scoring {
            threshold: Some(0.5),
            derived_metrics: vec![metric],
            ..Scoring::default()
        };
        let outcome = evaluate_expectations(
            &response,
            &exps,
            &scoring,
            None,
            None,
            &TransformContext::default(),
            None,
            &SnapshotContext::default(),
        )
        .await;
        // The combined score (0.5) meets the test threshold, but the derived
        // metric (0.5 < 0.7) vetoes the pass.
        assert_eq!(outcome.status, crate::report::TestStatus::Fail);
        assert_eq!(outcome.derived_metrics.len(), 1);
        assert_eq!(outcome.derived_metrics[0].name, "blended");
        assert_eq!(outcome.derived_metrics[0].passed, Some(false));
        let failure = outcome.failure.expect("failure payload");
        assert!(
            failure.message.contains("derived metric"),
            "{}",
            failure.message
        );
    }

    // Jury analytics as assertion targets. A provider that returns a
    // fixed sequence of judge replies (one per juror) so the test controls the
    // spread of scores and therefore the agreement, band, and escalation flag.
    use crate::eval::provider::{
        Capabilities, CompletionOptions, Cost, FinishReason, LlmProvider, Message, ProviderId,
        Response, ResponseContent, Usage,
    };
    use std::sync::Mutex;

    struct SequenceProvider(Mutex<std::collections::VecDeque<String>>);

    impl SequenceProvider {
        fn new(replies: &[&str]) -> Self {
            Self(Mutex::new(replies.iter().map(|s| s.to_string()).collect()))
        }
    }

    #[async_trait]
    impl LlmProvider for SequenceProvider {
        fn name(&self) -> ProviderId {
            ProviderId::Anthropic
        }
        fn supported_capabilities(&self) -> Capabilities {
            Capabilities::none()
        }
        async fn chat_completion(
            &self,
            _messages: &[Message],
            _options: &CompletionOptions,
        ) -> Result<Response, crate::eval::ProviderError> {
            let text = self.0.lock().unwrap().pop_front().unwrap_or_default();
            Ok(Response {
                content: ResponseContent::Text { text },
                finish_reason: FinishReason::Stop,
                usage: Usage::default(),
                model: "stub".into(),
            })
        }
        async fn count_tokens(
            &self,
            _text: &str,
            _model: &str,
        ) -> Result<u32, crate::eval::ProviderError> {
            Ok(0)
        }
        async fn embed(
            &self,
            _text: &str,
            _model: &str,
        ) -> Result<Vec<f32>, crate::eval::ProviderError> {
            Ok(vec![1.0])
        }
        fn estimate_cost(&self, _usage: &Usage, _model: &str) -> Cost {
            Cost::default()
        }
    }

    /// Build a three-juror jury assertion sharing one rubric.
    fn jury_assertion(target: &str) -> Assertion {
        asrt(
            target,
            Matcher::LlmJury(json!({
                "rubric": "grade the answer",
                "jurors": [{"model": "a"}, {"model": "b"}, {"model": "c"}],
                "quorum": 0.5,
            })),
        )
    }

    #[tokio::test]
    async fn jury_agreement_target_resolves_for_a_unanimous_jury() {
        // Three jurors agree at ~0.9: alpha is high, band is high, no escalate.
        let provider = SequenceProvider::new(&[
            r#"{"pass": true, "score": 0.9}"#,
            r#"{"pass": true, "score": 0.9}"#,
            r#"{"pass": true, "score": 0.9}"#,
        ]);
        let response = json!({"result": {"content": [{"text": "answer"}]}});
        let exps = vec![
            jury_assertion("result.content[0].text"),
            asrt("jury.agreement", Matcher::Schema(json!({"minimum": 0.7}))),
            asrt("jury.confidence", Matcher::Exact(json!("high"))),
            asrt("jury.escalate", Matcher::Exact(json!(false))),
        ];
        let outcome = evaluate_expectations(
            &response,
            &exps,
            &Scoring::default(),
            Some(&provider),
            None,
            &TransformContext::default(),
            None,
            &SnapshotContext::default(),
        )
        .await;
        assert_eq!(outcome.status, crate::report::TestStatus::Pass);
    }

    #[tokio::test]
    async fn jury_confidence_target_reads_low_for_a_split_jury() {
        // A wide split: alpha is low, band is low, escalate is true. A
        // `not: { exact: low }` confidence gate then fails.
        let provider = SequenceProvider::new(&[
            r#"{"pass": true, "score": 0.05}"#,
            r#"{"pass": false, "score": 0.95}"#,
            r#"{"pass": true, "score": 0.1}"#,
        ]);
        let response = json!({"result": {"content": [{"text": "answer"}]}});
        let exps = vec![
            jury_assertion("result.content[0].text"),
            asrt("jury.escalate", Matcher::Exact(json!(true))),
            asrt(
                "jury.confidence",
                Matcher::Not(Box::new(Matcher::Exact(json!("low")))),
            ),
        ];
        let outcome = evaluate_expectations(
            &response,
            &exps,
            &Scoring::default(),
            Some(&provider),
            None,
            &TransformContext::default(),
            None,
            &SnapshotContext::default(),
        )
        .await;
        // The `not low` confidence gate must fail on a low-agreement jury.
        assert_eq!(outcome.status, crate::report::TestStatus::Fail);
    }

    #[tokio::test]
    async fn llm_judge_grades_live_through_the_provider() {
        //  closer: the judge model comes from the executor default
        // (the agent's model). The provider returns a passing verdict, so the
        // assertion passes through the async evaluator rather than hitting the
        // old deferred stub.
        let provider = SequenceProvider::new(&[r#"{"pass": true, "score": 0.9}"#]);
        let raw = json!({"rubric": "mentions the city", "threshold": 0.5});
        let res = check_llm_judge(
            &json!("It is 72F in Denver"),
            &raw,
            Some(&provider),
            Some("a"),
            None,
        )
        .await;
        assert!(res.is_ok(), "judge should pass: {res:?}");
    }

    #[tokio::test]
    async fn llm_judge_uses_the_blocks_model_over_the_default() {
        // An explicit `model:` in the block wins over the executor default.
        let provider = SequenceProvider::new(&[r#"{"pass": true, "score": 0.8}"#]);
        let raw = json!({"rubric": "grade it", "model": "explicit-model"});
        let res = check_llm_judge(&json!("answer"), &raw, Some(&provider), None, None).await;
        assert!(res.is_ok(), "explicit model should be used: {res:?}");
    }

    #[tokio::test]
    async fn llm_judge_requires_a_provider() {
        let raw = json!({"rubric": "grade it", "model": "a"});
        let err = check_llm_judge(&json!("x"), &raw, None, None, None)
            .await
            .unwrap_err();
        assert!(err.contains("requires an LlmProvider"), "got: {err}");
    }

    #[tokio::test]
    async fn llm_judge_fails_clearly_when_no_model_is_available() {
        // No `model:` in the block and no executor default: fail with an
        // actionable message rather than calling the provider with no model.
        let provider = SequenceProvider::new(&[r#"{"pass": true}"#]);
        let raw = json!({"rubric": "grade it"});
        let err = check_llm_judge(&json!("x"), &raw, Some(&provider), None, None)
            .await
            .unwrap_err();
        assert!(err.contains("needs a model"), "got: {err}");
    }

    #[tokio::test]
    async fn jury_agreement_assertion_is_skipped_for_a_single_juror_jury() {
        // A single juror leaves alpha undefined; the jury.agreement assertion
        // is skipped (not failed), so the test passes on the jury verdict
        // alone.
        let provider = SequenceProvider::new(&[r#"{"pass": true, "score": 0.9}"#]);
        let response = json!({"result": {"content": [{"text": "answer"}]}});
        let exps = vec![
            asrt(
                "result.content[0].text",
                Matcher::LlmJury(json!({
                    "rubric": "grade",
                    "jurors": [{"model": "solo"}],
                    "quorum": 0.5,
                })),
            ),
            // This gate would fail if it ran (null is not >= 0.7), so a pass
            // proves the assertion was skipped.
            asrt("jury.agreement", Matcher::Schema(json!({"minimum": 0.7}))),
        ];
        let outcome = evaluate_expectations(
            &response,
            &exps,
            &Scoring::default(),
            Some(&provider),
            None,
            &TransformContext::default(),
            None,
            &SnapshotContext::default(),
        )
        .await;
        assert_eq!(outcome.status, crate::report::TestStatus::Pass);
    }

    #[tokio::test]
    async fn partially_skipped_test_passes_on_evaluated_assertions_only() {
        // WOR-1290: a skipped assertion is dropped from scoring, but as
        // long as at least one declared check evaluated, the test passes
        // on the evaluated checks alone.
        let provider = SequenceProvider::new(&[r#"{"pass": true, "score": 0.9}"#]);
        let response = json!({"result": {"content": [{"text": "answer"}], "isError": false}});
        let exps = vec![
            asrt(
                "result.content[0].text",
                Matcher::LlmJury(json!({
                    "rubric": "grade",
                    "jurors": [{"model": "solo"}],
                    "quorum": 0.5,
                })),
            ),
            // Single-juror jury: agreement is undefined, so this skips.
            asrt("jury.agreement", Matcher::Schema(json!({"minimum": 0.7}))),
            asrt("result.isError", Matcher::Exact(json!(false))),
        ];
        let outcome = evaluate_expectations(
            &response,
            &exps,
            &Scoring::default(),
            Some(&provider),
            None,
            &TransformContext::default(),
            None,
            &SnapshotContext::default(),
        )
        .await;
        assert_eq!(outcome.status, crate::report::TestStatus::Pass);
    }

    #[tokio::test]
    async fn skipped_assertions_do_not_affect_combined_score_weighting() {
        // The skipped jury.agreement assertion carries weight 100. If it
        // leaked into the score as a pass the score would be ~0.990; as a
        // fail, ~0.019. Dropping it leaves 2 of 3 evaluated items passing,
        // exactly 2/3.
        let provider = SequenceProvider::new(&[r#"{"pass": true, "score": 0.9}"#]);
        let response = json!({"result": {"content": [{"text": "answer"}], "isError": false}});
        let exps = vec![
            asrt(
                "result.content[0].text",
                Matcher::LlmJury(json!({
                    "rubric": "grade",
                    "jurors": [{"model": "solo"}],
                    "quorum": 0.5,
                })),
            ),
            weighted_named(
                "jury.agreement",
                Matcher::Schema(json!({"minimum": 0.7})),
                100.0,
                "agreement",
            ),
            asrt("result.isError", Matcher::Exact(json!(false))),
            asrt("result.missing", Matcher::Exact(json!("absent"))),
        ];
        let scoring = Scoring {
            threshold: Some(0.5),
            ..Scoring::default()
        };
        let outcome = evaluate_expectations(
            &response,
            &exps,
            &scoring,
            Some(&provider),
            None,
            &TransformContext::default(),
            None,
            &SnapshotContext::default(),
        )
        .await;
        assert_eq!(outcome.status, crate::report::TestStatus::Pass);
        let score = outcome.score.expect("thresholded test reports a score");
        assert!(
            (score - 2.0 / 3.0).abs() < 1e-9,
            "skipped weight leaked into the combined score: {score}"
        );
    }
}
