//! Comparative matrix-level assertions: max-score floor and select-best.
//!
//! Drives `mcptest run` against a deterministic stdio fixture server (the
//! built-in `mcptest mock`) with a two-model matrix. With no provider API key
//! in the env the agent runner falls back to the stub model, whose
//! `final_response` echoes the model label (`stub model response for agent ...
//! model <label>`).
//!
//! Each cell carries two assertions and a low per-cell `threshold:` so the
//! cell still PASSES its own verdict while scoring differently: one
//! assertion (`icontains: model`) holds for every cell, the other
//! (`icontains: claude`) holds only for the claude cell. The claude cell
//! therefore scores 1.0 and the gpt cell 0.5, but both clear the 0.4 cell
//! threshold and pass as rows. This isolates the matrix gate from the
//! per-row verdict so the max-score floor is the only thing that can fail
//! the run.
//!
//! Three behaviors are pinned:
//! - `max_score` passes when the best cell clears the floor.
//! - `max_score` fails (exit 1) when no cell clears the floor, even though
//!   every row passed its own threshold.
//! - `select_best` reports the top cell and never fails the run.

#![cfg(unix)]

use std::path::Path;

use assert_cmd::Command;
use predicates::prelude::*;

fn mcptest() -> Command {
    Command::cargo_bin("mcptest").expect("binary builds")
}

/// Write a `mcptest mock` manifest exposing a single `get_weather` tool, and
/// return its path. The suite points its stdio server at `mcptest mock
/// --tools-from <this>`, a native server that answers the handshake instantly.
/// An earlier shell-script fixture forked `sed`/`grep`/`printf` per JSON line
/// and blew past the 30s handshake budget when several of these tests ran in
/// parallel under load; the native mock removes that fork storm entirely.
fn write_weather_server(dir: &Path) -> std::path::PathBuf {
    let manifest = r#"mock_server:
  name: weather-fixture
  tools:
    - name: get_weather
      description: "Get the weather for a city"
      input_schema:
        type: object
        properties:
          city: { type: string }
        required: [city]
      response:
        content:
          - type: text
            text: "It is 72 F and sunny."
"#;
    let path = dir.join("get_weather.yaml");
    std::fs::write(&path, manifest).expect("write mock manifest");
    path
}

/// Strip provider credentials so the agent runner is forced down the
/// deterministic stub-model path regardless of the developer's shell.
fn strip_provider_keys(cmd: &mut Command) {
    for var in [
        "ANTHROPIC_API_KEY",
        "OPENAI_API_KEY",
        "GEMINI_API_KEY",
        "GOOGLE_API_KEY",
        "MISTRAL_API_KEY",
    ] {
        cmd.env_remove(var);
    }
}

/// Write a two-model agent suite. The stub model's reply echoes the model
/// label. The first assertion (`icontains: model`) holds for every cell;
/// the second (`icontains: <needle>`) holds only for cells whose label
/// contains `needle`. The optional `threshold_line` sets a per-cell
/// `threshold:` so a partially-scoring cell still passes its own verdict,
/// isolating the matrix gate from the per-row verdict. `matrix_block` is
/// spliced under the agent test so each test can vary the matrix assertion.
fn write_suite(
    dir: &Path,
    server: &Path,
    needle: &str,
    threshold_line: &str,
    matrix_block: &str,
) -> std::path::PathBuf {
    // Point the stdio server at the native `mcptest mock` with the manifest
    // `server`. `CARGO_BIN_EXE_mcptest` is the binary under test; the JSON
    // array keeps the path intact even if it contains spaces.
    let bin = env!("CARGO_BIN_EXE_mcptest");
    let yaml = format!(
        r#"servers:
  weather:
    command: ['{bin}', 'mock', '--tools-from', '{server}']
agents:
  - name: weather matrix
    models:
      - claude-sonnet-4-5
      - gpt-4o-mini
    servers: [weather]
    prompt: What is the weather in Sacramento?
{threshold_line}    expect:
      - target: final_response
        matcher:
          icontains: model
      - target: final_response
        matcher:
          icontains: {needle}
{matrix_block}
"#,
        server = server.display(),
    );
    let path = dir.join("mcptest.yaml");
    std::fs::write(&path, yaml).expect("write suite");
    path
}

#[test]
fn max_score_floor_passes_when_the_best_cell_clears_it() {
    let dir = tempfile::tempdir().unwrap();
    let server = write_weather_server(dir.path());
    // Needle `claude`: the claude cell matches both assertions (score 1.0),
    // the gpt cell only the first (score 0.5). Both clear the 0.4 cell
    // threshold and pass as rows, so the best cell (1.0) clearing the 0.8
    // floor is the only reason the run is green.
    let config = write_suite(
        dir.path(),
        &server,
        "claude",
        "    threshold: 0.4\n",
        "    matrix_assert:\n      max_score: 0.8\n",
    );

    let mut cmd = mcptest();
    strip_provider_keys(&mut cmd);
    cmd.current_dir(dir.path())
        .args(["run", "--allow-stub-provider", "--config"])
        .arg(&config)
        .assert()
        .code(0)
        .stdout(
            predicate::str::contains("matrix-assert [PASS] weather matrix")
                .and(predicate::str::contains("claude-sonnet-4-5")),
        );
}

#[test]
fn max_score_floor_fails_when_no_cell_clears_it() {
    let dir = tempfile::tempdir().unwrap();
    let server = write_weather_server(dir.path());
    // Needle `gemini`: neither label contains it, so every cell scores 0.5
    // (only the first assertion holds). Both rows still clear the 0.4 cell
    // threshold and PASS, yet the best cell (0.5) cannot clear the 0.8
    // floor. This proves the floor gates independently of the row verdict.
    let config = write_suite(
        dir.path(),
        &server,
        "gemini",
        "    threshold: 0.4\n",
        "    matrix_assert:\n      max_score: 0.8\n",
    );

    let mut cmd = mcptest();
    strip_provider_keys(&mut cmd);
    cmd.current_dir(dir.path())
        .args(["run", "--allow-stub-provider", "--config"])
        .arg(&config)
        .assert()
        // Exit 1: a max-score floor breach is a failing test outcome,
        // reusing the test-failure code like the tool-selection floor.
        .code(1)
        .stdout(
            predicate::str::contains("0 failed").and(predicate::str::contains(
                "matrix-assert [FAIL] weather matrix",
            )),
        );
}

#[test]
fn select_best_reports_the_top_cell_without_failing_the_run() {
    let dir = tempfile::tempdir().unwrap();
    let server = write_weather_server(dir.path());
    // `select_best` only, and NO per-cell threshold. The needle `model`
    // matches every cell, so both rows pass the legacy all-pass contract.
    // Because there is no threshold, the per-cell score rides back only via
    // the  `emit_score` path, exercising score-without-threshold end
    // to end. select_best reports the best cell and never gates, so the run
    // stays green.
    let config = write_suite(
        dir.path(),
        &server,
        "model",
        "",
        "    matrix_assert:\n      select_best: true\n",
    );

    let mut cmd = mcptest();
    strip_provider_keys(&mut cmd);
    cmd.current_dir(dir.path())
        .args(["run", "--allow-stub-provider", "--config"])
        .arg(&config)
        .assert()
        .code(0)
        .stdout(predicate::str::contains(
            "matrix-assert [BEST] weather matrix",
        ));
}
