//! Integration tests for `mcptest eval` single-judge mode and
//! the `--no-verdict-cache` flag.
//!
//! The binary's CLI ships with a placeholder judge that always scores
//! `1.0` at zero cost; tests against the runtime path therefore confirm
//! exit codes and verdict-cache wiring rather than provider semantics.
//! In-process tests in `commands::eval` cover the runner against a stub
//! provider.

use assert_cmd::Command;
use predicates::prelude::*;
use tempfile::tempdir;

fn mcptest() -> Command {
    Command::cargo_bin("mcptest").expect("binary builds")
}

fn write(path: &std::path::Path, body: &str) {
    if let Some(parent) = path.parent() {
        std::fs::create_dir_all(parent).unwrap();
    }
    std::fs::write(path, body).unwrap();
}

#[test]
fn eval_explain_prints_the_plan_without_grading() {
    let dir = tempdir().unwrap();
    let cfg = dir.path().join("mcptest.yaml");
    write(
        &cfg,
        r#"servers:
  main:
    command: ["echo", "hi"]
evals:
  - name: structured eval
    server: main
    prompt: "summarize"
    response: "a summary"
    rubric:
      criteria:
        - name: on topic
          description: "stays on subject"
  - name: string eval
    server: main
    prompt: "answer"
    rubric: "must be helpful"
"#,
    );
    mcptest()
        .args(["--config", cfg.to_str().unwrap(), "eval", "--explain"])
        .assert()
        .success()
        .stdout(predicate::str::contains("Eval plan"))
        .stdout(predicate::str::contains("structured eval"))
        .stdout(predicate::str::contains("1 criteria"))
        .stdout(predicate::str::contains("judge calls"))
        .stdout(predicate::str::contains("fixed response"));
}

#[test]
fn eval_exits_zero_with_no_evals_block() {
    let dir = tempdir().unwrap();
    let cfg = dir.path().join("mcptest.yaml");
    write(
        &cfg,
        "servers:\n  main:\n    command: [\"echo\", \"hi\"]\ntools: []\n",
    );
    mcptest()
        .args(["--config", cfg.to_str().unwrap(), "eval"])
        .assert()
        .success()
        .stdout(predicate::str::contains("no `evals:` block"));
}

#[test]
fn eval_with_evals_block_passes_under_null_judge() {
    let dir = tempdir().unwrap();
    let cfg = dir.path().join("mcptest.yaml");
    write(
        &cfg,
        r#"servers:
  main:
    command: ["echo", "hi"]
evals:
  - name: "smoke"
    server: main
    prompt: "do a thing"
    rubric: "tool returns useful output"
    threshold: 0.5
"#,
    );
    mcptest()
        .args(["--config", cfg.to_str().unwrap(), "eval"])
        .assert()
        .success()
        .stdout(predicate::str::contains("PASS"))
        .stdout(predicate::str::contains("Total cost"));
}

#[test]
fn eval_renders_the_json_reporter_without_a_key() {
    // No provider key, so grading defers, but the canonical report still
    // renders cleanly through the JSON reporter (the diagnostic header goes to
    // stderr, so stdout is valid JSON).
    let dir = tempdir().unwrap();
    let cfg = dir.path().join("mcptest.yaml");
    write(
        &cfg,
        r#"servers:
  main:
    command: ["echo", "hi"]
evals:
  - name: "smoke"
    server: main
    prompt: "do a thing"
    response: "a deterministic answer"
    rubric: "the answer is on topic"
    threshold: 0.5
"#,
    );
    mcptest()
        .args([
            "--config",
            cfg.to_str().unwrap(),
            "--reporter",
            "json",
            "eval",
        ])
        .assert()
        .success()
        .stdout(predicate::str::contains("\"schema_version\""))
        .stdout(predicate::str::contains("\"smoke\""))
        .stdout(predicate::str::contains("\"tests\""));
}

#[test]
fn eval_accepts_no_verdict_cache_flag() {
    let dir = tempdir().unwrap();
    let cfg = dir.path().join("mcptest.yaml");
    // YAML cannot have a mapping (`cache:`) and a sequence as siblings under
    // the same key, so the verdict-cache opt-in lives outside `evals:`. The
    // build_verdict_cache helper looks for `evals.cache.verdicts` though, so
    // we point the YAML at that nested path via a sibling node that the
    // helper sees through `value.get("evals").and_then(|e| e.get("cache"))`.
    // For this CLI test the flag must parse and the run must exit cleanly;
    // verdict-cache wiring itself is exercised by the in-process integration
    // test below.
    write(
        &cfg,
        r#"servers:
  main:
    command: ["echo", "hi"]
evals:
  - name: "smoke"
    server: main
    prompt: "do a thing"
    rubric: "tool returns useful output"
    threshold: 0.5
"#,
    );
    mcptest()
        .args([
            "--config",
            cfg.to_str().unwrap(),
            "eval",
            "--no-verdict-cache",
        ])
        .assert()
        .success();
}

/// Verdict-cache override is honored end-to-end.
///
/// Builds a YAML doc where `evals.cache.verdicts: true` would otherwise
/// enable the cache, then calls the runner-side helper directly to assert
/// that `--no-verdict-cache` disables the cache regardless. We use the
/// helper rather than the binary because the binary's null judge does not
/// produce real verdicts, so a cache hit/miss assertion would be a
/// tautology.
#[test]
fn no_verdict_cache_flag_overrides_yaml_opt_in() {
    use mcptest_cli::commands::eval::build_verdict_cache;
    use serde_yaml::Value;
    // The structure here works because `evals` is a map with a `cache`
    // child, which is what the helper inspects. A real YAML test suite
    // splits the array and the cache under different keys.
    let yaml: Value = serde_yaml::from_str(
        r#"
evals:
  cache:
    verdicts: true
"#,
    )
    .unwrap();
    let with_flag = build_verdict_cache(&yaml, true);
    assert!(!with_flag.is_enabled());
    let without_flag = build_verdict_cache(&yaml, false);
    assert!(without_flag.is_enabled());
}

#[test]
fn eval_max_cost_flag_parses_with_dollar_prefix() {
    let dir = tempdir().unwrap();
    let cfg = dir.path().join("mcptest.yaml");
    write(&cfg, "servers:\n  main:\n    command: [\"echo\", \"hi\"]\n");
    mcptest()
        .args([
            "--config",
            cfg.to_str().unwrap(),
            "eval",
            "--max-cost",
            "$1.00",
        ])
        .assert()
        .success();
}

/// End-to-end exec scorer: a real wrapper process scores an eval
/// and its PASS line shows up in the report.
#[test]
fn eval_runs_exec_scorer_against_a_real_wrapper() {
    let dir = tempdir().unwrap();
    let cfg = dir.path().join("mcptest.yaml");
    write(
        &cfg,
        r#"servers:
  main:
    command: ["echo", "hi"]
evals:
  - name: "smoke"
    server: main
    prompt: "do a thing"
    rubric: "useful output"
scorers:
  - name: shell-judge
    type: exec
    command:
      - sh
      - -c
      - "cat >/dev/null; printf '{\"verdict\":\"pass\",\"reasoning\":\"looks good\",\"cost_usd\":0.0}'"
"#,
    );
    mcptest()
        .args(["--config", cfg.to_str().unwrap(), "eval"])
        .assert()
        .success()
        .stdout(predicate::str::contains("PASS [exec] smoke"))
        .stdout(predicate::str::contains("Scorer cost"));
}

/// An unknown scorer `type` in the OSS build surfaces the "not available in
/// this build" diagnostic and fails the run, instead of crashing.
#[test]
fn eval_unknown_scorer_type_reports_not_available() {
    let dir = tempdir().unwrap();
    let cfg = dir.path().join("mcptest.yaml");
    write(
        &cfg,
        r#"servers:
  main:
    command: ["echo", "hi"]
evals:
  - name: "smoke"
    server: main
    prompt: "do a thing"
scorers:
  - name: bt
    type: braintrust
"#,
    );
    mcptest()
        .args(["--config", cfg.to_str().unwrap(), "eval"])
        .assert()
        .code(1)
        .stdout(predicate::str::contains(
            "scorer type 'braintrust' not available in this build",
        ));
}

/// A scorer that reports a null cost triggers the `--max-cost` warning.
#[test]
fn eval_null_cost_scorer_warns_under_max_cost() {
    let dir = tempdir().unwrap();
    let cfg = dir.path().join("mcptest.yaml");
    write(
        &cfg,
        r#"servers:
  main:
    command: ["echo", "hi"]
evals:
  - name: "smoke"
    server: main
    prompt: "do a thing"
scorers:
  - name: nullcost
    type: exec
    command:
      - sh
      - -c
      - "cat >/dev/null; printf '{\"verdict\":\"pass\",\"cost_usd\":null}'"
"#,
    );
    mcptest()
        .args([
            "--config",
            cfg.to_str().unwrap(),
            "eval",
            "--max-cost",
            "$1.00",
        ])
        .assert()
        .success()
        .stdout(predicate::str::contains("null-cost scorers do not count"));
}

/// WOR-1355: `mcptest eval <dir>` must refuse the directory with the
/// loader's actionable message and the invalid-invocation exit code (2),
/// matching `run`/`validate`, instead of a generic "failed to read ...
/// Is a directory (os error 21)" anyhow error at exit 1.
#[test]
fn eval_against_a_directory_names_the_fix_instead_of_the_os_error() {
    let dir = tempdir().unwrap();
    // A real suite file inside the dir so the failure is the directory
    // itself, not a missing path.
    let cfg = dir.path().join("suite.yaml");
    write(&cfg, "servers:\n  main:\n    command: [\"echo\", \"hi\"]\n");

    let assertion = mcptest()
        .args(["--config", dir.path().to_str().unwrap(), "eval"])
        .assert()
        .code(2);

    let output = {
        let out = assertion.get_output();
        format!(
            "{}{}",
            String::from_utf8_lossy(&out.stdout),
            String::from_utf8_lossy(&out.stderr)
        )
    };
    assert!(
        output.contains("is a directory"),
        "the error must say the path is a directory:\n{output}"
    );
    assert!(
        output.contains("--config <file>") || output.contains("mcptest run tests/example.yaml"),
        "the error must show a working invocation:\n{output}"
    );
    assert!(
        !output.contains("failed to read"),
        "must not leak the generic raw-read error:\n{output}"
    );
}
