//! Integration tests for `mcptest compliance run --from-suite` (WOR-1347).
//!
//! The bug: the suite loader read only `tools:`/`resources:`/`prompts:`,
//! so the `compliance:` test blocks parsed to nothing, the runner planned
//! zero checks, and the rubric failed every registry rule (grade F with
//! zero checks executed). These tests drive the compiled binary against
//! `mcptest mock` to prove the compliance checks now execute: the
//! `initialize` check evaluates the captured handshake, the `tools/list`
//! check issues the raw method, and the rubric scores the named rules as
//! passed.

use assert_cmd::Command;
use predicates::prelude::*;

fn mcptest() -> Command {
    Command::cargo_bin("mcptest").expect("binary builds")
}

/// Write a `mcptest mock` manifest exposing one `echo` tool, the live
/// server every compliance check in this file probes.
fn write_echo_manifest(dir: &std::path::Path) -> std::path::PathBuf {
    let manifest = r#"mock_server:
  name: echo
  tools:
    - name: echo
      description: "Echo"
      input_schema:
        type: object
      response:
        content:
          - type: text
            text: "hello"
"#;
    let path = dir.join("echo.yaml");
    std::fs::write(&path, manifest).expect("write mock manifest");
    path
}

/// Two-rule registry so the rubric verdict depends only on the two checks
/// the suite runs: both pass means grade A+ and zero failed MUSTs, while
/// the pre-fix behavior (zero checks executed) would grade F.
fn write_registry(dir: &std::path::Path) -> std::path::PathBuf {
    let yaml = r#"
version: 1
rules:
  - id: PROTO-002
    name: initialize returns negotiated capabilities
    category: PROTO
    severity: MUST
    applies_to: [stdio]
    spec_versions: [v2024-11-05]
    description: initialize response carries a capabilities object
    references: []
    section: lifecycle
    applicability: "true"
    spec_citation: spec
  - id: TOOL-001
    name: tools/list responds
    category: TOOL
    severity: MUST
    applies_to: [stdio]
    spec_versions: [v2024-11-05]
    description: tools/list returns the advertised tools
    references: []
    section: tools
    applicability: "true"
    spec_citation: spec
"#;
    let path = dir.join("registry.yml");
    std::fs::write(&path, yaml).expect("write registry");
    path
}

/// Suite whose `compliance:` block carries an initialize check and a
/// tools/list check against the mock server. The initialize entry also
/// declares `cache:`, a schema field nothing consumes yet, to exercise
/// the load-time warning.
fn write_suite(dir: &std::path::Path, manifest: &std::path::Path) -> std::path::PathBuf {
    let bin = env!("CARGO_BIN_EXE_mcptest");
    let yaml = format!(
        r#"servers:
  mock:
    command: ['{bin}', 'mock', '--tools-from', '{manifest}']
compliance:
  - name: PROTO-002 initialize returns negotiated capabilities
    server: mock
    check: "initialize"
    cache: never
    expect:
      - target: result.capabilities
        matcher:
          schema: {{ type: object }}
  - name: TOOL-001 tools/list responds
    server: mock
    check: "tools/list"
    expect:
      - target: result.tools
        matcher:
          schema: {{ type: array, minItems: 1 }}
"#,
        manifest = manifest.display(),
    );
    let path = dir.join("suite.yml");
    std::fs::write(&path, yaml).expect("write suite");
    path
}

#[test]
fn compliance_suite_checks_execute_and_score_their_rules_as_passed() {
    let dir = tempfile::tempdir().unwrap();
    let manifest = write_echo_manifest(dir.path());
    let registry = write_registry(dir.path());
    let suite = write_suite(dir.path(), &manifest);

    let assert = mcptest()
        .args([
            "compliance",
            "run",
            "--from-suite",
            suite.to_str().unwrap(),
            "--registry",
            registry.to_str().unwrap(),
            "--capabilities",
            "tools",
            "--format",
            "json",
        ])
        .assert()
        .code(0)
        .stderr(
            predicate::str::contains("declares `cache`").and(predicate::str::contains(
                "PROTO-002 initialize returns negotiated capabilities",
            )),
        );

    let stdout = String::from_utf8(assert.get_output().stdout.clone()).expect("utf8 stdout");
    let report: serde_json::Value = serde_json::from_str(&stdout).expect("JSON envelope");
    let results = report["results"].as_array().expect("results array");
    assert_eq!(results.len(), 2, "both compliance checks executed");
    for rule in ["PROTO-002", "TOOL-001"] {
        let row = results
            .iter()
            .find(|r| r["rule_id"] == rule)
            .unwrap_or_else(|| panic!("no result row for {rule}: {stdout}"));
        assert_eq!(row["passed"], true, "{rule} should pass: {stdout}");
    }
    // Pre-fix, zero checks executed and every MUST failed (grade F). Both
    // MUSTs passing with no SHOULD/MAY rules grades A+.
    assert_eq!(
        report["grade"], "A+",
        "rubric reflects the passes: {stdout}"
    );
    assert!(
        report["failed_must"].as_array().is_some_and(Vec::is_empty),
        "no failed MUSTs: {stdout}"
    );
}

#[test]
fn compliance_check_against_a_method_the_server_lacks_fails_its_rule() {
    // The mock implements no completion surface, so a completion/complete
    // check must surface the server's method-not-found error as a failed
    // rule, not a crash.
    let dir = tempfile::tempdir().unwrap();
    let manifest = write_echo_manifest(dir.path());
    let registry = write_registry(dir.path());
    let bin = env!("CARGO_BIN_EXE_mcptest");
    let yaml = format!(
        r#"servers:
  mock:
    command: ['{bin}', 'mock', '--tools-from', '{manifest}']
compliance:
  tests:
    - name: PROTO-002 initialize returns negotiated capabilities
      server: mock
      check: "initialize"
    - name: TOOL-001 tools/list responds
      server: mock
      check: "completion/complete"
"#,
        manifest = manifest.display(),
    );
    let suite = dir.path().join("suite.yml");
    std::fs::write(&suite, yaml).expect("write suite");

    let assert = mcptest()
        .args([
            "compliance",
            "run",
            "--from-suite",
            suite.to_str().unwrap(),
            "--registry",
            registry.to_str().unwrap(),
            "--capabilities",
            "tools",
            "--format",
            "json",
        ])
        .assert()
        .code(1);

    let stdout = String::from_utf8(assert.get_output().stdout.clone()).expect("utf8 stdout");
    let report: serde_json::Value = serde_json::from_str(&stdout).expect("JSON envelope");
    let results = report["results"].as_array().expect("results array");
    let failed = results
        .iter()
        .find(|r| r["rule_id"] == "TOOL-001")
        .expect("TOOL-001 row");
    assert_eq!(failed["passed"], false, "{stdout}");
    assert!(
        failed["message"]
            .as_str()
            .is_some_and(|m| m.contains("completion/complete")),
        "failure names the method: {stdout}"
    );
}

/// One-rule registry (TOOL-001 only) for the WOR-1282 skip/only tests,
/// so a parked check's rule id (PROTO-002) is absent from the registry
/// and the rubric verdict depends only on the check that actually runs.
fn write_tool_only_registry(dir: &std::path::Path) -> std::path::PathBuf {
    let yaml = r#"
version: 1
rules:
  - id: TOOL-001
    name: tools/list responds
    category: TOOL
    severity: MUST
    applies_to: [stdio]
    spec_versions: [v2024-11-05]
    description: tools/list returns the advertised tools
    references: []
    section: tools
    applicability: "true"
    spec_citation: spec
"#;
    let path = dir.join("registry.yml");
    std::fs::write(&path, yaml).expect("write registry");
    path
}

/// Pull the JSON envelope out of stdout, tolerating the WOR-1282 skip
/// lines that print ahead of the rendered scorecard.
fn json_envelope(stdout: &str) -> serde_json::Value {
    let start = stdout.find('{').expect("JSON envelope in stdout");
    serde_json::from_str(&stdout[start..]).expect("JSON envelope parses")
}

#[test]
fn explicit_skip_parks_the_check_and_prints_the_reason() {
    let dir = tempfile::tempdir().unwrap();
    let manifest = write_echo_manifest(dir.path());
    let registry = write_tool_only_registry(dir.path());
    let bin = env!("CARGO_BIN_EXE_mcptest");
    let yaml = format!(
        r#"servers:
  mock:
    command: ['{bin}', 'mock', '--tools-from', '{manifest}']
compliance:
  - name: PROTO-002 initialize returns negotiated capabilities
    server: mock
    check: "initialize"
    skip: "waiting on upstream fix"
  - name: TOOL-001 tools/list responds
    server: mock
    check: "tools/list"
"#,
        manifest = manifest.display(),
    );
    let suite = dir.path().join("suite.yml");
    std::fs::write(&suite, yaml).expect("write suite");

    let assert = mcptest()
        .env("CI", "false")
        .args([
            "compliance",
            "run",
            "--from-suite",
            suite.to_str().unwrap(),
            "--registry",
            registry.to_str().unwrap(),
            "--capabilities",
            "tools",
            "--format",
            "json",
        ])
        .assert()
        .code(0)
        .stdout(predicate::str::contains(
            "skip     PROTO-002 initialize returns negotiated capabilities (waiting on upstream fix)",
        ));

    let stdout = String::from_utf8(assert.get_output().stdout.clone()).expect("utf8 stdout");
    let report = json_envelope(&stdout);
    let results = report["results"].as_array().expect("results array");
    assert_eq!(
        results.len(),
        1,
        "only the live check produced a row: {stdout}"
    );
    assert_eq!(results[0]["rule_id"], "TOOL-001", "{stdout}");
    assert_eq!(results[0]["passed"], true, "{stdout}");
    assert!(
        results.iter().all(|r| r["rule_id"] != "PROTO-002"),
        "the skipped rule must not appear in the passed/failed rows: {stdout}"
    );
    assert!(
        report["failed_must"].as_array().is_some_and(Vec::is_empty),
        "the skipped rule is not a failure: {stdout}"
    );
}

#[test]
fn only_marker_focuses_the_run_and_reports_the_rest_as_implicitly_skipped() {
    let dir = tempfile::tempdir().unwrap();
    let manifest = write_echo_manifest(dir.path());
    let registry = write_tool_only_registry(dir.path());
    let bin = env!("CARGO_BIN_EXE_mcptest");
    let yaml = format!(
        r#"servers:
  mock:
    command: ['{bin}', 'mock', '--tools-from', '{manifest}']
compliance:
  - name: PROTO-002 initialize returns negotiated capabilities
    server: mock
    check: "initialize"
  - name: TOOL-001 tools/list responds
    server: mock
    check: "tools/list"
    only: true
"#,
        manifest = manifest.display(),
    );
    let suite = dir.path().join("suite.yml");
    std::fs::write(&suite, yaml).expect("write suite");

    let assert = mcptest()
        .env("CI", "false")
        .args([
            "compliance",
            "run",
            "--from-suite",
            suite.to_str().unwrap(),
            "--registry",
            registry.to_str().unwrap(),
            "--capabilities",
            "tools",
            "--format",
            "json",
        ])
        .assert()
        .code(0)
        .stdout(predicate::str::contains(
            "skip     PROTO-002 initialize returns negotiated capabilities (not in only set)",
        ))
        .stderr(
            predicate::str::contains("only: true is set")
                .and(predicate::str::contains("1 focused test(s)"))
                .and(predicate::str::contains("Remove `only:` before merging")),
        );

    let stdout = String::from_utf8(assert.get_output().stdout.clone()).expect("utf8 stdout");
    let report = json_envelope(&stdout);
    let results = report["results"].as_array().expect("results array");
    assert_eq!(
        results.len(),
        1,
        "only the focused check executed: {stdout}"
    );
    assert_eq!(results[0]["rule_id"], "TOOL-001", "{stdout}");
    assert_eq!(results[0]["passed"], true, "{stdout}");
}

#[test]
fn focused_suite_under_ci_true_refuses_with_exit_two_before_connecting() {
    // The server command points at a path that cannot spawn. If the run
    // ever tried to connect, the failure would surface as a connect
    // error, not the clean exit-2 refusal asserted here.
    let dir = tempfile::tempdir().unwrap();
    let registry = write_tool_only_registry(dir.path());
    let yaml = r#"servers:
  mock:
    command: ['/nonexistent/mcptest-never-spawns']
compliance:
  - name: TOOL-001 tools/list responds
    server: mock
    check: "tools/list"
    only: true
"#;
    let suite = dir.path().join("suite.yml");
    std::fs::write(&suite, yaml).expect("write suite");

    mcptest()
        .env("CI", "true")
        .args([
            "compliance",
            "run",
            "--from-suite",
            suite.to_str().unwrap(),
            "--registry",
            registry.to_str().unwrap(),
            "--format",
            "json",
        ])
        .assert()
        .code(2)
        .stdout(predicate::str::is_empty())
        .stderr(
            predicate::str::contains("contains `only: true` and CI=true is set").and(
                predicate::str::contains("remove the `only:` marker(s) before merging"),
            ),
        );
}

// WOR-1348: the marker universe spans the ride-along tool/resource/
// prompt tests in the same suite, not just the `compliance:` checks.

#[test]
fn skipped_tool_test_parks_prints_its_reason_and_never_spawns_its_server() {
    // The tool test carries `skip:` and is the ONLY entry referencing
    // the `dead` server, whose command cannot spawn. If the parked test
    // ever connected or executed, the run would fail with a connect
    // error instead of the clean exit 0 asserted here.
    let dir = tempfile::tempdir().unwrap();
    let manifest = write_echo_manifest(dir.path());
    let registry = write_tool_only_registry(dir.path());
    let bin = env!("CARGO_BIN_EXE_mcptest");
    let yaml = format!(
        r#"servers:
  mock:
    command: ['{bin}', 'mock', '--tools-from', '{manifest}']
  dead:
    command: ['/nonexistent/mcptest-never-spawns']
tools:
  - name: echo round-trips
    server: dead
    tool: echo
    args: {{ input: "hi" }}
    skip: "waiting on the echo fix"
    expect:
      - target: result.content
        matcher:
          schema: {{ type: array }}
compliance:
  - name: TOOL-001 tools/list responds
    server: mock
    check: "tools/list"
"#,
        manifest = manifest.display(),
    );
    let suite = dir.path().join("suite.yml");
    std::fs::write(&suite, yaml).expect("write suite");

    let assert = mcptest()
        .env("CI", "false")
        .args([
            "compliance",
            "run",
            "--from-suite",
            suite.to_str().unwrap(),
            "--registry",
            registry.to_str().unwrap(),
            "--capabilities",
            "tools",
            "--format",
            "json",
        ])
        .assert()
        .code(0)
        .stdout(predicate::str::contains(
            "skip     echo round-trips (waiting on the echo fix)",
        ));

    let stdout = String::from_utf8(assert.get_output().stdout.clone()).expect("utf8 stdout");
    let report = json_envelope(&stdout);
    let results = report["results"].as_array().expect("results array");
    assert_eq!(
        results.len(),
        1,
        "only the compliance check produced a row: {stdout}"
    );
    assert_eq!(results[0]["rule_id"], "TOOL-001", "{stdout}");
    assert_eq!(results[0]["passed"], true, "{stdout}");
    // The parked tool test's name must appear in no rule list of the
    // envelope: no result row, no failed-MUST entry, nothing.
    assert!(
        !report.to_string().contains("echo round-trips"),
        "the skipped tool test must not appear anywhere in the envelope: {stdout}"
    );
    assert!(
        report["failed_must"].as_array().is_some_and(Vec::is_empty),
        "the skipped tool test is not a failure: {stdout}"
    );
}

#[test]
fn only_on_a_compliance_check_implicitly_parks_the_ride_along_tool_test() {
    let dir = tempfile::tempdir().unwrap();
    let manifest = write_echo_manifest(dir.path());
    let registry = write_tool_only_registry(dir.path());
    let bin = env!("CARGO_BIN_EXE_mcptest");
    let yaml = format!(
        r#"servers:
  mock:
    command: ['{bin}', 'mock', '--tools-from', '{manifest}']
tools:
  - name: echo round-trips
    server: mock
    tool: echo
    args: {{ input: "hi" }}
    expect:
      - target: result.content
        matcher:
          schema: {{ type: array }}
compliance:
  - name: TOOL-001 tools/list responds
    server: mock
    check: "tools/list"
    only: true
"#,
        manifest = manifest.display(),
    );
    let suite = dir.path().join("suite.yml");
    std::fs::write(&suite, yaml).expect("write suite");

    let assert = mcptest()
        .env("CI", "false")
        .args([
            "compliance",
            "run",
            "--from-suite",
            suite.to_str().unwrap(),
            "--registry",
            registry.to_str().unwrap(),
            "--capabilities",
            "tools",
            "--format",
            "json",
        ])
        .assert()
        .code(0)
        .stdout(predicate::str::contains(
            "skip     echo round-trips (not in only set)",
        ))
        .stderr(
            predicate::str::contains("only: true is set")
                .and(predicate::str::contains("1 focused test(s)"))
                .and(predicate::str::contains(
                    "1 other test(s) implicitly skipped",
                )),
        );

    let stdout = String::from_utf8(assert.get_output().stdout.clone()).expect("utf8 stdout");
    let report = json_envelope(&stdout);
    let results = report["results"].as_array().expect("results array");
    assert_eq!(
        results.len(),
        1,
        "only the focused compliance check executed: {stdout}"
    );
    assert_eq!(results[0]["rule_id"], "TOOL-001", "{stdout}");
    assert_eq!(results[0]["passed"], true, "{stdout}");
}

#[test]
fn focused_tool_test_under_ci_true_refuses_with_exit_two_before_connecting() {
    // `only: true` rides on the plain tool test, not on a compliance
    // check. The server command cannot spawn, so a clean exit 2 proves
    // the refusal fires before any connect attempt.
    let dir = tempfile::tempdir().unwrap();
    let registry = write_tool_only_registry(dir.path());
    let yaml = r#"servers:
  mock:
    command: ['/nonexistent/mcptest-never-spawns']
tools:
  - name: echo round-trips
    server: mock
    tool: echo
    args: { input: "hi" }
    only: true
    expect:
      - target: result.content
        matcher:
          schema: { type: array }
compliance:
  - name: TOOL-001 tools/list responds
    server: mock
    check: "tools/list"
"#;
    let suite = dir.path().join("suite.yml");
    std::fs::write(&suite, yaml).expect("write suite");

    mcptest()
        .env("CI", "true")
        .args([
            "compliance",
            "run",
            "--from-suite",
            suite.to_str().unwrap(),
            "--registry",
            registry.to_str().unwrap(),
            "--format",
            "json",
        ])
        .assert()
        .code(2)
        .stdout(predicate::str::is_empty())
        .stderr(
            predicate::str::contains("contains `only: true` and CI=true is set").and(
                predicate::str::contains("remove the `only:` marker(s) before merging"),
            ),
        );
}

/// Regression guard for the WOR-1347 root cause: the documented example
/// suites must parse to a nonzero compliance-test list. Pre-fix they
/// parsed to nothing, which is exactly how the zero-checks F grade
/// happened.
#[test]
fn documented_example_suites_parse_nonzero_compliance_tests() {
    let examples = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("..")
        .join("..")
        .join("examples");
    for (file, expected) in [
        ("spec-version-pinning.yml", 2),
        ("compliance-grade-delta.yml", 1),
        ("official-conformance.yml", 3),
    ] {
        let raw = std::fs::read_to_string(examples.join(file))
            .unwrap_or_else(|e| panic!("read {file}: {e}"));
        let yaml: serde_yaml::Value = serde_yaml::from_str(&raw).expect("YAML parses");
        let suite = mcptest_core::suite::parse(&yaml)
            .unwrap_or_else(|e| panic!("typed parse of {file}: {e}"));
        assert_eq!(
            suite.compliance_tests.len(),
            expected,
            "{file} must yield its compliance tests"
        );
    }
}
