//! `mcptest conformance run` handler (WOR-945, WOR-1382).
//!
//! Two modes:
//!
//! * Without `--server`: load the resolved corpus and render a
//!   corpus-coverage scorecard. This counts documented requirements; it
//!   does not verify a server.
//! * With `--server <url|command>`: connect to the server, run the
//!   universal MUST probes (handshake, negotiated version, capabilities,
//!   `tools/list`), and let the tier reflect what the server actually
//!   did. Per-SEP corpus rows are reported as documented-but-not-probed
//!   until the per-requirement probe vocabulary lands (WOR-1404).

use std::fs;
use std::io::Write;

use anyhow::{Context, Result};
use serde::Serialize;

use super::conformance_probe;
use crate::cli::args::{ConformanceFormat, ConformanceRefreshArgs, ConformanceRunArgs};
use mcptest_core::conformance::{
    corpus, extended_probes, score_tier, tally_by_level, universal_probes, ProbeOutcome, Tier,
    TierInput,
};

/// `scope` value for a run that only counts documented requirements.
const SCOPE_COVERAGE: &str = "corpus-coverage";
/// `scope` value for a run that probed a live server.
const SCOPE_PROBE: &str = "live-probe";

/// Caveat for the corpus-coverage scorecard: the tier reflects the
/// documented corpus, not a server's observed behavior.
const COVERAGE_NOTE: &str = "corpus-coverage scorecard: counts documented requirements, \
NOT verified against a live server";
/// Caveat for the live-probe scorecard: only the universal MUST facts are
/// probed today; the rest of the corpus is documented, not verified.
const PROBE_NOTE: &str = "live-probe scorecard: universal MUST facts verified against the \
server; per-SEP corpus rows are documented, not individually probed yet (WOR-1404)";

/// Default upstream the auto-refresh fetches from. Mirrors the
/// `conformance refresh` clap defaults so `--auto-refresh` behaves like a
/// hand-run refresh.
const UPSTREAM_URL: &str = "https://github.com/modelcontextprotocol/conformance";
/// Default subdirectory mirrored by the auto-refresh.
const UPSTREAM_SOURCE_PATH: &str = "src/seps";

/// Wire shape of the JSON report.
#[derive(Debug, Serialize)]
struct RunReport {
    target_version: String,
    /// `corpus-coverage` (no server) or `live-probe` (a server was probed).
    scope: &'static str,
    corpus_source: corpus::CorpusSource,
    #[serde(rename = "must")]
    must_counts: BucketCounts,
    #[serde(rename = "should")]
    should_counts: BucketCounts,
    #[serde(rename = "may")]
    may_counts: BucketCounts,
    tier: Tier,
    badge: String,
    /// Per-check verdicts from a live probe. Empty in corpus-coverage mode.
    #[serde(skip_serializing_if = "Vec::is_empty")]
    probes: Vec<ProbeOutcome>,
    per_sep: Vec<PerSepReport>,
}

#[derive(Debug, Serialize, PartialEq, Eq)]
struct BucketCounts {
    passed: u32,
    total: u32,
}

#[derive(Debug, Serialize)]
struct PerSepReport {
    sep: String,
    tested: u32,
    passed: u32,
    excluded: u32,
}

/// One-line note matching a report's scope.
fn scope_note(scope: &str) -> &'static str {
    if scope == SCOPE_PROBE {
        PROBE_NOTE
    } else {
        COVERAGE_NOTE
    }
}

/// Per-SEP coverage rows plus the total tested-requirement count, shared by
/// both report shapes.
fn corpus_coverage(loaded: &corpus::LoadedCorpus) -> (Vec<PerSepReport>, u32) {
    let mut tested_total = 0u32;
    let mut per_sep = Vec::with_capacity(loaded.seps.len());
    for sep in &loaded.seps {
        let tested = sep
            .requirements
            .iter()
            .filter(|r| r.check.is_some())
            .count() as u32;
        let excluded = sep.excluded.len() as u32;
        tested_total += tested;
        per_sep.push(PerSepReport {
            sep: sep.sep.clone(),
            tested,
            passed: tested,
            excluded,
        });
    }
    (per_sep, tested_total)
}

/// Build the corpus-coverage report (no `--server`): every documented
/// requirement counts as a MUST that passed, matching the historical v1
/// scorecard.
fn coverage_report(loaded: &corpus::LoadedCorpus) -> RunReport {
    let (per_sep, tested) = corpus_coverage(loaded);
    let tier = score_tier(TierInput {
        must_passed: tested,
        must_total: tested,
        should_passed: 0,
        should_total: 0,
    });
    RunReport {
        target_version: loaded.spec_version.clone(),
        scope: SCOPE_COVERAGE,
        corpus_source: loaded.source,
        must_counts: BucketCounts {
            passed: tested,
            total: tested,
        },
        should_counts: BucketCounts {
            passed: 0,
            total: 0,
        },
        may_counts: BucketCounts {
            passed: 0,
            total: 0,
        },
        tier,
        badge: tier.badge().to_string(),
        probes: Vec::new(),
        per_sep,
    }
}

/// Build the live-probe report (`--server`): the tier reflects every probe run
/// against the server (universal MUSTs plus the per-SEP families). A per-SEP
/// row's `passed` counts the corpus checks actually probed and passed for that
/// SEP (matched by the `sep-<id>-` check-id prefix); rows with no probed check
/// stay at `passed: 0`, documented but not verified.
fn probe_report(loaded: &corpus::LoadedCorpus, outcomes: Vec<ProbeOutcome>) -> RunReport {
    let (mut per_sep, _tested) = corpus_coverage(loaded);
    for row in &mut per_sep {
        let prefix = format!("sep-{}-", row.sep);
        row.passed = outcomes
            .iter()
            .filter(|o| o.check.starts_with(&prefix) && o.result.passed())
            .count() as u32;
    }
    let (must_passed, must_total, should_passed, should_total) = tally_by_level(&outcomes);
    let tier = score_tier(TierInput {
        must_passed,
        must_total,
        should_passed,
        should_total,
    });
    RunReport {
        target_version: loaded.spec_version.clone(),
        scope: SCOPE_PROBE,
        corpus_source: loaded.source,
        must_counts: BucketCounts {
            passed: must_passed,
            total: must_total,
        },
        should_counts: BucketCounts {
            passed: should_passed,
            total: should_total,
        },
        may_counts: BucketCounts {
            passed: 0,
            total: 0,
        },
        tier,
        badge: tier.badge().to_string(),
        probes: outcomes,
        per_sep,
    }
}

/// Resolve the corpus, optionally auto-refreshing it. Returns the loaded
/// corpus on success, or an exit code the caller should return on a
/// version-not-found (with or without a failed auto-refresh).
fn resolve_corpus(
    args: &ConformanceRunArgs,
    target_version: &str,
    cache_dir: Option<&std::path::Path>,
) -> Result<std::result::Result<corpus::LoadedCorpus, i32>> {
    match corpus::load(target_version, args.corpus_dir.as_deref(), cache_dir) {
        Ok(c) => Ok(Ok(c)),
        Err(corpus::CorpusError::VersionNotFound {
            requested,
            searched,
        }) => {
            if args.auto_refresh {
                eprintln!("mcptest conformance: --auto-refresh: fetching spec `{requested}`.");
                let refresh = ConformanceRefreshArgs {
                    target_version: requested.clone(),
                    corpus_dir: args.corpus_dir.clone(),
                    url: UPSTREAM_URL.to_string(),
                    r#ref: None,
                    source_path: UPSTREAM_SOURCE_PATH.to_string(),
                    dry_run: false,
                };
                let refresh_code = super::conformance_refresh::refresh_command(refresh)?;
                if refresh_code != 0 {
                    eprintln!("mcptest conformance: auto-refresh failed for `{requested}`.");
                    // Surface the refresh's own exit code (3 for a network
                    // failure) rather than flattening it.
                    return Ok(Err(refresh_code));
                }
                return match corpus::load(&requested, args.corpus_dir.as_deref(), cache_dir) {
                    Ok(c) => Ok(Ok(c)),
                    Err(e) => Err(e.into()),
                };
            }
            eprintln!(
                "mcptest conformance: spec version `{requested}` not present at {searched}. \
                 Run `mcptest conformance refresh --target-version {requested}` to populate it, \
                 or pass --auto-refresh."
            );
            Ok(Err(2))
        }
        Err(e) => Err(e.into()),
    }
}

pub fn run_command(args: ConformanceRunArgs) -> Result<i32> {
    // The client lane scores a client-under-test against the universal MCP
    // handshake, which is not corpus-keyed, so it branches before the corpus is
    // resolved. `--client` and `--server` are mutually exclusive at the CLI.
    if args.client.is_some() {
        return super::conformance_client_run::run_client_lane(&args);
    }

    let cache_dir = super::conformance_paths::user_cache_dir();
    let target_version = match &args.target_version {
        Some(v) => v.clone(),
        None => corpus::latest_available(args.corpus_dir.as_deref(), cache_dir.as_deref())
            .context("could not resolve a target version")?,
    };
    let loaded = match resolve_corpus(&args, &target_version, cache_dir.as_deref())? {
        Ok(c) => c,
        Err(code) => return Ok(code),
    };

    let report = match &args.server {
        Some(server) => {
            let evidence = match conformance_probe::probe_server_evidence(server) {
                Ok(e) => e,
                Err(err) => {
                    eprintln!("mcptest conformance: could not reach `{server}`: {err:#}");
                    // Exit 4 ("server unreachable during run"), per the CLI
                    // contract in docs/conformance-cli.md.
                    return Ok(4);
                }
            };
            // Empty probe target: a server may legally negotiate a version
            // below the one we advertised, so require only that *some* version
            // was negotiated rather than a strict match.
            let mut outcomes = universal_probes("", &evidence);
            outcomes.extend(extended_probes(&evidence));
            probe_report(&loaded, outcomes)
        }
        None => coverage_report(&loaded),
    };

    let rendered = render(&report, args.format)?;
    write_output(&rendered, args.output.as_deref())?;
    // Per the CLI contract, `run` exits 0 once the run executed: a `Fail` tier
    // is reported in the badge and the JSON `tier`, not the exit code, so a
    // reachable-but-nonconformant server is distinguished from an unreachable
    // one (exit 4).
    Ok(0)
}

fn render(report: &RunReport, format: ConformanceFormat) -> Result<String> {
    match format {
        ConformanceFormat::Json => {
            serde_json::to_string_pretty(report).context("serializing JSON report")
        }
        ConformanceFormat::Pretty => Ok(render_pretty(report)),
        ConformanceFormat::Markdown => Ok(render_markdown(report)),
        ConformanceFormat::Html => Ok(render_html(report)),
    }
}

/// Format one probe outcome as `check: PASS|FAIL (reason)`.
fn probe_line(outcome: &ProbeOutcome) -> String {
    use mcptest_core::conformance::ProbeResult;
    match &outcome.result {
        ProbeResult::Passed => format!("    [PASS] {} ({})", outcome.check, outcome.level),
        ProbeResult::Failed { reason } => {
            format!("    [FAIL] {} ({}): {reason}", outcome.check, outcome.level)
        }
    }
}

fn render_pretty(report: &RunReport) -> String {
    let mut out = String::new();
    out.push_str(&format!(
        "conformance scorecard: {} ({})\n",
        report.target_version, report.badge
    ));
    out.push_str(&format!("  scope  : {}\n", scope_note(report.scope)));
    out.push_str(&format!(
        "  source : {:?}\n  must   : {} / {}\n  should : {} / {}\n  may    : {} / {}\n",
        serde_json::to_value(report.corpus_source).unwrap_or_default(),
        report.must_counts.passed,
        report.must_counts.total,
        report.should_counts.passed,
        report.should_counts.total,
        report.may_counts.passed,
        report.may_counts.total,
    ));
    if !report.probes.is_empty() {
        out.push_str("\n  probed checks:\n");
        for outcome in &report.probes {
            out.push_str(&probe_line(outcome));
            out.push('\n');
        }
    }
    out.push('\n');
    for sep in &report.per_sep {
        if report.scope == SCOPE_PROBE {
            out.push_str(&format!(
                "  SEP-{}: {} documented, {} excluded (not individually probed)\n",
                sep.sep, sep.tested, sep.excluded
            ));
        } else {
            out.push_str(&format!(
                "  SEP-{}: tested {}, passed {}, excluded {}\n",
                sep.sep, sep.tested, sep.passed, sep.excluded
            ));
        }
    }
    out
}

fn render_markdown(report: &RunReport) -> String {
    let mut out = String::new();
    out.push_str(&format!(
        "# mcptest conformance: {} ({})\n\n",
        report.target_version, report.badge
    ));
    out.push_str(&format!("> {}\n\n", scope_note(report.scope)));
    out.push_str(&format!(
        "- **Source:** `{}`\n- **MUST:** {} / {}\n- **SHOULD:** {} / {}\n- **MAY:** {} / {}\n\n",
        serde_json::to_value(report.corpus_source).unwrap_or_default(),
        report.must_counts.passed,
        report.must_counts.total,
        report.should_counts.passed,
        report.should_counts.total,
        report.may_counts.passed,
        report.may_counts.total,
    ));
    if !report.probes.is_empty() {
        out.push_str("## Probed checks\n\n");
        for outcome in &report.probes {
            out.push_str(&format!("- {}\n", probe_line(outcome).trim_start()));
        }
        out.push('\n');
    }
    out.push_str("| SEP | Tested | Passed | Excluded |\n");
    out.push_str("|---|---|---|---|\n");
    for sep in &report.per_sep {
        out.push_str(&format!(
            "| {} | {} | {} | {} |\n",
            sep.sep, sep.tested, sep.passed, sep.excluded
        ));
    }
    out
}

fn render_html(report: &RunReport) -> String {
    let mut out = String::new();
    out.push_str("<!doctype html><html><body>");
    out.push_str(&format!(
        "<h1>mcptest conformance: {} ({})</h1>",
        report.target_version, report.badge
    ));
    out.push_str(&format!("<p><em>{}</em></p>", scope_note(report.scope)));
    out.push_str(&format!(
        "<p>MUST {} / {}, SHOULD {} / {}, MAY {} / {}.</p>",
        report.must_counts.passed,
        report.must_counts.total,
        report.should_counts.passed,
        report.should_counts.total,
        report.may_counts.passed,
        report.may_counts.total
    ));
    if !report.probes.is_empty() {
        out.push_str("<ul>");
        for outcome in &report.probes {
            out.push_str(&format!("<li>{}</li>", probe_line(outcome).trim_start()));
        }
        out.push_str("</ul>");
    }
    out.push_str("<table><thead><tr><th>SEP</th><th>Tested</th><th>Passed</th><th>Excluded</th></tr></thead><tbody>");
    for sep in &report.per_sep {
        out.push_str(&format!(
            "<tr><td>{}</td><td>{}</td><td>{}</td><td>{}</td></tr>",
            sep.sep, sep.tested, sep.passed, sep.excluded
        ));
    }
    out.push_str("</tbody></table></body></html>");
    out
}

fn write_output(body: &str, out: Option<&std::path::Path>) -> Result<()> {
    match out {
        None => {
            let stdout = std::io::stdout();
            let mut handle = stdout.lock();
            handle
                .write_all(body.as_bytes())
                .context("writing stdout")?;
            if !body.ends_with('\n') {
                handle.write_all(b"\n").context("writing stdout newline")?;
            }
        }
        Some(path) => {
            fs::write(path, body).with_context(|| format!("writing {}", path.display()))?;
        }
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;
    use mcptest_core::conformance::{ConformanceEvidence, ProbeResult};

    fn embedded() -> corpus::LoadedCorpus {
        let version = corpus::list_versions_embedded()
            .into_iter()
            .next()
            .expect("embedded corpus has at least one version");
        corpus::load_from_embedded(&version).expect("load embedded corpus")
    }

    #[test]
    fn coverage_report_counts_documented_requirements_and_skips_probes() {
        let report = coverage_report(&embedded());
        assert_eq!(report.scope, SCOPE_COVERAGE);
        assert!(report.probes.is_empty());
        // The historical v1 invariant: every documented requirement counts as
        // a passed MUST, so passed == total.
        assert_eq!(report.must_counts.passed, report.must_counts.total);
        assert!(
            report.must_counts.total > 0,
            "embedded corpus has requirements"
        );
    }

    #[test]
    fn a_clean_probe_lands_in_tier_one_and_lists_the_probed_checks() {
        let evidence = ConformanceEvidence {
            initialize_ok: true,
            negotiated_version: Some("2025-06-18".to_string()),
            capabilities_advertised: true,
            tools_list_ok: Some(true),
            ..ConformanceEvidence::default()
        };
        let report = probe_report(&embedded(), universal_probes("", &evidence));
        assert_eq!(report.scope, SCOPE_PROBE);
        assert_eq!(report.tier, Tier::Tier1);
        assert_eq!(
            report.must_counts,
            BucketCounts {
                passed: 4,
                total: 4
            }
        );
        assert_eq!(report.probes.len(), 4);
        // Per-SEP rows are documented but not claimed as passed.
        assert!(report.per_sep.iter().all(|s| s.passed == 0));
    }

    #[test]
    fn extended_probes_mark_the_probed_sep_row_and_feed_the_should_tier() {
        use mcptest_core::conformance::{extended_probes, ResourceReadObservation};
        // A server with resources + prompts that returns a proper -32602 for a
        // missing resource: capability MUSTs pass and the SEP-2164 SHOULD passes.
        let evidence = ConformanceEvidence {
            initialize_ok: true,
            negotiated_version: Some("2025-06-18".to_string()),
            capabilities_advertised: true,
            tools_list_ok: Some(true),
            resources_list_ok: Some(true),
            prompts_list_ok: Some(true),
            bogus_resource_read: Some(ResourceReadObservation {
                is_error: true,
                error_code: Some(-32602),
                empty_contents: false,
            }),
            cancellation: None,
        };
        let mut outcomes = universal_probes("", &evidence);
        outcomes.extend(extended_probes(&evidence));
        let report = probe_report(&embedded(), outcomes);

        assert_eq!(report.tier, Tier::Tier1);
        // 4 universal + 2 capability (resources/prompts) + 1 SEP-2164 MUST.
        assert_eq!(report.must_counts.total, 7);
        assert_eq!(report.must_counts.passed, 7);
        // The SEP-2164 SHOULD (-32602) is tallied and passes.
        assert_eq!(report.should_counts.passed, 1);
        // The SEP-2164 row now reports its probed checks as passed (2 corpus ids).
        let sep_2164 = report
            .per_sep
            .iter()
            .find(|s| s.sep == "2164")
            .expect("SEP 2164 in corpus");
        assert_eq!(sep_2164.passed, 2, "both SEP-2164 corpus checks passed");
    }

    #[test]
    fn a_broken_handshake_probe_fails_the_tier() {
        let evidence = ConformanceEvidence::default(); // initialize_ok = false
        let report = probe_report(&embedded(), universal_probes("", &evidence));
        assert_eq!(report.tier, Tier::Fail);
        assert_eq!(report.badge, "F");
        assert!(report
            .probes
            .iter()
            .any(|p| matches!(p.result, ProbeResult::Failed { .. })));
    }

    #[test]
    fn the_scope_note_tracks_the_scope() {
        assert_eq!(scope_note(SCOPE_PROBE), PROBE_NOTE);
        assert_eq!(scope_note(SCOPE_COVERAGE), COVERAGE_NOTE);
    }

    #[test]
    fn pretty_render_of_a_probe_shows_the_checks_and_documented_seps() {
        let evidence = ConformanceEvidence {
            initialize_ok: true,
            negotiated_version: Some("2025-06-18".to_string()),
            capabilities_advertised: true,
            tools_list_ok: Some(true),
            ..ConformanceEvidence::default()
        };
        let report = probe_report(&embedded(), universal_probes("", &evidence));
        let pretty = render_pretty(&report);
        assert!(pretty.contains("probed checks:"));
        assert!(pretty.contains("[PASS] initialize-handshake"));
        assert!(pretty.contains("not individually probed"));
    }

    #[test]
    fn json_render_of_coverage_has_no_probes_key() {
        let report = coverage_report(&embedded());
        let json = render(&report, ConformanceFormat::Json).unwrap();
        assert!(!json.contains("\"probes\""));
        assert!(json.contains("\"scope\": \"corpus-coverage\""));
    }
}
