//! Canonical run output model + reporter trait.
//!
//! The runner produces a [`Report`] at the end of
//! every run. Reporters consume it and emit a format the user asked for.
//! Keeping the model in this crate (not the CLI crate) lets `mcptest report`
//! re-render previously saved JSON and lets future SDKs share the same
//! shape.
//!
//! Schema versioning: the on-disk format embeds [`Report::schema_version`].
//! v0.1 is the minimum-viable shape. v1 lands when the wire protocol
//! referenced by ADR-0017 is formalized.

use std::io::Write;

use serde::{Deserialize, Serialize};

use crate::compliance::RuleSeverity;
use crate::network::NetworkDiagnostic;
use crate::redaction::Redactor;

pub mod jury;
mod matrix_section;
pub mod metadata;
mod protocol_section;
pub mod reporters;
mod rollup;
pub mod run_store;
mod samples;
pub mod upload;

pub use jury::{Juror, JurorStatus, JurorTokens, JuryConsensus, Verdict};
pub use matrix_section::{MatrixCell, MatrixReport};
pub use metadata::{RedactField, RunMetadata, ServerPosture};
pub use protocol_section::{protocol_violation_entries, ProtocolViolationReport};
pub use rollup::MetricRollup;

/// Reporter that consumes the canonical [`Report`] and writes a
/// reporter-specific format to the supplied `Write` impl.
///
/// Reporters live in [`reporters`].
pub trait Reporter {
    /// Stable name matching the `--format` flag (`pretty`, `json`).
    fn name(&self) -> &'static str;

    /// Render `report` to `out`. Returns the underlying I/O error on
    /// failure.
    fn render(&self, report: &Report, out: &mut dyn Write) -> std::io::Result<()>;
}

/// One mcptest run, serializable to JSON for downstream consumption.
///
/// The shape is deliberately conservative.  will formalize the
/// canonical wire-protocol shape that the upload reporter and enterprise
/// collector consume; until then, this is the in-process model the report
/// subcommand round-trips.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct Report {
    /// Format version of this report, currently `"0.1"`. Will bump to
    /// `"v1"` when  lands.
    pub schema_version: String,

    /// `CARGO_PKG_VERSION` at run time.
    pub mcptest_version: String,

    /// RFC 3339 timestamp in UTC.
    pub generated_at: String,

    /// Stable run identifier (UUID v4 or ULID).
    pub run_id: String,

    /// Aggregate counters for the whole run.
    pub summary: RunSummary,

    /// One entry per test in execution order.
    pub tests: Vec<TestResult>,

    /// Aggregate LLM cost telemetry for the whole run. Absent
    /// when the run had no LLM-judge calls.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub cost: Option<RunCost>,

    /// Run-level metadata block. Identifies and locates the
    /// run in the operator's repo/CI/team context. Optional so existing
    /// fixtures and the legacy `mcptest report` round-trip path stay
    /// readable; the runner populates this on every fresh run.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub metadata: Option<RunMetadata>,

    /// Comparison-matrix section. Present only when a run fanned an agent
    /// test across a model/prompt matrix. Carried in the report (and so in
    /// the JSON artifact) so the `matrix` reporter and the cloud viewer can
    /// render the test-by-model grid from the same artifact every other
    /// reporter consumes, rather than re-deriving it from test names.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub matrix: Option<MatrixReport>,

    /// Protocol violations observed per server connection during the
    /// run (WOR-1311). Per run, not per test, because the evidence is
    /// recorded by the connection's pump and cannot be pinned to one
    /// test. Empty (and omitted from JSON) for a clean run so the
    /// legacy report shape is byte-identical.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub protocol_violations: Vec<ProtocolViolationReport>,
}

/// Aggregate LLM cost telemetry for a [`Report`].
///
/// Surfaces dollar totals plus per-call and per-test averages so the
/// reporter can print "this run cost $0.42 across 8 model calls (avg
/// $0.053 per call, $0.21 per test)" without re-deriving the math. The
/// "model call" unit spans both juror calls (the `mcptest eval` path) and
/// agent runs (the `mcptest run` path, WOR-1202); the field names keep their
/// `juror` spelling for on-disk compatibility.
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq)]
pub struct RunCost {
    /// Total dollars spent across every priced model call in the run.
    pub total_cost_usd: f64,
    /// Mean dollars per completed model call (juror call or agent run).
    pub cost_per_juror_avg: f64,
    /// Mean dollars per test that incurred a model cost.
    pub cost_per_test_avg: f64,
    /// Total priced model calls completed. Surfaced so consumers can multiply
    /// back to a total without depending on the avg's rounding.
    pub juror_calls: u64,
    /// Tests that incurred at least one priced model call. Denominator for
    /// `cost_per_test_avg`.
    pub tests_with_cost: u64,
    /// Mean dollars per *completed* (passing) task (WOR-1393): total cost over
    /// the number of passing tests, not the number attempted. `None` when the
    /// run did not compute it (no passing tests, or a path that does not track
    /// per-task completion). Distinct from `cost_per_test_avg`, which divides by
    /// tests attempted: a run that fails and retries costs more per completed
    /// task than per attempt.
    #[serde(default)]
    pub cost_per_completed_task: Option<f64>,
}

/// Aggregate counters for a [`Report`].
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct RunSummary {
    /// Number of tests attempted.
    pub total: usize,
    /// Tests that satisfied every expectation.
    pub passed: usize,
    /// Tests that failed at least one expectation.
    pub failed: usize,
    /// Tests that were declared but skipped (filter, blocked, etc.).
    pub skipped: usize,
    /// Tests whose verdict was inconclusive (a non-deterministic probe that
    /// neither confirmed nor refuted at the configured confidence). Counted
    /// separately so a flaky probe does not flake the suite. Zero and omitted
    /// from JSON for a run with no inconclusive verdict, so the report shape is
    /// unchanged for the common case.
    #[serde(default, skip_serializing_if = "is_zero")]
    pub inconclusive: usize,
    /// Wall-clock duration of the run in milliseconds.
    pub duration_ms: u64,
    /// Tests that returned a cached response. Reporters use this to
    /// surface a cold-vs-cached delta in the summary.
    #[serde(default)]
    pub cache_hits: usize,
    /// Verdicts served from the LLM verdict cache. Reported
    /// separately from `cache_hits` because the test-result cache and the
    /// verdict cache live in distinct namespaces (PRD section 9.2). Zero
    /// when no juries ran or when caching was disabled.
    #[serde(default)]
    pub verdict_cache_hits: u64,
    /// RFC 3339 timestamp when the run started, if known.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub started_at: Option<String>,
    /// RFC 3339 timestamp when the run ended, if known.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub ended_at: Option<String>,
}

impl RunSummary {
    /// Derive a summary from the tests it describes.
    ///
    /// This is the supported way to build a [`RunSummary`] (WOR-1317).
    /// Every count, including `total` and `cache_hits`, comes from one
    /// walk over `tests`, so `passed + failed + skipped + inconclusive
    /// == total == tests.len()` holds by construction. The fields stay
    /// public for serde and existing callers, but hand-rolled literals
    /// can drift from the test vec; prefer this constructor plus struct
    /// update syntax for the fields the tests cannot supply
    /// (`duration_ms`, `verdict_cache_hits`, timestamps).
    pub fn from_tests(tests: &[TestResult]) -> Self {
        let mut summary = Self {
            total: tests.len(),
            passed: 0,
            failed: 0,
            skipped: 0,
            inconclusive: 0,
            duration_ms: 0,
            cache_hits: 0,
            verdict_cache_hits: 0,
            started_at: None,
            ended_at: None,
        };
        for test in tests {
            match test.status {
                TestStatus::Pass => summary.passed += 1,
                TestStatus::Fail => summary.failed += 1,
                TestStatus::Skip => summary.skipped += 1,
                TestStatus::Inconclusive => summary.inconclusive += 1,
            }
            if test.cache_hit {
                summary.cache_hits += 1;
            }
        }
        summary
    }

    /// Check that the counters agree with the tests they describe.
    ///
    /// Reporters and the CLI exit-code path both assume the summary and
    /// [`Report::tests`] tell the same story, but a summary built by hand
    /// (or read from an external JSON artifact via `mcptest report`) can
    /// drift. This is the single place that spells out the invariant so
    /// consumers can reject a malformed artifact before trusting `total`.
    pub fn is_consistent(&self, tests: &[TestResult]) -> bool {
        let derived = Self::from_tests(tests);
        self.total == derived.total
            && self.passed == derived.passed
            && self.failed == derived.failed
            && self.skipped == derived.skipped
            && self.inconclusive == derived.inconclusive
    }
}

/// Outcome of a single test.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct TestResult {
    /// Human-readable name from the YAML `name:` field.
    pub name: String,
    /// Pass, fail, or skip.
    pub status: TestStatus,
    /// Wall-clock duration in milliseconds (1 for cache hits per
    /// convention;).
    pub duration_ms: u64,
    /// `true` when this test reused a cached response. The
    /// runner sets this so reporters can surface cold-vs-cached deltas
    /// without re-deriving the heuristic.
    #[serde(default)]
    pub cache_hit: bool,
    /// RFC 3339 timestamp when the test started, if known.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub started_at: Option<String>,
    /// RFC 3339 timestamp when the test ended, if known.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub ended_at: Option<String>,
    /// Populated only when `status = Fail`.
    pub failure: Option<TestFailure>,
    /// Source file path relative to the suite root, if known.
    pub file: Option<String>,
    /// One-based line number in `file`, if known.
    pub line: Option<u32>,
    /// Per-step breakdown for stepwise tests. Single-step
    /// tests leave this empty so the JSON shape stays compact.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub steps: Vec<TestStep>,
    /// LLM-judge consensus attached to this test, when an `llm-judge`
    /// matcher ran. Used by the pretty and JSON reporters to surface
    /// per-juror reasoning.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub consensus: Option<JuryConsensus>,
    /// Compliance rule identifier (`PROTO-001`, `EDGE-005`, etc.) when
    /// this test traces back to a registered compliance rule. SARIF and
    /// GitLab reporters use it to emit a per-rule definition and to map
    /// severity. Plain assertion tests leave this empty so the
    /// reporter falls back to a generic rule id.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub rule_id: Option<String>,
    /// RFC 2119 severity attached to the rule. Reporters use this to
    /// translate to SARIF `level` and GitLab `severity` consistently
    ///. `None` means the test is not a compliance check.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub severity: Option<RuleSeverity>,
    /// Combined weighted score for this test on a 0..1 scale.
    /// Present only when the test declared a `threshold:` (the score
    /// model is opt-in). Reporters surface it next to the pass/fail
    /// verdict so a 0.8-of-1.0 near-miss reads differently from a hard
    /// fail. Absent for legacy all-pass tests so their JSON is unchanged.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub score: Option<f64>,
    /// Derived metrics computed for this test. Each is a
    /// weighted aggregation over named assertion / assert-set scores.
    /// Empty (and omitted from JSON) when the test declared none.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub derived_metrics: Vec<DerivedMetricReport>,
    /// `true` when the test's tool is documented as returning a
    /// natural-language payload (a free-form string or a
    /// `content[].type == "text"` block). The enterprise policy
    /// evaluator reads this to target "NL-output tools must have
    /// an llm-judge or llm-jury matcher" rules. The runner sets
    /// this from the suite's `natural_language: true` test field
    /// (or `false`/omitted for plain tests). Omitted from JSON
    /// when false so the legacy report shape is unchanged.
    #[serde(default, skip_serializing_if = "is_false")]
    pub natural_language: bool,
}

#[allow(clippy::trivially_copy_pass_by_ref)]
fn is_false(b: &bool) -> bool {
    !*b
}

#[allow(clippy::trivially_copy_pass_by_ref)]
fn is_zero(n: &usize) -> bool {
    *n == 0
}

/// One derived metric in the run report.
///
/// Mirrors [`crate::suite::scoring::DerivedMetricResult`] but lives in the
/// report model so reporters and on-disk JSON do not depend on the suite
/// scoring types. The runner converts one into the other.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct DerivedMetricReport {
    /// Metric name from the YAML declaration.
    pub name: String,
    /// Aggregated numeric value.
    pub value: f64,
    /// `Some(true)`/`Some(false)` when the metric carried a threshold and
    /// gated the test; `None` when it was reported only.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub passed: Option<bool>,
}

/// One step inside a stepwise test.
///
/// The runner emits one of these per call recorded in the step list. The
/// reporters surface them under their parent test so a slow third step
/// stands out from uniformly slow steps. Cache-hit steps follow the same
/// 1ms convention as cache-hit tests so reporters do not have to special
/// case the cached branch.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct TestStep {
    /// Short human-readable label (for example the tool name or the
    /// `step:` field from the YAML).
    pub name: String,
    /// RFC 3339 timestamp when the step started, if known.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub started_at: Option<String>,
    /// RFC 3339 timestamp when the step ended, if known.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub ended_at: Option<String>,
    /// Wall-clock duration in milliseconds. Cache-hit steps emit `1` to
    /// mirror the test-level convention so reporter math does not divide
    /// by zero on the cached branch.
    pub duration_ms: u64,
    /// `true` when this step served from cache.
    #[serde(default)]
    pub cache_hit: bool,
    /// Optional per-step LLM-judge consensus (when the step itself ran an
    /// `llm-judge` matcher).
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub consensus: Option<JuryConsensus>,
}

/// Per-test status. New variants append at the end so consumers stay
/// forward-compatible.
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "kebab-case")]
#[non_exhaustive]
pub enum TestStatus {
    /// All expectations passed.
    Pass,
    /// At least one expectation failed.
    Fail,
    /// Skipped (filtered, blocked by hooks, etc.).
    Skip,
    /// A genuinely undecided verdict over a non-deterministic probe: the
    /// trials neither confirmed a pass nor a fail at the configured
    /// confidence (AgentAssay, arXiv:2603.02601). Reported distinctly and,
    /// by default, kept out of the failure count.
    Inconclusive,
}

/// Diagnostic payload attached to a failed [`TestResult`].
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct TestFailure {
    /// Single-line human summary.
    pub message: String,
    /// Optional structured diff rendered as text.
    pub diff: Option<String>,
    /// Network-layer diagnostic when the failure was a transport
    /// error rather than an assertion mismatch. The JSON
    /// reporter surfaces the layer as a kebab-case string so
    /// downstream tooling (CI dashboards, the cloud collector) can
    /// pivot on it without re-parsing the message.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub network_diagnostic: Option<NetworkDiagnostic>,
}

impl Report {
    /// Construct a 3-test fixture used by reporter tests. Two passing
    /// tests and one failing test with a diff. Stable across versions so
    /// snapshot tests can rely on it.
    pub fn sample() -> Self {
        let tests = vec![
            TestResult {
                name: "search returns at least one hit".into(),
                status: TestStatus::Pass,
                duration_ms: 80,
                cache_hit: false,
                started_at: Some("2026-05-15T22:14:03.000Z".into()),
                ended_at: Some("2026-05-15T22:14:03.080Z".into()),
                failure: None,
                file: Some("tests/search.yml".into()),
                line: Some(3),
                steps: Vec::new(),
                consensus: None,
                rule_id: None,
                severity: None,
                score: None,
                derived_metrics: Vec::new(),
                natural_language: false,
            },
            TestResult {
                name: "list_issues respects pagination".into(),
                status: TestStatus::Pass,
                duration_ms: 1,
                cache_hit: true,
                started_at: Some("2026-05-15T22:14:03.080Z".into()),
                ended_at: Some("2026-05-15T22:14:03.081Z".into()),
                failure: None,
                file: Some("tests/list_issues.yml".into()),
                line: Some(8),
                steps: Vec::new(),
                consensus: None,
                rule_id: None,
                severity: None,
                score: None,
                derived_metrics: Vec::new(),
                natural_language: false,
            },
            TestResult {
                name: "create_issue rejects empty title".into(),
                status: TestStatus::Fail,
                duration_ms: 169,
                cache_hit: false,
                started_at: Some("2026-05-15T22:14:03.081Z".into()),
                ended_at: Some("2026-05-15T22:14:03.250Z".into()),
                failure: Some(TestFailure {
                    message: "expected error but tool returned ok".into(),
                    diff: Some("  expected: error code -32602\n  actual:   ok with id=42".into()),
                    network_diagnostic: None,
                }),
                file: Some("tests/create_issue.yml".into()),
                line: Some(22),
                steps: Vec::new(),
                consensus: None,
                rule_id: None,
                severity: None,
                score: None,
                derived_metrics: Vec::new(),
                natural_language: false,
            },
        ];
        Self {
            schema_version: "0.1".into(),
            mcptest_version: crate::version::full().into(),
            generated_at: "2026-05-15T22:14:03Z".into(),
            run_id: "01JCQQYV4S3H6KX7M8RN2T5XZA".into(),
            summary: RunSummary {
                duration_ms: 250,
                started_at: Some("2026-05-15T22:14:03Z".into()),
                ended_at: Some("2026-05-15T22:14:03.250Z".into()),
                ..RunSummary::from_tests(&tests)
            },
            tests,
            cost: None,
            metadata: None,
            matrix: None,
            protocol_violations: Vec::new(),
        }
    }

    /// Construct a fixture exercising the comparison-matrix path.
    ///
    /// Two agent tests (`summarize`, `translate`) fanned across two models
    /// (`gpt-4o`, `claude-3-5-sonnet`). One cell fails (summarize on claude)
    /// so the matrix reporters have a pass cell, a fail cell with a
    /// drill-down, and non-trivial row/column pass rates to render. The body
    /// lives in the `matrix_section` sibling module to keep this module under
    /// the size ceiling.
    pub fn sample_matrix() -> Self {
        matrix_section::sample_matrix_report()
    }

    /// Construct a fixture exercising the stepwise path.
    ///
    /// Two tests: one stepwise three-step test (one step a cache hit) and
    /// one single-step test for the inline path. Reporter tests pin this
    /// shape so per-step renderings are stable across versions.
    pub fn sample_stepwise() -> Self {
        let tests = vec![
            TestResult {
                name: "create_issue then list and verify presence".into(),
                status: TestStatus::Pass,
                duration_ms: 18,
                cache_hit: false,
                started_at: Some("2026-05-15T22:14:03.000Z".into()),
                ended_at: Some("2026-05-15T22:14:03.018Z".into()),
                failure: None,
                file: Some("tests/stepwise.yml".into()),
                line: Some(3),
                steps: vec![
                    TestStep {
                        name: "create_issue".into(),
                        started_at: Some("2026-05-15T22:14:03.000Z".into()),
                        ended_at: Some("2026-05-15T22:14:03.012Z".into()),
                        duration_ms: 12,
                        cache_hit: false,
                        consensus: None,
                    },
                    TestStep {
                        name: "list_issues".into(),
                        started_at: Some("2026-05-15T22:14:03.012Z".into()),
                        ended_at: Some("2026-05-15T22:14:03.017Z".into()),
                        duration_ms: 5,
                        cache_hit: false,
                        consensus: None,
                    },
                    TestStep {
                        name: "list_issues (cached)".into(),
                        started_at: Some("2026-05-15T22:14:03.017Z".into()),
                        ended_at: Some("2026-05-15T22:14:03.018Z".into()),
                        duration_ms: 1,
                        cache_hit: true,
                        consensus: None,
                    },
                ],
                consensus: None,
                rule_id: None,
                severity: None,
                score: None,
                derived_metrics: Vec::new(),
                natural_language: false,
            },
            TestResult {
                name: "smoke test".into(),
                status: TestStatus::Pass,
                duration_ms: 3,
                cache_hit: false,
                started_at: Some("2026-05-15T22:14:03.018Z".into()),
                ended_at: Some("2026-05-15T22:14:03.021Z".into()),
                failure: None,
                file: Some("tests/smoke.yml".into()),
                line: Some(1),
                steps: Vec::new(),
                consensus: None,
                rule_id: None,
                severity: None,
                score: None,
                derived_metrics: Vec::new(),
                natural_language: false,
            },
        ];
        Self {
            schema_version: "0.1".into(),
            mcptest_version: crate::version::full().into(),
            generated_at: "2026-05-15T22:14:03Z".into(),
            run_id: "01JCQQYV4S3H6KX7M8RN2T5XZC".into(),
            summary: RunSummary {
                duration_ms: 21,
                started_at: Some("2026-05-15T22:14:03Z".into()),
                ended_at: Some("2026-05-15T22:14:03.021Z".into()),
                ..RunSummary::from_tests(&tests)
            },
            tests,
            cost: None,
            metadata: None,
            matrix: None,
            protocol_violations: Vec::new(),
        }
    }

    /// Return a copy of the report with every user-supplied string run
    /// through `redactor`. Use this once at the top of a reporter pipeline
    /// so every downstream reporter sees the same redacted view.
    ///
    /// Why this exists:  requires every output surface to share one
    /// redaction policy. Doing it here, on the `Report`, means reporter
    /// implementations stay simple (they print whatever they receive) and
    /// the audit checklist has a single integration point.
    ///
    /// The walked fields are `tests[i].name`, `tests[i].failure.message`,
    /// and `tests[i].failure.diff`. Summary counters, IDs, version
    /// strings, file paths, and line numbers are not user content, so
    /// they pass through unchanged.
    pub fn redacted(&self, redactor: &Redactor) -> Self {
        let tests = self
            .tests
            .iter()
            .map(|t| TestResult {
                name: redactor.redact_value(&t.name).into_owned(),
                status: t.status,
                duration_ms: t.duration_ms,
                cache_hit: t.cache_hit,
                started_at: t.started_at.clone(),
                ended_at: t.ended_at.clone(),
                failure: t.failure.as_ref().map(|f| TestFailure {
                    message: redactor.redact_value(&f.message).into_owned(),
                    diff: f
                        .diff
                        .as_ref()
                        .map(|d| redactor.redact_value(d).into_owned()),
                    network_diagnostic: f.network_diagnostic.clone(),
                }),
                file: t.file.clone(),
                line: t.line,
                steps: t
                    .steps
                    .iter()
                    .map(|step| TestStep {
                        name: redactor.redact_value(&step.name).into_owned(),
                        started_at: step.started_at.clone(),
                        ended_at: step.ended_at.clone(),
                        duration_ms: step.duration_ms,
                        cache_hit: step.cache_hit,
                        consensus: step.consensus.as_ref().map(|c| c.redacted(redactor)),
                    })
                    .collect(),
                consensus: t.consensus.as_ref().map(|c| c.redacted(redactor)),
                rule_id: t.rule_id.clone(),
                severity: t.severity,
                score: t.score,
                derived_metrics: t.derived_metrics.clone(),
                natural_language: false,
            })
            .collect();
        Self {
            schema_version: self.schema_version.clone(),
            mcptest_version: self.mcptest_version.clone(),
            generated_at: self.generated_at.clone(),
            run_id: self.run_id.clone(),
            summary: self.summary.clone(),
            tests,
            cost: self.cost,
            metadata: self.metadata.clone(),
            // Redact the matrix labels and row names with the same redactor so
            // the grid still joins to the (redacted) test names above.
            matrix: self.matrix.as_ref().map(|m| MatrixReport {
                cells: m
                    .cells
                    .iter()
                    .map(|c| MatrixCell {
                        agent: redactor.redact_value(&c.agent).into_owned(),
                        label: redactor.redact_value(&c.label).into_owned(),
                        rows: c
                            .rows
                            .iter()
                            .map(|r| redactor.redact_value(r).into_owned())
                            .collect(),
                    })
                    .collect(),
            }),
            // Violation examples carry frame snippets, which are server
            // payloads and so user content; the server names and kind
            // labels are not.
            protocol_violations: self
                .protocol_violations
                .iter()
                .map(|v| ProtocolViolationReport {
                    server: v.server.clone(),
                    kind: v.kind.clone(),
                    count: v.count,
                    example: redactor.redact_value(&v.example).into_owned(),
                })
                .collect(),
        }
    }

    /// Return up to `n` tests sorted by descending `duration_ms`.
    ///
    /// Reporters (pretty, HTML, Markdown) use this to surface a
    /// "slowest tests" callout. Returns an empty vector when
    /// the report has no tests; ties keep input order. The clone is
    /// cheap relative to the cost of rendering a reporter.
    pub fn slowest(&self, n: usize) -> Vec<&TestResult> {
        let mut indexed: Vec<&TestResult> = self.tests.iter().collect();
        indexed.sort_by_key(|t| std::cmp::Reverse(t.duration_ms));
        indexed.into_iter().take(n).collect()
    }

    /// Sum of `duration_ms` across tests that did not hit the cache.
    pub fn cold_duration_ms(&self) -> u64 {
        self.tests
            .iter()
            .filter(|t| !t.cache_hit)
            .map(|t| t.duration_ms)
            .sum()
    }

    /// Sum of `duration_ms` across cached tests.
    pub fn cached_duration_ms(&self) -> u64 {
        self.tests
            .iter()
            .filter(|t| t.cache_hit)
            .map(|t| t.duration_ms)
            .sum()
    }

    /// 90th-percentile `duration_ms` across tests in the run, used to
    /// highlight outliers in the HTML reporter. Returns 0 when the
    /// run is empty so callers do not have to guard against `Option`.
    pub fn p90_duration_ms(&self) -> u64 {
        if self.tests.is_empty() {
            return 0;
        }
        let mut sorted: Vec<u64> = self.tests.iter().map(|t| t.duration_ms).collect();
        sorted.sort_unstable();
        // Nearest-rank percentile, clamped so the 100% case picks the last
        // element rather than panicking on an out-of-bounds index.
        let idx = ((sorted.len() as f64) * 0.90).ceil() as usize;
        let idx = idx.saturating_sub(1).min(sorted.len() - 1);
        sorted[idx]
    }

    /// Construct an all-pass variant used by reporter tests that need to
    /// exercise the zero-failure branch. Single passing test.
    pub fn sample_all_pass() -> Self {
        let tests = vec![TestResult {
            name: "smoke test".into(),
            status: TestStatus::Pass,
            duration_ms: 12,
            cache_hit: false,
            started_at: None,
            ended_at: None,
            failure: None,
            file: Some("tests/smoke.yml".into()),
            line: Some(1),
            steps: Vec::new(),
            consensus: None,
            rule_id: None,
            severity: None,
            score: None,
            derived_metrics: Vec::new(),
            natural_language: false,
        }];
        Self {
            schema_version: "0.1".into(),
            mcptest_version: crate::version::full().into(),
            generated_at: "2026-05-15T22:14:03Z".into(),
            run_id: "01JCQQYV4S3H6KX7M8RN2T5XZB".into(),
            summary: RunSummary {
                duration_ms: 12,
                ..RunSummary::from_tests(&tests)
            },
            tests,
            cost: None,
            metadata: None,
            matrix: None,
            protocol_violations: Vec::new(),
        }
    }

    /// Construct a compliance-flavored fixture used by the SARIF and
    /// GitLab reporters. Every test traces back to a registered
    /// compliance rule so reporters can exercise the per-rule `rule_id`
    /// and severity branches without reaching outside this crate. The
    /// body lives in the `samples` sibling module to keep this module
    /// under the size ceiling.
    pub fn sample_full() -> Self {
        samples::sample_full_report()
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn sample_report_has_expected_summary() {
        let r = Report::sample();
        assert_eq!(r.summary.total, 3);
        assert_eq!(r.summary.passed, 2);
        assert_eq!(r.summary.failed, 1);
    }

    #[test]
    fn json_round_trip_is_lossless() {
        let r = Report::sample();
        let json = serde_json::to_string(&r).unwrap();
        let back: Report = serde_json::from_str(&json).unwrap();
        assert_eq!(r, back);
    }

    #[test]
    fn sample_all_pass_has_no_failures() {
        let r = Report::sample_all_pass();
        assert_eq!(r.summary.failed, 0);
        assert_eq!(r.summary.passed, 1);
    }

    #[test]
    fn redacted_replaces_token_shaped_test_name() {
        let mut r = Report::sample();
        r.tests[0].name = "ghp_abcdefghijklmnop1234".into();
        let out = r.redacted(&Redactor::new());
        assert_eq!(out.tests[0].name, "[REDACTED:token]");
    }

    #[test]
    fn redacted_preserves_non_secret_test_names() {
        let r = Report::sample();
        let out = r.redacted(&Redactor::new());
        assert_eq!(out.tests[0].name, "search returns at least one hit");
        assert_eq!(out.tests[1].name, "list_issues respects pagination");
        assert_eq!(out.tests[2].name, "create_issue rejects empty title");
    }

    #[test]
    fn redacted_does_not_change_summary_counts() {
        let mut r = Report::sample();
        r.tests[0].name = "ghp_abcdefghijklmnop1234".into();
        let out = r.redacted(&Redactor::new());
        assert_eq!(out.summary, r.summary);
    }

    #[test]
    fn redacted_redacts_failure_message_and_diff() {
        let mut r = Report::sample();
        r.tests[2].failure = Some(TestFailure {
            message: "ghp_abcdefghijklmnop1234".into(),
            diff: Some("pk-abcdefghijklmnop1234".into()),
            network_diagnostic: None,
        });
        let out = r.redacted(&Redactor::new());
        let failure = out.tests[2].failure.as_ref().expect("failure preserved");
        assert_eq!(failure.message, "[REDACTED:token]");
        assert_eq!(failure.diff.as_deref(), Some("[REDACTED:token]"));
    }

    #[test]
    fn redacted_leaves_file_and_line_untouched() {
        let r = Report::sample();
        let out = r.redacted(&Redactor::new());
        for (a, b) in r.tests.iter().zip(out.tests.iter()) {
            assert_eq!(a.file, b.file);
            assert_eq!(a.line, b.line);
        }
    }

    #[test]
    fn redacted_round_trip_via_pretty_reporter_hides_secret() {
        use crate::report::reporters::PrettyReporter;
        let mut r = Report::sample();
        let secret = "ghp_abcdefghijklmnop1234";
        r.tests[0].name = secret.into();
        let view = r.redacted(&Redactor::new());
        let mut buf = Vec::new();
        PrettyReporter::default().render(&view, &mut buf).unwrap();
        let s = String::from_utf8(buf).unwrap();
        assert!(!s.contains(secret));
        assert!(s.contains("[REDACTED:token]"));
    }

    #[test]
    fn redacted_round_trip_via_json_reporter_hides_secret() {
        use crate::report::reporters::JsonReporter;
        let mut r = Report::sample();
        let secret = "ghp_abcdefghijklmnop1234";
        r.tests[0].name = secret.into();
        let view = r.redacted(&Redactor::new());
        let mut buf = Vec::new();
        JsonReporter::default().render(&view, &mut buf).unwrap();
        let s = String::from_utf8(buf).unwrap();
        assert!(!s.contains(secret));
        assert!(s.contains("[REDACTED:token]"));
    }

    #[test]
    fn redacted_round_trip_via_sarif_reporter_hides_secret() {
        use crate::report::reporters::SarifReporter;
        let mut r = Report::sample();
        let secret = "ghp_abcdefghijklmnop1234";
        // SARIF only emits failed entries; place the secret on the failing
        // test so the smoke covers the path that ships data downstream.
        r.tests[2].name = secret.into();
        let view = r.redacted(&Redactor::new());
        let mut buf = Vec::new();
        SarifReporter::default().render(&view, &mut buf).unwrap();
        let s = String::from_utf8(buf).unwrap();
        assert!(!s.contains(secret));
        assert!(s.contains("[REDACTED:token]"));
    }

    #[test]
    fn metadata_round_trips_through_json_when_attached() {
        let mut r = Report::sample();
        r.metadata = Some(RunMetadata::sample());
        let s = serde_json::to_string(&r).expect("encode");
        let back: Report = serde_json::from_str(&s).expect("decode");
        assert_eq!(back.metadata, r.metadata);
    }

    #[test]
    fn metadata_is_omitted_from_json_when_absent() {
        let r = Report::sample();
        assert!(r.metadata.is_none());
        let value: serde_json::Value = serde_json::to_value(&r).expect("encode");
        assert!(value.get("metadata").is_none());
    }

    #[test]
    fn redacted_preserves_metadata_block() {
        let mut r = Report::sample();
        r.metadata = Some(RunMetadata::sample());
        let out = r.redacted(&Redactor::new());
        assert_eq!(out.metadata, r.metadata);
    }

    fn bare_test(name: &str, status: TestStatus, cache_hit: bool) -> TestResult {
        TestResult {
            name: name.into(),
            status,
            duration_ms: 5,
            cache_hit,
            started_at: None,
            ended_at: None,
            failure: None,
            file: None,
            line: None,
            steps: Vec::new(),
            consensus: None,
            rule_id: None,
            severity: None,
            score: None,
            derived_metrics: Vec::new(),
            natural_language: false,
        }
    }

    #[test]
    fn from_tests_derives_a_consistent_summary_across_every_status() {
        let tests = vec![
            bare_test("passes", TestStatus::Pass, false),
            bare_test("passes from cache", TestStatus::Pass, true),
            bare_test("fails", TestStatus::Fail, false),
            bare_test("skipped by filter", TestStatus::Skip, false),
            bare_test("probe stays undecided", TestStatus::Inconclusive, false),
        ];
        let summary = RunSummary::from_tests(&tests);
        assert_eq!(summary.total, 5);
        assert_eq!(summary.passed, 2);
        assert_eq!(summary.failed, 1);
        assert_eq!(summary.skipped, 1);
        assert_eq!(summary.inconclusive, 1);
        assert_eq!(summary.cache_hits, 1);
        assert_eq!(
            summary.passed + summary.failed + summary.skipped + summary.inconclusive,
            summary.total
        );
        assert!(summary.is_consistent(&tests));
    }

    #[test]
    fn is_consistent_rejects_a_summary_that_drifted_from_its_tests() {
        let r = Report::sample();
        assert!(r.summary.is_consistent(&r.tests));
        let mut drifted = r.summary.clone();
        drifted.passed += 1;
        assert!(!drifted.is_consistent(&r.tests));
        let mut wrong_total = r.summary.clone();
        wrong_total.total = 99;
        assert!(!wrong_total.is_consistent(&r.tests));
    }

    #[test]
    fn every_sample_fixture_summary_agrees_with_its_tests() {
        for r in [
            Report::sample(),
            Report::sample_all_pass(),
            Report::sample_stepwise(),
            Report::sample_full(),
            Report::sample_matrix(),
        ] {
            assert!(
                r.summary.is_consistent(&r.tests),
                "fixture {} has a drifted summary",
                r.run_id
            );
        }
    }

    fn sample_violation_entry() -> ProtocolViolationReport {
        ProtocolViolationReport {
            server: "github".into(),
            kind: "string-id echo".into(),
            count: 3,
            example: "server echoed numeric request id 1 as the string \"1\"".into(),
        }
    }

    #[test]
    fn protocol_violations_round_trip_through_the_json_reporter() {
        use crate::report::reporters::JsonReporter;
        let mut r = Report::sample();
        r.protocol_violations = vec![sample_violation_entry()];
        let mut buf = Vec::new();
        JsonReporter::default().render(&r, &mut buf).unwrap();
        let back: Report = serde_json::from_slice(&buf).unwrap();
        assert_eq!(back.protocol_violations, r.protocol_violations);
        assert_eq!(back, r);
    }

    #[test]
    fn protocol_violations_are_omitted_from_json_when_the_run_is_clean() {
        use crate::report::reporters::JsonReporter;
        let r = Report::sample();
        assert!(r.protocol_violations.is_empty());
        let mut buf = Vec::new();
        JsonReporter::default().render(&r, &mut buf).unwrap();
        let value: serde_json::Value = serde_json::from_slice(&buf).unwrap();
        assert!(
            value.get("protocol_violations").is_none(),
            "a clean run must keep the legacy JSON shape"
        );
    }

    #[test]
    fn redacted_scrubs_violation_examples_but_keeps_server_and_kind() {
        let mut r = Report::sample();
        let mut entry = sample_violation_entry();
        entry.example = "ghp_abcdefghijklmnop1234".into();
        r.protocol_violations = vec![entry];
        let out = r.redacted(&Redactor::new());
        assert_eq!(out.protocol_violations[0].example, "[REDACTED:token]");
        assert_eq!(out.protocol_violations[0].server, "github");
        assert_eq!(out.protocol_violations[0].kind, "string-id echo");
        assert_eq!(out.protocol_violations[0].count, 3);
    }

    #[test]
    fn an_inconclusive_verdict_survives_a_json_round_trip() {
        let mut r = Report::sample();
        r.tests[0].status = TestStatus::Inconclusive;
        r.summary.passed = 1;
        r.summary.inconclusive = 1;
        let json = serde_json::to_string(&r).expect("serializes");
        let back: Report = serde_json::from_str(&json).expect("deserializes");
        assert_eq!(back.tests[0].status, TestStatus::Inconclusive);
        assert_eq!(back.summary.inconclusive, 1);
        assert_eq!(back, r);
    }
}
