//! Reliability reporting beyond pass^k: power-analysis run-count and a
//! reliability-decay summary.
//!
//! Why this module exists: the pass^k floor in [`crate::eval::selection`]
//! tells you whether the runs you already made cleared a bar, but it does not
//! tell you how MANY runs you needed to trust the number, nor how the runs
//! decayed across the sequence. This module adds two deterministic, model-free
//! helpers on top of the existing pass^k surface. Both are pure arithmetic, so
//! a fixed input always yields a byte-stable result, and neither calls a model.
//!
//! The headline reliability metrics stay pass@k (optimistic: at least one of k
//! runs passed) and pass^k (pessimistic: all k runs passed). The curves below
//! are secondary detail that explains the headline.
//!
//! # Power-analysis run-count recommendation
//!
//! Question: how many runs N do I need so the estimated pass rate has a
//! confidence interval (the range that, at a stated confidence level, brackets
//! the true rate) no wider than a target half-width? Half-width is half the
//! full width of that interval, so a half-width of 5 percent means plus or
//! minus 5 percent.
//!
//! We use the standard normal-approximation (Wald) interval for a binomial
//! proportion. For an observed pass rate p over N runs the interval half-width
//! is:
//!
//! ```text
//! half_width = z * sqrt(p * (1 - p) / N)
//! ```
//!
//! where z is the standard-normal critical value for the confidence level.
//! The product p * (1 - p) is largest at p = 0.5 (worst case, value 0.25), so
//! to guarantee the target half-width for any observed rate we solve at
//! p = 0.5 and round up:
//!
//! ```text
//! N = ceil( (z / half_width)^2 * 0.25 )
//! ```
//!
//! Given instead a chosen N, the worst-case half-width the run budget buys is:
//!
//! ```text
//! half_width = z * sqrt(0.25 / N)
//! ```
//!
//! The z values for the common confidence levels are pinned constants:
//! 1.96 for 95 percent, 1.645 for 90 percent, 2.576 for 99 percent. Inputs are
//! integer percents (a 5 percent half-width is `5`, a 95 percent confidence
//! level is `95`) so the surface stays byte-stable and matches the integer
//! percent convention the rest of the eval modules use.
//!
//! # Reliability-decay summary
//!
//! Given the N per-run pass/fail outcomes of one multi-run test, three
//! secondary statistics, each spelled out in full:
//!
//! - Reliability Decay Curve: the cumulative pass^k as k goes 1..=N, reported
//!   as a small vector of integer percents. Entry k is the empirical pass^k
//!   over the first k runs, `(c_k / k)^k * 100`, where c_k is the count of
//!   passes among the first k runs. This is the maximum-likelihood estimate
//!   that all k independent runs pass given the observed rate. It decays as
//!   failures accumulate and is fully deterministic.
//! - Variance Amplification Factor: the observed spread of the per-run pass
//!   indicator, scaled to 0..=100. A perfectly stable run (all pass or all
//!   fail) has zero spread; a maximally flaky run (half pass, half fail) has
//!   the most. We define it as the population standard deviation of the per-run
//!   pass indicators (each 0 or 1) scaled by its theoretical maximum 0.5 and
//!   expressed as an integer percent: `round(std_dev / 0.5 * 100)`. It is 0
//!   when every run agrees and 100 at a perfect 50/50 split.
//! - Graceful Degradation Score: a 0..=100 score that is 100 when every run
//!   passed and decays as failures cluster LATE rather than early. We weight
//!   each run's pass by its position so a late failure costs more than an early
//!   one (an agent that degrades late is less graceful than one that fails fast
//!   and recovers). Concretely, with positions 1..=N and weight = position, the
//!   score is `round(100 * sum(weight_i * pass_i) / sum(weight_i))`. It is 100
//!   when all pass, 0 when all fail, and a late-clustered failure pattern
//!   scores lower than an early-clustered one with the same pass count.
//!
//! All three are exposed through [`ReliabilitySummary::resolve_target`] so the
//! same dot-path resolver the other eval reports use can read
//! `reliability.recommended_runs`, `reliability.graceful_degradation`, and
//! friends without a new matcher.

use serde::{Deserialize, Serialize};
use serde_json::{json, Value};

/// Standard-normal critical value (z) for 95 percent confidence.
const Z_95: f64 = 1.96;
/// Standard-normal critical value (z) for 90 percent confidence.
const Z_90: f64 = 1.645;
/// Standard-normal critical value (z) for 99 percent confidence.
const Z_99: f64 = 2.576;
/// Worst-case binomial variance numerator, p * (1 - p) maximized at p = 0.5.
const WORST_CASE_VARIANCE: f64 = 0.25;

/// Map a confidence level (an integer percent, e.g. `95`) to its z value.
///
/// Returns `None` for an unsupported level so the caller surfaces a typed error
/// rather than guessing a critical value. Only the three common levels are
/// pinned, matching the documented constants.
fn z_for_level(confidence_percent: u32) -> Option<f64> {
    match confidence_percent {
        95 => Some(Z_95),
        90 => Some(Z_90),
        99 => Some(Z_99),
        _ => None,
    }
}

/// Recommend a run count N for a target confidence-interval half-width.
///
/// `half_width_percent` is the target half-width as an integer percent (a value
/// of `5` means plus or minus 5 percent). `confidence_percent` is one of 90,
/// 95, or 99. Solves the worst-case (p = 0.5) Wald interval for N and rounds
/// up, so the returned N guarantees the half-width for any observed pass rate.
/// Returns `None` for an unsupported confidence level or a non-positive
/// half-width, since neither has a defined run count.
pub fn recommend_runs(half_width_percent: u32, confidence_percent: u32) -> Option<u32> {
    if half_width_percent == 0 {
        return None;
    }
    let z = z_for_level(confidence_percent)?;
    let half_width = f64::from(half_width_percent) / 100.0;
    let n = (z / half_width).powi(2) * WORST_CASE_VARIANCE;
    Some(n.ceil() as u32)
}

/// Report the worst-case half-width (as an integer percent) a chosen run count
/// buys at a confidence level.
///
/// The inverse of [`recommend_runs`]: given `runs`, returns the half-width of
/// the worst-case (p = 0.5) Wald interval, rounded to the nearest integer
/// percent. Returns `None` for an unsupported confidence level or zero runs
/// (an empty sample has no defined interval).
pub fn confidence_band(runs: u32, confidence_percent: u32) -> Option<u32> {
    if runs == 0 {
        return None;
    }
    let z = z_for_level(confidence_percent)?;
    let half_width = z * (WORST_CASE_VARIANCE / f64::from(runs)).sqrt();
    Some((half_width * 100.0).round() as u32)
}

/// Deterministic reliability-decay summary over the N per-run outcomes of one
/// multi-run test.
///
/// Serializable so reporters and the enterprise audit chain can persist the
/// secondary decay detail next to the headline pass@k / pass^k numbers. Every
/// percent field is an integer in `0..=100` to match the eval-module convention
/// and stay byte-stable across platforms.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ReliabilitySummary {
    /// Number of runs summarized (the test's `runs:` value).
    pub runs: usize,
    /// pass@k headline: at least one of the runs passed (optimistic).
    pub pass_at_k: bool,
    /// pass^k headline: every run passed (pessimistic). Integer percent in
    /// `0..=100`: 100 only when all runs passed.
    pub passhat_k_percent: u32,
    /// Reliability Decay Curve: empirical pass^k for k = 1..=N, each an integer
    /// percent. Entry index `k-1` is `(passes_in_first_k / k)^k * 100`.
    pub decay_curve: Vec<u32>,
    /// Variance Amplification Factor: per-run pass-indicator spread scaled to
    /// `0..=100` (0 when all runs agree, 100 at a perfect 50/50 split).
    pub variance_amplification: u32,
    /// Graceful Degradation Score in `0..=100`: 100 when all pass, lower when
    /// failures cluster late rather than early.
    pub graceful_degradation: u32,
}

impl ReliabilitySummary {
    /// One-line human-readable summary the CLI prints so the operator sees the
    /// headline and the secondary numbers together. Mirrors
    /// [`crate::eval::stability_aggregate::StabilitySummary::line`].
    pub fn line(&self) -> String {
        format!(
            "pass^k {}%, graceful {}%, variance-amp {}% over {} runs",
            self.passhat_k_percent,
            self.graceful_degradation,
            self.variance_amplification,
            self.runs
        )
    }

    /// Resolve a `reliability.<field>` dot-path target to a number, mirroring
    /// the `resolve_target` pattern the other eval reports expose.
    ///
    /// Returns `None` for an unknown target so a caller can distinguish "no such
    /// field" from a real value. Booleans are surfaced as `0`/`100` so a numeric
    /// matcher can gate them like the percent fields.
    pub fn resolve_target(&self, target: &str) -> Option<f64> {
        match target {
            "reliability.runs" => Some(self.runs as f64),
            "reliability.pass_at_k" => Some(if self.pass_at_k { 100.0 } else { 0.0 }),
            "reliability.passhat_k" => Some(f64::from(self.passhat_k_percent)),
            "reliability.variance_amplification" => Some(f64::from(self.variance_amplification)),
            "reliability.graceful_degradation" => Some(f64::from(self.graceful_degradation)),
            _ => None,
        }
    }

    /// Build the targets object the assertions resolve against.
    ///
    /// Shape: `{ "reliability": { "passhat_k": <num>, "graceful_degradation":
    /// <num>, ... } }`, so the executor's dot-path resolver walks
    /// `reliability.graceful_degradation` directly, the same way the
    /// `stability.*` and `tool_selection.*` targets resolve. Reusing the
    /// resolver means no new matcher and no new reporter.
    pub fn metrics_object(&self) -> Value {
        json!({
            "reliability": {
                "runs": self.runs,
                "pass_at_k": self.pass_at_k,
                "passhat_k": self.passhat_k_percent,
                "variance_amplification": self.variance_amplification,
                "graceful_degradation": self.graceful_degradation,
            }
        })
    }
}

/// Fold the N per-run pass/fail outcomes of a multi-run test into a
/// [`ReliabilitySummary`].
///
/// `outcomes[i]` is true when run `i` passed. An empty slice is trivially
/// reliable: pass^k and graceful degradation are 100, the curve is empty, and
/// variance amplification is 0, so a gate never fires on "no runs", matching
/// the empty-sample behavior of the other aggregates.
pub fn summarize(outcomes: &[bool]) -> ReliabilitySummary {
    let runs = outcomes.len();
    if runs == 0 {
        return ReliabilitySummary {
            runs: 0,
            pass_at_k: true,
            passhat_k_percent: 100,
            decay_curve: Vec::new(),
            variance_amplification: 0,
            graceful_degradation: 100,
        };
    }
    let passes = outcomes.iter().filter(|p| **p).count();
    ReliabilitySummary {
        runs,
        pass_at_k: passes > 0,
        passhat_k_percent: if passes == runs { 100 } else { 0 },
        decay_curve: decay_curve(outcomes),
        variance_amplification: variance_amplification(outcomes, passes),
        graceful_degradation: graceful_degradation(outcomes),
    }
}

/// Empirical pass^k for k = 1..=N as integer percents.
///
/// Entry index `k-1` is `(c_k / k)^k * 100` rounded down, where `c_k` is the
/// count of passes among the first k runs. This is the maximum-likelihood
/// estimate that all k independent runs pass given the observed rate, so it
/// decays as failures accumulate and is fully deterministic.
fn decay_curve(outcomes: &[bool]) -> Vec<u32> {
    let mut curve = Vec::with_capacity(outcomes.len());
    let mut passes = 0u32;
    for (idx, passed) in outcomes.iter().enumerate() {
        if *passed {
            passes += 1;
        }
        let k = (idx + 1) as u32;
        let rate = f64::from(passes) / f64::from(k);
        let estimate = rate.powi(k as i32) * 100.0;
        curve.push(estimate as u32);
    }
    curve
}

/// Variance Amplification Factor: per-run pass-indicator spread scaled to
/// `0..=100`.
///
/// Population standard deviation of the 0/1 pass indicators divided by its
/// theoretical maximum (0.5, reached at a 50/50 split), expressed as an integer
/// percent. 0 when every run agrees, 100 at a perfect split.
fn variance_amplification(outcomes: &[bool], passes: usize) -> u32 {
    let n = outcomes.len() as f64;
    let mean = passes as f64 / n;
    let variance = outcomes
        .iter()
        .map(|p| {
            let x = if *p { 1.0 } else { 0.0 };
            let d = x - mean;
            d * d
        })
        .sum::<f64>()
        / n;
    let scaled = variance.sqrt() / 0.5 * 100.0;
    scaled.round() as u32
}

/// Graceful Degradation Score in `0..=100`.
///
/// Position-weighted pass fraction with weight equal to position (1..=N), so a
/// late failure costs more than an early one: `100 * sum(pos_i * pass_i) /
/// sum(pos_i)`. 100 when all pass, 0 when all fail, and a late-clustered
/// failure scores below an early-clustered one with the same pass count.
fn graceful_degradation(outcomes: &[bool]) -> u32 {
    let mut weighted = 0u64;
    let mut total = 0u64;
    for (idx, passed) in outcomes.iter().enumerate() {
        let weight = (idx + 1) as u64;
        total += weight;
        if *passed {
            weighted += weight;
        }
    }
    if total == 0 {
        return 100;
    }
    ((weighted * 100) / total) as u32
}

#[cfg(test)]
mod tests {
    use super::*;

    fn outcomes(passes: &[bool]) -> Vec<bool> {
        passes.to_vec()
    }

    #[test]
    fn recommends_385_runs_for_five_percent_half_width_at_95_confidence() {
        // N = ceil((1.96 / 0.05)^2 * 0.25) = ceil(384.16) = 385.
        assert_eq!(recommend_runs(5, 95), Some(385));
    }

    #[test]
    fn tighter_half_width_needs_more_runs() {
        let loose = recommend_runs(10, 95).expect("supported");
        let tight = recommend_runs(2, 95).expect("supported");
        assert!(tight > loose, "loose {loose}, tight {tight}");
    }

    #[test]
    fn higher_confidence_needs_more_runs_for_same_half_width() {
        let at_90 = recommend_runs(5, 90).expect("supported");
        let at_99 = recommend_runs(5, 99).expect("supported");
        assert!(at_99 > at_90, "90 {at_90}, 99 {at_99}");
    }

    #[test]
    fn unsupported_confidence_level_has_no_recommendation() {
        assert_eq!(recommend_runs(5, 80), None);
    }

    #[test]
    fn zero_half_width_has_no_recommendation() {
        assert_eq!(recommend_runs(0, 95), None);
    }

    #[test]
    fn confidence_band_for_chosen_runs_matches_the_formula() {
        // half_width = 1.96 * sqrt(0.25 / 100) = 1.96 * 0.05 = 0.098 -> 10%.
        assert_eq!(confidence_band(100, 95), Some(10));
    }

    #[test]
    fn confidence_band_narrows_as_runs_grow() {
        let few = confidence_band(50, 95).expect("supported");
        let many = confidence_band(400, 95).expect("supported");
        assert!(many < few, "few {few}, many {many}");
    }

    #[test]
    fn confidence_band_undefined_for_zero_runs() {
        assert_eq!(confidence_band(0, 95), None);
    }

    #[test]
    fn all_pass_decay_curve_stays_at_full() {
        let summary = summarize(&outcomes(&[true, true, true, true]));
        assert_eq!(summary.passhat_k_percent, 100);
        assert!(summary.pass_at_k);
        // Every prefix is all-pass, so each empirical pass^k is 100.
        assert_eq!(summary.decay_curve, vec![100, 100, 100, 100]);
        assert_eq!(summary.graceful_degradation, 100);
        assert_eq!(summary.variance_amplification, 0);
    }

    #[test]
    fn late_clustered_failures_decay_the_curve() {
        // Three pass then one fail: prefix rates 1, 1, 1, 0.75.
        // Entry 4 = 0.75^4 * 100 = 31.6 -> 31.
        let summary = summarize(&outcomes(&[true, true, true, false]));
        assert_eq!(summary.decay_curve, vec![100, 100, 100, 31]);
        assert_eq!(summary.passhat_k_percent, 0);
        assert!(summary.pass_at_k);
    }

    #[test]
    fn late_failure_is_less_graceful_than_early_failure() {
        // Same pass count (3 of 4), failure early vs late.
        let early_fail = summarize(&outcomes(&[false, true, true, true]));
        let late_fail = summarize(&outcomes(&[true, true, true, false]));
        assert!(
            late_fail.graceful_degradation < early_fail.graceful_degradation,
            "late {} early {}",
            late_fail.graceful_degradation,
            early_fail.graceful_degradation
        );
    }

    #[test]
    fn stable_series_has_zero_variance_amplification() {
        let all_pass = summarize(&outcomes(&[true, true, true, true]));
        let all_fail = summarize(&outcomes(&[false, false, false, false]));
        assert_eq!(all_pass.variance_amplification, 0);
        assert_eq!(all_fail.variance_amplification, 0);
    }

    #[test]
    fn even_split_has_maximum_variance_amplification() {
        // 50/50 split: std_dev = 0.5, scaled = 100.
        let summary = summarize(&outcomes(&[true, false, true, false]));
        assert_eq!(summary.variance_amplification, 100);
    }

    #[test]
    fn all_fail_scores_zero_graceful_degradation() {
        let summary = summarize(&outcomes(&[false, false, false]));
        assert_eq!(summary.graceful_degradation, 0);
        assert!(!summary.pass_at_k);
        assert_eq!(summary.passhat_k_percent, 0);
    }

    #[test]
    fn empty_sample_is_trivially_reliable() {
        let summary = summarize(&[]);
        assert_eq!(summary.runs, 0);
        assert!(summary.pass_at_k);
        assert_eq!(summary.passhat_k_percent, 100);
        assert!(summary.decay_curve.is_empty());
        assert_eq!(summary.graceful_degradation, 100);
        assert_eq!(summary.variance_amplification, 0);
    }

    #[test]
    fn resolve_target_reads_each_field() {
        let summary = summarize(&outcomes(&[true, true, false]));
        assert_eq!(summary.resolve_target("reliability.runs"), Some(3.0));
        assert_eq!(summary.resolve_target("reliability.pass_at_k"), Some(100.0));
        assert!(summary
            .resolve_target("reliability.graceful_degradation")
            .is_some());
        assert!(summary
            .resolve_target("reliability.variance_amplification")
            .is_some());
        assert_eq!(summary.resolve_target("reliability.unknown"), None);
    }

    #[test]
    fn metrics_object_exposes_targets_through_the_executor_resolver() {
        use crate::executor::resolve_target;
        let summary = summarize(&outcomes(&[true, false, true]));
        let obj = summary.metrics_object();
        assert!(resolve_target(&obj, "reliability.graceful_degradation").is_number());
        assert!(resolve_target(&obj, "reliability.passhat_k").is_number());
    }

    #[test]
    fn line_pairs_the_headline_and_secondary_numbers() {
        let summary = ReliabilitySummary {
            runs: 5,
            pass_at_k: true,
            passhat_k_percent: 0,
            decay_curve: vec![100, 100, 66, 50, 40],
            variance_amplification: 80,
            graceful_degradation: 60,
        };
        assert_eq!(
            summary.line(),
            "pass^k 0%, graceful 60%, variance-amp 80% over 5 runs"
        );
    }
}
