//! Judge/jury certification record and the pure certify computation.
//!
//! Why this exists: model-graded assertions (`llm-judge`, `llm-jury`) can pass
//! or fail a release gate, but a judge that is poorly calibrated for a domain
//! produces verdicts no one should trust to gate CI. Certification is the
//! governance step in between: prove a judge configuration is calibrated enough
//! against a labeled anchor set before its verdict is allowed to gate. This
//! module reuses the existing calibration metrics ([`crate::eval::jury::calibration`])
//! and packages the result into a provenance-stamped [`CertificationRecord`]
//! that a later policy gate or evidence pack can read. The computation here is
//! pure and clock-free; the CLI layer stamps the timestamps and writes the file.

use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};

use crate::eval::jury::calibration::{
    brier_score, calibration_error, corrected_rate_ci, estimate_reliability, rogan_gladen,
    LabeledVerdict,
};

/// Schema version stamped on every certification record so a reader can route
/// on the format. Bumped only on a breaking change to the record shape.
pub const CERTIFICATION_SCHEMA_VERSION: &str = "mcptest.dev/judge-certification/v1";

/// Default maximum Expected Calibration Error a certified judge may carry,
/// matching the calibration check's default so the two surfaces agree.
pub const DEFAULT_MAX_ECE: f64 = 0.10;

/// Default maximum Brier score a certified judge may carry.
pub const DEFAULT_MAX_BRIER: f64 = 0.25;

/// Default minimum anchor-set size. One labeled verdict cannot calibrate a
/// judge; the floor keeps a near-empty set from certifying by accident.
pub const DEFAULT_MIN_SAMPLES: usize = 10;

/// Thresholds a judge must clear to be certified.
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
pub struct CertificationThresholds {
    /// Maximum allowed Expected Calibration Error.
    pub max_ece: f64,
    /// Maximum allowed Brier score.
    pub max_brier: f64,
    /// Minimum number of labeled verdicts in the anchor set.
    pub min_samples: usize,
}

impl Default for CertificationThresholds {
    fn default() -> Self {
        Self {
            max_ece: DEFAULT_MAX_ECE,
            max_brier: DEFAULT_MAX_BRIER,
            min_samples: DEFAULT_MIN_SAMPLES,
        }
    }
}

/// Optional trusted-set reliability inputs that unlock the Rogan-Gladen
/// prevalence correction. Supplied when the operator has a confusion matrix
/// from a separate trusted evaluation; omitted otherwise.
#[derive(Debug, Clone, Copy)]
pub struct ReliabilityInput {
    /// The judge's observed positive (correct/pass) rate on the wider set.
    pub observed_positive_rate: f64,
    /// True positives, false negatives, true negatives, false positives from
    /// the trusted set.
    pub tp: u32,
    /// False negatives.
    pub fn_: u32,
    /// True negatives.
    pub tn: u32,
    /// False positives.
    pub fp: u32,
}

/// The calibration metrics computed for a certification.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct CertificationMetrics {
    /// Number of labeled verdicts the metrics were computed over.
    pub samples: usize,
    /// Expected Calibration Error (lower is better).
    pub ece: f64,
    /// Brier score (lower is better).
    pub brier: f64,
    /// Rogan-Gladen prevalence-corrected positive rate, when reliability inputs
    /// were supplied.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub corrected_rate: Option<f64>,
    /// Lower bound of the Wald 95% interval around `corrected_rate`.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub corrected_rate_low: Option<f64>,
    /// Upper bound of the Wald 95% interval around `corrected_rate`.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub corrected_rate_high: Option<f64>,
}

/// The window during which a certification is considered valid.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ValidityWindow {
    /// RFC 3339 timestamp the certification was issued.
    pub valid_from: String,
    /// RFC 3339 timestamp the certification expires.
    pub valid_until: String,
}

/// A persisted certification record: the verdict, the metrics, and the
/// provenance needed to decide later whether it still applies (model, rubric
/// hash, anchor-set hash, validity window).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct CertificationRecord {
    /// Record format version.
    pub schema_version: String,
    /// Judge model identifier, when known.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub judge_model: Option<String>,
    /// Judge provider identifier, when known.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub judge_provider: Option<String>,
    /// `sha256:` digest of the rubric content, so a changed rubric invalidates
    /// the certification.
    pub rubric_hash: String,
    /// `sha256:` digest of the labels (anchor-set) content.
    pub anchor_set_hash: String,
    /// The computed calibration metrics.
    pub metrics: CertificationMetrics,
    /// The thresholds the metrics were judged against.
    pub thresholds: CertificationThresholds,
    /// True when every threshold was met.
    pub certified: bool,
    /// One line per failed threshold, empty when certified.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub failure_reasons: Vec<String>,
    /// Validity window, when the issuer stamped one.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub validity_window: Option<ValidityWindow>,
    /// mcptest version that issued the record.
    pub mcptest_version: String,
}

/// The outcome of evaluating an anchor set against the thresholds: the metrics,
/// the verdict, and the reasons any threshold failed.
#[derive(Debug, Clone, PartialEq)]
pub struct CertificationOutcome {
    /// The computed metrics.
    pub metrics: CertificationMetrics,
    /// True when every threshold was met.
    pub certified: bool,
    /// One line per failed threshold.
    pub failure_reasons: Vec<String>,
}

/// Evaluate a labeled anchor set against the certification thresholds.
///
/// Pure and clock-free: computes ECE and Brier over `samples` (and the
/// Rogan-Gladen corrected rate when `reliability` is given), then records a
/// failure reason for each threshold the result misses. `certified` is true
/// only when no threshold failed.
///
/// ```
/// use mcptest_core::checks::certification::{evaluate, CertificationThresholds};
/// use mcptest_core::eval::jury::calibration::LabeledVerdict;
///
/// let samples: Vec<LabeledVerdict> = (0..10)
///     .map(|i| LabeledVerdict { confidence: if i < 5 { 0.0 } else { 1.0 }, correct: i >= 5 })
///     .collect();
/// let outcome = evaluate(&samples, &CertificationThresholds::default(), None);
/// assert!(outcome.certified);
/// ```
pub fn evaluate(
    samples: &[LabeledVerdict],
    thresholds: &CertificationThresholds,
    reliability: Option<&ReliabilityInput>,
) -> CertificationOutcome {
    let ece = calibration_error(samples);
    let brier = brier_score(samples);
    let (corrected_rate, corrected_rate_low, corrected_rate_high) = match reliability {
        Some(r) => {
            let rel = estimate_reliability(r.tp, r.fn_, r.tn, r.fp);
            let corrected = rogan_gladen(r.observed_positive_rate, &rel);
            let trusted = r.tp + r.fn_ + r.tn + r.fp;
            let (low, high) = corrected_rate_ci(r.observed_positive_rate, &rel, trusted);
            (Some(corrected), Some(low), Some(high))
        }
        None => (None, None, None),
    };

    let mut failure_reasons = Vec::new();
    if samples.len() < thresholds.min_samples {
        failure_reasons.push(format!(
            "anchor set has {} verdict(s), below the required minimum of {}",
            samples.len(),
            thresholds.min_samples
        ));
    }
    if ece > thresholds.max_ece {
        failure_reasons.push(format!(
            "ECE {ece:.4} exceeds the maximum {:.4}",
            thresholds.max_ece
        ));
    }
    if brier > thresholds.max_brier {
        failure_reasons.push(format!(
            "Brier score {brier:.4} exceeds the maximum {:.4}",
            thresholds.max_brier
        ));
    }

    CertificationOutcome {
        metrics: CertificationMetrics {
            samples: samples.len(),
            ece,
            brier,
            corrected_rate,
            corrected_rate_low,
            corrected_rate_high,
        },
        certified: failure_reasons.is_empty(),
        failure_reasons,
    }
}

/// `sha256:`-prefixed hex digest of arbitrary bytes, the repo's hash format for
/// provenance (matches the evidence and fingerprint surfaces).
pub fn hash_bytes(bytes: &[u8]) -> String {
    let mut hasher = Sha256::new();
    hasher.update(bytes);
    format!("sha256:{:x}", hasher.finalize())
}

/// Assemble a [`CertificationRecord`] from an evaluated outcome plus its
/// provenance. The timestamps in `validity_window` are supplied by the caller
/// (the CLI layer owns the clock) so this stays pure and testable.
#[allow(clippy::too_many_arguments)]
pub fn build_record(
    judge_model: Option<String>,
    judge_provider: Option<String>,
    rubric_hash: String,
    anchor_set_hash: String,
    thresholds: CertificationThresholds,
    outcome: CertificationOutcome,
    validity_window: Option<ValidityWindow>,
    mcptest_version: String,
) -> CertificationRecord {
    CertificationRecord {
        schema_version: CERTIFICATION_SCHEMA_VERSION.to_string(),
        judge_model,
        judge_provider,
        rubric_hash,
        anchor_set_hash,
        metrics: outcome.metrics,
        thresholds,
        certified: outcome.certified,
        failure_reasons: outcome.failure_reasons,
        validity_window,
        mcptest_version,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn samples(spec: &[(f64, bool)]) -> Vec<LabeledVerdict> {
        spec.iter()
            .map(|&(confidence, correct)| LabeledVerdict {
                confidence,
                correct,
            })
            .collect()
    }

    fn calibrated_set() -> Vec<LabeledVerdict> {
        // Confident-and-correct or unconfident-and-wrong: well calibrated.
        let mut v = Vec::new();
        for _ in 0..10 {
            v.push(LabeledVerdict {
                confidence: 1.0,
                correct: true,
            });
            v.push(LabeledVerdict {
                confidence: 0.0,
                correct: false,
            });
        }
        v
    }

    #[test]
    fn a_calibrated_set_certifies() {
        let outcome = evaluate(&calibrated_set(), &CertificationThresholds::default(), None);
        assert!(outcome.certified, "reasons: {:?}", outcome.failure_reasons);
        assert!(outcome.failure_reasons.is_empty());
        assert_eq!(outcome.metrics.samples, 20);
    }

    #[test]
    fn an_overconfident_set_fails_calibration() {
        // Always maximally confident but wrong half the time: ECE and Brier high.
        let mut v = Vec::new();
        for i in 0..20 {
            v.push(LabeledVerdict {
                confidence: 1.0,
                correct: i % 2 == 0,
            });
        }
        let outcome = evaluate(&v, &CertificationThresholds::default(), None);
        assert!(!outcome.certified);
        assert!(outcome
            .failure_reasons
            .iter()
            .any(|r| r.contains("ECE") || r.contains("Brier")));
    }

    #[test]
    fn a_set_below_the_sample_floor_fails() {
        let outcome = evaluate(
            &samples(&[(1.0, true), (0.0, false)]),
            &CertificationThresholds::default(),
            None,
        );
        assert!(!outcome.certified);
        assert!(outcome
            .failure_reasons
            .iter()
            .any(|r| r.contains("below the required minimum")));
    }

    #[test]
    fn reliability_inputs_produce_a_corrected_rate() {
        let reliability = ReliabilityInput {
            observed_positive_rate: 0.5,
            tp: 90,
            fn_: 10,
            tn: 80,
            fp: 20,
        };
        let outcome = evaluate(
            &calibrated_set(),
            &CertificationThresholds::default(),
            Some(&reliability),
        );
        assert!(outcome.metrics.corrected_rate.is_some());
        assert!(outcome.metrics.corrected_rate_low.is_some());
        assert!(outcome.metrics.corrected_rate_high.is_some());
    }

    #[test]
    fn no_reliability_omits_the_corrected_rate() {
        let outcome = evaluate(&calibrated_set(), &CertificationThresholds::default(), None);
        assert!(outcome.metrics.corrected_rate.is_none());
    }

    #[test]
    fn hash_bytes_is_stable_and_prefixed() {
        let a = hash_bytes(b"rubric content");
        let b = hash_bytes(b"rubric content");
        assert_eq!(a, b);
        assert!(a.starts_with("sha256:"));
        assert_ne!(a, hash_bytes(b"different"));
    }

    #[test]
    fn build_record_round_trips_through_json() {
        let outcome = evaluate(&calibrated_set(), &CertificationThresholds::default(), None);
        let record = build_record(
            Some("claude-sonnet-4-5".to_string()),
            Some("anthropic".to_string()),
            hash_bytes(b"rubric"),
            hash_bytes(b"labels"),
            CertificationThresholds::default(),
            outcome,
            Some(ValidityWindow {
                valid_from: "2026-06-17T00:00:00Z".to_string(),
                valid_until: "2026-09-15T00:00:00Z".to_string(),
            }),
            "1.0.1".to_string(),
        );
        let json = serde_json::to_string(&record).unwrap();
        let back: CertificationRecord = serde_json::from_str(&json).unwrap();
        assert_eq!(record, back);
        assert!(back.certified);
        assert_eq!(back.schema_version, CERTIFICATION_SCHEMA_VERSION);
    }
}
