//! Rubric scoring: a weighted, multi-criterion judged metric.
//!
//! Our named alternative to a single free-form judge. A rubric is a list of
//! weighted criteria; each is judged 0..1 against the candidate (an agent
//! trace, a tool result, any value) through the same provider plumbing the
//! `llm-judge` matcher uses, and the metric is the weight-normalized average.
//! Per-criterion scores and reasons are returned so the report can show *why*
//! a rubric landed where it did, and the aggregate feeds the `calibration:`
//! block so a rubric's reliability can be measured.
//!
//! This module is the transport-free scoring core. Parsing the YAML `eval:`
//! block and folding the outcome into the run report live at the runner/CLI
//! layer.

use serde_json::Value;

use crate::eval::provider::LlmProvider;
use crate::eval::ProviderError;
use crate::matchers::llm_judge;

/// One labeled calibration anchor for a criterion: a sample response and the
/// score a human would give it. Passed to the judge as a few-shot example to
/// steer and stabilize its grading (ties into judge calibration).
#[derive(Debug, Clone, PartialEq, Default)]
pub struct CriterionExample {
    /// A sample candidate response.
    pub response: String,
    /// The expected normalized score in `[0, 1]` for that response.
    pub score: f64,
}

/// A deterministic predicate over the candidate that decides whether a
/// criterion applies. Evaluated with no model call, so it is cheap and stable.
#[derive(Debug, Clone, PartialEq)]
pub enum CriterionWhen {
    /// Applies only when the candidate text contains this substring.
    Contains(String),
    /// Applies only when the candidate text matches this regular expression.
    Regex(String),
}

impl CriterionWhen {
    /// Whether the criterion applies to `candidate` (the stringified candidate).
    /// An invalid regex matches nothing, so the criterion is skipped rather than
    /// erroring at grade time; the parser validates the pattern up front.
    pub fn applies(&self, candidate: &str) -> bool {
        match self {
            CriterionWhen::Contains(needle) => candidate.contains(needle.as_str()),
            CriterionWhen::Regex(pattern) => regex::Regex::new(pattern)
                .map(|re| re.is_match(candidate))
                .unwrap_or(false),
        }
    }
}

/// One weighted rubric criterion.
#[derive(Debug, Clone, PartialEq, Default)]
pub struct Criterion {
    /// Short label shown in the report.
    pub name: String,
    /// Relative weight. Weights are normalized across criteria, so only their
    /// ratios matter; a non-positive weight is treated as 1.0.
    pub weight: f64,
    /// What the judge scores this criterion against (becomes the judge rubric).
    pub description: String,
    /// When true, this criterion gates: if its score is below the rubric
    /// threshold the whole eval fails, regardless of the weighted average.
    pub required: bool,
    /// When true, this is a negative (guard) criterion: the description states
    /// something that must NOT hold. Its contribution to the average is
    /// inverted (a clean response scores 1.0), and a triggered guard (the bad
    /// thing is present, judged score >= 0.5) fails the eval like a required
    /// criterion.
    pub guard: bool,
    /// Labeled calibration anchors appended to the judge prompt as few-shot
    /// examples. Empty for an un-anchored criterion.
    pub examples: Vec<CriterionExample>,
    /// Optional predicate: when set, the criterion is judged only if it holds
    /// for the candidate. A skipped criterion does not enter the aggregate.
    pub when: Option<CriterionWhen>,
    /// Optional per-criterion gate threshold that overrides the rubric default
    /// for this criterion's `required`/`guard` gating. `None` uses the rubric
    /// threshold.
    pub threshold: Option<f64>,
}

/// A node in a decision-tree rubric. Either an `ask` node (a single
/// yes/no judgment that branches) or a `score` leaf (a deterministic verdict).
/// Each `ask` is one narrow judgment, which is more reliable and auditable than
/// one holistic score, and only the questions hit the model.
#[derive(Debug, Clone, PartialEq)]
pub enum RubricNode {
    /// A yes/no judgment. `yes`/`no` point at the next node down each branch.
    Ask {
        /// The question the judge answers about the candidate.
        question: String,
        /// Branch taken when the judge answers yes (score >= 0.5).
        yes: Box<RubricNode>,
        /// Branch taken when the judge answers no.
        no: Box<RubricNode>,
    },
    /// A leaf: a fixed score (and optional reason) with no model call.
    Leaf {
        /// Score in `[0, 1]` for this path.
        score: f64,
        /// Optional explanation surfaced in the report.
        reason: Option<String>,
    },
}

/// The native scale a rubric score is shown in. The judge always returns a
/// normalized 0..1 score, which stays the source of truth for gating, the
/// score-delta gate, and every machine reporter; the scale only changes the
/// value a human reads (and, for Likert, the units an author thinks in).
#[derive(Debug, Clone, PartialEq, Default)]
pub enum ScoreScale {
    /// Continuous 0..1. The native value equals the normalized value.
    #[default]
    Unit,
    /// Integer Likert band from `min` to `max` (for example 1..=5). A
    /// normalized 0.75 on a 1-5 scale displays as `4.0/5`.
    Likert {
        /// Lowest point on the scale; maps to a normalized 0.0.
        min: i64,
        /// Highest point on the scale; maps to a normalized 1.0.
        max: i64,
    },
    /// Boolean pass/fail: a normalized score `>= 0.5` displays as `pass`.
    Boolean,
    /// Letter grade A..F over evenly spaced normalized bands.
    Letter,
}

impl ScoreScale {
    /// Render a normalized 0..1 score in this scale's native units for human
    /// output. Reporters and the score-delta gate keep the normalized value;
    /// only the string a person reads changes.
    pub fn display(&self, normalized: f64) -> String {
        let n = normalized.clamp(0.0, 1.0);
        match self {
            ScoreScale::Unit => format!("{n:.2}"),
            ScoreScale::Likert { min, max } => {
                let native = *min as f64 + n * (max - min) as f64;
                format!("{native:.1}/{max}")
            }
            ScoreScale::Boolean => if n >= 0.5 { "pass" } else { "fail" }.to_string(),
            ScoreScale::Letter => match n {
                x if x >= 0.9 => "A",
                x if x >= 0.8 => "B",
                x if x >= 0.7 => "C",
                x if x >= 0.6 => "D",
                _ => "F",
            }
            .to_string(),
        }
    }
}

/// How per-criterion scores combine into the rubric aggregate.
#[derive(Debug, Clone, PartialEq, Default)]
pub enum AggregationMode {
    /// Weight-normalized average of per-criterion contributions (default).
    #[default]
    WeightedAverage,
    /// The worst (minimum) per-criterion contribution. One weak criterion caps
    /// the aggregate; weights do not enter the aggregate in this mode.
    Min,
    /// Walk the decision `tree`. Set automatically when a `tree:` is present;
    /// authors do not select it directly.
    Tree,
}

/// A parsed rubric metric. Either flat `criteria` or a decision `tree`; exactly
/// one is present after parsing.
#[derive(Debug, Clone, PartialEq)]
pub struct RubricConfig {
    /// The weighted criteria to score (empty when `tree` is set).
    pub criteria: Vec<Criterion>,
    /// A decision-tree rubric; `None` for the flat-criteria form.
    pub tree: Option<RubricNode>,
    /// Pass threshold on the 0..1 score. Default 0.7.
    pub threshold: f64,
    /// When true, the rubric passes only on a perfect 1.0 (binary mode).
    pub strict: bool,
    /// Native scale for human display of the score. Default [`ScoreScale::Unit`].
    pub scale: ScoreScale,
    /// How per-criterion scores combine into the aggregate. Default
    /// [`AggregationMode::WeightedAverage`].
    pub aggregation: AggregationMode,
    /// When true, each criterion's judge must cite a verbatim span from the
    /// candidate; a criterion with no evidence scores 0 and gates the eval.
    pub require_evidence: bool,
    /// When true (the default), a criterion anchored only at the extremes gets a
    /// synthetic mid-tier exemplar injected so the judge calibrates the middle
    /// of the scale, not just the ends (WOR-1370). Set false to pass the judge
    /// exactly the author's anchors.
    pub default_mid_anchor: bool,
}

impl Default for RubricConfig {
    fn default() -> Self {
        Self {
            criteria: Vec::new(),
            tree: None,
            // Mirrors rubric_parse::DEFAULT_RUBRIC_THRESHOLD; kept inline so the
            // engine module owns no parser dependency.
            threshold: 0.7,
            strict: false,
            scale: ScoreScale::Unit,
            aggregation: AggregationMode::WeightedAverage,
            require_evidence: false,
            default_mid_anchor: true,
        }
    }
}

/// One criterion's judged result.
#[derive(Debug, Clone)]
pub struct CriterionScore {
    /// The criterion's label.
    pub name: String,
    /// Judged score in `[0, 1]`.
    pub score: f64,
    /// The judge's one-sentence rationale, when it returned one.
    pub reason: Option<String>,
    /// A verbatim quote or span from the candidate that justifies the verdict,
    /// when the rubric set `require_evidence` and the judge cited one.
    pub evidence: Option<String>,
}

/// The rubric outcome: the aggregate score, whether it passed, and the
/// per-criterion breakdown.
#[derive(Debug, Clone)]
pub struct RubricOutcome {
    /// Weight-normalized aggregate score in `[0, 1]` (criteria mode) or the
    /// leaf score reached (tree mode).
    pub score: f64,
    /// Whether `score` met the threshold (or 1.0 in strict mode).
    pub passed: bool,
    /// Per-criterion scores and reasons, in declaration order (criteria mode).
    pub per_criterion: Vec<CriterionScore>,
    /// The yes/no path taken through a decision tree, e.g.
    /// `["called a tool? -> yes", "right day? -> no"]` (tree mode). Empty in
    /// criteria mode.
    pub path: Vec<String>,
    /// The reason from the leaf reached (tree mode), if any.
    pub leaf_reason: Option<String>,
    /// Estimated USD cost of every judge call this rubric made, summed from the
    /// provider's token usage. Zero for the deterministic stub provider.
    pub cost_usd: f64,
}

/// Build the single-criterion rubric for `mcp_task_completion`:
/// did the run accomplish the task it was given?
pub fn task_completion_rubric(task: &str, threshold: f64, strict: bool) -> RubricConfig {
    RubricConfig {
        criteria: vec![Criterion {
            name: "task completion".to_string(),
            weight: 1.0,
            description: format!(
                "The agent was asked to accomplish this task: \"{task}\". \
                 Judge whether the run, given its tool calls and final reply, \
                 actually accomplished the task. Score 1.0 when fully done, \
                 a partial value when partially done, 0.0 when not done."
            ),
            ..Criterion::default()
        }],
        threshold,
        strict,
        ..RubricConfig::default()
    }
}

/// Build the single-criterion rubric for `mcp_use`: were the right
/// tools called with the right arguments for the intent?
pub fn mcp_use_rubric(intent: &str, threshold: f64, strict: bool) -> RubricConfig {
    RubricConfig {
        criteria: vec![Criterion {
            name: "mcp use".to_string(),
            weight: 1.0,
            description: format!(
                "The user's intent was: \"{intent}\". Looking at the agent's tool \
                 calls in the trace, judge whether it selected appropriate tools \
                 and passed sensible arguments for that intent (no wrong tool, no \
                 missing or malformed arguments, no needless calls). Score 1.0 \
                 for ideal tool use, lower for misuse."
            ),
            ..Criterion::default()
        }],
        threshold,
        strict,
        ..RubricConfig::default()
    }
}

/// Build the single-criterion rubric for `argument_correctness`:
/// were the tool-call arguments right for the intent?
pub fn argument_correctness_rubric(intent: &str, threshold: f64, strict: bool) -> RubricConfig {
    RubricConfig {
        criteria: vec![Criterion {
            name: "argument correctness".to_string(),
            weight: 1.0,
            description: format!(
                "The user's intent was: \"{intent}\". For each tool call in the \
                 trace, judge whether its arguments are correct and complete for \
                 the intent: right values, no missing required field, no malformed \
                 or fabricated argument. Score 1.0 when every call's arguments are \
                 right, lower as arguments are wrong or missing."
            ),
            ..Criterion::default()
        }],
        threshold,
        strict,
        ..RubricConfig::default()
    }
}

/// Build the single-criterion rubric for `plan_quality`: was the
/// sequence of tool calls a sensible plan?
pub fn plan_quality_rubric(intent: &str, threshold: f64, strict: bool) -> RubricConfig {
    RubricConfig {
        criteria: vec![Criterion {
            name: "plan quality".to_string(),
            weight: 1.0,
            description: format!(
                "The user's intent was: \"{intent}\". Judge the agent's overall \
                 plan from its sequence of tool calls: are the calls in a sensible \
                 order, free of redundant or repeated calls, with no missing \
                 prerequisite step? Score 1.0 for a clean, efficient plan, lower \
                 for redundancy, bad ordering, or gaps."
            ),
            ..Criterion::default()
        }],
        threshold,
        strict,
        ..RubricConfig::default()
    }
}

/// Middle-of-scale score the synthetic mid-tier anchor lands on, so the judge
/// has a reference for partial credit, not just the extremes.
const MID_ANCHOR_SCORE: f64 = 0.5;

/// Lower and upper bounds of the "mid" band: an existing anchor in this range
/// means the author already calibrated the middle, so no synthetic one is added.
const MID_BAND: std::ops::RangeInclusive<f64> = 0.4..=0.6;

/// Append a synthetic mid-tier exemplar when a criterion's anchors span only the
/// extremes (WOR-1370): two or more anchors, none in the mid band. Anchored only
/// at the ends, a judge tends to push borderline answers to one pole, so a mid
/// reference pulls partial-credit grading toward the author's intent. An
/// un-anchored or already-mid-anchored criterion is returned untouched.
fn with_default_mid_anchor(examples: &[CriterionExample]) -> Vec<CriterionExample> {
    let mut out = examples.to_vec();
    if examples.len() < 2 || examples.iter().any(|e| MID_BAND.contains(&e.score)) {
        return out;
    }
    out.push(CriterionExample {
        response: "A partially correct response: it addresses the criterion but with a \
                   notable gap or error."
            .to_string(),
        score: MID_ANCHOR_SCORE,
    });
    out
}

/// Build the judge rubric text for a criterion: its description plus any
/// calibration anchors rendered as labeled few-shot examples. Anchors steer the
/// judge toward the author's scoring intent and reduce drift between models.
/// When `default_mid_anchor` is set, an extremes-only anchor set gains a
/// synthetic mid-tier exemplar so the middle of the scale is calibrated too.
fn judge_rubric_text(criterion: &Criterion, default_mid_anchor: bool) -> String {
    let examples = if default_mid_anchor {
        with_default_mid_anchor(&criterion.examples)
    } else {
        criterion.examples.clone()
    };
    if examples.is_empty() {
        return criterion.description.clone();
    }
    let mut text = criterion.description.clone();
    text.push_str("\n\nCalibration examples (grade the candidate consistently with these):");
    for ex in &examples {
        text.push_str(&format!(
            "\n- Response: \"{}\" -> score {:.2}",
            ex.response, ex.score
        ));
    }
    text
}

/// Score `candidate` against `config` by judging each criterion in turn and
/// taking the weight-normalized average. One judge call per criterion keeps
/// each judgment narrow (more reliable than one holistic score).
pub async fn evaluate_rubric(
    config: &RubricConfig,
    candidate: &Value,
    provider: &dyn LlmProvider,
    model: &str,
) -> Result<RubricOutcome, ProviderError> {
    if let Some(tree) = &config.tree {
        return walk_tree(tree, config, candidate, provider, model).await;
    }

    let mut total_weight = 0.0;
    let mut weighted_sum = 0.0;
    let mut total_cost = 0.0;
    let mut per_criterion = Vec::with_capacity(config.criteria.len());
    // A required criterion below threshold, or a triggered guard, fails the
    // eval regardless of the weighted average.
    let mut gate_failed = false;
    // Stringified once for the deterministic `when:` predicates.
    let candidate_text = match candidate {
        Value::String(s) => s.clone(),
        other => other.to_string(),
    };

    for criterion in &config.criteria {
        // A `when:` predicate that does not hold skips the criterion entirely:
        // no judge call, no weight, no contribution to the aggregate.
        if let Some(when) = &criterion.when {
            if !when.applies(&candidate_text) {
                continue;
            }
        }
        let weight = if criterion.weight > 0.0 {
            criterion.weight
        } else {
            1.0
        };
        let rubric_text = judge_rubric_text(criterion, config.default_mid_anchor);
        let (verdict, call_cost) = llm_judge::score_metered(
            &rubric_text,
            candidate,
            provider,
            model,
            config.require_evidence,
        )
        .await?;
        total_cost += call_cost;
        let evidence = verdict.evidence.filter(|e| !e.trim().is_empty());
        // When evidence is required but the judge cited none, the criterion
        // cannot be trusted: force it to zero and gate the eval so the failure
        // is auditable rather than silently averaged in.
        let missing_evidence = config.require_evidence && evidence.is_none();
        let raw = if missing_evidence {
            0.0
        } else {
            verdict.score.unwrap_or(0.0).clamp(0.0, 1.0)
        };
        // A guard criterion is negative: a high raw score means the bad thing
        // it describes is present, so invert its contribution to the average.
        let contribution = if criterion.guard { 1.0 - raw } else { raw };
        // A guard trips when the bad thing it describes is present; a required
        // criterion trips when its contribution is below its gate threshold
        // (the per-criterion override, else the rubric default).
        let gate_threshold = criterion.threshold.unwrap_or(config.threshold);
        let guard_triggered = criterion.guard && raw >= 0.5;
        let required_failed = criterion.required && contribution < gate_threshold;
        if guard_triggered || required_failed || missing_evidence {
            gate_failed = true;
        }
        weighted_sum += contribution * weight;
        total_weight += weight;
        per_criterion.push(CriterionScore {
            name: criterion.name.clone(),
            score: contribution,
            reason: verdict.reason,
            evidence,
        });
    }

    let score = if total_weight == 0.0 {
        // Every criterion was skipped by its `when:` predicate, so nothing
        // applied and nothing can fail: a vacuous pass.
        1.0
    } else {
        match config.aggregation {
            // The worst criterion caps the aggregate; weights do not enter here.
            AggregationMode::Min => per_criterion
                .iter()
                .map(|c| c.score)
                .reduce(f64::min)
                .unwrap_or(1.0),
            // Tree never reaches this path (it returns from walk_tree above), so
            // the remaining modes aggregate by the weight-normalized mean.
            AggregationMode::WeightedAverage | AggregationMode::Tree => weighted_sum / total_weight,
        }
    };
    let meets_average = if config.strict {
        score >= 1.0
    } else {
        score >= config.threshold
    };
    Ok(RubricOutcome {
        score,
        passed: meets_average && !gate_failed,
        per_criterion,
        path: Vec::new(),
        leaf_reason: None,
        cost_usd: total_cost,
    })
}

/// Walk a decision-tree rubric: at each `Ask` node, judge the
/// question yes/no (score >= 0.5 is yes) and descend the matching branch until
/// a `Leaf` is reached. The walk is a single path (one child per node), so it
/// is iterative, no async recursion. Records the path for the report.
async fn walk_tree(
    root: &RubricNode,
    config: &RubricConfig,
    candidate: &Value,
    provider: &dyn LlmProvider,
    model: &str,
) -> Result<RubricOutcome, ProviderError> {
    let mut node = root;
    let mut path = Vec::new();
    let mut total_cost = 0.0;
    loop {
        match node {
            RubricNode::Leaf { score, reason } => {
                let score = score.clamp(0.0, 1.0);
                let passed = if config.strict {
                    score >= 1.0
                } else {
                    score >= config.threshold
                };
                return Ok(RubricOutcome {
                    score,
                    passed,
                    per_criterion: Vec::new(),
                    path,
                    leaf_reason: reason.clone(),
                    cost_usd: total_cost,
                });
            }
            RubricNode::Ask { question, yes, no } => {
                // Evidence is a criteria-mode feature; a tree node is a yes/no
                // judgment, so it does not request a cited span.
                let (verdict, call_cost) =
                    llm_judge::score_metered(question, candidate, provider, model, false).await?;
                total_cost += call_cost;
                let is_yes = verdict.score.unwrap_or(0.0) >= 0.5;
                path.push(format!(
                    "{question} -> {}",
                    if is_yes { "yes" } else { "no" }
                ));
                node = if is_yes { yes } else { no };
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::eval::provider::{
        Capabilities, CompletionOptions, Cost, FinishReason, Message, ProviderId, Response,
        ResponseContent, Usage,
    };
    use async_trait::async_trait;
    use serde_json::json;
    use std::sync::Mutex;

    /// Returns canned judge replies in order, one per `score` call.
    struct QueueProvider {
        replies: Mutex<Vec<String>>,
    }

    impl QueueProvider {
        fn new(replies: &[&str]) -> Self {
            Self {
                replies: Mutex::new(replies.iter().rev().map(|s| s.to_string()).collect()),
            }
        }
    }

    #[async_trait]
    impl LlmProvider for QueueProvider {
        fn name(&self) -> ProviderId {
            ProviderId::Anthropic
        }
        fn supported_capabilities(&self) -> Capabilities {
            Capabilities::none()
        }
        async fn chat_completion(
            &self,
            _messages: &[Message],
            _options: &CompletionOptions,
        ) -> Result<Response, ProviderError> {
            let text = self.replies.lock().unwrap().pop().unwrap_or_default();
            Ok(Response {
                content: ResponseContent::Text { text },
                finish_reason: FinishReason::Stop,
                usage: Usage::default(),
                model: "stub".into(),
            })
        }
        async fn count_tokens(&self, _text: &str, _model: &str) -> Result<u32, ProviderError> {
            Ok(0)
        }
        async fn embed(&self, _text: &str, _model: &str) -> Result<Vec<f32>, ProviderError> {
            Ok(vec![1.0])
        }
        fn estimate_cost(&self, _usage: &Usage, _model: &str) -> Cost {
            Cost::default()
        }
    }

    fn crit(name: &str, weight: f64, description: &str) -> Criterion {
        Criterion {
            name: name.into(),
            weight,
            description: description.into(),
            ..Criterion::default()
        }
    }

    fn guard_crit(name: &str, description: &str) -> Criterion {
        Criterion {
            name: name.into(),
            weight: 1.0,
            description: description.into(),
            guard: true,
            ..Criterion::default()
        }
    }

    fn required_crit(name: &str, description: &str) -> Criterion {
        Criterion {
            name: name.into(),
            weight: 1.0,
            description: description.into(),
            required: true,
            ..Criterion::default()
        }
    }

    #[tokio::test]
    async fn weight_normalized_average_passes_above_threshold() {
        let config = RubricConfig {
            criteria: vec![crit("a", 1.0, "first"), crit("b", 1.0, "second")],
            tree: None,
            threshold: 0.7,
            strict: false,
            ..Default::default()
        };
        let provider = QueueProvider::new(&[r#"{"score": 1.0}"#, r#"{"score": 0.8}"#]);
        let out = evaluate_rubric(&config, &json!("x"), &provider, "stub")
            .await
            .unwrap();
        assert!((out.score - 0.9).abs() < 1e-9, "got {}", out.score);
        assert!(out.passed);
        assert_eq!(out.per_criterion.len(), 2);
        assert_eq!(out.per_criterion[0].name, "a");
    }

    #[tokio::test]
    async fn required_criterion_below_threshold_fails_despite_passing_average() {
        // The optional criterion carries enough weight that the average clears
        // the threshold, but the failed required criterion gates the eval.
        let config = RubricConfig {
            criteria: vec![crit("optional", 3.0, "nice"), required_crit("must", "req")],
            tree: None,
            threshold: 0.7,
            strict: false,
            ..Default::default()
        };
        let provider = QueueProvider::new(&[r#"{"score": 1.0}"#, r#"{"score": 0.6}"#]);
        let out = evaluate_rubric(&config, &json!("x"), &provider, "stub")
            .await
            .unwrap();
        // (1.0*3 + 0.6*1) / 4 = 0.9 average, but required 0.6 < 0.7 gates.
        assert!((out.score - 0.9).abs() < 1e-9, "got {}", out.score);
        assert!(!out.passed, "failed required criterion must gate the eval");
    }

    #[tokio::test]
    async fn triggered_guard_inverts_and_fails() {
        // The guard describes a bad outcome; a high raw score means it is
        // present, so the contribution inverts and the guard trips.
        let config = RubricConfig {
            criteria: vec![guard_crit("no secret leak", "the answer leaks a secret")],
            tree: None,
            threshold: 0.5,
            strict: false,
            ..Default::default()
        };
        let provider = QueueProvider::new(&[r#"{"score": 0.9}"#]);
        let out = evaluate_rubric(&config, &json!("x"), &provider, "stub")
            .await
            .unwrap();
        assert!((out.per_criterion[0].score - 0.1).abs() < 1e-9);
        assert!(!out.passed, "triggered guard must fail");
    }

    #[tokio::test]
    async fn clean_guard_passes() {
        let config = RubricConfig {
            criteria: vec![guard_crit("no secret leak", "the answer leaks a secret")],
            tree: None,
            threshold: 0.5,
            strict: false,
            ..Default::default()
        };
        let provider = QueueProvider::new(&[r#"{"score": 0.0}"#]);
        let out = evaluate_rubric(&config, &json!("x"), &provider, "stub")
            .await
            .unwrap();
        assert!((out.per_criterion[0].score - 1.0).abs() < 1e-9);
        assert!(out.passed, "untriggered guard contributes a clean 1.0");
    }

    #[tokio::test]
    async fn weights_shift_the_aggregate() {
        // The low score carries 3x the weight, so the average drops below 0.7.
        let config = RubricConfig {
            criteria: vec![crit("heavy", 3.0, "first"), crit("light", 1.0, "second")],
            tree: None,
            threshold: 0.7,
            strict: false,
            ..Default::default()
        };
        let provider = QueueProvider::new(&[r#"{"score": 0.4}"#, r#"{"score": 1.0}"#]);
        let out = evaluate_rubric(&config, &json!("x"), &provider, "stub")
            .await
            .unwrap();
        // (0.4*3 + 1.0*1) / 4 = 0.55
        assert!((out.score - 0.55).abs() < 1e-9, "got {}", out.score);
        assert!(!out.passed);
    }

    #[tokio::test]
    async fn strict_mode_requires_a_perfect_score() {
        let config = RubricConfig {
            criteria: vec![crit("a", 1.0, "first")],
            tree: None,
            threshold: 0.7,
            strict: true,
            ..Default::default()
        };
        let provider = QueueProvider::new(&[r#"{"score": 0.95}"#]);
        let out = evaluate_rubric(&config, &json!("x"), &provider, "stub")
            .await
            .unwrap();
        assert!(!out.passed, "strict mode must reject anything below 1.0");
    }

    #[tokio::test]
    async fn a_missing_score_counts_as_zero() {
        let config = RubricConfig {
            criteria: vec![crit("a", 1.0, "first")],
            tree: None,
            threshold: 0.5,
            strict: false,
            ..Default::default()
        };
        let provider = QueueProvider::new(&[r#"{"reason": "no score field"}"#]);
        let out = evaluate_rubric(&config, &json!("x"), &provider, "stub")
            .await
            .unwrap();
        assert_eq!(out.score, 0.0);
        assert!(!out.passed);
    }

    fn leaf(score: f64, reason: &str) -> Box<RubricNode> {
        Box::new(RubricNode::Leaf {
            score,
            reason: Some(reason.to_string()),
        })
    }

    #[tokio::test]
    async fn tree_descends_yes_branches_to_the_reward_leaf() {
        // called a tool? -> yes -> right city? -> yes -> 1.0
        let config = RubricConfig {
            criteria: Vec::new(),
            tree: Some(RubricNode::Ask {
                question: "did the agent call a tool?".to_string(),
                yes: Box::new(RubricNode::Ask {
                    question: "did it ask about the right city?".to_string(),
                    yes: leaf(1.0, "tool called for the right city"),
                    no: leaf(0.3, "tool called but wrong city"),
                }),
                no: leaf(0.0, "no tool call"),
            }),
            threshold: 0.7,
            strict: false,
            ..Default::default()
        };
        let provider = QueueProvider::new(&[r#"{"score": 1.0}"#, r#"{"score": 1.0}"#]);
        let out = evaluate_rubric(&config, &json!("trace"), &provider, "stub")
            .await
            .unwrap();
        assert_eq!(out.score, 1.0);
        assert!(out.passed);
        assert_eq!(out.path.len(), 2);
        assert!(out.path[0].ends_with("-> yes"));
        assert!(out.path[1].ends_with("-> yes"));
        assert_eq!(
            out.leaf_reason.as_deref(),
            Some("tool called for the right city")
        );
        assert!(out.per_criterion.is_empty());
    }

    #[tokio::test]
    async fn tree_takes_the_no_branch_and_fails() {
        let config = RubricConfig {
            criteria: Vec::new(),
            tree: Some(RubricNode::Ask {
                question: "did the agent call a tool?".to_string(),
                yes: leaf(1.0, "tool called"),
                no: leaf(0.0, "no tool call"),
            }),
            threshold: 0.7,
            strict: false,
            ..Default::default()
        };
        // score < 0.5 reads as "no".
        let provider = QueueProvider::new(&[r#"{"score": 0.1}"#]);
        let out = evaluate_rubric(&config, &json!("trace"), &provider, "stub")
            .await
            .unwrap();
        assert_eq!(out.score, 0.0);
        assert!(!out.passed);
        assert_eq!(out.path.len(), 1);
        assert!(out.path[0].ends_with("-> no"));
        assert_eq!(out.leaf_reason.as_deref(), Some("no tool call"));
    }

    #[tokio::test]
    async fn min_aggregation_reports_the_worst_criterion() {
        // The weighted average (0.7) would clear the threshold, but Min mode
        // reports the worst criterion (0.4) and fails the gate.
        let config = RubricConfig {
            criteria: vec![crit("strong", 1.0, "a"), crit("weak", 1.0, "b")],
            threshold: 0.7,
            aggregation: AggregationMode::Min,
            ..Default::default()
        };
        let provider = QueueProvider::new(&[r#"{"score": 1.0}"#, r#"{"score": 0.4}"#]);
        let out = evaluate_rubric(&config, &json!("x"), &provider, "stub")
            .await
            .unwrap();
        assert!(
            (out.score - 0.4).abs() < 1e-9,
            "min picks the worst criterion, got {}",
            out.score
        );
        assert!(!out.passed);
    }

    #[test]
    fn calibration_anchors_are_appended_to_the_judge_prompt() {
        let c = Criterion {
            name: "grounded".into(),
            weight: 1.0,
            description: "is it supported".into(),
            examples: vec![
                CriterionExample {
                    response: "good answer".into(),
                    score: 0.9,
                },
                CriterionExample {
                    response: "bad answer".into(),
                    score: 0.1,
                },
            ],
            ..Criterion::default()
        };
        // Render the author's anchors only (no synthetic mid) so the assertion
        // is about the explicit-anchor rendering.
        let text = judge_rubric_text(&c, false);
        assert!(text.contains("is it supported"));
        assert!(text.contains("Calibration examples"));
        assert!(text.contains("good answer"));
        assert!(text.contains("0.90"));
    }

    #[test]
    fn an_un_anchored_criterion_keeps_its_plain_description() {
        // The mid-anchor default never fires on an un-anchored criterion.
        assert_eq!(judge_rubric_text(&crit("a", 1.0, "plain"), true), "plain");
    }

    #[test]
    fn default_mid_anchor_injects_a_mid_exemplar_for_extreme_only_anchors() {
        let c = Criterion {
            name: "grounded".into(),
            weight: 1.0,
            description: "is it supported".into(),
            examples: vec![
                CriterionExample {
                    response: "good".into(),
                    score: 0.9,
                },
                CriterionExample {
                    response: "bad".into(),
                    score: 0.1,
                },
            ],
            ..Criterion::default()
        };
        let text = judge_rubric_text(&c, true);
        // The author's two anchors plus the synthetic mid-tier one.
        assert!(text.contains("0.90"));
        assert!(text.contains("0.10"));
        assert!(
            text.contains("0.50"),
            "a mid-tier anchor is injected: {text}"
        );
    }

    #[test]
    fn with_default_mid_anchor_only_fires_on_extreme_only_anchor_sets() {
        let ex = |score: f64| CriterionExample {
            response: "r".into(),
            score,
        };
        // Extremes only: a mid anchor is appended.
        let extremes = with_default_mid_anchor(&[ex(0.9), ex(0.1)]);
        assert_eq!(extremes.len(), 3);
        assert_eq!(extremes[2].score, 0.5);
        // Already has a mid anchor: untouched.
        assert_eq!(with_default_mid_anchor(&[ex(0.9), ex(0.5)]).len(), 2);
        // Fewer than two anchors: no calibration intent, untouched.
        assert_eq!(with_default_mid_anchor(&[ex(0.9)]).len(), 1);
        assert_eq!(with_default_mid_anchor(&[]).len(), 0);
    }

    #[tokio::test]
    async fn require_evidence_captures_the_cited_span() {
        let config = RubricConfig {
            criteria: vec![crit("grounded", 1.0, "is it supported")],
            threshold: 0.5,
            require_evidence: true,
            ..Default::default()
        };
        let provider =
            QueueProvider::new(&[r#"{"score": 1.0, "evidence": "per-tenant rate limits"}"#]);
        let out = evaluate_rubric(
            &config,
            &json!("added per-tenant rate limits"),
            &provider,
            "stub",
        )
        .await
        .unwrap();
        assert!(out.passed);
        assert_eq!(
            out.per_criterion[0].evidence.as_deref(),
            Some("per-tenant rate limits")
        );
    }

    #[tokio::test]
    async fn require_evidence_fails_a_criterion_with_no_citation() {
        let config = RubricConfig {
            criteria: vec![crit("grounded", 1.0, "is it supported")],
            threshold: 0.5,
            require_evidence: true,
            ..Default::default()
        };
        // A high score but no evidence field: the criterion is forced to 0 and
        // gates the eval so the unjustified verdict is auditable.
        let provider = QueueProvider::new(&[r#"{"score": 1.0}"#]);
        let out = evaluate_rubric(&config, &json!("x"), &provider, "stub")
            .await
            .unwrap();
        assert!(!out.passed, "no evidence gates the eval");
        assert_eq!(out.per_criterion[0].score, 0.0);
    }

    #[tokio::test]
    async fn when_predicate_skips_a_non_matching_criterion() {
        // The conditional criterion only applies when the candidate mentions
        // "error"; this candidate does not, so only the always-on criterion is
        // judged (one judge call drains from the queue).
        let config = RubricConfig {
            criteria: vec![
                crit("always", 1.0, "a"),
                Criterion {
                    name: "on errors".into(),
                    weight: 1.0,
                    description: "b".into(),
                    when: Some(CriterionWhen::Contains("error".into())),
                    ..Criterion::default()
                },
            ],
            threshold: 0.5,
            ..Default::default()
        };
        let provider = QueueProvider::new(&[r#"{"score": 0.9}"#]);
        let out = evaluate_rubric(&config, &json!("all good"), &provider, "stub")
            .await
            .unwrap();
        assert_eq!(
            out.per_criterion.len(),
            1,
            "the conditional one was skipped"
        );
        assert_eq!(out.per_criterion[0].name, "always");
        assert!((out.score - 0.9).abs() < 1e-9);
    }

    #[tokio::test]
    async fn per_criterion_threshold_overrides_the_rubric_default() {
        // The criterion scores 0.8, above the rubric threshold 0.5, but its own
        // gate threshold is 0.95, so the required gate fails the eval.
        let config = RubricConfig {
            criteria: vec![Criterion {
                name: "strict gate".into(),
                weight: 1.0,
                description: "a".into(),
                required: true,
                threshold: Some(0.95),
                ..Criterion::default()
            }],
            threshold: 0.5,
            ..Default::default()
        };
        let provider = QueueProvider::new(&[r#"{"score": 0.8}"#]);
        let out = evaluate_rubric(&config, &json!("x"), &provider, "stub")
            .await
            .unwrap();
        assert!(
            !out.passed,
            "0.8 clears the rubric 0.5 but not the criterion's own 0.95 gate"
        );
    }

    #[test]
    fn unit_scale_displays_a_two_decimal_fraction() {
        assert_eq!(ScoreScale::Unit.display(0.756), "0.76");
    }

    #[test]
    fn likert_scale_maps_normalized_into_its_native_band() {
        // 0.75 on a 1-5 Likert is 1 + 0.75 * 4 = 4.0.
        assert_eq!(ScoreScale::Likert { min: 1, max: 5 }.display(0.75), "4.0/5");
        assert_eq!(ScoreScale::Likert { min: 1, max: 5 }.display(0.0), "1.0/5");
    }

    #[test]
    fn boolean_scale_displays_pass_or_fail() {
        assert_eq!(ScoreScale::Boolean.display(0.9), "pass");
        assert_eq!(ScoreScale::Boolean.display(0.2), "fail");
    }

    #[test]
    fn letter_scale_displays_a_grade_band() {
        assert_eq!(ScoreScale::Letter.display(0.95), "A");
        assert_eq!(ScoreScale::Letter.display(0.72), "C");
        assert_eq!(ScoreScale::Letter.display(0.3), "F");
    }
}
