//! Reference: golden-set eval of a constrained-output classifier agent.
//! The deterministic test (FakeLlmClient) is the CI guarantee; the `#[ignore]`
//! test (Ollama) is the on-demand realism check.

use std::sync::Arc;

use klieo::App;
use klieo_core::test_utils::{FakeLlmClient, FakeLlmStep};
use klieo_core::SimpleAgent;
use klieo_eval::{
    extract_label, score_classification, ClassificationCase, ClassificationReport, LabelSpec,
};

const SYSTEM_PROMPT: &str = "Classify the text into exactly one regulatory domain. \
     Reply with ONLY one of: DORA | AIAct | GDPR | Other.";

fn label_specs() -> Vec<LabelSpec> {
    vec![
        LabelSpec::new("DORA", &[]),
        LabelSpec::new("AIAct", &["AI Act", "ai-act", "EU AI Act"]),
        LabelSpec::new("GDPR", &[]),
        LabelSpec::new("Other", &[]),
    ]
}

fn classifier() -> SimpleAgent {
    SimpleAgent::new("reg-domain", SYSTEM_PROMPT, vec![])
}

fn load_cases() -> Vec<ClassificationCase> {
    let raw = include_str!("../eval/reg-domain/cases.json");
    serde_json::from_str(raw).expect("golden set parses")
}

/// Reference consumer loop. Predictions align to `cases` by index; each case
/// runs in a fresh `AgentContext` (via `app.run`), so cases never share memory.
async fn run_golden(
    app: &App,
    agent: &SimpleAgent,
    cases: &[ClassificationCase],
    specs: &[LabelSpec],
) -> ClassificationReport {
    let mut predicted = Vec::with_capacity(cases.len());
    for case in cases {
        let output = app.run(agent, case.input.clone()).await.expect("agent run");
        predicted.push(extract_label(&output, specs));
    }
    score_classification(&predicted, cases)
}

async fn stub_app(steps: Vec<FakeLlmStep>) -> App {
    let stub = FakeLlmClient::new("stub").with_steps(steps);
    App::local()
        .llm(Arc::new(stub))
        .build()
        .await
        .expect("stub app builds")
}

#[tokio::test]
async fn deterministic_golden_eval_surfaces_misses() {
    let cases = load_cases();
    // Stub answers in cases.json order: a prose-wrapped hit, an alias hit, then a
    // wrong-label and an unparseable answer for the two final cases — so accuracy
    // lands below 1.0 and both miss shapes (wrong label, no label) are exercised.
    let steps = vec![
        FakeLlmStep::Text("DORA".into()),
        FakeLlmStep::Text("This concerns DORA.".into()),
        FakeLlmStep::Text("EU AI Act".into()),
        FakeLlmStep::Text("AIAct".into()),
        FakeLlmStep::Text("GDPR".into()),
        FakeLlmStep::Text("DORA".into()),
        FakeLlmStep::Text("Other".into()),
        FakeLlmStep::Text("not sure".into()),
    ];
    let app = stub_app(steps).await;
    let report = run_golden(&app, &classifier(), &cases, &label_specs()).await;

    assert_eq!(report.total, 8);
    assert_eq!(report.correct, 6);
    assert!(
        report.accuracy < 1.0,
        "accuracy {} should be < 1.0",
        report.accuracy
    );

    let wrong = report
        .misses
        .iter()
        .find(|m| m.expected == "GDPR")
        .expect("wrong-label miss");
    assert_eq!(wrong.got.as_deref(), Some("DORA"));

    let unparseable = report
        .misses
        .iter()
        .find(|m| m.expected == "Other")
        .expect("unparseable miss");
    assert_eq!(unparseable.got, None);
}

/// Realism check against a live Ollama (temp 0 via App::local defaults). `#[ignore]`
/// — needs a running Ollama with the default model pulled; run on demand, not in
/// every-push CI. The 0.6 floor is a smoke bar (catches a wholesale break), not a
/// quality gate.
#[tokio::test]
#[ignore = "needs a live Ollama (App::local); run on demand"]
async fn live_golden_eval_clears_smoke_floor() {
    let cases = load_cases();
    let app = App::local().build().await.expect("ollama app builds");
    let report = run_golden(&app, &classifier(), &cases, &label_specs()).await;
    assert!(
        report.accuracy >= 0.6,
        "live accuracy {} below smoke floor; misses: {:?}",
        report.accuracy,
        report.misses
    );
}
