//! FALSIFY-SHIP-012 — tokenizer-bpe-v1 GATE-BPE-003 evidence harness.
//!
//! Contract: contracts/tokenizer-bpe-v1.yaml GATE-BPE-003 / INV-BPE-003
//! Spec:     docs/specifications/aprender-train/ship-two-models-spec.md §5
//!
//! Gate rule: byte-exact round-trip (decode(encode(text)) == text) on a
//! held-out Python-like corpus. Ships with a fixed synthetic fixture
//! (SHIP-TWO-001 MODEL-2 does not yet have the 10K-doc holdout materialized;
//! see contracts/dataset-thestack-python-v1.yaml and task #91). This harness
//! is wired so swapping the fixture for the real 10K corpus is a data-only
//! change — no test rewrite required.
//!
//! Emits evidence JSON when APR_EVIDENCE_DIR is set, otherwise asserts
//! in-process and exits.

#![allow(clippy::unwrap_used, clippy::expect_used)]

use aprender::text::tokenize::BpeTokenizer;
use std::path::PathBuf;
use unicode_normalization::UnicodeNormalization;

/// Python-like held-out strings spanning: ASCII keywords, Unicode identifiers,
/// docstrings, numeric literals, byte strings, whitespace variants, emoji in
/// comments, multi-line. Expanded deliberately beyond ASCII to stress the
/// round-trip invariant per INV-BPE-003 / INV-BPE-005.
const HOLDOUT_CORPUS: &[&str] = &[
    "def fib(n):\n    return n if n < 2 else fib(n-1) + fib(n-2)",
    "class Café:\n    \"\"\"Stores café data with naïve fields.\"\"\"\n    pass",
    "x = 42",
    "y: float = 3.14159",
    "name = 'world'; print(f'hello, {name}')",
    "# emoji in comment 🚀 — shouldn't break round-trip",
    "data = b'\\x00\\xff\\x7f'",
    "if True:\n\tpass  # tab indent",
    "async def μ(α, β): return α + β",
    "s = 'composed café' + 'decomposed cafe\u{0301}'",
    "# combining marks: n\u{0303} vs ñ",
    "ids = [i for i in range(100) if i % 2 == 0]",
    "def greet(name: str) -> None: print(f'hi {name}')",
    "assert 1 + 1 == 2, 'math broken'",
    "from typing import List, Dict, Optional",
    "with open('file.txt', 'r', encoding='utf-8') as f: data = f.read()",
    "try:\n    x = 1/0\nexcept ZeroDivisionError:\n    pass",
    "lambda_fn = lambda x, y: x * y",
    "nested = {'a': [1, 2, 3], 'b': {'c': None}}",
    "# TODO: support aprender-train BPE when vocab config stabilizes",
];

/// Corpus to train on — distinct from holdout to enforce realism.
const TRAIN_CORPUS: &[&str] = &[
    "import sys",
    "import os",
    "import json",
    "def main(): pass",
    "return None",
    "for i in range(10): print(i)",
    "while True: break",
    "class A: pass",
    "class B(A): pass",
    "self.value = 42",
    "raise ValueError('bad input')",
    "yield from gen()",
    "async for item in stream: pass",
    "match x:\n    case 1: pass",
    "if x is None: return",
    "lst = [1, 2, 3, 4, 5]",
    "tpl = (1, 2, 3)",
    "dct = {'key': 'value'}",
    "s = 'hello world'",
    "f = lambda x: x + 1",
];

#[derive(serde::Serialize)]
struct Ship012Evidence {
    contract: &'static str,
    gate: &'static str,
    falsification: &'static str,
    tokenizer_algorithm: &'static str,
    train_corpus_size: usize,
    holdout_corpus_size: usize,
    vocab_size_trained: usize,
    vocab_size_requested: usize,
    total_bytes: usize,
    docs_passed: usize,
    docs_failed: usize,
    failing_doc_indices: Vec<usize>,
    nfc_idempotent: bool,
    passed: bool,
}

fn run_roundtrip() -> Ship012Evidence {
    let tokenizer = BpeTokenizer::train(TRAIN_CORPUS, 512).expect("BPE train");

    let mut passed = 0usize;
    let mut failed = 0usize;
    let mut failing_indices = Vec::new();
    let mut total_bytes = 0usize;

    for (i, doc) in HOLDOUT_CORPUS.iter().enumerate() {
        let normalized: String = doc.nfc().collect();
        total_bytes += normalized.len();

        let ids = tokenizer.encode(&normalized).expect("encode");
        let decoded = tokenizer.decode(&ids).expect("decode");

        if decoded == normalized {
            passed += 1;
        } else {
            failed += 1;
            failing_indices.push(i);
        }
    }

    let nfc_idempotent = HOLDOUT_CORPUS.iter().all(|doc| {
        let once: String = doc.nfc().collect();
        let twice: String = once.nfc().collect();
        once == twice
    });

    Ship012Evidence {
        contract: "C-TOK-BPE",
        gate: "GATE-BPE-003",
        falsification: "FALSIFY-SHIP-012",
        tokenizer_algorithm: "bpe",
        train_corpus_size: TRAIN_CORPUS.len(),
        holdout_corpus_size: HOLDOUT_CORPUS.len(),
        vocab_size_trained: tokenizer.vocab_size(),
        vocab_size_requested: 512,
        total_bytes,
        docs_passed: passed,
        docs_failed: failed,
        failing_doc_indices: failing_indices,
        nfc_idempotent,
        passed: failed == 0 && nfc_idempotent,
    }
}

#[test]
fn falsify_ship_012_tokenizer_roundtrip_byte_exact() {
    let evidence = run_roundtrip();

    if let Ok(dir) = std::env::var("APR_EVIDENCE_DIR") {
        let out_dir = PathBuf::from(&dir);
        std::fs::create_dir_all(&out_dir).expect("mkdir evidence dir");
        let path = out_dir.join("falsify-ship-012-tokenizer-roundtrip.json");
        let json = serde_json::to_string_pretty(&evidence).expect("serialize");
        std::fs::write(&path, json).expect("write evidence");
        eprintln!("FALSIFY-SHIP-012 evidence written: {}", path.display());
    }

    // Hard invariant: NFC idempotence (INV-BPE-005). Already-passing property
    // of unicode-normalization crate.
    assert!(
        evidence.nfc_idempotent,
        "INV-BPE-005: NFC must be idempotent"
    );

    // INV-BPE-003 (byte-exact round-trip) DISCHARGED by task #99:
    // aprender::text::tokenize::BpeTokenizer now runs byte-level GPT-2-style
    // BPE (see `bytes_to_unicode` in crate::text::bpe) so every byte is
    // recoverable via the inverse byte-encoder — no whitespace/indent loss.
    // Hard assertion from here on; if a regression reintroduces whitespace
    // splitting or drops the byte encoder, this will panic instead of
    // emitting a soft warning.
    assert_eq!(
        evidence.docs_failed, 0,
        "INV-BPE-003: byte-exact round-trip failed on {}/{} docs (indices={:?})",
        evidence.docs_failed, evidence.holdout_corpus_size, evidence.failing_doc_indices
    );
}

#[test]
fn falsify_ship_012_nfc_idempotence_only() {
    let evidence = run_roundtrip();
    assert!(
        evidence.nfc_idempotent,
        "INV-BPE-005: NFC must be idempotent on the holdout corpus"
    );
}

#[test]
fn falsify_ship_012_train_corpus_sanity() {
    assert!(
        TRAIN_CORPUS.len() >= 20,
        "train corpus must have enough variety to actually train"
    );
    assert!(
        HOLDOUT_CORPUS.len() >= 20,
        "holdout corpus must be non-trivial"
    );
    let train_set: std::collections::HashSet<_> = TRAIN_CORPUS.iter().collect();
    let holdout_set: std::collections::HashSet<_> = HOLDOUT_CORPUS.iter().collect();
    let overlap: Vec<_> = train_set.intersection(&holdout_set).collect();
    assert!(
        overlap.is_empty(),
        "train and holdout must not share docs verbatim; overlap: {:?}",
        overlap
    );
}
