// Copyright 2025-present Harīṣh Tummalachērla
// SPDX-License-Identifier: Apache-2.0

//! Inverted index: the classic "look up which docs contain this word" structure.
//!
//! The inverted index maps terms to posting lists. Each posting includes doc_id,
//! offset, field type, and a precomputed score. Posting lists are sorted by score
//! descending, so the best matches come first.
//!
//! # Lean Correspondence
//!
//! `SearchVerified/InvertedIndex.lean` - Posting list well-formedness
//!
//! # Key Invariants
//!
//! | Invariant                    | What it means |
//! |------------------------------|---------------|
//! | POSTING_LIST_SORTED_BY_SCORE | Best matches first (score DESC, doc_id ASC) |
//! | DOC_FREQ_CORRECT             | doc_freq = unique doc count |
//! | NON_EMPTY                    | Every term has at least one posting |
//! | SCORE_PRECOMPUTED            | No scoring at query time |

use crate::scoring::final_score;
use crate::types::{
    FieldBoundary, FieldType, IndexMode, InvertedIndex, Posting, PostingList, SearchDoc,
    UnifiedIndex,
};
use crate::util::normalize::normalize;
#[cfg(feature = "parallel")]
use rayon::prelude::*;
#[cfg(feature = "lean")]
use sorex_lean_macros::lean_verify;
use std::collections::HashMap;

/// Word boundary detection: checks if character is a word separator.
fn is_word_boundary(c: char) -> bool {
    !c.is_alphanumeric()
}

use std::collections::HashSet;
use std::sync::LazyLock;

/// Multilingual stop words loaded from data/stop_words.json.
///
/// These words are:
/// 1. Too common to be useful for search ranking
/// 2. Cause false positives in fuzzy matching (e.g., "land" → "and")
/// 3. Waste index space
///
/// The JSON file contains stop words for 20+ languages including:
/// English, Spanish, French, German, Portuguese, Italian, Dutch, Russian,
/// Polish, Swedish, Norwegian, Danish, Finnish, Turkish, Indonesian,
/// Arabic, Hindi, Chinese, Japanese, Korean (romanized forms).
static STOP_WORDS: LazyLock<HashSet<String>> = LazyLock::new(|| {
    let json_str = include_str!("../../data/stop_words.json");
    parse_stop_words_json(json_str)
});

/// Parse stop words from JSON, flattening all language arrays into a single set.
/// Normalizes words to match how input text is normalized (strips diacritics).
fn parse_stop_words_json(json_str: &str) -> HashSet<String> {
    let mut stop_words = HashSet::new();

    // Simple JSON parsing without external dependency
    // The JSON structure is: { "lang": ["word1", "word2", ...], ... }
    let mut in_array = false;
    let mut current_word = String::new();
    let mut in_string = false;

    for ch in json_str.chars() {
        match ch {
            '[' => in_array = true,
            ']' => in_array = false,
            '"' if in_array => {
                if in_string {
                    // End of string, normalize and add word
                    if !current_word.is_empty() {
                        // Normalize to strip diacritics (e.g., "tú" → "tu", "está" → "esta")
                        let normalized = normalize(&current_word);
                        if !normalized.is_empty() {
                            stop_words.insert(normalized);
                        }
                        // Also insert original for non-normalized lookups
                        stop_words.insert(current_word.clone());
                        current_word.clear();
                    }
                }
                in_string = !in_string;
            }
            _ if in_string => {
                current_word.push(ch);
            }
            _ => {}
        }
    }

    stop_words
}

/// Check if a word is a stop word.
///
/// Stop words are filtered during index construction to:
/// 1. Reduce index size
/// 2. Prevent low-quality fuzzy matches (e.g., "land" → "and")
/// 3. Improve search relevance
#[inline]
pub fn is_stop_word(word: &str) -> bool {
    STOP_WORDS.contains(word)
}

/// Tokenize text into (word, offset) pairs.
///
/// Returns normalized words with their byte offsets in the original text.
/// Only returns words at word boundaries (start of text or after non-alphanumeric).
///
/// # Lean Specification
///
/// This corresponds to the word extraction precondition in `build_complete`:
/// ```lean
/// (h_boundary : offset = 0 ∨ ¬ (texts[doc_id]).get! (offset - 1) |>.isAlphaNum)
/// ```
fn tokenize(text: &str) -> Vec<(String, usize)> {
    let mut tokens = Vec::new();
    let chars: Vec<char> = text.chars().collect();
    let mut i = 0;
    let mut byte_offset = 0;

    while i < chars.len() {
        // Skip non-alphanumeric characters
        while i < chars.len() && is_word_boundary(chars[i]) {
            byte_offset += chars[i].len_utf8();
            i += 1;
        }

        if i >= chars.len() {
            break;
        }

        // Record start of word
        let word_start = byte_offset;
        let word_char_start = i;

        // Collect word characters
        while i < chars.len() && !is_word_boundary(chars[i]) {
            byte_offset += chars[i].len_utf8();
            i += 1;
        }

        // Extract and normalize word
        let word: String = chars[word_char_start..i].iter().collect();
        let normalized = normalize(&word);

        // Skip empty words and stop words
        if !normalized.is_empty() && !is_stop_word(&normalized) {
            tokens.push((normalized, word_start));
        }
    }

    tokens
}

/// Build an inverted index from documents.
///
/// Creates a map from terms to posting lists, enabling O(1) term lookup.
///
/// # Lean Specification
///
/// The following properties are guaranteed (see `InvertedIndex.lean`):
/// - `posting_list_sorted`: Each posting list is sorted
/// - `doc_freq_correct`: Document frequencies are accurate
/// - `build_complete`: All words in documents are indexed
#[cfg_attr(
    feature = "lean",
    lean_verify(
        spec = "build_inverted_index",
        requires = "docs.len() > 0 ∧ docs.len() = texts.len()",
        ensures = "InvertedIndex.WellFormed result texts",
        properties = ["posting_list_sorted", "doc_freq_correct", "build_complete"]
    )
)]
pub fn build_inverted_index(texts: &[String], field_boundaries: &[FieldBoundary]) -> InvertedIndex {
    let mut terms: HashMap<String, Vec<Posting>> = HashMap::new();

    // Process each document
    for (doc_id, text) in texts.iter().enumerate() {
        let text_len = text.len();
        let tokens = tokenize(text);

        for (word, offset) in tokens {
            // Determine field type, section_id, and heading_level at this position
            let (field_type, section_id, heading_level) =
                get_field_info_for_inverted(doc_id, offset, field_boundaries);

            // INVARIANT: SCORE_PRECOMPUTED - compute score at index time
            let score = final_score(&field_type, offset, text_len);

            let posting = Posting {
                doc_id,
                offset,
                field_type,
                section_id,
                heading_level,
                score,
            };

            terms.entry(word).or_default().push(posting);
        }
    }

    // INVARIANT: POSTING_LIST_SORTED_BY_SCORE
    // Sort each posting list by (score DESC, doc_id ASC) for fast top-k retrieval
    for postings in terms.values_mut() {
        postings.sort_by(|a, b| {
            // Primary: score DESC (higher scores first)
            match b.score.partial_cmp(&a.score) {
                Some(std::cmp::Ordering::Equal) | None => {
                    // Secondary: doc_id ASC (for determinism)
                    a.doc_id.cmp(&b.doc_id)
                }
                Some(ord) => ord,
            }
        });
    }

    // Build final posting lists with doc_freq
    let mut final_terms: HashMap<String, PostingList> = HashMap::new();
    for (term, postings) in terms {
        // Calculate doc_freq (unique doc_ids)
        let mut doc_ids: Vec<usize> = postings.iter().map(|p| p.doc_id).collect();
        doc_ids.sort();
        doc_ids.dedup();
        let doc_freq = doc_ids.len();

        final_terms.insert(term, PostingList { postings, doc_freq });
    }

    InvertedIndex {
        terms: final_terms,
        total_docs: texts.len(),
    }
}

/// Get field type, section_id, and heading_level for a position (inverted index version).
/// Returns (field_type, section_id, heading_level) tuple.
///
/// When no boundary matches, defaults to (Content, None, 5) - regular content level.
/// Level 5 ensures content without boundaries ranks below title (0), sections (1-2),
/// and headings (3-4) in the bucketed ranking system.
fn get_field_info_for_inverted(
    doc_id: usize,
    offset: usize,
    boundaries: &[FieldBoundary],
) -> (FieldType, Option<String>, u8) {
    for b in boundaries {
        if b.doc_id == doc_id && offset >= b.start && offset < b.end {
            return (b.field_type, b.section_id.clone(), b.heading_level);
        }
    }
    // Default to Content with heading_level=5 (regular content, not title/heading)
    (FieldType::Content, None, 5)
}

/// Build an inverted index using parallel map-reduce.
///
/// This is faster for large corpora (100+ documents):
/// 1. **Map phase**: Parallel tokenization (one task per document)
/// 2. **Reduce phase**: Merge per-document posting maps into global index
///
/// For small corpora (<100 docs), use `build_inverted_index` instead.
#[cfg(feature = "parallel")]
pub fn build_inverted_index_parallel(
    texts: &[String],
    field_boundaries: &[FieldBoundary],
) -> InvertedIndex {
    // MAP PHASE: Parallel tokenization with precomputed scores
    let per_doc_terms: Vec<HashMap<String, Vec<Posting>>> = texts
        .par_iter()
        .enumerate()
        .map(|(doc_id, text)| {
            let text_len = text.len();
            let mut doc_terms: HashMap<String, Vec<Posting>> = HashMap::new();
            for (word, offset) in tokenize(text) {
                let (field_type, section_id, heading_level) =
                    get_field_info_for_inverted(doc_id, offset, field_boundaries);

                // INVARIANT: SCORE_PRECOMPUTED - compute score at index time
                let score = final_score(&field_type, offset, text_len);

                doc_terms.entry(word).or_default().push(Posting {
                    doc_id,
                    offset,
                    field_type,
                    section_id,
                    heading_level,
                    score,
                });
            }
            doc_terms
        })
        .collect();

    // REDUCE PHASE: Merge all per-document maps
    let mut terms: HashMap<String, Vec<Posting>> = HashMap::new();
    for doc_terms in per_doc_terms {
        for (term, postings) in doc_terms {
            terms.entry(term).or_default().extend(postings);
        }
    }

    // Build final posting lists: sort by score DESC, deduplicate, compute doc_freq
    let final_terms: HashMap<String, PostingList> = terms
        .into_par_iter()
        .map(|(term, mut postings)| {
            // INVARIANT: POSTING_LIST_SORTED_BY_SCORE
            // Sort by (score DESC, doc_id ASC) for fast top-k retrieval
            postings.sort_by(|a, b| {
                match b.score.partial_cmp(&a.score) {
                    Some(std::cmp::Ordering::Equal) | None => a.doc_id.cmp(&b.doc_id),
                    Some(ord) => ord,
                }
            });

            // Deduplicate by (doc_id, offset), keeping first (highest score) occurrence
            postings.dedup_by_key(|p| (p.doc_id, p.offset));

            // Calculate unique document IDs
            let mut doc_ids: Vec<usize> = postings.iter().map(|p| p.doc_id).collect();
            doc_ids.sort();
            doc_ids.dedup();

            (
                term,
                PostingList {
                    postings,
                    doc_freq: doc_ids.len(),
                },
            )
        })
        .collect();

    InvertedIndex {
        terms: final_terms,
        total_docs: texts.len(),
    }
}

/// Sequential version for non-parallel builds (WASM).
#[cfg(not(feature = "parallel"))]
pub fn build_inverted_index_parallel(
    texts: &[String],
    field_boundaries: &[FieldBoundary],
) -> InvertedIndex {
    let per_doc_terms: Vec<HashMap<String, Vec<Posting>>> = texts
        .iter()
        .enumerate()
        .map(|(doc_id, text)| {
            let text_len = text.len();
            let mut doc_terms: HashMap<String, Vec<Posting>> = HashMap::new();
            for (word, offset) in tokenize(text) {
                let (field_type, section_id, heading_level) =
                    get_field_info_for_inverted(doc_id, offset, field_boundaries);

                // INVARIANT: SCORE_PRECOMPUTED - compute score at index time
                let score = final_score(&field_type, offset, text_len);

                doc_terms.entry(word).or_default().push(Posting {
                    doc_id,
                    offset,
                    field_type,
                    section_id,
                    heading_level,
                    score,
                });
            }
            doc_terms
        })
        .collect();

    let mut terms: HashMap<String, Vec<Posting>> = HashMap::new();
    for doc_terms in per_doc_terms {
        for (term, postings) in doc_terms {
            terms.entry(term).or_default().extend(postings);
        }
    }

    // Build final posting lists: sort by score DESC, deduplicate, compute doc_freq
    let final_terms: HashMap<String, PostingList> = terms
        .into_iter()
        .map(|(term, mut postings)| {
            // INVARIANT: POSTING_LIST_SORTED_BY_SCORE
            // Sort by (score DESC, doc_id ASC) for fast top-k retrieval
            postings.sort_by(|a, b| {
                match b.score.partial_cmp(&a.score) {
                    Some(std::cmp::Ordering::Equal) | None => a.doc_id.cmp(&b.doc_id),
                    Some(ord) => ord,
                }
            });

            // Deduplicate by (doc_id, offset), keeping first (highest score) occurrence
            postings.dedup_by_key(|p| (p.doc_id, p.offset));

            // Calculate unique document IDs
            let mut doc_ids: Vec<usize> = postings.iter().map(|p| p.doc_id).collect();
            doc_ids.sort();
            doc_ids.dedup();

            (
                term,
                PostingList {
                    postings,
                    doc_freq: doc_ids.len(),
                },
            )
        })
        .collect();

    InvertedIndex {
        terms: final_terms,
        total_docs: texts.len(),
    }
}

/// Threshold configuration for index mode selection.
#[derive(Debug, Clone)]
pub struct IndexThresholds {
    /// Below this doc count, use suffix array only
    pub suffix_only_max_docs: usize,
    /// Below this total text size (bytes), use suffix array only
    pub suffix_only_max_bytes: usize,
    /// Above this doc count, use inverted index only (unless fuzzy needed)
    pub inverted_only_min_docs: usize,
}

impl Default for IndexThresholds {
    fn default() -> Self {
        Self {
            suffix_only_max_docs: 50,
            suffix_only_max_bytes: 100_000, // 100KB
            inverted_only_min_docs: 500,
        }
    }
}

/// Determine the best index mode based on content characteristics.
///
/// This is the build-time decision that determines which index(es) to create.
///
/// # Decision Logic
///
/// - Small content (few docs, small text): Suffix array only
///   - Simpler, good enough performance, supports all query types
/// - Large content with mostly exact queries: Inverted index only
///   - O(1) word lookup, efficient AND intersection
/// - Large content needing prefix/fuzzy: Hybrid
///   - Best of both, but larger index size
pub fn select_index_mode(
    doc_count: usize,
    total_text_bytes: usize,
    needs_prefix_matching: bool,
    needs_fuzzy_matching: bool,
    thresholds: &IndexThresholds,
) -> IndexMode {
    // Small content: suffix array is fine
    if doc_count <= thresholds.suffix_only_max_docs
        && total_text_bytes <= thresholds.suffix_only_max_bytes
    {
        return IndexMode::SuffixArrayOnly;
    }

    // Large content but needs prefix/fuzzy: must have suffix array
    if needs_prefix_matching || needs_fuzzy_matching {
        // Very large content: use hybrid
        if doc_count >= thresholds.inverted_only_min_docs {
            return IndexMode::Hybrid;
        }
        // Medium content: suffix array only is still ok
        return IndexMode::SuffixArrayOnly;
    }

    // Large content, exact matches only: inverted index
    if doc_count >= thresholds.inverted_only_min_docs {
        return IndexMode::InvertedIndexOnly;
    }

    // Medium content, no special needs: suffix array
    IndexMode::SuffixArrayOnly
}

/// Build a unified index based on content characteristics.
///
/// Automatically selects the best index mode and builds the appropriate indexes.
pub fn build_unified_index(
    docs: Vec<SearchDoc>,
    texts: Vec<String>,
    field_boundaries: Vec<FieldBoundary>,
    thresholds: &IndexThresholds,
    needs_prefix_matching: bool,
    needs_fuzzy_matching: bool,
) -> UnifiedIndex {
    let total_bytes: usize = texts.iter().map(|t| t.len()).sum();
    let mode = select_index_mode(
        docs.len(),
        total_bytes,
        needs_prefix_matching,
        needs_fuzzy_matching,
        thresholds,
    );

    let (suffix_array, lcp) = match mode {
        IndexMode::SuffixArrayOnly | IndexMode::Hybrid => {
            let suffix_index =
                crate::build_index(docs.clone(), texts.clone(), field_boundaries.clone());
            (Some(suffix_index.suffix_array), Some(suffix_index.lcp))
        }
        IndexMode::InvertedIndexOnly => (None, None),
    };

    let inverted_index = match mode {
        IndexMode::InvertedIndexOnly | IndexMode::Hybrid => {
            Some(build_inverted_index(&texts, &field_boundaries))
        }
        IndexMode::SuffixArrayOnly => None,
    };

    UnifiedIndex {
        docs,
        texts,
        field_boundaries,
        mode,
        suffix_array,
        lcp,
        inverted_index,
    }
}

/// Check if an inverted index is well-formed (debug assertion).
#[cfg(any(debug_assertions, test))]
#[allow(dead_code)]
pub fn check_inverted_index_well_formed(index: &InvertedIndex, texts: &[String]) -> bool {
    // Check total_docs
    if index.total_docs != texts.len() {
        return false;
    }

    for (term, list) in &index.terms {
        // Check non-empty
        if list.postings.is_empty() {
            return false;
        }

        // Check sortedness by (score DESC, doc_id ASC)
        for i in 1..list.postings.len() {
            let prev = &list.postings[i - 1];
            let curr = &list.postings[i];
            // Score should be descending (prev >= curr)
            // If equal score, doc_id should be ascending (prev <= curr)
            let score_ok = prev.score >= curr.score;
            let tiebreak_ok = prev.score != curr.score || prev.doc_id <= curr.doc_id;
            if !score_ok || !tiebreak_ok {
                return false;
            }
        }

        // Check doc_freq
        let mut doc_ids: Vec<usize> = list.postings.iter().map(|p| p.doc_id).collect();
        doc_ids.sort();
        doc_ids.dedup();
        if list.doc_freq != doc_ids.len() {
            return false;
        }

        // Check posting bounds
        let term_len = term.len();
        for posting in &list.postings {
            if posting.doc_id >= texts.len() {
                return false;
            }
            if posting.offset + term_len > texts[posting.doc_id].len() {
                return false;
            }
        }
    }

    true
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_tokenize_simple() {
        let tokens = tokenize("hello world");
        assert_eq!(tokens.len(), 2);
        assert_eq!(tokens[0].0, "hello");
        assert_eq!(tokens[0].1, 0);
        assert_eq!(tokens[1].0, "world");
        assert_eq!(tokens[1].1, 6);
    }

    #[test]
    fn test_tokenize_with_punctuation() {
        let tokens = tokenize("hello, world!");
        assert_eq!(tokens.len(), 2);
        assert_eq!(tokens[0].0, "hello");
        assert_eq!(tokens[1].0, "world");
    }

    #[test]
    fn test_tokenize_normalizes() {
        let tokens = tokenize("Hello WORLD");
        assert_eq!(tokens[0].0, "hello");
        assert_eq!(tokens[1].0, "world");
    }

    #[test]
    fn test_build_inverted_index_simple() {
        let texts = vec!["hello world".to_string(), "world peace".to_string()];
        let index = build_inverted_index(&texts, &[]);

        // Check "hello" appears in doc 0 only
        let hello = index.terms.get("hello").unwrap();
        assert_eq!(hello.postings.len(), 1);
        assert_eq!(hello.postings[0].doc_id, 0);
        assert_eq!(hello.doc_freq, 1);

        // Check "world" appears in both docs
        let world = index.terms.get("world").unwrap();
        assert_eq!(world.postings.len(), 2);
        assert_eq!(world.doc_freq, 2);
    }

    #[test]
    fn test_posting_list_sorted_by_score() {
        let texts = vec![
            "rust is great".to_string(),
            "rust programming".to_string(),
            "great rust code".to_string(),
        ];
        let index = build_inverted_index(&texts, &[]);

        let rust = index.terms.get("rust").unwrap();
        // Check sorted by score DESC, doc_id ASC
        for i in 1..rust.postings.len() {
            let prev = &rust.postings[i - 1];
            let curr = &rust.postings[i];
            // Score should be descending
            assert!(
                prev.score >= curr.score,
                "Posting list not sorted by score DESC at {}: {} < {}",
                i,
                prev.score,
                curr.score
            );
            // If equal score, doc_id should be ascending
            if prev.score == curr.score {
                assert!(
                    prev.doc_id <= curr.doc_id,
                    "Posting list not sorted by doc_id ASC at {}: {} > {}",
                    i,
                    prev.doc_id,
                    curr.doc_id
                );
            }
        }
    }

    #[test]
    fn test_title_beats_content_in_posting_order() {
        use crate::types::FieldBoundary;
        use crate::types::FieldType;

        // Doc 0: "rust" in content (at position 0)
        // Doc 1: "rust" in title (at position 0)
        let texts = vec![
            "rust programming language".to_string(),
            "rust is great".to_string(),
        ];

        // Mark doc 1's "rust" as title field
        let boundaries = vec![FieldBoundary {
            doc_id: 1,
            start: 0,
            end: 4, // "rust"
            field_type: FieldType::Title,
            section_id: None,
            heading_level: 0,
        }];

        let index = build_inverted_index(&texts, &boundaries);
        let rust = index.terms.get("rust").unwrap();

        // Title (doc 1) should come first due to higher score
        assert_eq!(rust.postings[0].doc_id, 1, "Title match should come first");
        assert_eq!(
            rust.postings[0].field_type,
            FieldType::Title,
            "First posting should be Title type"
        );

        // Content (doc 0) should come second
        assert_eq!(rust.postings[1].doc_id, 0, "Content match should come second");
        assert_eq!(
            rust.postings[1].field_type,
            FieldType::Content,
            "Second posting should be Content type"
        );

        // Verify scores
        assert!(
            rust.postings[0].score > rust.postings[1].score,
            "Title score ({}) should be greater than Content score ({})",
            rust.postings[0].score,
            rust.postings[1].score
        );
    }

    #[test]
    fn test_scores_are_precomputed() {
        let texts = vec!["hello world programming".to_string()];
        let index = build_inverted_index(&texts, &[]);

        let hello = index.terms.get("hello").unwrap();
        // Score should be precomputed (Content base = 1.0 + position bonus)
        // Position bonus for offset 0 in 23-char text = 0.5 * (1 - 0/23) = 0.5
        // Total = 1.5
        assert!(
            (hello.postings[0].score - 1.5).abs() < 0.01,
            "Score should be ~1.5 (Content base + position bonus), got {}",
            hello.postings[0].score
        );

        let world = index.terms.get("world").unwrap();
        // "world" at position 6 in 23-char text
        // Position bonus = 0.5 * (1 - 6/23) ≈ 0.37
        // Total ≈ 1.37
        assert!(
            world.postings[0].score > 1.0 && world.postings[0].score < 1.5,
            "Score should be between 1.0 and 1.5, got {}",
            world.postings[0].score
        );
    }

    #[test]
    fn test_index_mode_selection_small() {
        let thresholds = IndexThresholds::default();
        let mode = select_index_mode(10, 5000, false, false, &thresholds);
        assert_eq!(mode, IndexMode::SuffixArrayOnly);
    }

    #[test]
    fn test_index_mode_selection_large_exact() {
        let thresholds = IndexThresholds::default();
        let mode = select_index_mode(1000, 1_000_000, false, false, &thresholds);
        assert_eq!(mode, IndexMode::InvertedIndexOnly);
    }

    #[test]
    fn test_index_mode_selection_large_fuzzy() {
        let thresholds = IndexThresholds::default();
        let mode = select_index_mode(1000, 1_000_000, false, true, &thresholds);
        assert_eq!(mode, IndexMode::Hybrid);
    }

    #[test]
    fn test_check_inverted_index_well_formed() {
        let texts = vec!["hello world".to_string()];
        let index = build_inverted_index(&texts, &[]);
        assert!(check_inverted_index_well_formed(&index, &texts));
    }

    #[test]
    fn test_multiterm_and_with_presorted_postings() {
        // Test that multi-term AND queries work correctly with presorted posting lists
        // Both "hello" and "world" appear in doc 0, so AND should return doc 0
        // Only "hello" appears in doc 1, so it should NOT be in the AND result
        let texts = vec![
            "hello world".to_string(),      // doc 0: has both
            "hello there".to_string(),      // doc 1: only hello
            "goodbye world".to_string(),    // doc 2: only world
        ];
        let index = build_inverted_index(&texts, &[]);

        let hello = index.terms.get("hello").unwrap();
        let world = index.terms.get("world").unwrap();

        // Both posting lists should be sorted by score DESC
        assert!(!hello.postings.is_empty());
        assert!(!world.postings.is_empty());

        // Verify "hello" has docs 0 and 1
        let hello_docs: std::collections::HashSet<usize> =
            hello.postings.iter().map(|p| p.doc_id).collect();
        assert!(hello_docs.contains(&0));
        assert!(hello_docs.contains(&1));

        // Verify "world" has docs 0 and 2
        let world_docs: std::collections::HashSet<usize> =
            world.postings.iter().map(|p| p.doc_id).collect();
        assert!(world_docs.contains(&0));
        assert!(world_docs.contains(&2));

        // Intersection should be {0} (AND semantics)
        let intersection: std::collections::HashSet<usize> =
            hello_docs.intersection(&world_docs).cloned().collect();
        assert_eq!(intersection.len(), 1);
        assert!(intersection.contains(&0), "AND of 'hello' and 'world' should include doc 0");
    }

    #[test]
    fn test_multiterm_deduplication_within_doc() {
        // Test that a document appearing multiple times in a posting list
        // is properly deduplicated when scoring
        let texts = vec![
            // Title field with "rust" followed by content with "rust" and "rust"
            "rust programming rust language rust".to_string(),
        ];
        let boundaries = vec![
            crate::FieldBoundary {
                doc_id: 0,
                start: 0,
                end: 4,
                field_type: crate::FieldType::Title,
                section_id: None,
                heading_level: 0,
            },
            crate::FieldBoundary {
                doc_id: 0,
                start: 5,
                end: 35,
                field_type: crate::FieldType::Content,
                section_id: None,
                heading_level: 0,
            },
        ];
        let index = build_inverted_index(&texts, &boundaries);

        let rust = index.terms.get("rust").unwrap();

        // "rust" appears 3 times in doc 0, all postings should have doc_id = 0
        assert_eq!(rust.postings.len(), 3, "Should have 3 postings for 'rust'");

        // All postings should be for doc 0
        for posting in &rust.postings {
            assert_eq!(posting.doc_id, 0);
        }

        // First posting should be title (highest score), due to score DESC sorting
        assert_eq!(
            rust.postings[0].field_type,
            crate::FieldType::Title,
            "First posting should be Title (highest score)"
        );

        // Verify scores are sorted descending
        for i in 1..rust.postings.len() {
            assert!(
                rust.postings[i - 1].score >= rust.postings[i].score,
                "Scores should be sorted descending"
            );
        }
    }

    #[test]
    fn test_build_unified_index_suffix_only() {
        let docs = vec![crate::SearchDoc {
            id: 0,
            title: "Test".to_string(),
            excerpt: "".to_string(),
            href: "/test".to_string(),
            kind: "post".to_string(),
            category: None,
            author: None,
            tags: vec![],
        }];
        let texts = vec!["hello world".to_string()];
        let thresholds = IndexThresholds::default();

        let unified = build_unified_index(docs, texts, vec![], &thresholds, false, false);
        assert_eq!(unified.mode, IndexMode::SuffixArrayOnly);
        assert!(unified.suffix_array.is_some());
        assert!(unified.inverted_index.is_none());
    }
}
