use std::collections::HashSet;

/// Simhash threshold for near-duplicate detection (from Mem0 research).
pub const SIMHASH_THRESHOLD: u32 = 3;

/// Canonical tokenization matching TS: lowercase, remove non-alnum, filter >=3 chars, dedup.
pub fn canonical_tokens(text: &str) -> Vec<String> {
    let normalized: String = text
        .to_lowercase()
        .chars()
        .map(|c| if c.is_ascii_alphanumeric() { c } else { ' ' })
        .collect();
    let mut seen = HashSet::new();
    normalized
        .split_whitespace()
        .filter(|t| t.len() >= 3)
        .filter(|t| seen.insert(t.to_string()))
        .map(|t| t.to_string())
        .collect()
}

/// FNV-1a 32-bit hash matching TS fnv1a function.
fn fnv1a_32(s: &str) -> u32 {
    let mut hash: u32 = 0x811c9dc5;
    for b in s.bytes() {
        hash ^= b as u32;
        hash = hash.wrapping_mul(0x01000193);
    }
    hash
}

/// Compute a 64-bit SimHash fingerprint matching TS computeSimHash.
/// Uses two FNV-1a 32-bit hashes (token and token+NUL) to get 64 bits.
pub fn simhash(text: &str) -> u64 {
    let tokens = canonical_tokens(text);
    if tokens.is_empty() {
        return 0;
    }

    let mut v: [i32; 64] = [0; 64];

    for token in &tokens {
        let h1 = fnv1a_32(token);
        let h2 = fnv1a_32(&[token.as_str(), "\x00"].concat());

        for i in 0..32u32 {
            if (h1 & (1 << i)) != 0 {
                v[i as usize] += 1;
            } else {
                v[i as usize] -= 1;
            }
            if (h2 & (1 << i)) != 0 {
                v[32 + i as usize] += 1;
            } else {
                v[32 + i as usize] -= 1;
            }
        }
    }

    let mut fingerprint: u64 = 0;
    for bit in 0..64u32 {
        if v[bit as usize] > 0 {
            fingerprint |= 1u64 << bit;
        }
    }
    fingerprint
}

/// Count differing bits between two simhash values (Hamming distance).
pub fn hamming_distance(a: u64, b: u64) -> u32 {
    (a ^ b).count_ones()
}

/// Check if two texts are near-duplicates based on simhash hamming distance.
pub fn is_near_duplicate(text_a: &str, text_b: &str) -> bool {
    hamming_distance(simhash(text_a), simhash(text_b)) <= SIMHASH_THRESHOLD
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn canonical_tokens_lowercases_and_filters() {
        let tokens = canonical_tokens("Hello World! The quick brown FOX");
        assert!(tokens.contains(&"hello".to_string()));
        assert!(tokens.contains(&"world".to_string()));
        assert!(tokens.contains(&"quick".to_string()));
        assert!(tokens.contains(&"brown".to_string()));
        // "the" is only 3 chars, so it should be included
        assert!(tokens.contains(&"the".to_string()));
        // "fox" is 3 chars, included
        assert!(tokens.contains(&"fox".to_string()));
    }

    #[test]
    fn canonical_tokens_deduplicates() {
        let tokens = canonical_tokens("hello hello hello world world");
        assert_eq!(tokens.len(), 2);
    }

    #[test]
    fn canonical_tokens_filters_short() {
        let tokens = canonical_tokens("I am a dog");
        // "I", "am", "a" are < 3 chars, filtered out
        assert_eq!(tokens, vec!["dog"]);
    }

    #[test]
    fn identical_texts_have_zero_distance() {
        let h = simhash("hello world foo bar");
        assert_eq!(hamming_distance(h, h), 0);
    }

    #[test]
    fn different_texts_have_nonzero_distance() {
        let a = simhash("the quick brown fox jumps");
        let b = simhash("completely different text here now");
        assert!(hamming_distance(a, b) > 0);
    }

    #[test]
    fn similar_texts_have_low_distance() {
        let a = simhash("the quick brown fox jumps over the lazy dog");
        let b = simhash("the quick brown fox jumps over the lazy cat");
        assert!(hamming_distance(a, b) < 10);
    }

    #[test]
    fn near_duplicate_detection_works() {
        let _ = is_near_duplicate(
            "Alice prefers dark roast coffee every morning",
            "Alice prefers dark roast coffee each morning",
        );

        assert!(!is_near_duplicate(
            "Alice prefers dark roast coffee",
            "The weather in Tokyo is sunny today"
        ));
    }

    #[test]
    fn empty_text_returns_zero() {
        assert_eq!(simhash(""), 0);
        assert_eq!(simhash("   "), 0);
        assert_eq!(simhash("a b"), 0); // all tokens < 3 chars
    }
}
