use crate::fact::MemoFact;

/// Simple Jaccard similarity on space-tokenised key+value.
fn jaccard(a: &str, b: &str) -> f64 {
    let sa: std::collections::HashSet<&str> = a.split_whitespace().collect();
    let sb: std::collections::HashSet<&str> = b.split_whitespace().collect();
    let intersection = sa.intersection(&sb).count();
    let union = sa.union(&sb).count();
    if union == 0 {
        return 0.0;
    }
    intersection as f64 / union as f64
}

pub fn merge_duplicates(facts: &mut Vec<MemoFact>, threshold: f64) {
    let mut to_remove = std::collections::HashSet::new();
    for i in 0..facts.len() {
        if to_remove.contains(&i) {
            continue;
        }
        for j in (i + 1)..facts.len() {
            if to_remove.contains(&j) {
                continue;
            }
            let combined_a = format!("{} {}", facts[i].key, facts[i].value);
            let combined_b = format!("{} {}", facts[j].key, facts[j].value);
            if jaccard(&combined_a, &combined_b) >= threshold {
                // Keep the one with higher confidence
                if facts[i].confidence >= facts[j].confidence {
                    to_remove.insert(j);
                } else {
                    to_remove.insert(i);
                    break;
                }
            }
        }
    }
    let mut idx = 0;
    facts.retain(|_| {
        let keep = !to_remove.contains(&idx);
        idx += 1;
        keep
    });
}
