// WIP
pub struct Terms {
    list: Vec<String>,
    repeated: HashSet<String>
}

impl Terms {
    pub fn new() -> Self {
        let list: Vec<String> = Vec::new();
        let repeated: HashSet<String> = HashSet::new();

        Self {
            list: list,
            repeated: repeated
        }
    }

    pub fn add_to_list(&mut self, word: &String) {
        self.list.push(word.clone());
    }

    pub fn add_to_repeated(&mut self, word: &String) {
        self.repeated.insert(word.clone());
    }

    pub fn is_repeated_often(&self, key: &str) -> bool {
        self.list.clone().into_iter().collect::<Vec<_>>().iter().filter(|&x| &x.to_string() == &key.to_string()).count() > 2
    }
}

fn scrape_file(path: PathBuf, dictionary: &Dictionary) {
    let file = path.display().to_string();
    let file_type = file.split(".").last().unwrap();
    let file_type_is_accepted = ACCEPTED_FILE_TYPES.contains(&file_type);

    if !file_type_is_accepted {
        return;
    }

    let source = fs::read_to_string(file).unwrap();
    let mut terms = Terms::new();

    if !is_english(source.as_str()) {
        return;
    }

    for line in source.lines() {
        for word in line.split_whitespace() {
            let target = word.to_lowercase();
            let target = sanitize(target);

            let is_in_dictionary = &dictionary.get(&&*target).is_some();

            if !is_in_dictionary {
                terms.add_to_list(&target);

                if terms.is_repeated_often(&target) {
                    terms.add_to_repeated(&word.to_string());
                }
            }
        }
    }

    let mut words: Vec<String> = terms.repeated.into_iter().collect();

    words.sort();

    println!("📄 {}: {} words found", path.display().to_string().green().bold(), words.len());

    let mut new_file = fs::OpenOptions::new()
        .write(true)
        .create(true)
        .append(true)
        .open("scraped.txt")
        .unwrap();

    for word in words {
        new_file.write_all(word.as_bytes()).unwrap();
        new_file.write_all("📄 {}: {} words found", path.display().to_string()).unwrap();
    }
}

fn scrape_for_words(folder: &str, dictionary: &Dictionary) {
    check_path(folder, scrape_file, &dictionary);
}