use std::borrow::Cow;
use std::collections::HashSet;

use reqwest::Url;
use select::document::Document;
use select::predicate::{Attr, Name};
use url::Host;

use crate::article::{
    ArticleContent, ArticleUrl, ALLOWED_FILE_EXT, BAD_DOMAINS, BAD_SEGMENTS, GOOD_SEGMENTS,
};
use crate::clean::{DefaultDocumentCleaner, DocumentCleaner};
use crate::date::{ArticleDate, RE_DATE_SEGMENTS_M_D_Y, RE_DATE_SEGMENTS_Y_M_D};

use crate::category::Category;
use crate::nlp::CATEGORY_STOPWORDS;
use crate::video::VideoNode;
use crate::Language;
use crate::extract_meta::meta_content;
use crate::extract_title::title;
use crate::extract_pb_date::publishing_date;
use crate::extract_authors::authors;
use crate::extract_node::article_node;
use crate::extract_favicon::favicon;
use crate::extract_meta_language::meta_language;
use crate::extract_thumbnail::meta_thumbnail_url;
use crate::extract_top_img::meta_img_url;
use crate::extract_urls::{all_urls, image_urls};
use crate::extract_base_url::base_url;
use crate::extract_meta_data::meta_data;
use crate::extract_canonical::canonical_link;
use crate::extract_videos::videos;

pub(crate) struct NodeValueQuery<'a> {
    pub name: Name<&'a str>,
    /// The name of the attribute that holds the `value` of the `attribute` to
    /// check against.
    pub attr: Attr<&'a str, &'a str>,
    /// The name of the attribute that holds the requested value.
    pub content_name: &'a str,
}

impl<'a> NodeValueQuery<'a> {
    pub fn new(
        name: Name<&'a str>,
        attr: Attr<&'a str, &'a str>,
        content_name: &'a str,
    ) -> NodeValueQuery<'a> {
        NodeValueQuery {
            name,
            attr,
            content_name,
        }
    }
}

// Re-export MetaNode from extract_meta_data module
pub use crate::extract_meta_data::MetaNode;

/// Used to retrieve all valuable information from a
/// [`select::document::Document`].
pub trait Extractor {
    /// Extract the article title using advanced heuristics.
    ///
    /// Extraction priority:
    /// 1. Try all known meta tags (TITLE_META_INFO) such as og:title, twitter:title, dc.title, etc.
    /// 2. If not found, try the longest <h1> element (must be >2 words).
    /// 3. If not found, try the <title> tag.
    /// 4. If all above fail, apply advanced heuristics:
    ///    - Compare filtered versions of <title>, <h1>, and meta tag values (case-insensitive, alphanumeric only)
    ///    - Prefer <h1> if it matches <title> or meta tag after filtering
    ///    - If <title> contains <h1> and meta tag, and <h1> is longer, use <h1>
    ///    - If <title> starts with meta tag, use meta tag
    ///    - Otherwise, split <title> on common delimiters and pick the best part
    ///    - Prefer <h1> if final candidate matches after filtering
    /// 5. Always postprocess the result using MOTLEY_REPLACEMENT and TITLE_REPLACEMENTS for cleanup.
    fn title<'a>(&self, doc: &'a Document) -> Option<Cow<'a, str>> {
        return title(doc);
    }

    /// Extract all the listed authors for the article.
    fn authors<'a>(&self, doc: &'a Document) -> Vec<Cow<'a, str>> {
        return authors(doc);
    }

    /// When the article was published (and last updated).
    fn publishing_date(&self, doc: &Document, base_url: Option<&Url>) -> Option<ArticleDate> {
        return publishing_date(doc, base_url);
    }

    /// Extract the favicon from a website.
    fn favicon(&self, doc: &Document, base_url: &Url) -> Option<Url> {
        favicon(doc, base_url)
    }

    /// Finds the href in the `<base>` tag.
    fn base_url(&self, doc: &Document) -> Option<Url> {
        base_url(doc)
    }

    /// Extract content language from meta tag.
    fn meta_language(&self, doc: &Document) -> Option<Language> {
        meta_language(doc)
    }

    /// Finds all `<meta>` nodes in the document.
    fn meta_data<'a>(&self, doc: &'a Document) -> Vec<MetaNode<'a>> {
        meta_data(doc)
    }


    /// Extract a given meta content form document.
    fn meta_content<'a, 'b>(
        &self,
        doc: &'a Document,
        attr: Attr<&'b str, &'b str>,
    ) -> Option<Cow<'a, str>> {
        return meta_content(doc, attr);
    }

    /// Extract the thumbnail for the article.
    fn meta_thumbnail_url(&self, doc: &Document, base_url: Option<&Url>) -> Option<Url> {
        meta_thumbnail_url(doc, base_url)
    }

    /// Extract the 'top img' as specified by the website.
    fn meta_img_url(&self, doc: &Document, base_url: Option<&Url>) -> Option<Url> {
        meta_img_url(doc, base_url)
    }

    /// Returns meta type of article, open graph protocol
    fn meta_type<'a>(&self, doc: &'a Document) -> Option<Cow<'a, str>> {
        self.meta_content(doc, Attr("property", "og:type"))
    }

    /// Returns site name of article, open graph protocol.
    fn meta_site_name<'a>(&self, doc: &'a Document) -> Option<Cow<'a, str>> {
        self.meta_content(doc, Attr("property", "og:site_name"))
    }

    /// If the article has meta description set in the source, use that
    fn meta_description<'a>(&self, doc: &'a Document) -> Option<Cow<'a, str>> {
        [("property", "description"), ("name", "description")]
            .iter()
            .filter_map(|(k, v)| self.meta_content(doc, Attr(k, v)))
            .map(|desc| {
                // Clean non-breaking spaces from description
                match desc {
                    Cow::Borrowed(s) => Cow::Owned(s.replace('\u{a0}', " ")),
                    Cow::Owned(s) => Cow::Owned(s.replace('\u{a0}', " "))
                }
            })
            .next()
    }

    /// If the article has meta keywords set in the source, use that.
    fn meta_keywords<'a>(&self, doc: &'a Document) -> Vec<Cow<'a, str>> {
        for (k, v) in &[
            ("property", "keywords"),
            ("name", "news_keywords"),
            ("name", "keywords"),
        ] {
            if let Some(keywords) = self.meta_content(doc, Attr(k, v)) {
                return match keywords {
                    Cow::Owned(s) => s
                        .split(',')
                        .map(str::trim)
                        .map(ToString::to_string)
                        .map(Cow::Owned)
                        .collect(),
                    Cow::Borrowed(s) => s.split(',').map(str::trim).map(Cow::Borrowed).collect(),
                };
            }
        }
        Vec::new()
    }

    /// Get the full text of the article.
    fn text<'a>(&self, doc: &'a Document, lang: Language) -> Option<Cow<'a, str>> {
        self.text_with_cleaner(doc, lang, DefaultDocumentCleaner)
    }

    /// Get the full text of the article with a designated `DocumentCleaner`
    fn text_with_cleaner<'a, T: DocumentCleaner>(
        &self,
        doc: &'a Document,
        lang: Language,
        cleaner: T,
    ) -> Option<Cow<'a, str>> {
        article_node(doc, lang)
            .map(|n| cleaner.clean_node_text(*n).into())
    }

    /// Extract the `href` attribute for all `<a>` tags of the document.
    fn all_urls<'a>(&self, doc: &'a Document) -> Vec<Cow<'a, str>> {
        all_urls(doc)
    }

    /// Finds all urls from the document that might point to an article.
    fn article_urls(&self, doc: &Document, base_url: Option<&Url>) -> Vec<ArticleUrl> {
        let options = Url::options().base_url(base_url);
        let mut uniques = HashSet::new();
        let q = doc
            .find(Name("a"))
            .filter_map(|n| {
                if let Some(href) = n.attr("href").map(str::trim) {
                    Some((href, n.as_text().map(str::trim)))
                } else {
                    None
                }
            })
            .filter(|(href, _)| uniques.insert(*href))
            .filter_map(|(link, title)| {
                options
                    .parse(link)
                    .map(|url| ArticleUrl::new_with_title(url, title))
                    .ok()
            });
        if let Some(base_url) = base_url {
            q.filter(|article| Self::is_article(article, base_url))
                .collect()
        } else {
            q.collect()
        }
    }

    /// Extract all of the images of the document.
    fn image_urls(&self, doc: &Document, base_url: Option<&Url>) -> Vec<Url> {
        image_urls(doc, base_url)
    }

    /// First, perform basic format and domain checks like making sure the
    /// format of the url.
    ///
    ///
    /// We also filter out articles with a subdomain or first degree path on a
    /// registered bad keyword.
    fn is_article(article: &ArticleUrl, base_url: &Url) -> bool {
        if article.url.path().starts_with('#') {
            return false;
        }
        if !is_valid_domain(&article.url, base_url) {
            return false;
        }
        let mut path_segments = Vec::new();
        if let Some(segments) = article.url.path_segments() {
            for segment in segments {
                let segment = segment.to_lowercase().to_lowercase();
                if BAD_SEGMENTS.contains(&segment.as_str()) {
                    return false;
                }
                if !segment.is_empty() {
                    path_segments.push(segment);
                }
            }

            if path_segments.is_empty() {
                return false;
            }

            let last_segment = path_segments.remove(path_segments.len() - 1);

            // check file extensions
            let mut iter = last_segment.rsplitn(2, '.');
            let after = iter.next();
            if let Some(segment) = iter.next() {
                let extension = after.unwrap().to_lowercase();
                if ALLOWED_FILE_EXT.contains(&extension.as_str()) {
                    // assumption that the article slug is also the filename like `some-title.html`
                    if segment.len() > 10 {
                        path_segments.push(segment.to_string());
                    }
                } else if extension.chars().all(char::is_numeric) {
                    // prevents false negatives for articles with a trailing id like `1.343234`
                    path_segments.push(last_segment);
                } else {
                    return false;
                }
            } else {
                path_segments.push(last_segment);
            }
        } else {
            return false;
        }

        // check if the url has a news slug title
        if let Some(last_segment) = path_segments.last() {
            let (dash_count, underscore_count) = count_dashes_and_underscores(last_segment);

            if dash_count > 4 || underscore_count > 4 {
                if let Some(Host::Domain(domain)) = article.url.host() {
                    if let Some(domain) = domain.split('.').rev().nth(1) {
                        let delim = if underscore_count > dash_count {
                            '_'
                        } else {
                            '-'
                        };
                        if last_segment.split(delim).all(|s| s != domain) {
                            return true;
                        }
                    } else {
                        return true;
                    }
                } else {
                    return true;
                }
            }
        }

        // check for dates
        if RE_DATE_SEGMENTS_Y_M_D.is_match(article.url.path()) {
            return true;
        }
        if RE_DATE_SEGMENTS_M_D_Y.is_match(article.url.path()) {
            return true;
        }

        // allow good paths
        if path_segments.len() > 1 {
            for segment in path_segments.iter() {
                if GOOD_SEGMENTS.contains(&segment.as_str()) {
                    return true;
                }
            }
        }

        false
    }

    fn is_category(category: &Category, base_url: &Url) -> bool {
        if category.url.path().starts_with("/#") {
            return false;
        }

        if let Some(segments) = category.url.path_segments() {
            for (i, segment) in segments.enumerate() {
                if i > 0 {
                    return false;
                }
                if CATEGORY_STOPWORDS.contains(&segment) || segment == "index.html" {
                    return false;
                }
            }
        }

        if category.url.scheme() != base_url.scheme() {
            return false;
        }

        is_valid_domain(&category.url, base_url)
    }

    /// Finds all of the top level urls, assuming that these are the category
    /// urls.
    fn categories(&self, doc: &Document, base_url: &Url) -> Vec<Category> {
        let options = Url::options().base_url(Some(base_url));
        let category_urls: HashSet<_> = self
            .all_urls(doc)
            .into_iter()
            .filter_map(|url| options.parse(&*url).ok())
            .map(|mut url| {
                url.set_query(None);
                url
            })
            .collect();

        category_urls
            .into_iter()
            .map(Category::new)
            .filter(|cat| Self::is_category(cat, base_url))
            .collect()
    }

    /// Gathers all items for an article from the document.
    fn article_content<'a>(
        &self,
        doc: &'a Document,
        base_url: Option<&Url>,
        lang: Option<Language>,
    ) -> ArticleContent<'a> {
        let mut builder = ArticleContent::builder()
            .authors(self.authors(doc))
            .keywords(self.meta_keywords(doc));

        let lang = if let Some(meta_lang) = self.meta_language(doc) {
            builder = builder.language(meta_lang.clone());
            meta_lang
        } else {
            lang.unwrap_or_default()
        };

        if let Some(txt_node) = article_node(doc, lang) {
            builder = builder
                .videos(
                    txt_node
                        .videos()
                        .into_iter()
                        .filter_map(|x| x.get_src_url(base_url))
                        .filter_map(|url| url.ok())
                        .collect(),
                )
                .references(txt_node.references())
                .text(txt_node.clean_text().into())
                .images(txt_node.images(base_url));
        }

        if let Some(description) = self.meta_description(doc) {
            builder = builder.description(description);
        }
        if let Some(thumbnail) = self.meta_thumbnail_url(doc, base_url) {
            builder = builder.thumbnail(thumbnail);
        }
        if let Some(title) = self.title(doc) {
            builder = builder.title(title);
        }
        if let Some(date) = self.publishing_date(doc, base_url) {
            builder = builder.publishing_date(date);
        }
        if let Some(img) = self.meta_img_url(doc, base_url) {
            builder = builder.top_image(img);
        }
        builder.build()
    }

    /// Return the article's canonical URL
    ///
    /// Gets the first available value of:
    ///   1. The rel=canonical tag
    ///   2. The og:url tag
    fn canonical_link(&self, doc: &Document) -> Option<Url> {
        canonical_link(doc)
    }

    /// All video content in the article.
    fn videos<'a>(&self, doc: &'a Document, lang: Option<Language>) -> Vec<VideoNode<'a>> {
        videos(doc, lang)
    }
}

fn count_dashes_and_underscores<T: AsRef<str>>(s: T) -> (usize, usize) {
    let s = s.as_ref();
    s.chars().fold((0, 0), |(dashes, unders), c| {
        if c == '-' {
            (dashes + 1, unders)
        } else if c == '_' {
            (dashes, unders + 1)
        } else {
            (dashes, unders)
        }
    })
}

/// Checks whether the `url`'s domain contains at least the `base_url` domain as
/// a subdomain.
fn is_valid_domain(url: &Url, base_url: &Url) -> bool {
    // check for subdomains
    if let Some(Host::Domain(domain)) = url.host() {
        let base_subdomains = base_url.domain().map(|x| x.split('.').collect::<Vec<_>>());

        if let Some(parent_domains) = &base_subdomains {
            let candidate_domains: Vec<_> = domain.split('.').collect();

            if parent_domains.iter().all(|d| candidate_domains.contains(d)) {
                // check for mobile
                if candidate_domains.iter().any(|d| *d == "m" || *d == "i") {
                    return false;
                }
                if candidate_domains
                    .iter()
                    .all(|d| !CATEGORY_STOPWORDS.contains(d) && !BAD_DOMAINS.contains(d))
                {
                    return true;
                }
            }
        }
    } else {
        // check host equality
        return base_url.host() == url.host();
    }
    false
}

/// An Extractor that only uses the default implementation in the `Extractor`
/// trait.
#[derive(Debug, Default)]
pub struct DefaultExtractor;

impl Extractor for DefaultExtractor {}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn detect_articles() {
        macro_rules! assert_articles {
            ($base:expr => $($url:expr,)*) => {
                let base_url = Url::parse($base).unwrap();
                $(
                    let article = ArticleUrl::new(Url::parse($url).unwrap());
                    assert!(DefaultExtractor::is_article(&article, &base_url));
                )*
            };
        }

        assert_articles!(
               "https://extrablatt.com" =>
               "https://extrablatt.com/politics/live-news/some-title-12-05-2019/index.html",
               "https://extrablatt.com/2019/12/04/us/politics/some-title.html",
               "https://www.extrablatt.com/graphics/2019/investigations/some-title/",
               "https://extrablatt.com/2019/12/06/uk/some-longer-title-with-dashes/index.html",
               "https://www.extrablatt.com/politik/some-longer-title-with-dashes-interview-1.347823?reduced=true",
               "https://www.extrablatt.com/auto/some_longer_title_with_underscores_1300105.html",
                "https://extrablatt.com/hmm-some-very-long-title-speparated",
        );
    }
}
