mod context;

use self::context::Context;
use crate::{
    config::Config,
    document_output::DocumentOutput,
    document_type::DocumentType,
    element::Element,
    element_output::ElementOutput,
    error::{Error, ItemError},
    html_parser::HtmlParser,
    http_client::{HttpClient, ROBOTS_PATH},
    item_output::ItemOutput,
    request::Request,
    response::Response,
    robot_list::RobotList,
};
use alloc::{collections::BTreeSet, sync::Arc};
use core::{iter, str};
use futures::{Stream, StreamExt, future::try_join_all};
use muffy_document::html::Node;
use sitemaps::{Sitemaps, siteindex::SiteIndex, sitemap::Sitemap};
use std::collections::HashMap;
use tokio::{spawn, sync::mpsc::channel, task::JoinHandle};
use tokio_stream::wrappers::ReceiverStream;
use url::Url;

type ElementFuture = (Element, Vec<JoinHandle<Result<ItemOutput, ItemError>>>);

const JOB_CAPACITY: usize = 1 << 16;
const JOB_COMPLETION_BUFFER: usize = 1 << 8;

const DOCUMENT_SCHEMES: &[&str] = &["http", "https"];
const FRAGMENT_ATTRIBUTES: &[&str] = &["id", "name"];
const META_LINK_PROPERTIES: &[&str] = &[
    "og:image",
    "og:audio",
    "og:video",
    "og:image:url",
    "og:image:secure_url",
    "twitter:image",
];
const LINK_ORIGIN_RELATIONS: &[&str] = &["dns-prefetch", "preconnect"];

/// A web validator.
pub struct WebValidator(Arc<WebValidatorInner>);

struct WebValidatorInner {
    http_client: HttpClient,
    html_parser: HtmlParser,
}

impl WebValidator {
    /// Creates a web validator.
    pub fn new(http_client: HttpClient, html_parser: HtmlParser) -> Self {
        Self(
            WebValidatorInner {
                http_client,
                html_parser,
            }
            .into(),
        )
    }

    fn cloned(&self) -> Self {
        Self(self.0.clone())
    }

    /// Validates websites recursively.
    pub async fn validate(
        &self,
        config: &Config,
    ) -> Result<impl Stream<Item = Result<DocumentOutput, Error>> + use<>, Error> {
        let (sender, receiver) = channel(JOB_CAPACITY);
        let context = Arc::new(Context::new(sender, config.clone()));

        try_join_all(config.roots().map(|url| {
            self.cloned()
                .validate_link(context.clone(), url.into(), None)
        }))
        .await?;

        Ok(ReceiverStream::new(receiver)
            .map(Box::into_pin)
            .buffer_unordered(JOB_COMPLETION_BUFFER))
    }

    async fn validate_link(
        self,
        context: Arc<Context>,
        url: String,
        document_type: Option<DocumentType>,
    ) -> Result<ItemOutput, ItemError> {
        let url = Url::parse(&url)?;

        if context
            .config()
            .ignored_links()
            .any(|pattern| pattern.is_match(url.as_str()))
        {
            return Ok(ItemOutput::new());
        } else if document_type != Some(DocumentType::Robots) {
            let _ = Box::into_pin(Box::new(self.cloned().validate_link(
                context.clone(),
                url.join(ROBOTS_PATH)?.into(),
                Some(DocumentType::Robots),
            )))
            .await;
        }

        let mut document_url = url.clone();
        // We keep this fragment removal not configurable as otherwise we might have a
        // lot more requests for the same HTML pages, which makes crawling
        // unacceptably inefficient.
        document_url.set_fragment(None);

        let site = context.config().site(&url);
        let Some(response) = self
            .0
            .http_client
            .get(
                &Request::new(document_url, site.headers().clone())
                    .set_max_age(site.cache().max_age())
                    .set_max_redirects(site.max_redirects())
                    .set_retry(site.retry().clone())
                    .set_site_id(site.id().cloned())
                    .set_timeout(site.timeout()),
            )
            .await?
        else {
            return Ok(ItemOutput::default());
        };

        if !context
            .config()
            .site(&url)
            .status()
            .accepted(response.status())
        {
            return Err(ItemError::HttpStatus(response.status()));
        }

        let Some(document_type) = Self::validate_document_type(&response, document_type)? else {
            return Ok(ItemOutput::new().with_response(response));
        };

        if let Some(fragment) = url.fragment()
            && document_type == DocumentType::Html
            && !site.fragments_ignored()
            && !self.has_html_element(&response, fragment).await?
        {
            return Err(ItemError::HtmlElementNotFound(fragment.into()));
        }

        if url
            .host_str()
            .map(|host| {
                context
                    .config()
                    .sites()
                    .get(host)
                    .map(|sites| {
                        sites.iter().any(|(path, config)| {
                            url.path().starts_with(path) && config.recursive()
                        })
                    })
                    .unwrap_or_default()
            })
            .unwrap_or_default()
            && context
                .documents()
                .insert_async(response.url().to_string())
                .await
                .is_ok()
        {
            let handle = spawn({
                let context = context.clone();
                let response = response.clone();

                async move {
                    self.validate_document(context, response, document_type)
                        .await
                }
            });

            context
                .job_sender()
                .send(Box::new(async move { handle.await? }))
                .await
                .unwrap();
        }

        Ok(ItemOutput::new().with_response(response))
    }

    async fn validate_document(
        &self,
        context: Arc<Context>,
        response: Arc<Response>,
        document_type: DocumentType,
    ) -> Result<DocumentOutput, Error> {
        let futures = match document_type {
            DocumentType::Html => self.validate_html(&context, &response).await?,
            DocumentType::Robots => self.validate_robots(&context, &response)?,
            DocumentType::Sitemap => self.validate_sitemap(&context, &response)?,
        };
        let (elements, futures) = futures.into_iter().unzip::<_, _, Vec<_>, Vec<_>>();

        Ok(DocumentOutput::new(
            response.url().clone(),
            elements
                .into_iter()
                .zip(try_join_all(futures.into_iter().map(try_join_all)).await?)
                .map(|(element, results)| ElementOutput::new(element, results))
                .collect(),
        ))
    }

    async fn validate_element_link(
        self,
        context: Arc<Context>,
        url: String,
        base: Arc<Url>,
        document_type: Option<DocumentType>,
    ) -> Result<ItemOutput, ItemError> {
        let url = Url::parse(&Self::normalize_url(&url)).or_else(|_| base.join(&url))?;

        if !DOCUMENT_SCHEMES.contains(&url.scheme()) {
            Ok(ItemOutput::new())
        } else if context.config().site(&url).scheme().accepted(url.scheme()) {
            self.validate_link(context, url.to_string(), document_type)
                .await
        } else if context
            .config()
            .ignored_links()
            .any(|pattern| pattern.is_match(url.as_str()))
        {
            Ok(ItemOutput::new())
        } else {
            Err(ItemError::InvalidScheme(url.scheme().into()))
        }
    }

    fn normalize_url(url: &str) -> String {
        url.split_whitespace().collect()
    }

    async fn validate_html(
        &self,
        context: &Arc<Context>,
        response: &Arc<Response>,
    ) -> Result<Vec<ElementFuture>, Error> {
        let mut futures = vec![];
        let document = self.0.html_parser.parse(response).await?;
        let base = document
            .base()
            .map(|href| response.url().join(href))
            .transpose()?
            .unwrap_or_else(|| response.url().clone())
            .into();

        for node in document.children() {
            self.validate_html_element(context, &base, node, &mut futures)?;
        }

        Ok(futures)
    }

    fn validate_html_element(
        &self,
        context: &Arc<Context>,
        base: &Arc<Url>,
        node: &Node,
        futures: &mut Vec<ElementFuture>,
    ) -> Result<(), Error> {
        if let Node::Element(element) = &node {
            if let Some(future) = self.validate_html_element_content(context, base, element) {
                futures.push(future);
            }

            for node in element.children() {
                self.validate_html_element(context, base, node, futures)?;
            }
        }

        Ok(())
    }

    fn validate_html_element_content(
        &self,
        context: &Arc<Context>,
        base: &Arc<Url>,
        element: &muffy_document::html::Element,
    ) -> Option<ElementFuture> {
        let attributes = HashMap::<_, _>::from_iter(element.attributes());
        let mut links = vec![];

        match element.name() {
            "base" => {}
            "link" => {
                if !attributes
                    .get("rel")
                    .map(|rel| LINK_ORIGIN_RELATIONS.contains(rel))
                    .unwrap_or_default()
                    && let Some(value) = attributes.get("href")
                {
                    links.push((
                        vec![("href", value)],
                        vec![(
                            value.to_string(),
                            (attributes.get("rel") == Some(&"sitemap"))
                                .then_some(DocumentType::Sitemap),
                        )],
                    ));
                }
            }
            "meta" => {
                if let Some(content) = attributes.get("content")
                    && let Some(property) = attributes.get("property")
                    && META_LINK_PROPERTIES.contains(property)
                {
                    links.push((
                        vec![("property", property), ("content", content)],
                        vec![(content.to_string(), None)],
                    ));
                }
            }
            _ => {
                if let Some(value) = attributes.get("href") {
                    links.push((vec![("href", value)], vec![(value.to_string(), None)]));
                }

                if let Some(value) = attributes.get("src") {
                    links.push((vec![("src", value)], vec![(value.to_string(), None)]));
                }

                if let Some(value) = attributes.get("srcset") {
                    links.push((
                        vec![("srcset", value)],
                        Self::parse_srcset(value)
                            .map(|url| (url.into(), None))
                            .collect(),
                    ));
                }
            }
        }

        let validation_result =
            if let Some(config) = context.config().site(base).validation().html() {
                muffy_validation::validate_html_element(
                    element,
                    config.ignored_attributes(),
                    config.ignored_elements(),
                )
            } else {
                Ok(())
            };

        let mut items = links
            .iter()
            .flat_map(|(_, links)| {
                links.iter().map(|(link, document_type)| {
                    spawn(self.cloned().validate_element_link(
                        context.clone(),
                        link.to_string(),
                        base.clone(),
                        *document_type,
                    ))
                })
            })
            .collect::<Vec<_>>();

        if let Err(error) = &validation_result {
            match error {
                muffy_validation::MarkupError::UnknownTag(_) => {
                    items.push(spawn({
                        let error = error.clone();
                        async move { Err(ItemError::HtmlValidation(error)) }
                    }));
                }
                muffy_validation::MarkupError::InvalidElement {
                    attributes,
                    children,
                } => {
                    for name in attributes.keys() {
                        items.push(spawn({
                            let error = muffy_validation::MarkupError::InvalidElement {
                                attributes: [(
                                    name.clone(),
                                    [muffy_validation::AttributeError::NotAllowed].into(),
                                )]
                                .into(),
                                children: Default::default(),
                            };
                            async move { Err(ItemError::HtmlValidation(error)) }
                        }));
                    }

                    for name in children.keys() {
                        items.push(spawn({
                            let error = muffy_validation::MarkupError::InvalidElement {
                                attributes: Default::default(),
                                children: [(
                                    name.clone(),
                                    [muffy_validation::ChildError::NotAllowed].into(),
                                )]
                                .into(),
                            };
                            async move { Err(ItemError::HtmlValidation(error)) }
                        }));
                    }
                }
            }
        }

        if items.is_empty() {
            None
        } else {
            Some((
                Element::new(
                    element.name().into(),
                    links
                        .iter()
                        .flat_map(|(attributes, _)| attributes.iter().map(|(name, _)| *name))
                        .chain(
                            if let Err(muffy_validation::MarkupError::InvalidElement {
                                attributes,
                                ..
                            }) = &validation_result
                            {
                                attributes.keys().map(AsRef::as_ref).collect::<Vec<_>>()
                            } else {
                                Default::default()
                            },
                        )
                        .collect::<BTreeSet<_>>()
                        .into_iter()
                        .filter_map(|name| {
                            attributes
                                .get(name)
                                .map(|value| (name.to_string(), value.to_string()))
                        })
                        .collect(),
                ),
                items,
            ))
        }
    }

    fn validate_robots(
        &self,
        context: &Arc<Context>,
        response: &Arc<Response>,
    ) -> Result<Vec<ElementFuture>, Error> {
        Ok(RobotList::parse(str::from_utf8(response.body())?)
            .sitemaps()
            .map(|url| {
                (
                    Element::new("sitemap".into(), vec![]),
                    vec![spawn(self.cloned().validate_link(
                        context.clone(),
                        url.to_owned(),
                        Some(DocumentType::Sitemap),
                    ))],
                )
            })
            .collect::<Vec<_>>())
    }

    fn validate_sitemap(
        &self,
        context: &Arc<Context>,
        response: &Arc<Response>,
    ) -> Result<Vec<ElementFuture>, Error> {
        Ok(match SiteIndex::read_from(response.body()) {
            Ok(site_index) if !site_index.entries.is_empty() => site_index
                .entries
                .iter()
                .map(|entry| {
                    (
                        Element::new("loc".into(), vec![]),
                        vec![spawn(self.cloned().validate_link(
                            context.clone(),
                            entry.loc.clone(),
                            Some(DocumentType::Sitemap),
                        ))],
                    )
                })
                .collect::<Vec<_>>(),
            _ => {
                let sitemap = Sitemap::read_from(response.body())?;

                sitemap
                    .entries
                    .iter()
                    .map(|entry| {
                        (
                            Element::new("loc".into(), vec![]),
                            vec![spawn(self.cloned().validate_link(
                                context.clone(),
                                entry.loc.clone(),
                                None,
                            ))],
                        )
                    })
                    .collect::<Vec<_>>()
            }
        })
    }

    fn validate_document_type(
        response: &Response,
        document_type: Option<DocumentType>,
    ) -> Result<Option<DocumentType>, ItemError> {
        let Some(value) = response.headers().get("content-type") else {
            return Ok(document_type);
        };
        let Some(value) = value.as_bytes().split(|byte| *byte == b';').next() else {
            return Ok(document_type);
        };
        let value = str::from_utf8(value)?;

        Ok(match document_type {
            Some(DocumentType::Html) => {
                if value != "text/html" {
                    return Err(ItemError::ContentTypeInvalid {
                        actual: value.into(),
                        expected: "text/html",
                    });
                }

                document_type
            }
            Some(DocumentType::Robots) => {
                if value != "text/plain" {
                    return Err(ItemError::ContentTypeInvalid {
                        actual: value.into(),
                        expected: "text/plain",
                    });
                }

                document_type
            }
            Some(DocumentType::Sitemap) => {
                if !value.ends_with("/xml") {
                    return Err(ItemError::ContentTypeInvalid {
                        actual: value.into(),
                        expected: "*/xml",
                    });
                }

                document_type
            }
            None => (value == "text/html").then_some(DocumentType::Html),
        })
    }

    async fn has_html_element(
        &self,
        response: &Arc<Response>,
        id: &str,
    ) -> Result<bool, ItemError> {
        Ok(self
            .0
            .html_parser
            .parse(response)
            .await?
            .children()
            .any(|node| Self::has_html_element_in_node(node, id)))
    }

    fn has_html_element_in_node(node: &Node, id: &str) -> bool {
        if let Node::Element(element) = &node {
            element
                .attributes()
                .any(|(name, value)| FRAGMENT_ATTRIBUTES.contains(&name) && value == id)
                || element
                    .children()
                    .any(|node| Self::has_html_element_in_node(node, id))
        } else {
            false
        }
    }

    fn parse_srcset(srcset: &str) -> impl Iterator<Item = &str> + '_ {
        let mut rest = srcset;

        iter::from_fn(move || {
            rest = rest.trim_start_matches(|char: char| char.is_whitespace() || char == ',');

            let url =
                rest[..rest.find(char::is_whitespace).unwrap_or(rest.len())].trim_end_matches(',');

            rest = &rest[rest[url.len()..]
                .find(',')
                .map_or(rest.len(), |index| url.len() + index)..];

            (!url.is_empty()).then_some(url)
        })
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::{
        Metrics, MokaCache, SchemeConfig,
        config::{Config, MarkupConfig, SiteConfig},
        html_parser::HtmlParser,
        http_client::{BareHttpClient, StubHttpClient, build_stub_response},
        timer::StubTimer,
    };
    use futures::{Stream, StreamExt};
    use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
    use indoc::indoc;
    use pretty_assertions::assert_eq;
    use regex::Regex;
    use url::Url;

    async fn validate(
        client: impl BareHttpClient + 'static,
        url: &str,
    ) -> Result<impl Stream<Item = Result<DocumentOutput, Error>>, Error> {
        let url = Url::parse(url).unwrap();

        WebValidator::new(
            HttpClient::new(client, StubTimer::new(), Box::new(MokaCache::new(0))),
            HtmlParser::new(MokaCache::new(0)),
        )
        .validate(&Config::new(
            vec![url.to_string()],
            Default::default(),
            [(
                url.host_str().unwrap_or_default().into(),
                [(
                    "".into(),
                    SiteConfig::default()
                        .set_recursive(true)
                        .set_max_redirects(1 << 32)
                        .set_validation(
                            crate::ValidationConfig::default()
                                .set_html(Some(MarkupConfig::default())),
                        )
                        .into(),
                )]
                .into(),
            )]
            .into(),
        ))
        .await
    }

    async fn collect_metrics(
        documents: &mut (impl Stream<Item = Result<DocumentOutput, Error>> + Unpin),
    ) -> (Metrics, Metrics) {
        let mut document_metrics = Metrics::default();
        let mut element_metrics = Metrics::default();

        while let Some(document) = documents.next().await {
            let document = document.unwrap();

            document_metrics.add(document.metrics().has_error());
            element_metrics.merge(&document.metrics());
        }

        (document_metrics, element_metrics)
    }

    #[tokio::test]
    async fn validate_site() {
        let mut documents = validate(
            StubHttpClient::new(
                [
                    build_stub_response(
                        "https://foo.com/robots.txt",
                        StatusCode::OK,
                        Default::default(),
                        Default::default(),
                    ),
                    build_stub_response(
                        "https://foo.com",
                        StatusCode::OK,
                        HeaderMap::from_iter([(
                            HeaderName::from_static("content-type"),
                            HeaderValue::from_static("text/html"),
                        )]),
                        Default::default(),
                    ),
                ]
                .into_iter()
                .collect(),
            ),
            "https://foo.com",
        )
        .await
        .unwrap();

        assert_eq!(
            collect_metrics(&mut documents).await,
            (Metrics::new(2, 0), Metrics::new(0, 0))
        );
    }

    #[tokio::test]
    async fn validate_document_not_found() {
        let result = validate(
            StubHttpClient::new(
                [
                    build_stub_response(
                        "https://foo.com/robots.txt",
                        StatusCode::OK,
                        Default::default(),
                        Default::default(),
                    ),
                    build_stub_response(
                        "https://foo.com",
                        StatusCode::NOT_FOUND,
                        Default::default(),
                        Default::default(),
                    ),
                ]
                .into_iter()
                .collect(),
            ),
            "https://foo.com",
        )
        .await;

        assert!(matches!(
            result,
            Err(Error::Item(ItemError::HttpStatus(StatusCode::NOT_FOUND)))
        ));
    }

    #[tokio::test]
    async fn validate_two_documents() {
        let html_headers = HeaderMap::from_iter([(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("text/html"),
        )]);
        let mut documents = validate(
            StubHttpClient::new(
                [
                    build_stub_response(
                        "https://foo.com/robots.txt",
                        StatusCode::OK,
                        Default::default(),
                        Default::default(),
                    ),
                    build_stub_response(
                        "https://foo.com",
                        StatusCode::OK,
                        html_headers.clone(),
                        r#"<a href="https://foo.com/bar"/>" "#.as_bytes().to_vec(),
                    ),
                    build_stub_response(
                        "https://foo.com/bar",
                        StatusCode::OK,
                        html_headers,
                        Default::default(),
                    ),
                ]
                .into_iter()
                .collect(),
            ),
            "https://foo.com",
        )
        .await
        .unwrap();

        assert_eq!(
            collect_metrics(&mut documents).await,
            (Metrics::new(3, 0), Metrics::new(1, 0))
        );
    }

    #[tokio::test]
    async fn validate_base_element() {
        let html_headers = HeaderMap::from_iter([(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("text/html"),
        )]);
        let mut documents = validate(
            StubHttpClient::new(
                [
                    build_stub_response(
                        "https://foo.com/robots.txt",
                        StatusCode::OK,
                        Default::default(),
                        Default::default(),
                    ),
                    build_stub_response(
                        "https://foo.com",
                        StatusCode::OK,
                        html_headers.clone(),
                        indoc! {r#"
                            <html>
                                <head>
                                    <base href="https://foo.com/foo/" />
                                </head>
                                <body>
                                    <a href="bar" />
                                </body>
                            </html>
                        "#}
                        .as_bytes()
                        .to_vec(),
                    ),
                    build_stub_response(
                        "https://foo.com/foo/bar",
                        StatusCode::OK,
                        html_headers,
                        Default::default(),
                    ),
                ]
                .into_iter()
                .collect(),
            ),
            "https://foo.com",
        )
        .await
        .unwrap();

        assert_eq!(
            collect_metrics(&mut documents).await,
            (Metrics::new(3, 0), Metrics::new(1, 0))
        );
    }

    #[tokio::test]
    async fn validate_base_element_with_relative_href() {
        let html_headers = HeaderMap::from_iter([(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("text/html"),
        )]);
        let mut documents = validate(
            StubHttpClient::new(
                [
                    build_stub_response(
                        "https://foo.com/robots.txt",
                        StatusCode::OK,
                        Default::default(),
                        Default::default(),
                    ),
                    build_stub_response(
                        "https://foo.com",
                        StatusCode::OK,
                        html_headers.clone(),
                        indoc! {r#"
                            <html>
                                <head>
                                    <base href="/foo/" />
                                </head>
                                <body>
                                    <a href="bar" />
                                </body>
                            </html>
                        "#}
                        .as_bytes()
                        .to_vec(),
                    ),
                    build_stub_response(
                        "https://foo.com/foo/bar",
                        StatusCode::OK,
                        html_headers,
                        Default::default(),
                    ),
                ]
                .into_iter()
                .collect(),
            ),
            "https://foo.com",
        )
        .await
        .unwrap();

        assert_eq!(
            collect_metrics(&mut documents).await,
            (Metrics::new(3, 0), Metrics::new(1, 0))
        );
    }

    #[tokio::test]
    async fn validate_base_element_without_href() {
        let html_headers = HeaderMap::from_iter([(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("text/html"),
        )]);
        let mut documents = validate(
            StubHttpClient::new(
                [
                    build_stub_response(
                        "https://foo.com/robots.txt",
                        StatusCode::OK,
                        Default::default(),
                        Default::default(),
                    ),
                    build_stub_response(
                        "https://foo.com",
                        StatusCode::OK,
                        html_headers.clone(),
                        indoc! {r#"
                            <html>
                                <head>
                                    <base />
                                </head>
                                <body>
                                    <a href="bar" />
                                </body>
                            </html>
                        "#}
                        .as_bytes()
                        .to_vec(),
                    ),
                    build_stub_response(
                        "https://foo.com/bar",
                        StatusCode::OK,
                        html_headers,
                        Default::default(),
                    ),
                ]
                .into_iter()
                .collect(),
            ),
            "https://foo.com",
        )
        .await
        .unwrap();

        assert_eq!(
            collect_metrics(&mut documents).await,
            (Metrics::new(3, 0), Metrics::new(1, 0))
        );
    }

    #[tokio::test]
    async fn validate_base_element_with_invalid_href() {
        let html_headers = HeaderMap::from_iter([(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("text/html"),
        )]);
        let mut documents = validate(
            StubHttpClient::new(
                [
                    build_stub_response(
                        "https://foo.com/robots.txt",
                        StatusCode::OK,
                        Default::default(),
                        Default::default(),
                    ),
                    build_stub_response(
                        "https://foo.com",
                        StatusCode::OK,
                        html_headers.clone(),
                        indoc! {r#"
                            <html>
                                <head>
                                    <base href="::::" />
                                </head>
                                <body>
                                    <a href="bar" />
                                </body>
                            </html>
                        "#}
                        .as_bytes()
                        .to_vec(),
                    ),
                    build_stub_response(
                        "https://foo.com/bar",
                        StatusCode::OK,
                        html_headers,
                        Default::default(),
                    ),
                ]
                .into_iter()
                .collect(),
            ),
            "https://foo.com",
        )
        .await
        .unwrap();

        assert_eq!(
            collect_metrics(&mut documents).await,
            (Metrics::new(3, 0), Metrics::new(1, 0))
        );
    }

    #[tokio::test]
    async fn validate_multiple_base_elements() {
        let html_headers = HeaderMap::from_iter([(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("text/html"),
        )]);
        let mut documents = validate(
            StubHttpClient::new(
                [
                    build_stub_response(
                        "https://foo.com/robots.txt",
                        StatusCode::OK,
                        Default::default(),
                        Default::default(),
                    ),
                    build_stub_response(
                        "https://foo.com",
                        StatusCode::OK,
                        html_headers.clone(),
                        indoc! {r#"
                            <html>
                                <head>
                                    <base href="https://foo.com/foo/" />
                                    <base href="https://foo.com/ignored/" />
                                </head>
                                <body>
                                    <a href="bar" />
                                </body>
                            </html>
                        "#}
                        .as_bytes()
                        .to_vec(),
                    ),
                    build_stub_response(
                        "https://foo.com/foo/bar",
                        StatusCode::OK,
                        html_headers,
                        Default::default(),
                    ),
                ]
                .into_iter()
                .collect(),
            ),
            "https://foo.com",
        )
        .await
        .unwrap();

        assert_eq!(
            collect_metrics(&mut documents).await,
            (Metrics::new(3, 0), Metrics::new(1, 0))
        );
    }

    #[tokio::test]
    async fn validate_two_links_in_document() {
        let html_headers = HeaderMap::from_iter([(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("text/html"),
        )]);
        let mut documents = validate(
            StubHttpClient::new(
                [
                    build_stub_response(
                        "https://foo.com/robots.txt",
                        StatusCode::OK,
                        Default::default(),
                        Default::default(),
                    ),
                    build_stub_response(
                        "https://foo.com",
                        StatusCode::OK,
                        html_headers.clone(),
                        r#"
                        <a href="https://foo.com/bar"/>
                        <a href="https://foo.com/baz"/>
                    "#
                        .as_bytes()
                        .to_vec(),
                    ),
                    build_stub_response(
                        "https://foo.com/bar",
                        StatusCode::OK,
                        html_headers.clone(),
                        Default::default(),
                    ),
                    build_stub_response(
                        "https://foo.com/baz",
                        StatusCode::OK,
                        html_headers,
                        Default::default(),
                    ),
                ]
                .into_iter()
                .collect(),
            ),
            "https://foo.com",
        )
        .await
        .unwrap();

        assert_eq!(
            collect_metrics(&mut documents).await,
            (Metrics::new(4, 0), Metrics::new(2, 0))
        );
    }

    #[tokio::test]
    async fn validate_links_recursively() {
        let html_headers = HeaderMap::from_iter([(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("text/html"),
        )]);
        let mut documents = validate(
            StubHttpClient::new(
                [
                    build_stub_response(
                        "https://foo.com/robots.txt",
                        StatusCode::OK,
                        Default::default(),
                        Default::default(),
                    ),
                    build_stub_response(
                        "https://foo.com",
                        StatusCode::OK,
                        html_headers.clone(),
                        r#"<a href="https://foo.com/bar"/>"#.as_bytes().to_vec(),
                    ),
                    build_stub_response(
                        "https://foo.com/bar",
                        StatusCode::OK,
                        html_headers.clone(),
                        r#"<a href="https://foo.com"/>"#.as_bytes().to_vec(),
                    ),
                ]
                .into_iter()
                .collect(),
            ),
            "https://foo.com",
        )
        .await
        .unwrap();

        assert_eq!(
            collect_metrics(&mut documents).await,
            (Metrics::new(3, 0), Metrics::new(2, 0))
        );
    }

    #[tokio::test]
    async fn validate_fragment_for_html() {
        let html_headers = HeaderMap::from_iter([(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("text/html"),
        )]);
        let mut documents = validate(
            StubHttpClient::new(
                [
                    build_stub_response(
                        "https://foo.com/robots.txt",
                        StatusCode::OK,
                        Default::default(),
                        Default::default(),
                    ),
                    build_stub_response(
                        "https://foo.com",
                        StatusCode::OK,
                        html_headers.clone(),
                        indoc!(
                            r#"
                            <a href="https://foo.com#foo"/>
                            <div id="foo" />
                        "#
                        )
                        .as_bytes()
                        .into(),
                    ),
                ]
                .into_iter()
                .collect(),
            ),
            "https://foo.com",
        )
        .await
        .unwrap();

        assert_eq!(
            collect_metrics(&mut documents).await,
            (Metrics::new(2, 0), Metrics::new(1, 0))
        );
    }

    #[tokio::test]
    async fn validate_srcset() {
        let html_headers = HeaderMap::from_iter([(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("text/html"),
        )]);
        let image_headers = HeaderMap::from_iter([(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("image/png"),
        )]);

        let mut documents = validate(
            StubHttpClient::new(
                [
                    build_stub_response(
                        "https://foo.com/robots.txt",
                        StatusCode::OK,
                        Default::default(),
                        Default::default(),
                    ),
                    build_stub_response(
                        "https://foo.com",
                        StatusCode::OK,
                        html_headers.clone(),
                        indoc!(
                            r#"
                            <img src="/foo.png" srcset="/bar.png, /baz.png 2x, /qux.png 800w">
                            "#
                        )
                        .as_bytes()
                        .into(),
                    ),
                    build_stub_response(
                        "https://foo.com/foo.png",
                        StatusCode::OK,
                        image_headers.clone(),
                        Default::default(),
                    ),
                    build_stub_response(
                        "https://foo.com/bar.png",
                        StatusCode::OK,
                        image_headers.clone(),
                        Default::default(),
                    ),
                    build_stub_response(
                        "https://foo.com/baz.png",
                        StatusCode::OK,
                        image_headers.clone(),
                        Default::default(),
                    ),
                    build_stub_response(
                        "https://foo.com/qux.png",
                        StatusCode::OK,
                        image_headers.clone(),
                        Default::default(),
                    ),
                ]
                .into_iter()
                .collect(),
            ),
            "https://foo.com",
        )
        .await
        .unwrap();

        assert_eq!(
            collect_metrics(&mut documents).await,
            (Metrics::new(2, 0), Metrics::new(4, 0))
        );
    }

    #[tokio::test]
    async fn validate_meta_element_with_link_property() {
        let html_headers = HeaderMap::from_iter([(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("text/html"),
        )]);
        let image_headers = HeaderMap::from_iter([(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("image/png"),
        )]);
        let mut documents = validate(
            StubHttpClient::new(
                [
                    build_stub_response(
                        "https://foo.com/robots.txt",
                        StatusCode::OK,
                        Default::default(),
                        Default::default(),
                    ),
                    build_stub_response(
                        "https://foo.com",
                        StatusCode::OK,
                        html_headers.clone(),
                        indoc!(
                            r#"
                            <meta property="og:image" content="https://foo.com/og.png" />
                            "#
                        )
                        .as_bytes()
                        .into(),
                    ),
                    build_stub_response(
                        "https://foo.com/og.png",
                        StatusCode::OK,
                        image_headers,
                        Default::default(),
                    ),
                ]
                .into_iter()
                .collect(),
            ),
            "https://foo.com",
        )
        .await
        .unwrap();

        assert_eq!(
            collect_metrics(&mut documents).await,
            (Metrics::new(2, 0), Metrics::new(1, 0))
        );
    }

    #[tokio::test]
    async fn validate_meta_element_with_non_link_property() {
        let html_headers = HeaderMap::from_iter([(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("text/html"),
        )]);
        let mut documents = validate(
            StubHttpClient::new(
                [
                    build_stub_response(
                        "https://foo.com/robots.txt",
                        StatusCode::OK,
                        Default::default(),
                        Default::default(),
                    ),
                    build_stub_response(
                        "https://foo.com",
                        StatusCode::OK,
                        html_headers,
                        indoc!(
                            r#"
                            <meta property="og:title" content="https://foo.com/ignored" />
                            "#
                        )
                        .as_bytes()
                        .into(),
                    ),
                ]
                .into_iter()
                .collect(),
            ),
            "https://foo.com",
        )
        .await
        .unwrap();

        assert_eq!(
            collect_metrics(&mut documents).await,
            (Metrics::new(2, 0), Metrics::new(0, 0))
        );
    }

    #[tokio::test]
    async fn validate_document_not_belonging_to_roots() {
        let html_headers = HeaderMap::from_iter([(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("text/html"),
        )]);
        let mut documents = validate(
            StubHttpClient::new(
                [
                    build_stub_response(
                        "https://foo.com/robots.txt",
                        StatusCode::OK,
                        Default::default(),
                        Default::default(),
                    ),
                    build_stub_response(
                        "https://foo.com",
                        StatusCode::OK,
                        html_headers.clone(),
                        r#"<a href="https://bar.com" />"#.as_bytes().into(),
                    ),
                    build_stub_response(
                        "https://bar.com/robots.txt",
                        StatusCode::OK,
                        Default::default(),
                        Default::default(),
                    ),
                    build_stub_response(
                        "https://bar.com",
                        StatusCode::OK,
                        html_headers,
                        Default::default(),
                    ),
                ]
                .into_iter()
                .collect(),
            ),
            "https://foo.com",
        )
        .await
        .unwrap();

        assert_eq!(
            collect_metrics(&mut documents).await,
            (Metrics::new(2, 0), Metrics::new(1, 0))
        );
    }

    #[tokio::test]
    async fn validate_missing_fragment_for_html() {
        let html_headers = HeaderMap::from_iter([(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("text/html"),
        )]);
        let mut documents = validate(
            StubHttpClient::new(
                [
                    build_stub_response(
                        "https://foo.com/robots.txt",
                        StatusCode::OK,
                        Default::default(),
                        Default::default(),
                    ),
                    build_stub_response(
                        "https://foo.com",
                        StatusCode::OK,
                        html_headers.clone(),
                        r#"<a href="https://foo.com#foo"/>"#.as_bytes().to_vec(),
                    ),
                ]
                .into_iter()
                .collect(),
            ),
            "https://foo.com",
        )
        .await
        .unwrap();

        assert_eq!(
            collect_metrics(&mut documents).await,
            (Metrics::new(1, 1), Metrics::new(0, 1))
        );
    }

    #[tokio::test]
    async fn validate_ignored_fragment_for_html() {
        let url = Url::parse("https://foo.com").unwrap();
        let mut documents = WebValidator::new(
            HttpClient::new(
                StubHttpClient::new(
                    [
                        build_stub_response(
                            "https://foo.com/robots.txt",
                            StatusCode::OK,
                            Default::default(),
                            Default::default(),
                        ),
                        build_stub_response(
                            url.as_str(),
                            StatusCode::OK,
                            HeaderMap::from_iter([(
                                HeaderName::from_static("content-type"),
                                HeaderValue::from_static("text/html"),
                            )]),
                            r#"<a href="https://foo.com#foo"/>"#.as_bytes().to_vec(),
                        ),
                    ]
                    .into_iter()
                    .collect(),
                ),
                StubTimer::new(),
                Box::new(MokaCache::new(0)),
            ),
            HtmlParser::new(MokaCache::new(0)),
        )
        .validate(&Config::new(
            vec![url.as_str().into()],
            Default::default(),
            [(
                url.host_str().unwrap_or_default().into(),
                [(
                    "".into(),
                    SiteConfig::default()
                        .set_recursive(true)
                        .set_fragments_ignored(true)
                        .into(),
                )]
                .into(),
            )]
            .into(),
        ))
        .await
        .unwrap();

        assert_eq!(
            collect_metrics(&mut documents).await,
            (Metrics::new(2, 0), Metrics::new(1, 0))
        );
    }

    #[tokio::test]
    async fn validate_link_with_whitespaces() {
        let html_headers = HeaderMap::from_iter([(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("text/html"),
        )]);
        let mut documents = validate(
            StubHttpClient::new(
                [
                    build_stub_response(
                        "https://foo.com/robots.txt",
                        StatusCode::OK,
                        Default::default(),
                        Default::default(),
                    ),
                    build_stub_response(
                        "https://foo.com",
                        StatusCode::OK,
                        html_headers.clone(),
                        r#"<a href="https:/  /foo. com/ bar"/>"#.as_bytes().to_vec(),
                    ),
                    build_stub_response(
                        "https://foo.com/bar",
                        StatusCode::OK,
                        html_headers.clone(),
                        Default::default(),
                    ),
                ]
                .into_iter()
                .collect(),
            ),
            "https://foo.com",
        )
        .await
        .unwrap();

        assert_eq!(
            collect_metrics(&mut documents).await,
            (Metrics::new(3, 0), Metrics::new(1, 0))
        );
    }

    #[tokio::test]
    async fn validate_scheme() {
        let url = Url::parse("https://foo.com").unwrap();
        let mut documents = WebValidator::new(
            HttpClient::new(
                StubHttpClient::new(
                    [
                        build_stub_response(
                            "https://foo.com/robots.txt",
                            StatusCode::OK,
                            Default::default(),
                            Default::default(),
                        ),
                        build_stub_response(
                            url.as_str().into(),
                            StatusCode::OK,
                            HeaderMap::from_iter([(
                                HeaderName::from_static("content-type"),
                                HeaderValue::from_static("text/html"),
                            )]),
                            r#"
                                <a href="http://foo.com"/>
                            "#
                            .as_bytes()
                            .to_vec(),
                        ),
                    ]
                    .into_iter()
                    .collect(),
                ),
                StubTimer::new(),
                Box::new(MokaCache::new(0)),
            ),
            HtmlParser::new(MokaCache::new(0)),
        )
        .validate(&Config::new(
            vec![url.as_str().into()],
            SiteConfig::default().into(),
            [(
                url.host_str().unwrap_or_default().into(),
                [(
                    "".into(),
                    SiteConfig::default()
                        .set_scheme(SchemeConfig::new(["https".into()].into()))
                        .set_recursive(true)
                        .into(),
                )]
                .into_iter()
                .collect(),
            )]
            .into_iter()
            .collect(),
        ))
        .await
        .unwrap();

        assert_eq!(
            collect_metrics(&mut documents).await,
            (Metrics::new(1, 1), Metrics::new(0, 1))
        );
    }

    #[tokio::test]
    async fn validate_ignored_link() {
        let url = Url::parse("https://foo.com").unwrap();
        let html_headers = HeaderMap::from_iter([(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("text/html"),
        )]);
        let mut documents = WebValidator::new(
            HttpClient::new(
                StubHttpClient::new(
                    [
                        build_stub_response(
                            "https://foo.com/robots.txt",
                            StatusCode::OK,
                            Default::default(),
                            Default::default(),
                        ),
                        build_stub_response(
                            url.as_str().into(),
                            StatusCode::OK,
                            html_headers.clone(),
                            r#"
                                <a href="https://foo.com/bar"/>
                            "#
                            .as_bytes()
                            .to_vec(),
                        ),
                        build_stub_response(
                            "https://foo.com/bar",
                            StatusCode::OK,
                            html_headers,
                            Default::default(),
                        ),
                    ]
                    .into_iter()
                    .collect(),
                ),
                StubTimer::new(),
                Box::new(MokaCache::new(0)),
            ),
            HtmlParser::new(MokaCache::new(0)),
        )
        .validate(
            &Config::new(
                vec![url.as_str().into()],
                Default::default(),
                [(
                    url.host_str().unwrap_or_default().into(),
                    [("".into(), SiteConfig::default().set_recursive(true).into())]
                        .into_iter()
                        .collect(),
                )]
                .into_iter()
                .collect(),
            )
            .set_ignored_links(vec![Regex::new("bar").unwrap()]),
        )
        .await
        .unwrap();

        assert_eq!(
            collect_metrics(&mut documents).await,
            (Metrics::new(2, 0), Metrics::new(1, 0))
        );
    }

    #[tokio::test]
    async fn validate_ignored_link_with_invalid_scheme() {
        let url = Url::parse("https://foo.com").unwrap();
        let html_headers = HeaderMap::from_iter([(
            HeaderName::from_static("content-type"),
            HeaderValue::from_static("text/html"),
        )]);
        let mut documents = WebValidator::new(
            HttpClient::new(
                StubHttpClient::new(
                    [
                        build_stub_response(
                            "https://foo.com/robots.txt",
                            StatusCode::OK,
                            Default::default(),
                            Default::default(),
                        ),
                        build_stub_response(
                            url.as_str().into(),
                            StatusCode::OK,
                            html_headers.clone(),
                            r#"
                                <a href="http://foo.com/bar"/>
                            "#
                            .as_bytes()
                            .to_vec(),
                        ),
                    ]
                    .into_iter()
                    .collect(),
                ),
                StubTimer::new(),
                Box::new(MokaCache::new(0)),
            ),
            HtmlParser::new(MokaCache::new(0)),
        )
        .validate(
            &Config::new(
                vec![url.as_str().into()],
                Default::default(),
                [(
                    url.host_str().unwrap_or_default().into(),
                    [(
                        "".into(),
                        SiteConfig::default()
                            .set_scheme(SchemeConfig::new(["https".into()].into()))
                            .set_recursive(true)
                            .into(),
                    )]
                    .into_iter()
                    .collect(),
                )]
                .into_iter()
                .collect(),
            )
            .set_ignored_links(vec![Regex::new("bar").unwrap()]),
        )
        .await
        .unwrap();

        assert_eq!(
            collect_metrics(&mut documents).await,
            (Metrics::new(2, 0), Metrics::new(1, 0))
        );
    }

    mod srcset {
        use super::*;
        use pretty_assertions::assert_eq;

        #[test]
        fn parse_url() {
            assert_eq!(
                WebValidator::parse_srcset("/foo.png").collect::<Vec<_>>(),
                ["/foo.png"]
            );
        }

        #[test]
        fn parse_url_with_descriptor() {
            assert_eq!(
                WebValidator::parse_srcset("/foo.png 2x").collect::<Vec<_>>(),
                ["/foo.png"]
            );
        }

        #[test]
        fn parse_multiple_urls() {
            assert_eq!(
                WebValidator::parse_srcset("/foo.png, /bar.png 2x, /baz.png 800w")
                    .collect::<Vec<_>>(),
                ["/foo.png", "/bar.png", "/baz.png"]
            );
        }

        #[test]
        fn skip_empty_entry() {
            assert_eq!(
                WebValidator::parse_srcset("/foo.png,, /bar.png").collect::<Vec<_>>(),
                ["/foo.png", "/bar.png"]
            );
        }

        #[test]
        fn skip_trailing_comma() {
            assert_eq!(
                WebValidator::parse_srcset("/foo.png,").collect::<Vec<_>>(),
                ["/foo.png"]
            );
        }

        #[test]
        fn parse_data_url() {
            assert_eq!(
                WebValidator::parse_srcset("data:image/png;base64,abc 2x, /bar.png")
                    .collect::<Vec<_>>(),
                ["data:image/png;base64,abc", "/bar.png"]
            );
        }

        #[test]
        fn parse_urls_with_width_descriptors() {
            assert_eq!(
                WebValidator::parse_srcset("small.jpg 500w, medium.jpg 1000w, large.jpg 1500w")
                    .collect::<Vec<_>>(),
                ["small.jpg", "medium.jpg", "large.jpg"]
            );
        }
    }

    mod sitemap {
        use super::*;
        use pretty_assertions::assert_eq;

        async fn validate_sitemap(content_type: &'static str) {
            let html_headers = HeaderMap::from_iter([(
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("text/html"),
            )]);

            let mut documents = validate(
                StubHttpClient::new(
                    [
                        build_stub_response(
                            "https://foo.com/robots.txt",
                            StatusCode::OK,
                            Default::default(),
                            Default::default(),
                        ),
                        build_stub_response(
                            "https://foo.com",
                            StatusCode::OK,
                            html_headers.clone(),
                            r#"<link rel="sitemap" href="https://foo.com/sitemap.xml"/>"#
                                .as_bytes()
                                .to_vec(),
                        ),
                        build_stub_response(
                            "https://foo.com/sitemap.xml",
                            StatusCode::OK,
                            HeaderMap::from_iter([(
                                HeaderName::from_static("content-type"),
                                HeaderValue::from_static(content_type),
                            )]),
                            r#"
                            <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
                                <url>
                                    <loc>https://foo.com/</loc>
                                    <lastmod>1970-01-01</lastmod>
                                    <changefreq>daily</changefreq>
                                    <priority>1</priority>
                                </url>
                                <url>
                                    <loc>https://foo.com/bar</loc>
                                    <lastmod>1970-01-01</lastmod>
                                    <changefreq>daily</changefreq>
                                    <priority>1</priority>
                                </url>
                            </urlset>
                            "#
                            .as_bytes()
                            .to_vec(),
                        ),
                        build_stub_response(
                            "https://foo.com/bar",
                            StatusCode::OK,
                            html_headers.clone(),
                            Default::default(),
                        ),
                    ]
                    .into_iter()
                    .collect(),
                ),
                "https://foo.com",
            )
            .await
            .unwrap();

            assert_eq!(
                collect_metrics(&mut documents).await,
                (Metrics::new(4, 0), Metrics::new(3, 0))
            );
        }

        #[tokio::test]
        async fn validate_sitemap_in_text_xml() {
            validate_sitemap("text/xml").await;
        }

        #[tokio::test]
        async fn validate_sitemap_in_application_xml() {
            validate_sitemap("application/xml").await;
        }

        async fn validate_sitemap_index(content_type: &'static str) {
            let html_headers = HeaderMap::from_iter([(
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("text/html"),
            )]);

            let mut documents = validate(
                StubHttpClient::new(
                    [
                        build_stub_response(
                            "https://foo.com/robots.txt",
                            StatusCode::OK,
                            Default::default(),
                            Default::default(),
                        ),
                        build_stub_response(
                            "https://foo.com",
                            StatusCode::OK,
                            html_headers.clone(),
                            r#"<link rel="sitemap" href="https://foo.com/sitemap-index.xml"/>"#
                                .as_bytes()
                                .to_vec(),
                        ),
                        build_stub_response(
                            "https://foo.com/sitemap-index.xml",
                            StatusCode::OK,
                            HeaderMap::from_iter([(
                                HeaderName::from_static("content-type"),
                                HeaderValue::from_static(content_type),
                            )]),
                            r#"
                        <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
                            <sitemap>
                                <loc>https://foo.com/sitemap-0.xml</loc>
                                <lastmod>1970-01-01T00:00:00+00:00</lastmod>
                            </sitemap>
                        </sitemapindex>
                        "#
                            .as_bytes()
                            .to_vec(),
                        ),
                        build_stub_response(
                            "https://foo.com/sitemap-0.xml",
                            StatusCode::OK,
                            HeaderMap::from_iter([(
                                HeaderName::from_static("content-type"),
                                HeaderValue::from_static(content_type),
                            )]),
                            r#"
                        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
                            <url>
                                <loc>https://foo.com/</loc>
                                <lastmod>1970-01-01</lastmod>
                                <changefreq>daily</changefreq>
                                <priority>1</priority>
                            </url>
                            <url>
                                <loc>https://foo.com/bar</loc>
                                <lastmod>1970-01-01</lastmod>
                                <changefreq>daily</changefreq>
                                <priority>1</priority>
                            </url>
                        </urlset>
                        "#
                            .as_bytes()
                            .to_vec(),
                        ),
                        build_stub_response(
                            "https://foo.com/bar",
                            StatusCode::OK,
                            html_headers.clone(),
                            Default::default(),
                        ),
                    ]
                    .into_iter()
                    .collect(),
                ),
                "https://foo.com",
            )
            .await
            .unwrap();

            assert_eq!(
                collect_metrics(&mut documents).await,
                (Metrics::new(5, 0), Metrics::new(4, 0))
            );
        }

        #[tokio::test]
        async fn validate_sitemap_index_in_text_xml() {
            validate_sitemap_index("text/xml").await;
        }

        #[tokio::test]
        async fn validate_sitemap_index_in_application_xml() {
            validate_sitemap_index("application/xml").await;
        }
    }

    mod robots {
        use super::*;
        use pretty_assertions::assert_eq;

        #[tokio::test]
        async fn ignore_link_with_robots_txt() {
            let html_headers = HeaderMap::from_iter([(
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("text/html"),
            )]);
            let mut documents = validate(
                StubHttpClient::new(
                    [
                        build_stub_response(
                            "https://foo.com/robots.txt",
                            StatusCode::OK,
                            Default::default(),
                            indoc!(
                                "
                            User-agent: *
                            Disallow: /bar
                            "
                            )
                            .as_bytes()
                            .to_vec(),
                        ),
                        build_stub_response(
                            "https://foo.com",
                            StatusCode::OK,
                            html_headers.clone(),
                            r#"<a href="https://foo.com/bar"/>"#.as_bytes().to_vec(),
                        ),
                        build_stub_response(
                            "https://foo.com/bar",
                            StatusCode::OK,
                            html_headers.clone(),
                            Default::default(),
                        ),
                    ]
                    .into_iter()
                    .collect(),
                ),
                "https://foo.com",
            )
            .await
            .unwrap();

            assert_eq!(
                collect_metrics(&mut documents).await,
                (Metrics::new(2, 0), Metrics::new(1, 0))
            );
        }

        #[tokio::test]
        async fn handle_missing_robots_txt() {
            let html_headers = HeaderMap::from_iter([(
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("text/html"),
            )]);
            let mut documents = validate(
                StubHttpClient::new(
                    [
                        build_stub_response(
                            "https://foo.com/robots.txt",
                            StatusCode::NOT_FOUND,
                            Default::default(),
                            Default::default(),
                        ),
                        build_stub_response(
                            "https://foo.com",
                            StatusCode::OK,
                            html_headers.clone(),
                            r#"<a href="https://foo.com/bar"/>"#.as_bytes().to_vec(),
                        ),
                        build_stub_response(
                            "https://foo.com/bar",
                            StatusCode::OK,
                            html_headers.clone(),
                            Default::default(),
                        ),
                    ]
                    .into_iter()
                    .collect(),
                ),
                "https://foo.com",
            )
            .await
            .unwrap();

            assert_eq!(
                collect_metrics(&mut documents).await,
                (Metrics::new(2, 0), Metrics::new(1, 0))
            );
        }

        #[tokio::test]
        async fn handle_redirected_robots_txt() {
            let html_headers = HeaderMap::from_iter([(
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("text/html"),
            )]);
            let mut documents = validate(
                StubHttpClient::new(
                    [
                        build_stub_response(
                            "https://foo.com/robots.txt",
                            StatusCode::PERMANENT_REDIRECT,
                            HeaderMap::from_iter([(
                                HeaderName::from_static("location"),
                                HeaderValue::from_static("/foo/robots.txt"),
                            )]),
                            Default::default(),
                        ),
                        build_stub_response(
                            "https://foo.com/foo/robots.txt",
                            StatusCode::OK,
                            Default::default(),
                            Default::default(),
                        ),
                        build_stub_response(
                            "https://foo.com",
                            StatusCode::OK,
                            html_headers.clone(),
                            Default::default(),
                        ),
                    ]
                    .into_iter()
                    .collect(),
                ),
                "https://foo.com",
            )
            .await
            .unwrap();

            assert_eq!(
                collect_metrics(&mut documents).await,
                (Metrics::new(2, 0), Metrics::new(0, 0))
            );
        }

        #[tokio::test]
        async fn handle_sitemap_link() {
            let html_headers = HeaderMap::from_iter([(
                HeaderName::from_static("content-type"),
                HeaderValue::from_static("text/html"),
            )]);

            let mut documents = validate(
                StubHttpClient::new(
                    [
                        build_stub_response(
                            "https://foo.com/robots.txt",
                            StatusCode::OK,
                            Default::default(),
                            indoc!(
                                "
                                User-agent: *
                                Allow: /

                                Sitemap: https://foo.com/sitemap.xml
                                "
                            )
                            .as_bytes()
                            .to_vec(),
                        ),
                        build_stub_response(
                            "https://foo.com/sitemap.xml",
                            StatusCode::OK,
                            HeaderMap::from_iter([(
                                HeaderName::from_static("content-type"),
                                HeaderValue::from_static("application/xml"),
                            )]),
                            r#"
                            <?xml version="1.0" encoding="UTF-8"?>
                            <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
                                <url>
                                    <loc>https://foo.com/bar</loc>
                                </url>
                            </urlset>
                            "#
                            .as_bytes()
                            .to_vec(),
                        ),
                        build_stub_response(
                            "https://foo.com",
                            StatusCode::OK,
                            html_headers.clone(),
                            Default::default(),
                        ),
                        build_stub_response(
                            "https://foo.com/bar",
                            StatusCode::OK,
                            html_headers,
                            Default::default(),
                        ),
                    ]
                    .into_iter()
                    .collect(),
                ),
                "https://foo.com",
            )
            .await
            .unwrap();

            assert_eq!(
                collect_metrics(&mut documents).await,
                (Metrics::new(4, 0), Metrics::new(2, 0))
            );
        }
    }
}
