pub use config::CrawlerConfig;
pub use error::CrawlerError;
use futures::{StreamExt, stream::FuturesUnordered};
use log::{debug, warn};
use reqwest::Client;
use scraper::Html;
use std::{
    collections::{HashSet, VecDeque},
    future::Future,
    time::Duration,
};
use tokio::time::sleep;
use url::Url;

#[cfg(feature = "checkpoints")]
pub mod checkpoint;
mod config;
mod error;
#[cfg(feature = "headless")]
pub mod headless;
#[cfg(feature = "sitemap")]
pub mod sitemap;
pub mod utils;

#[cfg(feature = "url-filtering")]
pub type UrlFilter = Box<dyn Fn(&Url) -> bool + Send + Sync>;
#[cfg(feature = "url-priority")]
pub type UrlPriority = Box<dyn Fn(&Url) -> i32 + Send + Sync>;

/// Pages and URLs collected by a crawl run.
pub struct CrawlResult {
    /// URLs accepted into the crawl queue, including pages skipped from resume state.
    pub crawled_urls: Vec<Url>,
    /// Raw HTML strings. Parse with `Html::parse_document` when needed.
    pub pages: Vec<(Url, String)>,
}

/// Common interface for crawler implementations.
pub trait Crawl: Send + Sync + 'static {
    /// Run the crawl and return fetched pages plus accepted URLs.
    fn crawl(&self) -> impl Future<Output = Result<CrawlResult, CrawlerError>> + Send;
}

/// HTTP crawler driven by [`CrawlerConfig`].
pub struct Crawler {
    config: CrawlerConfig,
    client: Client,
}

impl Crawler {
    /// Build a crawler with a configured reqwest client.
    pub fn new(config: CrawlerConfig) -> Result<Self, CrawlerError> {
        let client = Client::builder()
            .user_agent(&config.user_agent)
            .timeout(Duration::from_secs(30))
            .build()?;

        Ok(Self { config, client })
    }

    #[cfg(feature = "sitemap")]
    async fn try_seed_from_sitemap(
        &self,
        start_url: &Url,
        visited: &mut HashSet<Url>,
        queue: &mut VecDeque<Url>,
    ) {
        debug!("📍 Checking for sitemap...");
        match sitemap::fetch_sitemap_urls(&self.client, start_url).await {
            Ok(sitemap_urls) if !sitemap_urls.is_empty() => {
                debug!("✅ Found {} URLs in sitemap(s)", sitemap_urls.len());
                self.seed_from_sitemap(sitemap_urls, start_url, visited, queue);
            }
            Ok(_) => {
                debug!("⚠️  No sitemap found, falling back to traditional crawling");
                self.enqueue_if_new(start_url.clone(), visited, queue);
            }
            Err(e) => {
                warn!(
                    "⚠️  Error fetching sitemap: {}, falling back to traditional crawling",
                    e
                );
                self.enqueue_if_new(start_url.clone(), visited, queue);
            }
        }
    }

    #[cfg(feature = "sitemap")]
    fn seed_from_sitemap(
        &self,
        sitemap_urls: impl IntoIterator<Item = Url>,
        start_url: &Url,
        visited: &mut HashSet<Url>,
        queue: &mut VecDeque<Url>,
    ) {
        sitemap_urls
            .into_iter()
            .filter(|url| self.passes_filter(url))
            .for_each(|url| self.enqueue_if_new(url, visited, queue));
        self.enqueue_if_new(start_url.clone(), visited, queue);
    }

    fn passes_filter(&self, url: &Url) -> bool {
        #[cfg(not(feature = "url-filtering"))]
        let _ = url;
        #[cfg(feature = "url-filtering")]
        let result = self.config.url_filter.as_ref().is_none_or(|f| f(url));
        #[cfg(not(feature = "url-filtering"))]
        let result = true;
        result
    }

    async fn fetch_page(&self, url: &Url) -> Result<String, CrawlerError> {
        let response = self
            .client
            .get(url.as_str())
            .send()
            .await?
            .error_for_status()?;

        Ok(response.text().await?)
    }

    async fn fetch_page_with_retry(&self, url: &Url) -> Result<String, CrawlerError> {
        let err = match self.fetch_page(url).await {
            Ok(s) => return Ok(s),
            Err(e) => e,
        };

        if !utils::is_not_found_error(&err) || !utils::has_duplicate_segments(url) {
            return Err(err);
        }

        let Some(deduped_url) = utils::deduplicate_url_segments(url) else {
            return Err(err);
        };

        warn!(
            "   🔄 404 on {}, retrying with deduplicated: {}",
            url, deduped_url
        );
        match self.fetch_page(&deduped_url).await {
            Ok(s) => {
                debug!("   ✅ Success with deduplicated URL");
                Ok(s)
            }
            Err(_) => {
                debug!("   ❌ Deduplicated URL also failed");
                Err(err)
            }
        }
    }

    async fn crawl_page(
        &self,
        current_url: Url,
        crawl_root_path: String,
    ) -> Result<(Url, String, Vec<Url>), (Url, CrawlerError)> {
        let html_str = self
            .fetch_page_with_retry(&current_url)
            .await
            .map_err(|error| (current_url.clone(), error))?;

        let html = Html::parse_document(&html_str);
        let links = utils::extract_links(&html, &current_url)
            .into_iter()
            .filter(|link| link != &current_url)
            .filter(|link| is_within_scope(link.path(), &crawl_root_path))
            .collect();

        Ok((current_url, html_str, links))
    }

    fn enqueue_if_new(&self, url: Url, visited: &mut HashSet<Url>, queue: &mut VecDeque<Url>) {
        if !visited.insert(url.clone()) {
            return;
        }

        #[cfg(feature = "url-priority")]
        if let Some(priority) = self.config.url_priority.as_ref() {
            let score = priority(&url);
            let position = queue
                .iter()
                .position(|queued| priority(queued) < score)
                .unwrap_or(queue.len());
            queue.insert(position, url);
            return;
        }

        queue.push_back(url);
    }
}

impl Crawl for Crawler {
    async fn crawl(&self) -> Result<CrawlResult, CrawlerError> {
        let mut visited = self.config.initial_visited.clone();
        let mut queue = VecDeque::new();
        let mut pages = vec![];
        let mut in_flight = FuturesUnordered::new();
        let max_concurrency = self.config.max_concurrency.max(1);
        let mut launched_requests = 0usize;

        let start_url = self
            .config
            .start_url
            .clone()
            .ok_or(CrawlerError::NoStartUrl)?;

        debug!("🕷️  Starting crawl from: {}", start_url);
        #[cfg(feature = "sitemap")]
        self.try_seed_from_sitemap(&start_url, &mut visited, &mut queue)
            .await;
        #[cfg(not(feature = "sitemap"))]
        self.enqueue_if_new(start_url.clone(), &mut visited, &mut queue);

        let crawl_root_path = normalize_scope_path(start_url.path());

        loop {
            while in_flight.len() < max_concurrency
                && pages.len() + in_flight.len() < self.config.max_pages
            {
                let Some(current_url) = queue.pop_front() else {
                    break;
                };

                if launched_requests > 0 {
                    sleep(Duration::from_millis(self.config.delay_ms)).await;
                }

                debug!(
                    "📄 [{}/{} fetched | {} accepted | {} in flight] Crawling: {}",
                    pages.len() + in_flight.len() + 1,
                    self.config.max_pages,
                    visited.len(),
                    in_flight.len() + 1,
                    current_url
                );

                in_flight.push(self.crawl_page(current_url, crawl_root_path.clone()));
                launched_requests += 1;
            }

            if pages.len() >= self.config.max_pages {
                debug!("⚠️  Reached max pages limit ({})", self.config.max_pages);
                break;
            }

            let Some(result) = in_flight.next().await else {
                break;
            };

            let (current_url, html_str, links) = match result {
                Ok(result) => result,
                Err((current_url, error)) => {
                    warn!("❌ Failed to fetch {}: {}", current_url, error);
                    continue;
                }
            };

            for link in links {
                if !self.passes_filter(&link) {
                    continue;
                }
                self.enqueue_if_new(link, &mut visited, &mut queue);
            }

            pages.push((current_url.clone(), html_str));

            if let Some(callback) = self.config.on_page_crawled.as_ref() {
                let page_html = pages
                    .last()
                    .map(|(_, html)| html.as_str())
                    .unwrap_or_default();
                callback(
                    &current_url,
                    page_html,
                    pages.len(),
                    queue.len(),
                    visited.len(),
                );
            }

            debug!(
                "  Queue: {} | Accepted: {} | Fetched: {}",
                queue.len(),
                visited.len(),
                pages.len()
            );
        }

        debug!("✅ Crawl complete! Crawled {} pages", pages.len());
        Ok(CrawlResult {
            crawled_urls: visited.into_iter().collect(),
            pages,
        })
    }
}

fn normalize_scope_path(path: &str) -> String {
    let trimmed = path.trim_end_matches('/');
    if trimmed.is_empty() {
        "/".to_string()
    } else {
        trimmed.to_string()
    }
}

fn is_within_scope(path: &str, scope_root: &str) -> bool {
    if scope_root == "/" {
        return path.starts_with('/');
    }
    path == scope_root || path.starts_with(&format!("{scope_root}/"))
}

#[cfg(test)]
mod tests;
