use std::collections::HashMap;

use chrono::{DateTime, Utc};
use scraper::{ElementRef, Html, Selector};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use url::Url;

use crate::archive::stable_id;
use crate::errors::Result;

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ParsedPage {
    pub archive_id: String,
    pub page_id: String,
    pub source_url: String,
    pub canonical_url: Option<String>,
    pub title: String,
    pub archived_at: DateTime<Utc>,
    pub content_hash: String,
    pub language: Option<String>,
    pub metadata: PageMetadata,
    pub crawl_depth: usize,
    pub headings: Vec<Heading>,
    pub blocks: Vec<ContentBlock>,
    pub links: Vec<LinkRef>,
    pub assets: Vec<AssetRef>,
    pub code_blocks: Vec<CodeBlockRef>,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct Heading {
    pub level: u8,
    pub text: String,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct ContentBlock {
    #[serde(rename = "type")]
    pub block_type: BlockType,
    pub text: String,
    pub language: Option<String>,
    pub source_location: Option<String>,
    pub asset_id: Option<String>,
    pub confidence: f32,
    pub heading_level: Option<u8>,
    pub heading_path: Vec<String>,
    pub order_index: usize,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum BlockType {
    Paragraph,
    Heading,
    Code,
    Image,
    Table,
    List,
    Quote,
    Ocr,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct LinkRef {
    pub url: String,
    pub text: String,
}

#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
pub struct PageMetadata {
    pub description: Option<String>,
    pub meta_tags: Vec<MetaRef>,
    pub link_relations: Vec<LinkRelationRef>,
    pub json_ld: Vec<StructuredDataRef>,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct MetaRef {
    pub key: String,
    pub content: String,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct LinkRelationRef {
    pub rel: String,
    pub href: String,
    pub media_type: Option<String>,
    pub title: Option<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct StructuredDataRef {
    pub format: String,
    pub raw: String,
    pub parsed: Option<Value>,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct AssetRef {
    pub asset_id: String,
    pub url: String,
    pub local_path: Option<String>,
    pub alt_text: Option<String>,
    pub caption: Option<String>,
    pub dimensions: Option<ImageDimensions>,
    pub hash: Option<String>,
    pub appears_in: Vec<String>,
    pub ocr_text: Option<String>,
    pub ocr_status: Option<String>,
}

#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
pub struct ImageDimensions {
    pub width: u32,
    pub height: u32,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct CodeBlockRef {
    pub code_block_id: String,
    pub language: Option<String>,
    pub source_location: Option<String>,
    pub order_index: usize,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PageJson {
    pub archive_id: String,
    pub page_id: String,
    pub source_url: String,
    pub canonical_url: Option<String>,
    pub title: String,
    pub archived_at: DateTime<Utc>,
    pub content_hash: String,
    pub language: Option<String>,
    pub crawl_depth: usize,
    pub metadata: PageMetadata,
    pub headings: Vec<Heading>,
    pub blocks: Vec<ContentBlock>,
    pub links: Vec<LinkRef>,
    pub assets: Vec<AssetRef>,
    pub code_blocks: Vec<CodeBlockRef>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct JsonlChunk {
    pub archive_id: String,
    pub page_id: String,
    pub chunk_id: String,
    pub source_url: String,
    pub title: String,
    pub heading_path: Vec<String>,
    pub content_type: String,
    pub text: String,
    pub code_language: Option<String>,
    pub token_estimate: usize,
    pub asset_refs: Vec<String>,
    pub order_index: usize,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
struct MarkdownFrontmatter<'a> {
    title: &'a str,
    source_url: &'a str,
    canonical_url: Option<&'a str>,
    archived_at: DateTime<Utc>,
    content_hash: &'a str,
    language: Option<&'a str>,
    description: Option<&'a str>,
    crawl_depth: usize,
    assets: Vec<&'a str>,
    code_blocks: Vec<&'a str>,
}

pub fn parse_html(
    archive_id: &str,
    page_id: &str,
    source_url: &Url,
    html: &str,
    content_hash: &str,
    crawl_depth: usize,
) -> ParsedPage {
    let document = Html::parse_document(html);
    let metadata = extract_page_metadata(&document, source_url);
    let title = first_text(&document, "title")
        .or_else(|| first_text(&document, "h1"))
        .or_else(|| metadata_content(&metadata, &["og:title", "twitter:title", "title"]))
        .unwrap_or_else(|| source_url.to_string());
    let canonical_url = document
        .select(&selector("link[rel=canonical]"))
        .next()
        .and_then(|element| element.value().attr("href"))
        .and_then(|href| normalize_link(source_url, href))
        .map(|url| url.to_string());
    let language = document
        .select(&selector("html"))
        .next()
        .and_then(|element| element.value().attr("lang"))
        .map(str::to_string);

    let links = extract_links(&document, source_url);
    let root = content_root(&document);
    let mut blocks = Vec::new();
    let mut headings = Vec::new();
    let mut heading_stack: Vec<(u8, String)> = Vec::new();
    let mut assets_by_url: HashMap<String, AssetRef> = HashMap::new();
    let mut code_blocks = Vec::new();

    for element in root.descendent_elements() {
        let name = element.value().name();
        if should_skip_element(&element) {
            continue;
        }

        match name {
            "h1" | "h2" | "h3" | "h4" | "h5" | "h6" => {
                if has_ancestor_named(&element, &["h1", "h2", "h3", "h4", "h5", "h6"]) {
                    continue;
                }
                let text = normalize_ws(&element_text(&element));
                if text.is_empty() {
                    continue;
                }
                let level = name[1..].parse::<u8>().unwrap_or(1);
                while heading_stack
                    .last()
                    .map(|(last_level, _)| *last_level >= level)
                    .unwrap_or(false)
                {
                    heading_stack.pop();
                }
                heading_stack.push((level, text.clone()));
                headings.push(Heading {
                    level,
                    text: text.clone(),
                });
                push_block(
                    &mut blocks,
                    BlockType::Heading,
                    text,
                    None,
                    Some(format!("{}[{}]", name, headings.len())),
                    None,
                    1.0,
                    Some(level),
                    heading_path(&heading_stack),
                );
            }
            "p" => {
                if has_ancestor_named(&element, &["li", "blockquote", "table"])
                    || descendant_contains(&element, &["pre"])
                {
                    continue;
                }
                let text = normalize_ws(&element_text(&element));
                if !text.is_empty() {
                    push_block(
                        &mut blocks,
                        BlockType::Paragraph,
                        text,
                        None,
                        Some("p".to_string()),
                        None,
                        1.0,
                        None,
                        heading_path(&heading_stack),
                    );
                }
            }
            "pre" => {
                if has_ancestor_named(&element, &["pre"]) {
                    continue;
                }
                let code = element.text().collect::<Vec<_>>().join("");
                if code.trim().is_empty() {
                    continue;
                }
                let language = detect_code_language(&element);
                let source_location = Some("pre".to_string());
                let order_index = blocks.len();
                let code_block_id = format!(
                    "code-{}",
                    stable_id(&format!("{}:{}", page_id, order_index))
                );
                code_blocks.push(CodeBlockRef {
                    code_block_id,
                    language: language.clone(),
                    source_location: source_location.clone(),
                    order_index,
                });
                push_block(
                    &mut blocks,
                    BlockType::Code,
                    code,
                    language,
                    source_location,
                    None,
                    1.0,
                    None,
                    heading_path(&heading_stack),
                );
            }
            "img" => {
                if has_ancestor_named(&element, &["svg"]) {
                    continue;
                }
                let image_urls = element_image_urls(&element, source_url);
                if image_urls.is_empty() {
                    continue;
                }
                let alt_text = element
                    .value()
                    .attr("alt")
                    .map(str::trim)
                    .filter(|s| !s.is_empty())
                    .map(str::to_string);
                let caption = figure_caption(&element);
                for url in image_urls {
                    let url_string = url.to_string();
                    let asset_id = format!("asset-{}", stable_id(&url_string));
                    let block_text = caption
                        .clone()
                        .or_else(|| alt_text.clone())
                        .unwrap_or_else(|| url_string.clone());
                    let source_location = Some("img".to_string());
                    let block_ref = format!("block-{}", blocks.len());
                    assets_by_url
                        .entry(url_string.clone())
                        .and_modify(|asset| asset.appears_in.push(block_ref.clone()))
                        .or_insert_with(|| AssetRef {
                            asset_id: asset_id.clone(),
                            url: url_string.clone(),
                            local_path: None,
                            alt_text: alt_text.clone(),
                            caption: caption.clone(),
                            dimensions: None,
                            hash: None,
                            appears_in: vec![block_ref],
                            ocr_text: None,
                            ocr_status: None,
                        });
                    push_block(
                        &mut blocks,
                        BlockType::Image,
                        block_text,
                        None,
                        source_location,
                        Some(asset_id),
                        1.0,
                        None,
                        heading_path(&heading_stack),
                    );
                }
            }
            "source" => {
                if has_ancestor_named(&element, &["svg"]) {
                    continue;
                }
                for url in srcset_urls(
                    source_url,
                    element.value().attr("srcset").unwrap_or_default(),
                ) {
                    let url_string = url.to_string();
                    let asset_id = format!("asset-{}", stable_id(&url_string));
                    let block_ref = format!("block-{}", blocks.len());
                    assets_by_url
                        .entry(url_string.clone())
                        .and_modify(|asset| asset.appears_in.push(block_ref.clone()))
                        .or_insert_with(|| AssetRef {
                            asset_id,
                            url: url_string,
                            local_path: None,
                            alt_text: None,
                            caption: None,
                            dimensions: None,
                            hash: None,
                            appears_in: vec![block_ref],
                            ocr_text: None,
                            ocr_status: None,
                        });
                }
            }
            "ul" | "ol" => {
                if has_ancestor_named(&element, &["ul", "ol"]) {
                    continue;
                }
                let list = extract_list(&element, name == "ol");
                if !list.is_empty() {
                    push_block(
                        &mut blocks,
                        BlockType::List,
                        list,
                        None,
                        Some(name.to_string()),
                        None,
                        1.0,
                        None,
                        heading_path(&heading_stack),
                    );
                }
            }
            "table" => {
                if has_ancestor_named(&element, &["table"]) {
                    continue;
                }
                let table = extract_table(&element);
                if !table.is_empty() {
                    push_block(
                        &mut blocks,
                        BlockType::Table,
                        table,
                        None,
                        Some("table".to_string()),
                        None,
                        1.0,
                        None,
                        heading_path(&heading_stack),
                    );
                }
            }
            "blockquote" => {
                if has_ancestor_named(&element, &["blockquote"]) {
                    continue;
                }
                let text = normalize_ws(&element_text(&element));
                if !text.is_empty() {
                    push_block(
                        &mut blocks,
                        BlockType::Quote,
                        text,
                        None,
                        Some("blockquote".to_string()),
                        None,
                        1.0,
                        None,
                        heading_path(&heading_stack),
                    );
                }
            }
            _ => {}
        }
    }

    add_external_resource_assets(&document, source_url, &mut assets_by_url);

    ParsedPage {
        archive_id: archive_id.to_string(),
        page_id: page_id.to_string(),
        source_url: source_url.to_string(),
        canonical_url,
        title,
        archived_at: Utc::now(),
        content_hash: content_hash.to_string(),
        language,
        metadata,
        crawl_depth,
        headings,
        blocks,
        links,
        assets: assets_by_url.into_values().collect(),
        code_blocks,
    }
}

impl ParsedPage {
    pub fn to_page_json(&self) -> PageJson {
        PageJson {
            archive_id: self.archive_id.clone(),
            page_id: self.page_id.clone(),
            source_url: self.source_url.clone(),
            canonical_url: self.canonical_url.clone(),
            title: self.title.clone(),
            archived_at: self.archived_at,
            content_hash: self.content_hash.clone(),
            language: self.language.clone(),
            crawl_depth: self.crawl_depth,
            metadata: self.metadata.clone(),
            headings: self.headings.clone(),
            blocks: self.blocks.clone(),
            links: self.links.clone(),
            assets: self.assets.clone(),
            code_blocks: self.code_blocks.clone(),
        }
    }

    pub fn rewrite_asset_local_paths(&mut self, local_paths: &HashMap<String, String>) {
        for asset in &mut self.assets {
            if let Some(path) = local_paths.get(&asset.asset_id) {
                asset.local_path = Some(path.clone());
            }
        }
    }

    pub fn set_asset_ocr(&mut self, asset_id: &str, status: String, text: Option<String>) {
        if let Some(asset) = self
            .assets
            .iter_mut()
            .find(|asset| asset.asset_id == asset_id)
        {
            asset.ocr_status = Some(status.clone());
            asset.ocr_text = text.clone();
        }
        if let Some(text) = text.filter(|text| !text.trim().is_empty()) {
            let heading_path = self
                .blocks
                .iter()
                .rev()
                .find(|block| {
                    block.block_type == BlockType::Image
                        && block.asset_id.as_deref() == Some(asset_id)
                })
                .map(|block| block.heading_path.clone())
                .unwrap_or_default();
            push_block(
                &mut self.blocks,
                BlockType::Ocr,
                text,
                None,
                Some(format!("ocr:{asset_id}")),
                Some(asset_id.to_string()),
                0.55,
                None,
                heading_path,
            );
        }
    }

    pub fn add_source_asset_code(
        &mut self,
        asset_id: &str,
        source_url: &str,
        text: String,
        language: Option<String>,
    ) {
        let heading_path = vec!["Downloaded assets".to_string(), source_url.to_string()];
        push_block(
            &mut self.blocks,
            BlockType::Heading,
            format!("Downloaded source asset: {source_url}"),
            None,
            Some(format!("asset:{asset_id}")),
            Some(asset_id.to_string()),
            1.0,
            Some(3),
            heading_path.clone(),
        );

        let order_index = self.blocks.len();
        self.code_blocks.push(CodeBlockRef {
            code_block_id: format!(
                "code-{}",
                stable_id(&format!("{}:{asset_id}:{order_index}", self.page_id))
            ),
            language: language.clone(),
            source_location: Some(format!("asset:{source_url}")),
            order_index,
        });
        push_block(
            &mut self.blocks,
            BlockType::Code,
            text,
            language,
            Some(format!("asset:{source_url}")),
            Some(asset_id.to_string()),
            1.0,
            None,
            heading_path,
        );
    }

    pub fn to_markdown(&self) -> Result<String> {
        let frontmatter = MarkdownFrontmatter {
            title: &self.title,
            source_url: &self.source_url,
            canonical_url: self.canonical_url.as_deref(),
            archived_at: self.archived_at,
            content_hash: &self.content_hash,
            language: self.language.as_deref(),
            description: self.metadata.description.as_deref(),
            crawl_depth: self.crawl_depth,
            assets: self
                .assets
                .iter()
                .map(|asset| asset.asset_id.as_str())
                .collect(),
            code_blocks: self
                .code_blocks
                .iter()
                .map(|code| code.code_block_id.as_str())
                .collect(),
        };
        let yaml = serde_yaml::to_string(&frontmatter)?;
        let mut markdown = String::new();
        markdown.push_str("---\n");
        markdown.push_str(yaml.trim_start_matches("---\n"));
        markdown.push_str("---\n\n");
        markdown.push_str(&format!("<!-- Source: {} -->\n\n", self.source_url));

        for block in &self.blocks {
            append_markdown_block(&mut markdown, self, block);
        }

        markdown.push_str(&format!("---\n\nArchived from: {}\n", self.source_url));
        Ok(markdown)
    }

    pub fn to_basic_markdown(&self) -> String {
        let mut markdown = String::new();
        markdown.push_str(&format!("# {}\n\n", self.title.trim()));
        markdown.push_str(&format!("Source: {}\n\n", self.source_url));
        if let Some(canonical_url) = self.canonical_url.as_deref() {
            markdown.push_str(&format!("Canonical: {canonical_url}\n\n"));
        }
        if let Some(description) = self.metadata.description.as_deref() {
            markdown.push_str(&format!("Description: {}\n\n", description.trim()));
        }

        markdown.push_str("## Content\n\n");
        for block in &self.blocks {
            append_markdown_block(&mut markdown, self, block);
        }

        append_basic_metadata(&mut markdown, self);
        append_basic_links(&mut markdown, self);
        append_basic_assets(&mut markdown, self);
        markdown
    }

    pub fn to_plain_text(&self) -> String {
        let mut text = String::new();
        text.push_str(&self.title);
        text.push('\n');
        text.push_str(&self.source_url);
        text.push('\n');
        for block in &self.blocks {
            text.push_str(&block.text);
            text.push_str("\n\n");
        }
        text
    }

    pub fn chunks(&self) -> Vec<JsonlChunk> {
        self.blocks
            .iter()
            .filter(|block| !block.text.trim().is_empty())
            .map(|block| {
                let content_type = match block.block_type {
                    BlockType::Paragraph => "paragraph",
                    BlockType::Heading => "heading",
                    BlockType::Code => "code",
                    BlockType::Image => "image",
                    BlockType::Table => "table",
                    BlockType::List => "list",
                    BlockType::Quote => "quote",
                    BlockType::Ocr => "ocr",
                }
                .to_string();
                JsonlChunk {
                    archive_id: self.archive_id.clone(),
                    page_id: self.page_id.clone(),
                    chunk_id: format!("{}-{:05}", self.page_id, block.order_index),
                    source_url: self.source_url.clone(),
                    title: self.title.clone(),
                    heading_path: block.heading_path.clone(),
                    content_type,
                    text: block.text.clone(),
                    code_language: block.language.clone(),
                    token_estimate: estimate_tokens(&block.text),
                    asset_refs: block.asset_id.iter().cloned().collect(),
                    order_index: block.order_index,
                }
            })
            .collect()
    }
}

pub fn estimate_tokens(text: &str) -> usize {
    (text.chars().count() / 4).max(1)
}

fn append_markdown_block(markdown: &mut String, page: &ParsedPage, block: &ContentBlock) {
    match block.block_type {
        BlockType::Heading => {
            let level = block.heading_level.unwrap_or(2).clamp(1, 6) as usize;
            markdown.push_str(&format!("{} {}\n\n", "#".repeat(level), block.text.trim()));
        }
        BlockType::Paragraph => {
            markdown.push_str(block.text.trim());
            markdown.push_str("\n\n");
        }
        BlockType::Code => {
            let fence = code_fence_for(&block.text);
            let language = block.language.as_deref().unwrap_or("");
            markdown.push_str(&format!("{fence}{language}\n"));
            markdown.push_str(block.text.trim_end_matches('\n'));
            markdown.push('\n');
            markdown.push_str(&format!("{fence}\n\n"));
        }
        BlockType::Image => {
            let path = block
                .asset_id
                .as_ref()
                .and_then(|id| page.assets.iter().find(|asset| &asset.asset_id == id))
                .and_then(|asset| asset.local_path.as_deref().or(Some(asset.url.as_str())))
                .unwrap_or(block.text.as_str());
            markdown.push_str(&format!(
                "![{}]({})\n\n",
                escape_markdown_alt(&block.text),
                path
            ));
        }
        BlockType::Table | BlockType::List => {
            markdown.push_str(&block.text);
            markdown.push_str("\n\n");
        }
        BlockType::Quote => {
            for line in block.text.lines() {
                markdown.push_str(&format!("> {}\n", line.trim()));
            }
            markdown.push('\n');
        }
        BlockType::Ocr => {
            let asset = block.asset_id.as_deref().unwrap_or("unknown-image");
            markdown.push_str(&format!("### OCR-derived text for `{asset}`\n\n"));
            markdown
                .push_str("> Confidence: lower than verified HTML code. Review before reuse.\n\n");
            let fence = code_fence_for(&block.text);
            markdown.push_str(&format!("{fence}\n{}\n{fence}\n\n", block.text.trim_end()));
        }
    }
}

fn append_basic_metadata(markdown: &mut String, page: &ParsedPage) {
    markdown.push_str("## Page Data\n\n");
    markdown.push_str(&format!("- Archive ID: `{}`\n", page.archive_id));
    markdown.push_str(&format!("- Page ID: `{}`\n", page.page_id));
    markdown.push_str(&format!("- Archived at: {}\n", page.archived_at));
    markdown.push_str(&format!("- Content hash: `{}`\n", page.content_hash));
    markdown.push_str(&format!("- Crawl depth: {}\n", page.crawl_depth));
    if let Some(language) = page.language.as_deref() {
        markdown.push_str(&format!("- Language: `{language}`\n"));
    }
    markdown.push('\n');

    if !page.metadata.meta_tags.is_empty() {
        markdown.push_str("### Meta Tags\n\n");
        for tag in &page.metadata.meta_tags {
            markdown.push_str(&format!("- `{}`: {}\n", tag.key, tag.content));
        }
        markdown.push('\n');
    }

    if !page.metadata.link_relations.is_empty() {
        markdown.push_str("### Link Relations\n\n");
        for link in &page.metadata.link_relations {
            markdown.push_str(&format!("- `{}`: {}", link.rel, link.href));
            if let Some(media_type) = link.media_type.as_deref() {
                markdown.push_str(&format!(" ({media_type})"));
            }
            if let Some(title) = link.title.as_deref() {
                markdown.push_str(&format!(" - {title}"));
            }
            markdown.push('\n');
        }
        markdown.push('\n');
    }

    if !page.metadata.json_ld.is_empty() {
        markdown.push_str("### JSON-LD Structured Data\n\n");
        for item in &page.metadata.json_ld {
            let fence = code_fence_for(&item.raw);
            markdown.push_str(&format!("{fence}json\n{}\n{fence}\n\n", item.raw.trim()));
        }
    }
}

fn append_basic_links(markdown: &mut String, page: &ParsedPage) {
    if page.links.is_empty() {
        return;
    }
    markdown.push_str("## Links\n\n");
    for link in &page.links {
        let label = if link.text.trim().is_empty() {
            link.url.as_str()
        } else {
            link.text.as_str()
        };
        markdown.push_str(&format!("- [{}]({})\n", escape_link_label(label), link.url));
    }
    markdown.push('\n');
}

fn append_basic_assets(markdown: &mut String, page: &ParsedPage) {
    if page.assets.is_empty() {
        return;
    }
    markdown.push_str("## Assets\n\n");
    for asset in &page.assets {
        markdown.push_str(&format!("- `{}`\n", asset.asset_id));
        markdown.push_str(&format!("  - Source: {}\n", asset.url));
        if let Some(local_path) = asset.local_path.as_deref() {
            markdown.push_str(&format!("  - Local: {local_path}\n"));
        }
        if let Some(alt_text) = asset.alt_text.as_deref() {
            markdown.push_str(&format!("  - Alt: {alt_text}\n"));
        }
        if let Some(caption) = asset.caption.as_deref() {
            markdown.push_str(&format!("  - Caption: {caption}\n"));
        }
        if let Some(dimensions) = asset.dimensions {
            markdown.push_str(&format!(
                "  - Dimensions: {}x{}\n",
                dimensions.width, dimensions.height
            ));
        }
        if let Some(hash) = asset.hash.as_deref() {
            markdown.push_str(&format!("  - Hash: `{hash}`\n"));
        }
        if let Some(status) = asset.ocr_status.as_deref() {
            markdown.push_str(&format!("  - OCR: {status}\n"));
        }
    }
    markdown.push('\n');
}

fn extract_page_metadata(document: &Html, base: &Url) -> PageMetadata {
    let meta_tags = extract_meta_tags(document);
    let description = meta_content_from_tags(
        &meta_tags,
        &[
            "description",
            "og:description",
            "twitter:description",
            "dc.description",
        ],
    );
    PageMetadata {
        description,
        meta_tags,
        link_relations: extract_link_relations(document, base),
        json_ld: extract_json_ld(document),
    }
}

fn extract_meta_tags(document: &Html) -> Vec<MetaRef> {
    let mut tags = Vec::new();
    for element in document.select(&selector("meta")) {
        let key = element
            .value()
            .attr("name")
            .or_else(|| element.value().attr("property"))
            .or_else(|| element.value().attr("http-equiv"))
            .or_else(|| element.value().attr("itemprop"))
            .or_else(|| element.value().attr("charset"))
            .map(str::trim)
            .filter(|value| !value.is_empty());
        let content = element
            .value()
            .attr("content")
            .or_else(|| element.value().attr("charset"))
            .map(str::trim)
            .filter(|value| !value.is_empty());
        let (Some(key), Some(content)) = (key, content) else {
            continue;
        };
        tags.push(MetaRef {
            key: key.to_ascii_lowercase(),
            content: html_escape::decode_html_entities(content).to_string(),
        });
    }
    tags.dedup_by(|a, b| a.key == b.key && a.content == b.content);
    tags
}

fn extract_link_relations(document: &Html, base: &Url) -> Vec<LinkRelationRef> {
    let mut relations = Vec::new();
    for element in document.select(&selector("link[rel][href]")) {
        let Some(rel) = element.value().attr("rel").map(str::trim) else {
            continue;
        };
        let Some(href) = element.value().attr("href").map(str::trim) else {
            continue;
        };
        if rel.is_empty() || href.is_empty() {
            continue;
        }
        relations.push(LinkRelationRef {
            rel: rel.to_ascii_lowercase(),
            href: normalize_link_relation_href(base, href),
            media_type: element
                .value()
                .attr("type")
                .map(str::trim)
                .filter(|value| !value.is_empty())
                .map(str::to_string),
            title: element
                .value()
                .attr("title")
                .map(str::trim)
                .filter(|value| !value.is_empty())
                .map(str::to_string),
        });
    }
    relations.dedup_by(|a, b| a.rel == b.rel && a.href == b.href);
    relations
}

fn extract_json_ld(document: &Html) -> Vec<StructuredDataRef> {
    let mut items = Vec::new();
    for element in document.select(&selector("script[type]")) {
        let kind = element
            .value()
            .attr("type")
            .unwrap_or_default()
            .to_ascii_lowercase();
        if !kind.contains("ld+json") {
            continue;
        }
        let raw = element
            .text()
            .collect::<Vec<_>>()
            .join("")
            .trim()
            .to_string();
        if raw.is_empty() {
            continue;
        }
        items.push(StructuredDataRef {
            format: "json-ld".to_string(),
            parsed: serde_json::from_str::<Value>(&raw).ok(),
            raw,
        });
    }
    items
}

fn add_external_resource_assets(
    document: &Html,
    base: &Url,
    assets_by_url: &mut HashMap<String, AssetRef>,
) {
    for element in document.select(&selector("script[src]")) {
        add_resource_asset(
            assets_by_url,
            base,
            element.value().attr("src"),
            "script",
            element.value().attr("type"),
        );
    }
    for element in document.select(&selector("link[rel][href]")) {
        let rel = element.value().attr("rel").unwrap_or_default();
        if !is_asset_link_relation(rel) {
            continue;
        }
        add_resource_asset(
            assets_by_url,
            base,
            element.value().attr("href"),
            rel,
            element
                .value()
                .attr("type")
                .or_else(|| element.value().attr("title")),
        );
    }
    for (selector_text, attr, kind) in [
        ("video[src]", "src", "video"),
        ("audio[src]", "src", "audio"),
        ("track[src]", "src", "track"),
        ("iframe[src]", "src", "iframe"),
        ("embed[src]", "src", "embed"),
        ("object[data]", "data", "object"),
        ("source[src]", "src", "source"),
    ] {
        for element in document.select(&selector(selector_text)) {
            add_resource_asset(
                assets_by_url,
                base,
                element.value().attr(attr),
                kind,
                element
                    .value()
                    .attr("type")
                    .or_else(|| element.value().attr("title")),
            );
        }
    }
    for element in document.select(&selector("source[srcset]")) {
        for url in srcset_urls(base, element.value().attr("srcset").unwrap_or_default()) {
            insert_resource_asset(
                assets_by_url,
                url,
                "source srcset",
                element.value().attr("type"),
            );
        }
    }
}

fn add_resource_asset(
    assets_by_url: &mut HashMap<String, AssetRef>,
    base: &Url,
    raw_url: Option<&str>,
    kind: &str,
    label: Option<&str>,
) {
    let Some(url) = raw_url.and_then(|raw| normalize_link(base, raw)) else {
        return;
    };
    insert_resource_asset(assets_by_url, url, kind, label);
}

fn insert_resource_asset(
    assets_by_url: &mut HashMap<String, AssetRef>,
    url: Url,
    kind: &str,
    label: Option<&str>,
) {
    let url_string = url.to_string();
    let asset_id = format!("asset-{}", stable_id(&url_string));
    let block_ref = format!("resource:{kind}");
    assets_by_url
        .entry(url_string.clone())
        .and_modify(|asset| {
            if !asset
                .appears_in
                .iter()
                .any(|existing| existing == &block_ref)
            {
                asset.appears_in.push(block_ref.clone());
            }
        })
        .or_insert_with(|| AssetRef {
            asset_id,
            url: url_string,
            local_path: None,
            alt_text: label
                .map(str::trim)
                .filter(|value| !value.is_empty())
                .map(str::to_string),
            caption: Some(format!("{kind} resource")),
            dimensions: None,
            hash: None,
            appears_in: vec![block_ref],
            ocr_text: None,
            ocr_status: None,
        });
}

fn is_asset_link_relation(rel: &str) -> bool {
    rel.split_whitespace().any(|part| {
        matches!(
            part.to_ascii_lowercase().as_str(),
            "stylesheet"
                | "preload"
                | "modulepreload"
                | "manifest"
                | "icon"
                | "shortcut"
                | "apple-touch-icon"
                | "alternate"
        )
    })
}

fn metadata_content(metadata: &PageMetadata, keys: &[&str]) -> Option<String> {
    meta_content_from_tags(&metadata.meta_tags, keys)
}

fn meta_content_from_tags(tags: &[MetaRef], keys: &[&str]) -> Option<String> {
    tags.iter()
        .find(|tag| keys.iter().any(|key| tag.key.eq_ignore_ascii_case(key)))
        .map(|tag| tag.content.clone())
        .filter(|content| !content.trim().is_empty())
}

fn normalize_link_relation_href(base: &Url, raw: &str) -> String {
    normalize_link(base, raw)
        .map(|url| url.to_string())
        .or_else(|| base.join(raw).ok().map(|url| url.to_string()))
        .unwrap_or_else(|| raw.to_string())
}

fn first_text(document: &Html, selector_text: &str) -> Option<String> {
    document
        .select(&selector(selector_text))
        .next()
        .map(|element| normalize_ws(&element_text(&element)))
        .filter(|text| !text.is_empty())
}

fn content_root<'a>(document: &'a Html) -> ElementRef<'a> {
    for selector_text in ["article", "main", "[role=main]", "body"] {
        if let Some(element) = document.select(&selector(selector_text)).next() {
            return element;
        }
    }
    document.root_element()
}

fn extract_links(document: &Html, base: &Url) -> Vec<LinkRef> {
    let mut links = Vec::new();
    for element in document.select(&selector("a[href]")) {
        let Some(href) = element.value().attr("href") else {
            continue;
        };
        let Some(url) = normalize_link(base, href) else {
            continue;
        };
        let text = normalize_ws(&element_text(&element));
        links.push(LinkRef {
            url: url.to_string(),
            text,
        });
    }
    links.sort_by(|a, b| a.url.cmp(&b.url));
    links.dedup_by(|a, b| a.url == b.url);
    links
}

pub fn normalize_link(base: &Url, raw: &str) -> Option<Url> {
    let trimmed = raw.trim();
    if trimmed.is_empty()
        || trimmed.starts_with('#')
        || trimmed.starts_with("mailto:")
        || trimmed.starts_with("tel:")
        || trimmed.starts_with("javascript:")
        || trimmed.starts_with("data:")
    {
        return None;
    }
    let mut url = base.join(trimmed).ok()?;
    if !matches!(url.scheme(), "http" | "https") {
        return None;
    }
    url.set_fragment(None);
    if (url.scheme() == "http" && url.port() == Some(80))
        || (url.scheme() == "https" && url.port() == Some(443))
    {
        let _ = url.set_port(None);
    }
    if url.path().is_empty() {
        url.set_path("/");
    }
    if url.query().is_some() {
        let mut pairs = url
            .query_pairs()
            .map(|(key, value)| (key.into_owned(), value.into_owned()))
            .collect::<Vec<_>>();
        pairs.sort();
        url.set_query(None);
        let mut serializer = url.query_pairs_mut();
        for (key, value) in pairs {
            serializer.append_pair(&key, &value);
        }
    }
    Some(url)
}

fn element_image_urls(element: &ElementRef<'_>, base: &Url) -> Vec<Url> {
    let mut urls = Vec::new();
    for attr in [
        "src",
        "data-src",
        "data-original",
        "data-lazy-src",
        "data-url",
    ] {
        if let Some(url) = element
            .value()
            .attr(attr)
            .and_then(|value| normalize_link(base, value))
        {
            urls.push(url);
        }
    }
    if let Some(srcset) = element.value().attr("srcset") {
        urls.extend(srcset_urls(base, srcset));
    }
    urls.sort();
    urls.dedup();
    urls
}

fn srcset_urls(base: &Url, srcset: &str) -> Vec<Url> {
    let mut urls = srcset
        .split(',')
        .filter_map(|candidate| candidate.split_whitespace().next())
        .filter_map(|candidate| normalize_link(base, candidate))
        .collect::<Vec<_>>();
    urls.sort();
    urls.dedup();
    urls
}

fn should_skip_element(element: &ElementRef<'_>) -> bool {
    has_ancestor_named(
        element,
        &[
            "script", "style", "noscript", "template", "nav", "footer", "header", "aside",
        ],
    )
}

fn has_ancestor_named(element: &ElementRef<'_>, names: &[&str]) -> bool {
    let mut parent = element.parent();
    while let Some(node) = parent {
        if let Some(parent_element) = ElementRef::wrap(node) {
            if names.contains(&parent_element.value().name()) {
                return true;
            }
        }
        parent = node.parent();
    }
    false
}

fn descendant_contains(element: &ElementRef<'_>, names: &[&str]) -> bool {
    element
        .descendent_elements()
        .skip(1)
        .any(|descendant| names.contains(&descendant.value().name()))
}

fn figure_caption(element: &ElementRef<'_>) -> Option<String> {
    let mut parent = element.parent();
    while let Some(node) = parent {
        let Some(parent_element) = ElementRef::wrap(node) else {
            parent = node.parent();
            continue;
        };
        if parent_element.value().name() == "figure" {
            return parent_element
                .select(&selector("figcaption"))
                .next()
                .map(|caption| normalize_ws(&element_text(&caption)))
                .filter(|caption| !caption.is_empty());
        }
        parent = node.parent();
    }
    None
}

fn extract_list(element: &ElementRef<'_>, ordered: bool) -> String {
    let mut lines = Vec::new();
    let mut index = 1usize;
    for child in element.child_elements() {
        if child.value().name() != "li" {
            continue;
        }
        let text = normalize_ws(&element_text(&child));
        if text.is_empty() {
            continue;
        }
        if ordered {
            lines.push(format!("{index}. {text}"));
            index += 1;
        } else {
            lines.push(format!("- {text}"));
        }
    }
    lines.join("\n")
}

fn extract_table(element: &ElementRef<'_>) -> String {
    let mut rows = Vec::new();
    for row in element.select(&selector("tr")) {
        let cells = row
            .child_elements()
            .filter(|cell| matches!(cell.value().name(), "th" | "td"))
            .map(|cell| normalize_ws(&element_text(&cell)).replace('|', "\\|"))
            .collect::<Vec<_>>();
        if !cells.is_empty() {
            rows.push(cells);
        }
    }
    if rows.is_empty() {
        return String::new();
    }
    let width = rows.iter().map(Vec::len).max().unwrap_or(0);
    let mut markdown = String::new();
    for (index, row) in rows.iter().enumerate() {
        markdown.push('|');
        for column in 0..width {
            markdown.push(' ');
            markdown.push_str(row.get(column).map(String::as_str).unwrap_or(""));
            markdown.push_str(" |");
        }
        markdown.push('\n');
        if index == 0 {
            markdown.push('|');
            for _ in 0..width {
                markdown.push_str(" --- |");
            }
            markdown.push('\n');
        }
    }
    markdown.trim_end().to_string()
}

fn element_text(element: &ElementRef<'_>) -> String {
    element.text().collect::<Vec<_>>().join(" ")
}

fn normalize_ws(input: &str) -> String {
    input.split_whitespace().collect::<Vec<_>>().join(" ")
}

fn heading_path(stack: &[(u8, String)]) -> Vec<String> {
    stack.iter().map(|(_, text)| text.clone()).collect()
}

#[allow(clippy::too_many_arguments)]
fn push_block(
    blocks: &mut Vec<ContentBlock>,
    block_type: BlockType,
    text: String,
    language: Option<String>,
    source_location: Option<String>,
    asset_id: Option<String>,
    confidence: f32,
    heading_level: Option<u8>,
    heading_path: Vec<String>,
) {
    let order_index = blocks.len();
    blocks.push(ContentBlock {
        block_type,
        text,
        language,
        source_location,
        asset_id,
        confidence,
        heading_level,
        heading_path,
        order_index,
    });
}

fn detect_code_language(element: &ElementRef<'_>) -> Option<String> {
    let mut classes = Vec::new();
    collect_classes(element, &mut classes);
    for child in element.descendent_elements() {
        if child.value().name() == "code" {
            collect_classes(&child, &mut classes);
        }
    }
    classes.into_iter().find_map(language_from_class)
}

fn collect_classes(element: &ElementRef<'_>, classes: &mut Vec<String>) {
    if let Some(class_attr) = element.value().attr("class") {
        classes.extend(class_attr.split_whitespace().map(str::to_string));
    }
}

fn language_from_class(class: String) -> Option<String> {
    let normalized = class.to_ascii_lowercase();
    for prefix in ["language-", "lang-", "highlight-source-", "brush:"] {
        if let Some(language) = normalized.strip_prefix(prefix) {
            return Some(normalize_language(language.trim_matches(';')));
        }
    }
    match normalized.as_str() {
        "csharp" | "cs" => Some("csharp".to_string()),
        "rust" | "rs" => Some("rust".to_string()),
        "python" | "py" => Some("python".to_string()),
        "javascript" | "js" => Some("javascript".to_string()),
        "typescript" | "ts" => Some("typescript".to_string()),
        "html" | "xml" | "css" | "json" | "toml" | "yaml" | "yml" | "bash" | "sh" => {
            Some(normalize_language(&normalized))
        }
        _ => None,
    }
}

fn normalize_language(language: &str) -> String {
    match language {
        "cs" => "csharp".to_string(),
        "py" => "python".to_string(),
        "js" => "javascript".to_string(),
        "ts" => "typescript".to_string(),
        "rs" => "rust".to_string(),
        "yml" => "yaml".to_string(),
        other => other.to_string(),
    }
}

fn code_fence_for(code: &str) -> String {
    let mut longest = 0usize;
    let mut current = 0usize;
    for ch in code.chars() {
        if ch == '`' {
            current += 1;
            longest = longest.max(current);
        } else {
            current = 0;
        }
    }
    "`".repeat(longest.max(3) + 1)
}

fn escape_markdown_alt(text: &str) -> String {
    text.replace('[', "\\[").replace(']', "\\]")
}

fn escape_link_label(text: &str) -> String {
    escape_markdown_alt(text).replace('\n', " ")
}

fn selector(input: &str) -> Selector {
    Selector::parse(input).expect("static selector should parse")
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn normalizes_urls() {
        let base = Url::parse("https://example.com/a/b/index.html").unwrap();
        let normalized = normalize_link(&base, "../c/?b=2&a=1#section").unwrap();
        assert_eq!(normalized.as_str(), "https://example.com/a/c/?a=1&b=2");
    }

    #[test]
    fn preserves_code_block_indentation() {
        let html = r#"<article><h1>Code</h1><pre><code class="language-csharp">for (int i = 0; i < 3; i++) {
    Debug.Log(i);
}</code></pre></article>"#;
        let url = Url::parse("https://example.com/tutorial").unwrap();
        let page = parse_html("a", "p", &url, html, "hash", 0);
        let code = page
            .blocks
            .iter()
            .find(|block| block.block_type == BlockType::Code)
            .unwrap();
        assert!(code.text.contains("    Debug.Log(i);"));
        assert_eq!(code.language.as_deref(), Some("csharp"));
    }

    #[test]
    fn rewrites_asset_paths() {
        let html = r#"<article><p>Intro</p><img src="img/code.png" alt="code"></article>"#;
        let url = Url::parse("https://example.com/tutorial/").unwrap();
        let mut page = parse_html("a", "p", &url, html, "hash", 0);
        let asset_id = page.assets[0].asset_id.clone();
        page.rewrite_asset_local_paths(&HashMap::from([(
            asset_id.clone(),
            "../../raw/assets/a.png".to_string(),
        )]));
        assert_eq!(
            page.assets[0].local_path.as_deref(),
            Some("../../raw/assets/a.png")
        );
        assert!(page
            .to_markdown()
            .unwrap()
            .contains("../../raw/assets/a.png"));
    }

    #[test]
    fn chunks_do_not_split_code_blocks() {
        let html = r#"<article><pre><code>fn main() {
    println!("hi");
}</code></pre></article>"#;
        let url = Url::parse("https://example.com/tutorial/").unwrap();
        let page = parse_html("a", "p", &url, html, "hash", 0);
        let chunks = page.chunks();
        assert_eq!(chunks.len(), 1);
        assert!(chunks[0].text.contains("println!"));
        assert_eq!(chunks[0].content_type, "code");
    }
}
