pub mod cache;
pub mod diff;
pub mod extract;
pub mod fetch;
pub mod fmt;
pub mod mcp;

use extract::Filter;
use serde::Serialize;

#[derive(Default, Clone)]
pub struct Opts {
	pub headers: Vec<String>,
	pub agent: String,
	pub max_time: u64,
	pub cookie: Option<String>,
	pub select: Option<String>,
	pub filters: Vec<Filter>,
	pub grep: Option<String>,
	pub no_meta: bool,
	pub max_tokens: Option<usize>,
	pub raw: bool,
	pub cache: bool,
	pub cache_ttl: u64,
	pub diff: bool,
}

#[derive(Serialize)]
pub struct Page {
	#[serde(skip_serializing_if = "String::is_empty")]
	pub title: String,
	pub url: String,
	#[serde(skip_serializing_if = "String::is_empty")]
	pub lang: String,
	#[serde(skip_serializing_if = "String::is_empty")]
	pub desc: String,
	pub body: String,
	pub tokens: usize,
	pub truncated: bool,
}

pub fn process(url: &str, opts: &Opts) -> Result<Page, String> {
	let html = fetch_html(url, opts)?;

	if opts.raw {
		return Ok(Page {
			title: String::new(),
			url: url.to_string(),
			lang: String::new(),
			desc: String::new(),
			tokens: estimate_tokens(&html),
			body: html,
			truncated: false,
		});
	}

	let doc = extract::run(&html, url, opts.select.as_deref(), &opts.filters);
	let mut body = fmt::render_body(&doc);

	if opts.diff {
		let old = cache::get_stale(url).unwrap_or_default();
		let new_body = body.clone();
		body = diff::lines(&old, &new_body);
		cache::put(url, &new_body);
	} else if opts.cache && url.starts_with("http") {
		cache::put(url, &body);
	}

	if let Some(pat) = &opts.grep {
		body = grep(&body, pat);
	}

	let mut truncated = false;
	if let Some(budget) = opts.max_tokens {
		let (t, did) = truncate_to_tokens(&body, budget);
		body = t;
		truncated = did;
	}

	let tokens = estimate_tokens(&body);

	Ok(Page {
		title: if opts.no_meta {
			String::new()
		} else {
			doc.title
		},
		url: doc.url,
		lang: if opts.no_meta {
			String::new()
		} else {
			doc.lang
		},
		desc: if opts.no_meta {
			String::new()
		} else {
			doc.desc
		},
		body,
		tokens,
		truncated,
	})
}

fn fetch_html(url: &str, opts: &Opts) -> Result<String, String> {
	let res = fetch::get(url, opts)?;
	Ok(res.body)
}

pub fn crawl(seed: &str, depth: u8, max_pages: usize, opts: &Opts) -> Vec<Result<Page, String>> {
	let mut seen = std::collections::HashSet::new();
	let mut queue: Vec<(String, u8)> = vec![(seed.to_string(), 0)];
	let mut out = Vec::new();

	while let Some((url, d)) = queue.pop() {
		if out.len() >= max_pages || !seen.insert(url.clone()) {
			continue;
		}
		let html = match fetch_html(&url, opts) {
			Ok(h) => h,
			Err(e) => {
				out.push(Err(e));
				continue;
			}
		};
		if d < depth {
			let base = origin(&url);
			for link in extract::out_links(&html) {
				if link.starts_with(&base) && !seen.contains(&link) {
					queue.push((link, d + 1));
				}
			}
		}
		let doc = extract::run(&html, &url, opts.select.as_deref(), &opts.filters);
		let body = fmt::render_body(&doc);
		let tokens = estimate_tokens(&body);
		out.push(Ok(Page {
			title: doc.title,
			url: doc.url,
			lang: doc.lang,
			desc: doc.desc,
			body,
			tokens,
			truncated: false,
		}));
	}
	out
}

fn origin(url: &str) -> String {
	url.splitn(4, '/').take(3).collect::<Vec<_>>().join("/")
}

fn grep(body: &str, pat: &str) -> String {
	let lc = pat.to_lowercase();
	body.lines()
		.filter(|l| l.to_lowercase().contains(&lc))
		.collect::<Vec<_>>()
		.join("\n")
}

pub fn estimate_tokens(s: &str) -> usize {
	let mut n: usize = 0;
	for ch in s.chars() {
		if ch.is_ascii() {
			n += 1;
		} else {
			n += 3;
		}
	}
	n / 4 + 1
}

fn truncate_to_tokens(s: &str, budget: usize) -> (String, bool) {
	if estimate_tokens(s) <= budget {
		return (s.to_string(), false);
	}
	let mut out = String::new();
	let mut cost: usize = 0;
	for line in s.lines() {
		let lc = estimate_tokens(line);
		if cost + lc > budget {
			break;
		}
		if !out.is_empty() {
			out.push('\n');
		}
		out.push_str(line);
		cost += lc;
	}
	(out, true)
}

#[cfg(test)]
mod tests {
	use super::*;

	// ---- estimate_tokens ----

	#[test]
	fn tokens_empty_is_one() {
		assert_eq!(estimate_tokens(""), 1); // 0 / 4 + 1
	}

	#[test]
	fn tokens_ascii_weight() {
		// 8 ascii chars -> 8/4 + 1 = 3
		assert_eq!(estimate_tokens("abcdefgh"), 3);
	}

	#[test]
	fn tokens_cjk_costs_more_than_ascii() {
		// non-ascii counts as 3 each
		assert!(estimate_tokens("あいう") > estimate_tokens("abc"));
	}

	// ---- truncate_to_tokens ----

	#[test]
	fn truncate_under_budget_is_untouched() {
		let (out, cut) = truncate_to_tokens("short line", 1000);
		assert_eq!(out, "short line");
		assert!(!cut);
	}

	#[test]
	fn truncate_over_budget_keeps_whole_lines_within_budget() {
		let text = "line one here\nline two here\nline three here";
		let (out, cut) = truncate_to_tokens(text, 4);
		assert!(cut);
		assert!(estimate_tokens(&out) <= 4);
		// every retained line is an intact original line
		assert!(out.lines().all(|l| text.lines().any(|t| t == l)));
	}

	// ---- grep ----

	#[test]
	fn grep_is_case_insensitive() {
		let body = "Install guide\nUsage\ninstallation notes";
		assert_eq!(grep(body, "install"), "Install guide\ninstallation notes");
	}

	#[test]
	fn grep_no_match_is_empty() {
		assert_eq!(grep("a\nb", "zzz"), "");
	}

	// ---- origin ----

	#[test]
	fn origin_extracts_scheme_and_host() {
		assert_eq!(
			origin("https://example.com/path/to/x"),
			"https://example.com"
		);
		assert_eq!(origin("http://a.b.co/"), "http://a.b.co");
	}

	// ---- process (end-to-end over a local file) ----

	fn write_html(name: &str, html: &str) -> String {
		let p = std::env::temp_dir().join(name);
		std::fs::write(&p, html).unwrap();
		p.to_str().unwrap().to_string()
	}

	#[test]
	fn process_renders_flq_body_with_meta() {
		let path = write_html(
			"flq-proc-meta.html",
			r#"<html lang="en"><head><title>T</title></head>
			<body><h1>Head</h1><p>a paragraph</p></body></html>"#,
		);
		let page = process(&path, &Opts::default()).unwrap();
		assert_eq!(page.title, "T");
		assert_eq!(page.lang, "en");
		assert!(page.body.contains("# Head"));
		assert!(page.body.contains("a paragraph"));
		assert!(page.tokens > 0);
		assert!(!page.truncated);
		let _ = std::fs::remove_file(&path);
	}

	#[test]
	fn process_no_meta_strips_frontmatter_fields() {
		let path = write_html(
			"flq-proc-nometa.html",
			r#"<html lang="en"><head><title>T</title></head><body><p>body text</p></body></html>"#,
		);
		let opts = Opts {
			no_meta: true,
			..Default::default()
		};
		let page = process(&path, &opts).unwrap();
		assert_eq!(page.title, "");
		assert_eq!(page.lang, "");
		assert!(page.body.contains("body text"));
		let _ = std::fs::remove_file(&path);
	}

	#[test]
	fn process_raw_passes_html_through() {
		let html = "<html><body><p>hi</p></body></html>";
		let path = write_html("flq-proc-raw.html", html);
		let opts = Opts {
			raw: true,
			..Default::default()
		};
		let page = process(&path, &opts).unwrap();
		assert_eq!(page.body, html);
		let _ = std::fs::remove_file(&path);
	}

	#[test]
	fn process_max_tokens_truncates() {
		let mut html = String::from("<html><body>");
		for i in 0..200 {
			html.push_str(&format!(
				"<p>paragraph number {i} with some filler text</p>"
			));
		}
		html.push_str("</body></html>");
		let path = write_html("flq-proc-trunc.html", &html);
		let opts = Opts {
			max_tokens: Some(20),
			..Default::default()
		};
		let page = process(&path, &opts).unwrap();
		assert!(page.truncated);
		assert!(page.tokens <= 20);
		let _ = std::fs::remove_file(&path);
	}
}
