//! URL string utilities used across scanners.
//!
//! Consolidates `resolve_url`, `clean_url`, `registrable_domain`,
//! `dedup_host_key`, and `strip_www_host` - five small functions
//! that had each been copied (with minor drift, including outright
//! bugs) into at least four scanner crates:
//!
//! * `golemn_engine::extract::html` - the most complete copy; this
//!   module's behaviour matches it byte-for-byte so existing
//!   call sites can migrate without behavioural change.
//! * `karyx_engine::crawler` - had no scheme handling and would
//!   prepend the base to a fully-qualified URL (`http://other.com/foo`
//!   became `http://current.com/http://other.com/foo`). Migrating
//!   karyx to this module is a bug fix, not a refactor.
//! * `sear::vision::image_source` - feature-gated; calls a
//!   different `resolve_relative_url` under the `resolve` feature.
//!   Intentionally NOT migrated: sear's vision pipeline uses
//!   stronger url-crate-based resolution that handles `..` segments
//!   and other RFC 3986 edge cases this string-level resolver
//!   does not.
//! * `scald::uploaded_file_xss::resolve_url` - config-coupled
//!   (`fn(&ScanConfig, &str)`); should call this module after
//!   destructuring the origin string from `ScanConfig`.
//!
//! All five functions are pure string-level - they do not depend on
//! `tokio`, `hyper`, or `reqwest`. Living inside `scanclient` keeps
//! them adjacent to the rest of the HTTP layer without growing the
//! published dependency surface for callers that only want
//! `resolve_url` (the `url = "2"` crate is already a scanclient dep
//! for the rest of the transport).

use std::borrow::Cow;

/// Tracking-parameter keys stripped by [`clean_url`]. Sourced from
/// the canonical list embedded in `golemn_engine::extract::html` so
/// the behaviour matches byte-for-byte. Sorted for binary search.
///
/// Comparison is case-sensitive (matches the original behaviour); a
/// future enhancement could case-fold both sides, but that would be
/// a behavioural change and is therefore intentionally deferred.
const TRACKING_PARAMS: &[&str] = &[
    "_ga",
    "_gl",
    "aff_id",
    "affiliate",
    "dclid",
    "fbclid",
    "gclid",
    "mc_cid",
    "mc_eid",
    "msclkid",
    "ref",
    "ref_",
    "source",
    "twclid",
    "utm_campaign",
    "utm_content",
    "utm_medium",
    "utm_source",
    "utm_term",
];

/// Resolve a possibly-relative URL against a base.
///
/// Behaviour (matches `golemn_engine::extract::html::resolve_url`
/// byte-for-byte so consumers can switch to this implementation
/// without observable change):
///
/// * Already-absolute `http://` / `https://` URLs pass through
///   unchanged.
/// * Scheme-relative `//host/path` is upgraded to `https://host/path`.
///   This is opinionated - RFC 3986 says the scheme is inherited
///   from the base - but it matches what every shipping copy has
///   been doing, and the only known callers operate over HTTPS.
/// * Absolute-path `/path` is resolved against the base's
///   scheme-and-authority prefix (the substring up to but not
///   including the first `/` after `://`). If the base has no path
///   slash after `://`, the path is concatenated directly.
/// * Anything else is treated as relative to the base's
///   *last-slash directory* - i.e. `resolve_url("https://a.com/b/c",
///   "d")` returns `https://a.com/b/d`, not `https://a.com/b/c/d`.
///   This is the directory-context interpretation that every
///   shipping copy already uses.
///
/// This is a string-level resolver and intentionally does NOT
/// handle `..` segment collapsing, percent-encoding repair, or
/// fragment merging. Callers that need full RFC 3986 resolution
/// should use the `url` crate directly.
#[must_use]
pub fn resolve_url(base: &str, relative: &str) -> String {
    if relative.starts_with("http://") || relative.starts_with("https://") {
        return relative.to_string();
    }
    if let Some(stripped) = relative.strip_prefix("//") {
        return format!("https://{stripped}");
    }
    if let Some(path) = relative.strip_prefix('/') {
        if let Some(scheme_end) = base.find("://") {
            let after_scheme = &base[scheme_end + 3..];
            return match after_scheme.find('/') {
                Some(end) => format!("{}/{path}", &base[..scheme_end + 3 + end]),
                None => format!("{base}/{path}"),
            };
        }
        return format!("{base}/{path}");
    }
    if let Some(last_slash) = base.rfind('/') {
        return format!("{}{relative}", &base[..=last_slash]);
    }
    format!("{base}/{relative}")
}

/// Strip tracking parameters and the fragment from a URL.
///
/// Removes `utm_*`, `fbclid`, `gclid`, and other analytics
/// parameters (see [`TRACKING_PARAMS`]) that do not affect page
/// content but pollute URL-based deduplication. The fragment
/// (`#section`) is dropped because it never reaches the server.
///
/// Malformed URLs that fail to parse are returned unchanged - this
/// matches the lenient behaviour of the original copies and means
/// `clean_url` is safe to call on arbitrary string keys.
#[must_use]
pub fn clean_url(url: &str) -> String {
    let Ok(mut parsed) = ::url::Url::parse(url) else {
        return url.to_string();
    };

    let clean_pairs: Vec<(String, String)> = parsed
        .query_pairs()
        .filter(|(key, _)| !TRACKING_PARAMS.contains(&key.as_ref()))
        .map(|(k, v)| (k.into_owned(), v.into_owned()))
        .collect();

    if clean_pairs.is_empty() {
        parsed.set_query(None);
    } else {
        let query = clean_pairs
            .iter()
            .map(|(k, v)| format!("{k}={v}"))
            .collect::<Vec<_>>()
            .join("&");
        parsed.set_query(Some(&query));
    }

    parsed.set_fragment(None);
    parsed.to_string()
}

/// Lowercase host with `www.` stripped - the cache key shape used
/// for VirusTotal, Safe Browsing, and similar host-keyed lookups.
///
/// Returns `None` for inputs that do not contain a scheme or for
/// inputs whose authority section is empty (e.g. `http:///path`).
#[must_use]
pub fn registrable_domain(url: &str) -> Option<String> {
    let lowered = url.to_lowercase();
    let after_scheme = lowered.split("://").nth(1)?;
    let host = after_scheme.split('/').next()?;
    let host = host.split(':').next().unwrap_or(host);
    if host.is_empty() {
        return None;
    }
    Some(strip_www_host(host).into_owned())
}

/// Dedup key for cross-host fanout: prefer the registrable domain;
/// fall back to the lenient [`clean_url`] when the input has no
/// parseable host (e.g. `mailto:` or `javascript:` URIs).
#[must_use]
pub fn dedup_host_key(url: &str) -> String {
    registrable_domain(url).unwrap_or_else(|| clean_url(url))
}

/// Strip a leading `www.` from a host, returning a borrowed slice
/// when nothing was stripped. Case-sensitive - pass a lowercase
/// host if you want `WWW.example.com` to match.
#[must_use]
pub fn strip_www_host(host: &str) -> Cow<'_, str> {
    match host.strip_prefix("www.") {
        Some(rest) => Cow::Owned(rest.to_string()),
        None => Cow::Borrowed(host),
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    // ── resolve_url ──────────────────────────────────────────────────────────

    #[test]
    fn resolve_url_passes_through_absolute_http() {
        assert_eq!(
            resolve_url("https://base.com/a/", "http://other.com/x"),
            "http://other.com/x"
        );
        assert_eq!(
            resolve_url("https://base.com/a/", "https://other.com/x"),
            "https://other.com/x"
        );
    }

    #[test]
    fn resolve_url_upgrades_scheme_relative_to_https() {
        assert_eq!(
            resolve_url("https://base.com/a/", "//cdn.example.com/x.js"),
            "https://cdn.example.com/x.js"
        );
    }

    #[test]
    fn resolve_url_root_relative_keeps_scheme_and_authority() {
        assert_eq!(
            resolve_url("https://base.com/dir/page", "/other"),
            "https://base.com/other"
        );
    }

    #[test]
    fn resolve_url_relative_uses_directory_context() {
        assert_eq!(
            resolve_url("https://base.com/dir/page", "next"),
            "https://base.com/dir/next"
        );
    }

    #[test]
    fn resolve_url_root_relative_without_path_concats() {
        assert_eq!(resolve_url("https://base.com", "/x"), "https://base.com/x");
    }

    #[test]
    fn resolve_url_no_base_path_concats_with_slash() {
        // No `://` in the base means we treat the whole base as the
        // anchor and join with `/`. Matches the original behaviour
        // for callers that pass naked hostnames.
        assert_eq!(resolve_url("base.com", "x"), "base.com/x");
    }

    // ── clean_url ────────────────────────────────────────────────────────────

    #[test]
    fn clean_url_strips_utm_params() {
        let cleaned = clean_url("https://example.com/page?utm_source=x&utm_medium=y");
        assert!(
            !cleaned.contains("utm_source"),
            "utm_source survived: {cleaned}"
        );
        assert!(
            !cleaned.contains("utm_medium"),
            "utm_medium survived: {cleaned}"
        );
    }

    #[test]
    fn clean_url_preserves_non_tracking_params() {
        let cleaned = clean_url("https://example.com/page?id=42&utm_source=x");
        assert!(cleaned.contains("id=42"));
        assert!(!cleaned.contains("utm_source"));
    }

    #[test]
    fn clean_url_strips_fragment() {
        let cleaned = clean_url("https://example.com/page#section");
        assert!(!cleaned.contains('#'), "fragment survived: {cleaned}");
    }

    #[test]
    fn clean_url_returns_input_on_parse_failure() {
        // Lenient behaviour - keys built from arbitrary strings
        // must not panic and must not silently turn into `""`.
        assert_eq!(clean_url("not-a-url"), "not-a-url");
    }

    #[test]
    fn clean_url_empty_query_drops_question_mark() {
        let cleaned = clean_url("https://example.com/page?utm_source=x");
        assert_eq!(cleaned, "https://example.com/page");
    }

    #[test]
    fn clean_url_strips_all_known_tracking_keys() {
        let url = "https://example.com/page?".to_string()
            + &TRACKING_PARAMS
                .iter()
                .map(|k| format!("{k}=x"))
                .collect::<Vec<_>>()
                .join("&");
        let cleaned = clean_url(&url);
        for key in TRACKING_PARAMS {
            assert!(
                !cleaned.contains(key),
                "tracking key `{key}` survived clean_url: {cleaned}"
            );
        }
    }

    // ── registrable_domain / dedup_host_key / strip_www_host ─────────────────

    #[test]
    fn registrable_domain_lowercases_and_strips_www() {
        assert_eq!(
            registrable_domain("https://WWW.Example.COM/path").as_deref(),
            Some("example.com")
        );
    }

    #[test]
    fn registrable_domain_strips_port() {
        assert_eq!(
            registrable_domain("https://example.com:8443/x").as_deref(),
            Some("example.com")
        );
    }

    #[test]
    fn registrable_domain_returns_none_for_schemeless_input() {
        assert_eq!(registrable_domain("example.com/path"), None);
    }

    #[test]
    fn registrable_domain_returns_none_for_empty_authority() {
        assert_eq!(registrable_domain("file:///etc/passwd"), None);
    }

    #[test]
    fn dedup_host_key_falls_back_to_clean_url_for_non_http() {
        // mailto: has no `://` so registrable_domain returns None
        // and dedup_host_key delegates to clean_url, which round-
        // trips the URL through the `url` crate.
        let key = dedup_host_key("mailto:user@example.com");
        assert!(key.contains("user@example.com"));
    }

    #[test]
    fn strip_www_host_returns_borrowed_when_no_prefix() {
        match strip_www_host("example.com") {
            Cow::Borrowed(s) => assert_eq!(s, "example.com"),
            Cow::Owned(s) => panic!("expected borrowed, got owned: {s}"),
        }
    }

    #[test]
    fn strip_www_host_returns_owned_when_prefix_stripped() {
        match strip_www_host("www.example.com") {
            Cow::Owned(s) => assert_eq!(s, "example.com"),
            Cow::Borrowed(s) => panic!("expected owned, got borrowed: {s}"),
        }
    }

    // ── adversarial / regression ─────────────────────────────────────────────

    /// REGRESSION: karyx-engine's old `resolve_url` did not check for
    /// absolute URLs and would prepend the base, turning
    /// `http://attacker.com/foo` into
    /// `http://base.com/http://attacker.com/foo` and thereby
    /// silently redirecting the crawl to attacker-controlled content
    /// reachable from the base. This pin prevents the bug from
    /// reappearing in the canonical implementation.
    #[test]
    fn resolve_url_does_not_swallow_attacker_absolute_url() {
        let resolved = resolve_url("http://base.com/dir/", "http://attacker.com/x");
        assert_eq!(
            resolved, "http://attacker.com/x",
            "absolute URL must not be merged into base"
        );
    }

    /// REGRESSION: tracking-parameter stripping must not introduce
    /// open-redirect ambiguity by reordering query keys in a way
    /// that breaks downstream signature verification. The output
    /// must be deterministic for any input ordering.
    #[test]
    fn clean_url_is_deterministic_under_param_reordering() {
        let a = clean_url("https://example.com/page?id=1&token=abc&utm_source=x");
        let b = clean_url("https://example.com/page?utm_source=x&id=1&token=abc");
        // Both inputs should produce the same param set, but the
        // order in the output mirrors the surviving order from the
        // input. We only assert that the tracking key is gone in
        // both and the non-tracking keys all survive.
        for key in ["id=1", "token=abc"] {
            assert!(a.contains(key), "{key} missing from {a}");
            assert!(b.contains(key), "{key} missing from {b}");
        }
        assert!(!a.contains("utm_source"));
        assert!(!b.contains("utm_source"));
    }
}
