use crate::proxy::config::ProxyConfig;
use anyhow::Result;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ScraperConfig {
    pub name: String,
    pub version: Option<String>,
    pub base_url: String,
    pub description: Option<String>,
    pub request: RequestConfig,
    pub proxy: Option<ProxyConfig>,
    pub selectors: Vec<SelectorConfig>,
    pub pagination: Option<PaginationConfig>,
    pub rate_limit: Option<RateLimitConfig>,
    pub output: Option<OutputConfig>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RequestConfig {
    pub method: String,
    pub headers: Option<HashMap<String, String>>,
    pub timeout_secs: Option<u64>,
    pub follow_redirects: Option<bool>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SelectorConfig {
    pub name: String,
    pub css: String,
    pub attribute: String,
    pub required: Option<bool>,
    pub transform: Option<String>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PaginationConfig {
    pub enabled: bool,
    pub next_selector: Option<String>,
    pub max_pages: Option<u32>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RateLimitConfig {
    pub requests_per_second: u32,
    pub burst_size: u32,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OutputConfig {
    pub table: String,
    pub deduplicate_by: Option<String>,
}

pub fn load_scraper_configs(dir: &str) -> Result<HashMap<String, ScraperConfig>> {
    let mut configs = HashMap::new();
    let pattern = format!("{}/*.yaml", dir);
    let paths = glob::glob(&pattern).map_err(|e| anyhow::anyhow!("Invalid glob pattern: {}", e))?;
    for path in paths {
        match path {
            Ok(path) => {
                let content = std::fs::read_to_string(&path)?;
                let config: ScraperConfig = serde_yaml::from_str(&content)?;
                configs.insert(config.name.clone(), config);
            }
            Err(e) => {
                tracing::warn!("Skipping unreadable config path: {}", e);
            }
        }
    }
    Ok(configs)
}
