use std::collections::HashMap;
use std::collections::HashSet;
use std::env;
use std::fs;
use std::fs::File;
use std::io;
use std::io::Read;
use std::io::Write;
use std::path::Path;
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;

use rayon::prelude::*;
use serde::{Deserialize, Deserializer, Serialize, Serializer};

use crate::audit_report::AuditReport;
use crate::cli::CvssFilter;
use crate::count_report::CountReport;
use crate::dep_manifest::DepManifest;
use crate::env_marker::EnvMarkerState;
use crate::exe_search::find_exe;
use crate::package::Package;
use crate::package_match::match_str;
use crate::path_shared::PathShared;
use crate::scan_report::ScanReport;
use crate::site_customize::install_validation;
use crate::site_customize::uninstall_validation;
use crate::unpack_report::UnpackReport;
use crate::ureq_client::UreqClient;
use crate::util::exe_path_normalize;
use crate::util::hash_paths;
use crate::util::logger;
use crate::util::path_is_component;
use crate::util::path_normalize;
use crate::util::path_within_duration;
use crate::util::vecs_equal_as_sets;
use crate::util::Anchor;
use crate::util::CacheConfig;
use crate::util::FlagCacheRefresh;
use crate::util::FlagLog;
use crate::util::FlagRetainPassing;
use crate::util::ResultDynError;
use crate::util::ScanConfig;
use crate::util::DURATION_0;
use crate::validation_report::ValidationFlags;
use crate::validation_report::ValidationReport;

//------------------------------------------------------------------------------

/// Given a path to a Python binary, call out to Python to get all known site packages; some site packages may not exist; we do not filter them here. This will include "dist-packages" on Linux. If `config.force_usite` is false, we use site.ENABLE_USER_SITE to determine if we should include the user site packages; if `config.force_usite` is true, we always include usite.
/// Calling Python using `-S` disables loading site so that we can mock sitecustomize.py (which fetter might customize). We then call `site.main()` to force proper initialization.
const PY_SITE_PACKAGES: &str = "import sys;import site;import types;sys.modules['fetter_validate'] = types.ModuleType('fetter_validate');site.main();print(site.ENABLE_USER_SITE);print(\"\\n\".join(site.getsitepackages()));print(site.getusersitepackages())";
fn get_site_package_dirs(
    executable: &Path,
    config: &ScanConfig,
    log: FlagLog,
) -> Vec<PathShared> {
    match Command::new(executable)
        .arg("-S") // disable site on startup
        .arg("-c")
        .arg(PY_SITE_PACKAGES)
        .output()
    {
        Ok(output) => {
            let mut paths = Vec::new();
            let mut usite_enabled = false;

            let lines = std::str::from_utf8(&output.stdout)
                .expect("Failed to convert to UTF-8")
                .trim()
                .lines();
            for (i, line) in lines.enumerate() {
                if i == 0 {
                    usite_enabled = line.trim() == "True";
                } else {
                    paths.push(PathShared::from(line.trim()));
                }
            }
            // if necessary, remove the usite
            if !config.force_usite && !usite_enabled {
                let _p = paths.pop();
            }
            paths
        }
        Err(e) => {
            logger!(
                log,
                module_path!(),
                "Failed to execute command with {:?}: {}",
                executable,
                e
            );

            Vec::with_capacity(0)
        }
    }
}

// Given a package directory, collect the name of all packages.
fn get_packages(site_packages: &Path) -> Vec<Package> {
    let mut packages = Vec::new();
    if let Ok(entries) = fs::read_dir(site_packages) {
        for entry in entries.flatten() {
            let file_path = entry.path();
            if let Some(package) = Package::from_file_path(&file_path) {
                packages.push(package);
            }
        }
    }
    packages
}

//------------------------------------------------------------------------------

// The result of a file-system scan.
#[derive(Clone, Debug)]
pub struct ScanFS {
    // NOTE: these attributes are used by reporters
    /// A mapping of exe path to site packages paths
    pub exe_to_sites: HashMap<PathShared, Vec<PathShared>>,
    /// A mapping of Package tp a site package paths
    pub package_to_sites: HashMap<Package, Vec<PathShared>>,

    // A mapping of site package to exe paths
    pub site_to_exes: HashMap<PathShared, Vec<PathShared>>,
    /// Optionally populate EnvMarkerState for all exe, only if env markers are found
    pub exe_to_ems: Option<HashMap<PathShared, EnvMarkerState>>,
    /// Scan configuration
    config: ScanConfig,
    /// Store the hash of the un-normalized exe inputs for cache lookup.
    exes_hash: String,
}

impl PartialEq for ScanFS {
    fn eq(&self, other: &Self) -> bool {
        if self.config != other.config || self.exes_hash != other.exes_hash {
            return false;
        }
        if self.exe_to_ems != other.exe_to_ems {
            return false;
        }

        if self.site_to_exes.len() != other.site_to_exes.len() {
            return false;
        }
        for (key, vec1) in &self.site_to_exes {
            if let Some(vec2) = other.site_to_exes.get(key) {
                if !vecs_equal_as_sets(vec1, vec2) {
                    return false;
                }
            } else {
                return false;
            }
        }

        if self.exe_to_sites.len() != other.exe_to_sites.len() {
            return false;
        }
        for (key, vec1) in &self.exe_to_sites {
            if let Some(vec2) = other.exe_to_sites.get(key) {
                if !vecs_equal_as_sets(vec1, vec2) {
                    return false;
                }
            } else {
                return false;
            }
        }
        if self.package_to_sites.len() != other.package_to_sites.len() {
            return false;
        }
        for (key, vec1) in &self.package_to_sites {
            if let Some(vec2) = other.package_to_sites.get(key) {
                if !vecs_equal_as_sets(vec1, vec2) {
                    return false;
                }
            } else {
                return false;
            }
        }
        true
    }
}

struct PathIndexer {
    path_to_index: HashMap<PathShared, usize>,
    paths: Vec<String>,
}

impl PathIndexer {
    fn new() -> Self {
        Self {
            path_to_index: HashMap::new(),
            paths: Vec::new(),
        }
    }

    fn get_index<S: serde::ser::Error>(&mut self, p: &PathShared) -> Result<usize, S> {
        if let Some(&idx) = self.path_to_index.get(p) {
            Ok(idx)
        } else {
            let idx = self.paths.len();
            let s = p
                .as_path()
                .to_str()
                .ok_or_else(|| S::custom("Invalid UTF-8 in path"))?;
            self.path_to_index.insert(p.clone(), idx);
            self.paths.push(s.to_string());
            Ok(idx)
        }
    }
}

impl Serialize for ScanFS {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        // Collect and sort by keys for stable ordering
        let mut exe_to_sites: Vec<_> = self.exe_to_sites.iter().collect();
        exe_to_sites.sort_by_key(|(k, _)| *k);

        let mut package_to_sites: Vec<_> = self.package_to_sites.iter().collect();
        package_to_sites.sort_by_key(|(k, _)| *k);

        let mut site_to_exes: Vec<_> = self.site_to_exes.iter().collect();
        site_to_exes.sort_by_key(|(k, _)| k.to_string());

        let mut pi = PathIndexer::new();

        let exe_to_sites_idx: Result<Vec<_>, S::Error> = exe_to_sites
            .into_iter()
            .map(|(exe, sites)| {
                let exe_i = pi.get_index(exe)?;
                let site_idx: Result<Vec<_>, S::Error> =
                    sites.iter().map(|s| pi.get_index(s)).collect();
                Ok((exe_i, site_idx?))
            })
            .collect();

        let package_to_sites_idx: Result<Vec<_>, S::Error> = package_to_sites
            .into_iter()
            .map(|(pkg, sites)| {
                let site_idx: Result<Vec<_>, S::Error> =
                    sites.iter().map(|s| pi.get_index(s)).collect();
                Ok((pkg, site_idx?))
            })
            .collect();

        let site_to_exe_idx: Result<Vec<_>, S::Error> = site_to_exes
            .into_iter()
            .map(|(site, exes)| {
                let site_i = pi.get_index(site)?;
                // sort exe paths by string for deterministic output
                let mut exes_sorted: Vec<_> = exes.iter().collect();
                exes_sorted.sort_by_key(|p| p.to_string());
                let exe_idx: Result<Vec<_>, S::Error> =
                    exes_sorted.into_iter().map(|e| pi.get_index(e)).collect();
                Ok((site_i, exe_idx?))
            })
            .collect();

        // Serialize as tuple of sorted vectors with dictionary
        let data = (
            pi.paths,
            exe_to_sites_idx?,
            package_to_sites_idx?,
            site_to_exe_idx?,
            self.config.force_usite,
            self.config.all_users,
            &self.exes_hash,
        );
        data.serialize(serializer)
    }
}

/// Flattened data representation used for serialization with path dictionary.
type ScanFSData = (
    Vec<String>,
    Vec<(usize, Vec<usize>)>,
    Vec<(Package, Vec<usize>)>,
    Vec<(usize, Vec<usize>)>,
    bool,
    bool,
    String,
);

impl<'de> Deserialize<'de> for ScanFS {
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
    where
        D: Deserializer<'de>,
    {
        let (
            paths,
            exe_to_sites_idx,
            package_to_sites_idx,
            site_to_exe_idx,
            force_usite,
            all_users,
            exes_hash,
        ): ScanFSData = Deserialize::deserialize(deserializer)?;

        // let paths: Vec<PathBuf> = paths.into_iter().map(PathBuf::from).collect();
        let ps: Vec<PathShared> = paths.into_iter().map(PathShared::from).collect();

        let exe_to_sites: HashMap<PathShared, Vec<PathShared>> = exe_to_sites_idx
            .into_iter()
            .map(|(exe_i, site_is)| {
                let exe = ps[exe_i].clone();
                let sites = site_is.into_iter().map(|i| ps[i].clone()).collect();
                (exe, sites)
            })
            .collect();

        let package_to_sites: HashMap<Package, Vec<PathShared>> = package_to_sites_idx
            .into_iter()
            .map(|(pkg, site_is)| {
                let sites = site_is.into_iter().map(|i| ps[i].clone()).collect();
                (pkg, sites)
            })
            .collect();

        let site_to_exes: HashMap<PathShared, Vec<PathShared>> = site_to_exe_idx
            .into_iter()
            .map(|(site_i, exe_is)| {
                let site = ps[site_i].clone();
                let exes: Vec<PathShared> =
                    exe_is.into_iter().map(|i| ps[i].clone()).collect();
                (site, exes)
            })
            .collect();

        Ok(ScanFS {
            exe_to_sites,
            package_to_sites,
            site_to_exes,
            exe_to_ems: None,
            config: ScanConfig::new(force_usite, all_users),
            exes_hash,
        })
    }
}

impl ScanFS {
    /// Main entry point for creating a ScanFS. All public creation should go through this interface.
    fn from_exe_to_sites(
        exe_to_sites: HashMap<PathShared, Vec<PathShared>>,
        config: &ScanConfig,
        exes_hash: String,
    ) -> ResultDynError<Self> {
        // Some site packages will be repeated; let them be processed more than once here, as it seems easier than filtering them out
        let site_to_packages = exe_to_sites
            .par_iter()
            .flat_map(|(_, site_packages)| {
                site_packages.par_iter().map(|site_package_path| {
                    let packages = get_packages(site_package_path.as_path());
                    (site_package_path.clone(), packages)
                })
            })
            .collect::<HashMap<PathShared, Vec<Package>>>();

        let mut site_to_exes: HashMap<PathShared, Vec<PathShared>> = HashMap::new();
        for (exe, sites) in &exe_to_sites {
            for site in sites {
                site_to_exes
                    .entry(site.clone())
                    .or_default()
                    .push(exe.clone());
            }
        }

        let mut package_to_sites: HashMap<Package, Vec<PathShared>> = HashMap::new();
        for (site_package_path, packages) in site_to_packages.iter() {
            for package in packages {
                package_to_sites
                    .entry(package.clone())
                    .or_default()
                    .push(site_package_path.clone());
            }
        }

        Ok(ScanFS {
            exe_to_sites,
            package_to_sites,
            site_to_exes,
            exe_to_ems: None,
            config: *config,
            exes_hash,
        })
    }

    /// Create a ScanFS from a cache: exes provided here should be pre-normalization.
    pub(crate) fn from_cache(
        exes: &[PathBuf],
        config: &ScanConfig,
        cache_config: &CacheConfig,
        log: FlagLog,
    ) -> ResultDynError<Self> {
        if cache_config.duration == DURATION_0 {
            Err("Cache disabled by duration".into())
        } else {
            let exes_hash = hash_paths(exes, config);

            let cache_fp = cache_config
                .directory
                .join(format!("scan_fs_{exes_hash}"))
                .with_extension("json");

            if path_within_duration(&cache_fp, cache_config.duration) {
                logger!(log, module_path!(), "Loading ScanFS cache: {:?}", cache_fp);

                let mut file = File::open(cache_fp)?;
                let mut contents = String::new();
                file.read_to_string(&mut contents)?;
                let data: ScanFS = serde_json::from_str(&contents)?;
                Ok(data)
            } else if cache_fp.exists() {
                // NOTE: could remove cache_fp to clean up
                Err("Cache expired".into())
            } else {
                Err("Cache file does not exist".into())
            }
        }
    }

    /// Given a Vec of PathBuf to executables, use them to collect site packages. In this function, provided PathBuf are normalized to absolute paths, and if a PathBuf is "*", a system-wide path search will be conducted.
    pub(crate) fn from_exes(
        exes: &Vec<PathBuf>,
        config: &ScanConfig,
        log: FlagLog,
    ) -> ResultDynError<Self> {
        let path_wild = PathBuf::from("*");
        let mut exes_norm = Vec::new();
        for e in exes {
            if path_is_component(e) && *e == path_wild {
                exes_norm.extend(find_exe(config.all_users));
            } else {
                exes_norm.push(exe_path_normalize(e)?);
            }
        }

        let exe_to_sites: HashMap<PathShared, Vec<PathShared>> = exes_norm
            .into_par_iter()
            .map(|exe| {
                let dirs = get_site_package_dirs(&exe, config, log);
                (PathShared::from(exe), dirs)
            })
            .collect();

        let exes_hash = hash_paths(exes, config);
        Self::from_exe_to_sites(exe_to_sites, config, exes_hash)
    }

    /// Alternative constructor from in-memory objects, only for testing. Here we provide notional exe and site paths, and focus just on collecting Packages.
    #[cfg(test)]
    pub(crate) fn from_exe_site_packages(
        exe: PathBuf,
        site: PathBuf,
        packages: Vec<Package>,
    ) -> ResultDynError<Self> {
        let mut exe_to_sites = HashMap::new();
        let site_shared = PathShared::from_path_buf(site);

        exe_to_sites.insert(PathShared::from(exe.clone()), vec![site_shared.clone()]);
        let exes = vec![exe];

        let mut site_to_exes: HashMap<PathShared, Vec<PathShared>> = HashMap::new();
        for (exe, sites) in &exe_to_sites {
            for site in sites {
                site_to_exes
                    .entry(site.clone())
                    .or_default()
                    .push(exe.clone());
            }
        }

        let mut package_to_sites = HashMap::new();
        for package in packages {
            package_to_sites
                .entry(package)
                .or_insert_with(Vec::new)
                .push(site_shared.clone());
        }
        let config = ScanConfig::new(false, false);
        let exes_hash = hash_paths(&exes, &config);

        Ok(ScanFS {
            exe_to_sites,
            package_to_sites,
            site_to_exes,
            exe_to_ems: None,
            config,
            exes_hash,
        })
    }

    //--------------------------------------------------------------------------

    // If not set, optionally load EnvMarkerState for each exe
    pub(crate) fn load_env_marker_state(&mut self, log: FlagLog) {
        logger!(log, module_path!(), "Fetching EnvMarkerState");

        if self.exe_to_ems.is_none() {
            let ems_map: HashMap<PathShared, EnvMarkerState> = self
                .exe_to_sites
                .keys()
                .cloned()
                .collect::<Vec<_>>()
                .into_par_iter()
                .map(|exe| {
                    (
                        exe.clone(),
                        EnvMarkerState::from_exe(exe.as_path()).unwrap(),
                    )
                })
                .collect();

            self.exe_to_ems = Some(ems_map);
        }
    }

    // searching
    pub(crate) fn search_by_match(
        &self,
        pattern: &str,
        case_insensitive: bool,
    ) -> Vec<Package> {
        // take ownership of Package in the result of get_packages
        let matched = self
            .get_packages()
            .into_par_iter()
            .filter(|package| {
                match_str(pattern, package.to_string().as_str(), case_insensitive)
            })
            .collect();
        matched
    }

    /// Return sorted packages.
    pub fn get_packages(&self) -> Vec<Package> {
        let mut packages: Vec<Package> = self.package_to_sites.keys().cloned().collect();
        packages.sort();
        packages
    }

    //--------------------------------------------------------------------------

    pub(crate) fn to_cache(
        &self,
        cache_config: &CacheConfig,
        log: FlagLog,
    ) -> ResultDynError<()> {
        // use hash of exes observed at initialization
        let cache_fp = cache_config
            .directory
            .join(format!("scan_fs_{}", self.exes_hash))
            .with_extension("json");

        // only write if cache does not exist or it is out of duration
        if !cache_fp.exists() || !path_within_duration(&cache_fp, cache_config.duration) {
            logger!(log, module_path!(), "Writing ScanFS cache: {:?}", cache_fp);

            let json = serde_json::to_string(self)?;
            let mut file = File::create(cache_fp)?;
            file.write_all(json.as_bytes())?;
            Ok(())
        } else {
            logger!(log, module_path!(), "Keeping ScanFS cache {:?}", cache_fp);
            Ok(())
        }
    }

    //--------------------------------------------------------------------------

    /// Validate this scan against the provided DepManifest.
    #[allow(clippy::wrong_self_convention)]
    pub(crate) fn to_validation_report(
        &mut self,
        dm: DepManifest,
        vf: ValidationFlags,
        ignore: Option<&HashSet<String>>,
        log: FlagLog,
    ) -> ValidationReport {
        if dm.env_marker_active {
            self.load_env_marker_state(log);
        }
        let packages = self.get_packages();

        ValidationReport::from_components(
            &packages,
            &self.package_to_sites,
            &self.site_to_exes,
            &self.exe_to_ems,
            &dm,
            &vf,
            ignore,
        )
    }

    #[allow(clippy::too_many_arguments)]
    pub(crate) fn to_audit_report(
        &self,
        pattern: &str,
        client: Arc<dyn UreqClient>,
        case_insensitive: bool,
        cache_refresh: FlagCacheRefresh,
        cache_config: &CacheConfig,
        log: FlagLog,
        filter_cvss: CvssFilter,
        retain_passing: FlagRetainPassing,
    ) -> AuditReport {
        let packages = self.search_by_match(pattern, case_insensitive);
        // NOTE: packages is sorted upstream
        AuditReport::from_packages(
            client,
            &packages,
            cache_refresh,
            cache_config,
            log,
            filter_cvss,
            retain_passing,
        )
    }

    /// The `count` Boolean determine what type of UnpackReport is returned
    pub(crate) fn to_unpack_report(
        &self,
        pattern: &str,
        case_insensitive: bool,
        count: bool,
    ) -> UnpackReport {
        let mut packages = self.search_by_match(pattern, case_insensitive);
        packages.sort();
        let package_to_sites = packages
            .iter()
            .map(|p| (p.clone(), self.package_to_sites.get(p).unwrap().clone()))
            .collect();

        UnpackReport::from_package_to_sites(count, &package_to_sites)
    }

    /// Given an `anchor`, produce a DepManifest based ont the packages observed in this scan.
    pub(crate) fn to_dep_manifest(
        &self,
        anchor: Anchor,
    ) -> Result<DepManifest, Box<dyn std::error::Error>> {
        DepManifest::from_packages(self.package_to_sites.keys(), anchor)
    }

    pub(crate) fn to_scan_report(&self) -> ScanReport {
        ScanReport::from_package_to_sites(&self.package_to_sites)
    }

    pub(crate) fn to_count_report(&self) -> CountReport {
        CountReport::from_scan_fs(self)
    }

    pub(crate) fn to_search_report(
        &self,
        pattern: &str,
        case_insensitive: bool,
    ) -> ScanReport {
        let packages = self.search_by_match(pattern, case_insensitive);
        // println!("packages: {:?}", packages);
        ScanReport::from_packages(&packages, &self.package_to_sites)
    }

    pub(crate) fn to_purge_pattern(
        &self,
        pattern: &Option<String>,
        case_insensitive: bool,
        log: FlagLog,
    ) -> io::Result<()> {
        let packages = match pattern {
            Some(p) => self.search_by_match(p, case_insensitive),
            None => self.package_to_sites.keys().cloned().collect(),
        };
        // packages.sort();
        let package_to_sites = packages
            .iter()
            .map(|p| (p.clone(), self.package_to_sites.get(p).unwrap().clone()))
            .collect();

        let sr = UnpackReport::from_package_to_sites(false, &package_to_sites);
        sr.remove(log)
    }

    #[allow(clippy::wrong_self_convention)]
    pub(crate) fn to_purge_invalid(
        &mut self,
        dm: DepManifest,
        vf: ValidationFlags,
        log: FlagLog,
    ) -> io::Result<()> {
        let vr = self.to_validation_report(dm, vf, None, log);
        let packages: Vec<Package> = vr
            .records
            .iter()
            .filter_map(|r| r.package.as_ref().cloned())
            .collect();
        // packages.sort();
        let package_to_sites = packages
            .iter()
            .map(|p| (p.clone(), self.package_to_sites.get(p).unwrap().clone()))
            .collect();

        let sr = UnpackReport::from_package_to_sites(false, &package_to_sites);
        sr.remove(log)
    }

    pub(crate) fn site_validate_install(
        &self,
        bound: &Path,
        bound_options: &Option<Vec<String>>,
        ignore: &[String],
        vf: &ValidationFlags,
        exit_else_warn: Option<i32>,
        log: FlagLog,
    ) -> ResultDynError<()> {
        if self.exe_to_sites.len() > 1 {
            return Err(format!("site-install will not operate on multiple ({}) Python environments; use `-e` to specify a single Python environment.", self.exe_to_sites.len()).into());
        }
        let ba = path_normalize(bound, true)?;
        // only a single exe, so no need to parallelize
        for (exe, sites) in &self.exe_to_sites {
            // NOTE: taking the first site, but might prioritize by some other criteria
            if let Some(site) = sites.first() {
                install_validation(
                    exe.as_path(),
                    &ba,
                    bound_options,
                    ignore,
                    vf,
                    exit_else_warn,
                    site,
                    env::current_dir().ok(), // as option type
                    log,
                )?;
            }
        }
        Ok(())
    }

    pub(crate) fn site_validate_uninstall(&self, log: FlagLog) -> ResultDynError<()> {
        if self.exe_to_sites.len() > 1 {
            return Err(format!("site-install will not operate on multiple ({}) Python environments; use `-e` to specify a single Python environment.", self.exe_to_sites.len()).into());
        }
        for sites in self.exe_to_sites.values() {
            if let Some(site) = sites.first() {
                uninstall_validation(site, log)?;
            }
        }
        Ok(())
    }
}

//------------------------------------------------------------------------------
#[cfg(test)]
mod tests {
    use super::*;
    use std::fs::File;
    use tempfile::tempdir;
    // use rand::seq::SliceRandom;
    // use rand::rng;

    #[test]
    fn test_get_site_package_dirs_a() {
        let p1 = Path::new("python3");
        let config_force = ScanConfig::new(true, false);
        let paths1 = get_site_package_dirs(p1, &config_force, FlagLog(false));
        assert!(!paths1.is_empty());
        let config_no_force = ScanConfig::new(false, false);
        let paths2 = get_site_package_dirs(p1, &config_no_force, FlagLog(false));
        assert!(paths1.len() >= paths2.len());
    }
    #[test]
    fn test_from_exe_to_sites_a() {
        let fp_dir = tempdir().unwrap();
        let fp_exe = fp_dir.path().join("python");
        let _ = File::create(fp_exe.clone()).unwrap();

        let fp_sp = fp_dir.path().join("site-packages");
        fs::create_dir(fp_sp.clone()).unwrap();

        let fp_p1 = fp_sp.join("numpy-1.19.1.dist-info");
        fs::create_dir(&fp_p1).unwrap();

        let fp_p2 = fp_sp.join("foo-3.0.dist-info");
        fs::create_dir(&fp_p2).unwrap();

        let mut exe_to_sites = HashMap::<PathShared, Vec<PathShared>>::new();
        exe_to_sites.insert(
            PathShared::from(fp_exe.clone()),
            vec![PathShared::from_path_buf(fp_sp.to_path_buf())],
        );
        let config = ScanConfig::new(false, false);
        let mut sfs =
            ScanFS::from_exe_to_sites(exe_to_sites, &config, "".to_string()).unwrap();
        assert_eq!(sfs.package_to_sites.len(), 2);

        let dm1 = DepManifest::try_from_iter(vec!["numpy >= 1.19", "foo==3"]).unwrap();
        assert_eq!(dm1.len(), 2);
        let invalid1 = sfs.to_validation_report(
            dm1,
            ValidationFlags {
                permit_superset: false,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        assert_eq!(invalid1.len(), 0);

        let dm2 = DepManifest::try_from_iter(vec!["numpy >= 2", "foo==3"]).unwrap();
        let invalid2 = sfs.to_validation_report(
            dm2,
            ValidationFlags {
                permit_superset: false,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        assert_eq!(invalid2.len(), 1);
    }
    //--------------------------------------------------------------------------
    #[test]
    fn from_exe_site_packages_a() {
        let exe = PathBuf::from("/usr/bin/python3");
        let site = PathBuf::from("/usr/lib/python3.8/site-packages");
        let packages = vec![
            Package::from_name_version_durl("numpy", "1.19.3", None).unwrap(),
            Package::from_name_version_durl("numpy", "1.20.1", None).unwrap(),
            Package::from_name_version_durl("numpy", "2.1.1", None).unwrap(),
            Package::from_name_version_durl("requests", "0.7.6", None).unwrap(),
            Package::from_name_version_durl("requests", "2.32.3", None).unwrap(),
            Package::from_name_version_durl("flask", "3.0.3", None).unwrap(),
            Package::from_name_version_durl("flask", "1.1.3", None).unwrap(),
        ];
        let sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();
        assert_eq!(sfs.package_to_sites.len(), 7);
        // sfs.report();
        let dm = sfs.to_dep_manifest(Anchor::Lower).unwrap();
        assert_eq!(dm.len(), 3);
    }

    //--------------------------------------------------------------------------
    #[test]
    fn test_validation_a() {
        let exe = PathBuf::from("/usr/bin/python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("numpy", "1.19.3", None).unwrap(),
            Package::from_name_version_durl("requests", "0.7.6", None).unwrap(),
            Package::from_name_version_durl("flask", "1.1.3", None).unwrap(),
        ];
        let dm = DepManifest::try_from_iter(
            ["numpy>1.19", "requests==0.7.6", "flask> 1"].iter(),
        )
        .unwrap();

        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();
        let vr = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: false,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        assert_eq!(vr.len(), 0);
    }
    #[test]
    fn test_validation_b() {
        let exe = PathBuf::from("/usr/bin/python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("numpy", "1.19.3", None).unwrap(),
            Package::from_name_version_durl("requests", "0.7.6", None).unwrap(),
            Package::from_name_version_durl("flask", "1.1.3", None).unwrap(),
        ];
        let dm = DepManifest::try_from_iter(
            ["numpy>1.19", "requests==0.7.6", "flask> 2"].iter(),
        )
        .unwrap();

        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();
        let vr = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: false,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );

        let json = serde_json::to_string(&vr.to_validation_digest()).unwrap();
        assert_eq!(
            json,
            r#"[{"package":"flask-1.1.3","dependency":"flask>2","explain":"Misdefined","sites":["/usr/lib/python3/site-packages"]}]"#
        );
    }
    #[test]
    fn test_validation_c() {
        let exe = PathBuf::from("/usr/bin/python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("numpy", "1.19.3", None).unwrap(),
            Package::from_name_version_durl("requests", "0.7.6", None).unwrap(),
            Package::from_name_version_durl("flask", "1.1.3", None).unwrap(),
        ];
        let dm = DepManifest::try_from_iter(
            ["numpy>2", "requests==0.7.1", "flask> 2,<3"].iter(),
        )
        .unwrap();

        let mut sfs =
            ScanFS::from_exe_site_packages(exe.clone(), site, packages).unwrap();
        let vr = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: false,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        assert_eq!(
            sfs.exe_to_sites.get(&PathShared::from(exe)).unwrap()[0].strong_count(),
            8
        );
        let json = serde_json::to_string(&vr.to_validation_digest()).unwrap();
        assert_eq!(
            json,
            r#"[{"package":"flask-1.1.3","dependency":"flask>2,<3","explain":"Misdefined","sites":["/usr/lib/python3/site-packages"]},{"package":"numpy-1.19.3","dependency":"numpy>2","explain":"Misdefined","sites":["/usr/lib/python3/site-packages"]},{"package":"requests-0.7.6","dependency":"requests==0.7.1","explain":"Misdefined","sites":["/usr/lib/python3/site-packages"]}]"#
        );
    }

    #[test]
    fn test_validation_d() {
        let exe = PathBuf::from("/usr/bin/python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("numpy", "1.19.3", None).unwrap(),
            Package::from_name_version_durl("requests", "0.7.6", None).unwrap(),
            Package::from_name_version_durl("flask", "1.1.3", None).unwrap(),
        ];
        let dm = DepManifest::try_from_iter(["numpy>2", "flask> 2,<3"].iter()).unwrap();

        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();

        let vr = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: true,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        let json = serde_json::to_string(&vr.to_validation_digest()).unwrap();
        assert_eq!(
            json,
            r#"[{"package":"flask-1.1.3","dependency":"flask>2,<3","explain":"Misdefined","sites":["/usr/lib/python3/site-packages"]},{"package":"numpy-1.19.3","dependency":"numpy>2","explain":"Misdefined","sites":["/usr/lib/python3/site-packages"]}]"#
        );
    }
    #[test]
    fn test_validation_e() {
        let exe = PathBuf::from("/usr/bin/python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("numpy", "1.19.3", None).unwrap(),
            Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
            Package::from_name_version_durl("flask", "1.1.3", None).unwrap(),
        ];
        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();

        // hyphen / underscore are normalized
        let dm = DepManifest::try_from_iter(
            ["numpy==1.19.3", "flask>1,<2", "static_frame==2.13.0"].iter(),
        )
        .unwrap();
        let vr = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: false,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        assert_eq!(vr.len(), 0);
    }
    #[test]
    fn test_validation_f() {
        let exe = PathBuf::from("/usr/bin/python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("numpy", "1.19.3", None).unwrap(),
            Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
        ];
        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();

        // hyphen / underscore are normalized
        let dm = DepManifest::try_from_iter(
            ["numpy==1.19.3", "flask>1,<2", "static_frame==2.13.0"].iter(),
        )
        .unwrap();
        let vr = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: false,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        assert_eq!(vr.len(), 1);
        let json = serde_json::to_string(&vr.to_validation_digest()).unwrap();
        assert_eq!(
            json,
            r#"[{"package":null,"dependency":"flask>1,<2","explain":"Missing","sites":null}]"#
        );
    }
    #[test]
    fn test_validation_g() {
        let exe = PathBuf::from("/usr/bin/python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("numpy", "1.19.3", None).unwrap(),
            Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
        ];
        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();
        let dm = DepManifest::try_from_iter(["numpy==1.19.3"].iter()).unwrap();
        let vr1 = sfs.to_validation_report(
            dm.clone(),
            ValidationFlags {
                permit_superset: false,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        assert_eq!(vr1.len(), 1);
        let json = serde_json::to_string(&vr1.to_validation_digest()).unwrap();
        assert_eq!(
            json,
            r#"[{"package":"static-frame-2.13.0","dependency":null,"explain":"Unrequired","sites":["/usr/lib/python3/site-packages"]}]"#
        );

        let vr2 = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: true,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        assert_eq!(vr2.len(), 0);
    }
    #[test]
    fn test_validation_h() {
        let exe = PathBuf::from("/usr/bin/python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("numpy", "1.19.3", None).unwrap(),
            Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
        ];
        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();

        // hyphen / underscore are normalized
        let dm = DepManifest::try_from_iter(
            ["numpy==1.19.3", "flask>1,<2", "static_frame==2.13.0"].iter(),
        )
        .unwrap();
        let vr1 = sfs.to_validation_report(
            dm.clone(),
            ValidationFlags {
                permit_superset: false,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        let json = serde_json::to_string(&vr1.to_validation_digest()).unwrap();
        assert_eq!(
            json,
            r#"[{"package":null,"dependency":"flask>1,<2","explain":"Missing","sites":null}]"#
        );

        let vr2 = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: false,
                permit_subset: true,
            },
            None,
            FlagLog(false),
        );
        assert_eq!(vr2.len(), 0);
    }

    //--------------------------------------------------------------------------

    #[test]
    fn test_validation_evn_marker_a() {
        let exe = PathBuf::from("python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let mut packages =
            vec![
                Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
            ];
        if env::consts::OS == "macos" {
            packages
                .push(Package::from_name_version_durl("numpy", "1.19.3", None).unwrap());
        }
        if env::consts::OS == "linux" {
            packages.push(Package::from_name_version_durl("numpy", "2.1", None).unwrap());
        }
        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();

        let dm = DepManifest::try_from_iter(
            [
                "numpy==1.19.3; platform_system == 'Darwin'",
                "numpy==2.1; platform_system == 'Linux'",
                "static_frame==2.13.0",
            ]
            .iter(),
        )
        .unwrap();
        let vr = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: false,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        let json = serde_json::to_string(&vr.to_validation_digest()).unwrap();
        assert_eq!(json, r#"[]"#);
    }

    #[test]
    fn test_validation_evn_marker_b1() {
        let exe = PathBuf::from("python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
            Package::from_name_version_durl("numpy", "2.0", None).unwrap(),
        ];

        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();

        // this DM means that we only need NumPy if Python < 3,
        let dm = DepManifest::try_from_iter(
            [
                "numpy==1.19.3; python_version < '3.0'",
                "static_frame==2.13.0",
            ]
            .iter(),
        )
        .unwrap();

        let vr = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: false,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        let json = serde_json::to_string(&vr.to_validation_digest()).unwrap();
        assert_eq!(
            json,
            r#"[{"package":"numpy-2.0","dependency":null,"explain":"Unrequired","sites":["/usr/lib/python3/site-packages"]}]"#
        );
    }

    #[test]
    fn test_validation_evn_marker_b2() {
        let exe = PathBuf::from("python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
            Package::from_name_version_durl("numpy", "2.0", None).unwrap(),
        ];

        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();

        // this DM means that we only need NumPy if Python < 3,
        let dm = DepManifest::try_from_iter(
            [
                "numpy==1.19.3; python_version < '3.0'",
                "static_frame==2.13.0",
            ]
            .iter(),
        )
        .unwrap();

        let vr = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: true,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        let json = serde_json::to_string(&vr.to_validation_digest()).unwrap();
        assert_eq!(json, r#"[]"#);
    }

    #[test]
    fn test_validation_evn_marker_b3() {
        let exe = PathBuf::from("python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
            Package::from_name_version_durl("numpy", "2.0", None).unwrap(),
        ];

        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();

        // this DM means that we only need NumPy if Python < 3,
        let dm = DepManifest::try_from_iter(
            [
                "numpy==1.19.3; python_version < '3.0'",
                "static_frame==2.13.0; python_version >= '3.0'",
            ]
            .iter(),
        )
        .unwrap();

        let vr = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: true,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        let json = serde_json::to_string(&vr.to_validation_digest()).unwrap();
        assert_eq!(json, r#"[]"#);
    }

    #[test]
    fn test_validation_evn_marker_b4() {
        let exe = PathBuf::from("python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
            Package::from_name_version_durl("numpy", "2.0", None).unwrap(),
        ];

        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();

        // this DM means that we only need NumPy if Python < 3,
        let dm = DepManifest::try_from_iter(
            [
                "numpy==1.19.3; python_version < '3.0'",
                "static_frame==2.13.0; python_version >= '20.0'",
            ]
            .iter(),
        )
        .unwrap();

        let vr = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: false,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        let json = serde_json::to_string(&vr.to_validation_digest()).unwrap();
        assert_eq!(
            json,
            r#"[{"package":"numpy-2.0","dependency":null,"explain":"Unrequired","sites":["/usr/lib/python3/site-packages"]},{"package":"static-frame-2.13.0","dependency":null,"explain":"Unrequired","sites":["/usr/lib/python3/site-packages"]}]"#
        );
    }

    #[test]
    fn test_validation_evn_marker_b5() {
        let exe = PathBuf::from("python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
            Package::from_name_version_durl("numpy", "2.0", None).unwrap(),
        ];

        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();

        // this DM means that we only need NumPy if Python < 3,
        let dm = DepManifest::try_from_iter(
            [
                "numpy==1.19.3; python_version < '3.0'",
                "static_frame==2.13.0; python_version >= '20.0'",
            ]
            .iter(),
        )
        .unwrap();

        let vr = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: false,
                permit_subset: true,
            },
            None,
            FlagLog(false),
        );
        let json = serde_json::to_string(&vr.to_validation_digest()).unwrap();
        assert_eq!(
            json,
            r#"[{"package":"numpy-2.0","dependency":null,"explain":"Unrequired","sites":["/usr/lib/python3/site-packages"]},{"package":"static-frame-2.13.0","dependency":null,"explain":"Unrequired","sites":["/usr/lib/python3/site-packages"]}]"#
        );
    }

    #[test]
    fn test_validation_evn_marker_b6() {
        let exe = PathBuf::from("python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
            Package::from_name_version_durl("numpy", "2.0", None).unwrap(),
        ];

        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();

        // this DM means that we only need NumPy if Python < 3,
        let dm = DepManifest::try_from_iter(
            [
                "numpy==1.19.3; python_version < '3.0'",
                "static_frame==2.13.0; python_version >= '20.0'",
            ]
            .iter(),
        )
        .unwrap();

        let vr = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: true,
                permit_subset: true,
            },
            None,
            FlagLog(false),
        );
        let json = serde_json::to_string(&vr.to_validation_digest()).unwrap();
        assert_eq!(json, r#"[]"#);
    }

    #[test]
    fn test_validation_evn_marker_c1() {
        let exe = PathBuf::from("python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
            Package::from_name_version_durl("numpy", "1.19.3", None).unwrap(),
        ];

        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();

        let dm = DepManifest::try_from_iter(
            [
                "numpy==1.2; python_version > '20'",
                "numpy==1.19.3; python_version > '3.0' and python_version < '20'",
                "numpy==2.0; python_version < '3.0'",
                "static_frame==2.13.0",
            ]
            .iter(),
        )
        .unwrap();

        let vr = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: false,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        let json = serde_json::to_string(&vr.to_validation_digest()).unwrap();
        assert_eq!(json, r#"[]"#);
    }

    #[test]
    fn test_validation_evn_marker_c2() {
        let exe = PathBuf::from("python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
            Package::from_name_version_durl("numpy", "1.19.3", None).unwrap(),
        ];

        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();

        let dm = DepManifest::try_from_iter(
            [
                "numpy==1.2; python_version > '20'",
                "numpy==1.19.1; python_version > '3.0' and python_version < '20'",
                "numpy==2.0; python_version < '3.0'",
                "static_frame==2.13.0",
            ]
            .iter(),
        )
        .unwrap();

        let vr = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: false,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        let json = serde_json::to_string(&vr.to_validation_digest()).unwrap();
        assert_eq!(
            json,
            r#"[{"package":"numpy-1.19.3","dependency":"numpy==1.19.1; python_version > '3.0' and python_version < '20'","explain":"Misdefined","sites":["/usr/lib/python3/site-packages"]}]"#
        );
    }

    #[test]
    fn test_validation_evn_marker_c3() {
        let exe = PathBuf::from("python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages =
            vec![
                Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
            ];

        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();

        let dm = DepManifest::try_from_iter(
            [
                "numpy==1.2; python_version > '20'",
                "numpy==1.19.1; python_version > '3.0' and python_version < '20'",
                "numpy==2.0; python_version < '3.0'",
                "static_frame==2.13.0",
            ]
            .iter(),
        )
        .unwrap();

        let vr = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: false,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        let json = serde_json::to_string(&vr.to_validation_digest()).unwrap();
        assert_eq!(
            json,
            r#"[{"package":null,"dependency":"numpy==1.19.1; python_version > '3.0' and python_version < '20'","explain":"Missing","sites":null}]"#
        );
    }

    #[test]
    fn test_validation_evn_marker_c4() {
        let exe = PathBuf::from("python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages =
            vec![
                Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
            ];

        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();

        let dm = DepManifest::try_from_iter(
            [
                "numpy==1.2; python_version > '20'",
                "numpy==1.19.1; python_version > '3.0' and python_version < '20'",
                "numpy==2.0; python_version < '3.0'",
                "static_frame==2.13.0",
            ]
            .iter(),
        )
        .unwrap();

        let vr = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: false,
                permit_subset: true,
            },
            None,
            FlagLog(false),
        );
        let json = serde_json::to_string(&vr.to_validation_digest()).unwrap();
        assert_eq!(json, r#"[]"#);
    }

    #[test]
    fn test_validation_evn_marker_c5() {
        let exe = PathBuf::from("python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
            Package::from_name_version_durl("numpy", "1.19.3", None).unwrap(),
        ];

        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();

        let dm = DepManifest::try_from_iter(
            [
                "numpy==1.2; python_version > '20'",
                "numpy==2.0; python_version < '3.0'",
                "static_frame==2.13.0",
            ]
            .iter(),
        )
        .unwrap();

        let vr = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: false,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        let json = serde_json::to_string(&vr.to_validation_digest()).unwrap();
        assert_eq!(
            json,
            r#"[{"package":"numpy-1.19.3","dependency":null,"explain":"Unrequired","sites":["/usr/lib/python3/site-packages"]}]"#
        );
    }

    #[test]
    fn test_validation_evn_marker_c6() {
        let exe = PathBuf::from("python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
            Package::from_name_version_durl("numpy", "1.19.3", None).unwrap(),
        ];

        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();

        let dm = DepManifest::try_from_iter(
            [
                "numpy==1.2; python_version > '20'",
                "numpy==2.0; python_version < '3.0'",
                "static_frame==2.13.0",
            ]
            .iter(),
        )
        .unwrap();

        let vr = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: true,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        let json = serde_json::to_string(&vr.to_validation_digest()).unwrap();
        assert_eq!(json, r#"[]"#);
    }

    #[test]
    fn test_validation_evn_marker_d1() {
        let exe = PathBuf::from("python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
            Package::from_name_version_durl("numpy", "2.0", None).unwrap(),
        ];

        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();

        let dm = DepManifest::try_from_iter(
            ["numpy==1.2", "numpy==2.0", "static_frame==2.13.0"].iter(),
        )
        .unwrap();

        let vr = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: true,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        let json = serde_json::to_string(&vr.to_validation_digest()).unwrap();
        assert_eq!(json, r#"[]"#);
    }

    #[test]
    fn test_validation_evn_marker_d2() {
        let exe = PathBuf::from("python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
            Package::from_name_version_durl("numpy", "2.2", None).unwrap(),
        ];

        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();

        let dm = DepManifest::try_from_iter(
            ["numpy==1.2", "numpy==2.0", "static_frame==2.13.0"].iter(),
        )
        .unwrap();

        let vr = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: true,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        let json = serde_json::to_string(&vr.to_validation_digest()).unwrap();
        assert_eq!(
            json,
            r#"[{"package":null,"dependency":"numpy==1.2","explain":"Missing","sites":null},{"package":null,"dependency":"numpy==2.0","explain":"Missing","sites":null}]"#
        );
    }

    #[test]
    fn test_validation_evn_marker_e() {
        let exe = PathBuf::from("python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![];
        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();

        let content = r#"
# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand.

[[package]]
name = "pathlib2"
version = "2.3.7.post1"
description = "Object-oriented filesystem paths"
optional = false
python-versions = "*"
groups = ["main"]
markers = "sys_platform == \"win32\""
files = [
    {file = "pathlib2-2.3.7.post1-py2.py3-none-any.whl", hash = "sha256:5266a0fd000452f1b3467d782f079a4343c63aaa119221fbdc4e39577489ca5b"},
    {file = "pathlib2-2.3.7.post1.tar.gz", hash = "sha256:9fe0edad898b83c0c3e199c842b27ed216645d2e177757b2dd67384d4113c641"},
]

[package.dependencies]
six = "*"

[[package]]
name = "six"
version = "1.17.0"
description = "Python 2 and 3 compatibility utilities"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
groups = ["main"]
markers = "sys_platform == \"win32\""
files = [
    {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"},
    {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"},
]

[metadata]
lock-version = "2.1"
python-versions = ">=3.13"
content-hash = "f05bd817b200790c9d7fdfecc11143473da90202f39a4a185ba66e28b04e079a"
"#;

        let fp_dir = tempdir().unwrap();
        let fp_lock = fp_dir.path().join("poetry.lock");
        let mut file = File::create(&fp_lock).unwrap();
        write!(file, "{}", content).unwrap();
        let dm = DepManifest::from_path(&fp_lock, None).unwrap();

        let vr = sfs.to_validation_report(
            dm,
            ValidationFlags {
                permit_superset: false,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        let json = serde_json::to_string(&vr.to_validation_digest()).unwrap();
        assert_eq!(json, r#"[]"#);
    }

    //--------------------------------------------------------------------------
    #[test]
    fn test_validation_ignore_a() {
        let exe = PathBuf::from("/usr/bin/python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("numpy", "1.19.3", None).unwrap(),
            Package::from_name_version_durl("static-frame", "2.13.2", None).unwrap(),
            Package::from_name_version_durl("pip", "23.1.2", None).unwrap(),
        ];
        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();

        // hyphen / underscore are normalized
        let dm =
            DepManifest::try_from_iter(["numpy==1.19.3", "static_frame>=2.13.0"].iter())
                .unwrap();

        let vr1 = sfs.to_validation_report(
            dm.clone(),
            ValidationFlags {
                permit_superset: false,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        let json1 = serde_json::to_string(&vr1.to_validation_digest()).unwrap();
        assert_eq!(
            json1,
            r#"[{"package":"pip-23.1.2","dependency":null,"explain":"Unrequired","sites":["/usr/lib/python3/site-packages"]}]"#
        );

        let mut ignore: HashSet<String> = HashSet::new();
        ignore.insert("pip".to_string());

        let vr2 = sfs.to_validation_report(
            dm.clone(),
            ValidationFlags {
                permit_superset: false,
                permit_subset: false,
            },
            Some(&ignore),
            FlagLog(false),
        );
        let json2 = serde_json::to_string(&vr2.to_validation_digest()).unwrap();
        assert_eq!(json2, r#"[]"#);
    }

    #[test]
    fn test_validation_ignore_b() {
        let exe = PathBuf::from("/usr/bin/python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("numpy", "1.19.3", None).unwrap(),
            Package::from_name_version_durl("static-frame", "2.13.2", None).unwrap(),
        ];
        let mut sfs = ScanFS::from_exe_site_packages(exe, site, packages).unwrap();

        // hyphen / underscore are normalized
        let dm = DepManifest::try_from_iter(
            ["numpy==1.19.3", "static_frame>=2.13.0", "pip==23.1.2"].iter(),
        )
        .unwrap();

        let vr1 = sfs.to_validation_report(
            dm.clone(),
            ValidationFlags {
                permit_superset: false,
                permit_subset: false,
            },
            None,
            FlagLog(false),
        );
        let json1 = serde_json::to_string(&vr1.to_validation_digest()).unwrap();
        assert_eq!(
            json1,
            r#"[{"package":null,"dependency":"pip==23.1.2","explain":"Missing","sites":null}]"#
        );

        let mut ignore: HashSet<String> = HashSet::new();
        ignore.insert("pip".to_string());

        let vr2 = sfs.to_validation_report(
            dm.clone(),
            ValidationFlags {
                permit_superset: false,
                permit_subset: false,
            },
            Some(&ignore),
            FlagLog(false),
        );
        let json2 = serde_json::to_string(&vr2.to_validation_digest()).unwrap();
        assert_eq!(json2, r#"[]"#);
    }

    //--------------------------------------------------------------------------
    #[test]
    fn test_search_a() {
        let exe = PathBuf::from("/usr/bin/python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("numpy", "1.19.3", None).unwrap(),
            Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
            Package::from_name_version_durl("flask", "1.1.3", None).unwrap(),
        ];
        let sfs = ScanFS::from_exe_site_packages(exe, site, packages.clone()).unwrap();
        let matched = sfs.search_by_match("*.3", true);
        assert_eq!(matched, vec![packages[2].clone(), packages[0].clone()]);
    }

    #[test]
    fn test_search_b() {
        let exe = PathBuf::from("/usr/bin/python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("numpy", "1.19.3", None).unwrap(),
            Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
            Package::from_name_version_durl("flask", "1.1.3", None).unwrap(),
        ];
        let sfs = ScanFS::from_exe_site_packages(exe, site, packages.clone()).unwrap();
        let matched = sfs.search_by_match("*frame*", true);
        assert_eq!(matched, vec![packages[1].clone()]);
    }

    //--------------------------------------------------------------------------

    #[test]
    fn test_serialize_a() {
        let exe = PathBuf::from("/usr/bin/python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("numpy", "1.19.3", None).unwrap(),
            Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
            Package::from_name_version_durl("flask", "1.1.3", None).unwrap(),
        ];
        let sfs = ScanFS::from_exe_site_packages(exe, site, packages.clone()).unwrap();
        let json = serde_json::to_string(&sfs).unwrap();
        assert_eq!(json, "[[\"/usr/bin/python3\",\"/usr/lib/python3/site-packages\"],[[0,[1]]],[[{\"name\":\"flask\",\"key\":\"flask\",\"version\":\"1.1.3\",\"direct_url\":null},[1]],[{\"name\":\"numpy\",\"key\":\"numpy\",\"version\":\"1.19.3\",\"direct_url\":null},[1]],[{\"name\":\"static-frame\",\"key\":\"static_frame\",\"version\":\"2.13.0\",\"direct_url\":null},[1]]],[[1,[0]]],false,false,\"c0b5205c57aa54df7dcbddb79399a81accfe1198a5a7842cb99ac806fb1524a7\"]");

        let sfsd: ScanFS = serde_json::from_str(&json).unwrap();
        assert_eq!(sfsd.exe_to_sites.len(), 1);
        assert_eq!(sfsd.package_to_sites.len(), 3);
    }

    #[test]
    fn test_serialize_b() {
        let exe1 = PathBuf::from("/usr/bin/python3");
        let exe2 = PathBuf::from("/opt/venv/bin/python");
        let site1 = PathBuf::from("/usr/lib/python3/site-packages");
        let site2 = PathBuf::from("/opt/venv/lib/python3.9/site-packages");

        let pkg_numpy = Package::from_name_version_durl("numpy", "1.21.0", None).unwrap();
        let pkg_pandas =
            Package::from_name_version_durl("pandas", "1.3.0", None).unwrap();
        let pkg_flask = Package::from_name_version_durl("flask", "2.0.1", None).unwrap();
        let pkg_requests =
            Package::from_name_version_durl("requests", "2.25.1", None).unwrap();

        let mut sfs = ScanFS {
            exe_to_sites: HashMap::new(),
            package_to_sites: HashMap::new(),
            site_to_exes: HashMap::new(),
            exe_to_ems: None,
            config: ScanConfig::new(false, false),
            exes_hash: "hash".to_string(),
        };

        // Populate exe_to_sites
        sfs.exe_to_sites
            .insert(exe1.clone().into(), vec![site1.clone().into()]);
        sfs.exe_to_sites.insert(
            exe2.clone().into(),
            vec![site1.clone().into(), site2.clone().into()],
        );

        // Populate package_to_sites
        sfs.package_to_sites.insert(
            pkg_numpy.clone(),
            vec![site1.clone().into(), site2.clone().into()],
        );
        sfs.package_to_sites
            .insert(pkg_pandas.clone(), vec![site2.clone().into()]);
        sfs.package_to_sites
            .insert(pkg_flask.clone(), vec![site1.clone().into()]);
        sfs.package_to_sites
            .insert(pkg_requests.clone(), vec![site2.clone().into()]);

        // Populate site_to_exes
        let exes1 = vec![PathShared::from(exe1.to_path_buf())];
        let exes2 = vec![PathShared::from(exe2.to_path_buf())];
        sfs.site_to_exes.insert(site1.clone().into(), exes1);
        sfs.site_to_exes.insert(site2.clone().into(), exes2);

        let json = serde_json::to_string(&sfs).unwrap();
        let expected_json = r#"[["/opt/venv/bin/python","/usr/lib/python3/site-packages","/opt/venv/lib/python3.9/site-packages","/usr/bin/python3"],[[0,[1,2]],[3,[1]]],[[{"name":"flask","key":"flask","version":"2.0.1","direct_url":null},[1]],[{"name":"numpy","key":"numpy","version":"1.21.0","direct_url":null},[1,2]],[{"name":"pandas","key":"pandas","version":"1.3.0","direct_url":null},[2]],[{"name":"requests","key":"requests","version":"2.25.1","direct_url":null},[2]]],[[2,[0]],[1,[3]]],false,false,"hash"]"#;
        assert_eq!(json, expected_json);

        let sfsd: ScanFS = serde_json::from_str(&json).unwrap();
        // Check deserialized sizes
        assert_eq!(sfsd.exe_to_sites.len(), 2);
        assert_eq!(sfsd.package_to_sites.len(), 4);
        assert_eq!(sfsd.site_to_exes.len(), 2);

        // Check exe_to_sites keys
        assert!(sfsd.exe_to_sites.contains_key(&exe1.clone().into()));
        assert!(sfsd.exe_to_sites.contains_key(&exe2.clone().into()));

        // Check that numpy maps to both sites
        let numpy_sites = sfsd.package_to_sites.get(&pkg_numpy).unwrap();
        assert_eq!(numpy_sites.len(), 2);

        // Check site_to_exes mapping
        assert_eq!(
            sfsd.site_to_exes.get(&site1.into()).and_then(|v| v.first()),
            Some(PathShared::from(&exe1)).as_ref()
        );
        assert_eq!(
            sfsd.site_to_exes.get(&site2.into()).and_then(|v| v.first()),
            Some(PathShared::from(&exe2)).as_ref()
        );
    }

    #[test]
    fn test_to_hash_a() {
        let exe = PathBuf::from("/usr/bin/python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("numpy", "1.19.3", None).unwrap(),
            Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
            Package::from_name_version_durl("flask", "1.1.3", None).unwrap(),
        ];
        let sfs = ScanFS::from_exe_site_packages(exe, site, packages.clone()).unwrap();
        assert_eq!(
            sfs.exes_hash,
            "c0b5205c57aa54df7dcbddb79399a81accfe1198a5a7842cb99ac806fb1524a7"
        );
    }

    #[test]
    fn test_to_hash_b() {
        let exe = PathBuf::from("/usr/local/bin/python3");
        let site = PathBuf::from("/usr/lib/python3/site-packages");
        let packages = vec![
            Package::from_name_version_durl("numpy", "1.19.3", None).unwrap(),
            Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap(),
            Package::from_name_version_durl("flask", "1.1.3", None).unwrap(),
        ];
        let sfs = ScanFS::from_exe_site_packages(exe, site, packages.clone()).unwrap();
        assert_eq!(
            sfs.exes_hash,
            "2a8da48d3b320314e74fa1b06b1b1d03330f4fa801b03ac81e57409726169a94"
        );
    }

    #[test]
    fn test_site_install_a() {
        let site_shared1 = PathShared::from("foo");
        let site_shared2 = PathShared::from("bar");
        let exe1 = PathBuf::from("a");
        let exe2 = PathBuf::from("b");

        let p1 = Package::from_name_version_durl("numpy", "1.19.3", None).unwrap();
        let p2 = Package::from_name_version_durl("static-frame", "2.13.0", None).unwrap();
        let p3 = Package::from_name_version_durl("flask", "1.1.3", None).unwrap();

        let mut exe_to_sites = HashMap::new();
        exe_to_sites.insert(exe1.clone().into(), vec![site_shared1.clone()]);
        exe_to_sites.insert(exe2.clone().into(), vec![site_shared2.clone()]);

        let exes = vec![exe1.clone(), exe2.clone()];

        let mut package_to_sites = HashMap::new();
        package_to_sites.insert(p1, vec![site_shared1.clone()]);
        package_to_sites.insert(p2, vec![site_shared1.clone()]);
        package_to_sites.insert(p3, vec![site_shared1.clone(), site_shared2.clone()]);

        let mut site_to_exes: HashMap<PathShared, Vec<PathShared>> = HashMap::new();
        let exes1 = vec![PathShared::from(exe1.to_path_buf())];
        let exes2 = vec![PathShared::from(exe2.to_path_buf())];
        site_to_exes.insert(site_shared1.clone(), exes1);
        site_to_exes.insert(site_shared2.clone(), exes2);

        let config = ScanConfig::new(false, false);
        let exes_hash = hash_paths(&exes, &config);
        let sfs = ScanFS {
            exe_to_sites,
            package_to_sites,
            site_to_exes,
            exe_to_ems: None,
            config,
            exes_hash,
        };

        let bound = PathBuf::from("foo");
        let bound_options = None;
        let vf = ValidationFlags {
            permit_superset: false,
            permit_subset: false,
        };
        let ignore = vec![];
        // ensure this retruns an error when multiple exe are defined
        assert!(sfs
            .site_validate_install(
                &bound,
                &bound_options,
                &ignore,
                &vf,
                None,
                FlagLog(false)
            )
            .is_err());
    }

    #[test]
    fn test_from_exes_a() {
        let exe1 = PathBuf::from("a");
        let exe2 = PathBuf::from("b");
        let exes = vec![exe1, exe2];
        let config = ScanConfig::new(false, false);
        let post = ScanFS::from_exes(&exes, &config, FlagLog(false));
        // error for bad exe
        assert!(post.is_err());
    }

    #[test]
    fn test_from_exes_equality_comparison() {
        let exe1 = PathBuf::from("/usr/bin/python3");
        let exe2 = PathBuf::from("/usr/bin/python3");

        let exes = vec![exe1.clone(), exe2.clone()];

        let config = ScanConfig::new(false, false);
        let scan1 = ScanFS::from_exes(&exes, &config, FlagLog(false))
            .expect("Failed to build ScanFS from the first ordering");
        let scan2 = ScanFS::from_exes(&exes, &config, FlagLog(false))
            .expect("Failed to build ScanFS from the shuffled executables");

        let json1 = serde_json::to_string(&scan1).expect("Failed to serialize scan1");
        let json2 = serde_json::to_string(&scan2).expect("Failed to serialize scan2");

        assert_eq!(
            json1, json2,
            "Non-deterministic serialization! ScanFS outputs differ between runs.
             json1: {}
             json2: {}",
            json1, json2
        );
    }
}
