//
// Syd: rock-solid unikernel
// src/syd-oci.rs: syd's OCI container runtime
//
// Copyright (c) 2024 Ali Polatel <alip@chesswob.org>
// Based in part upon youki which is:
//     Copyright (c) 2021 youki team
//     SPDX-License-Identifier: Apache-2.0
//
// SPDX-License-Identifier: GPL-3.0-or-later

// Exception: We don't use FxHash{Map,Set} here instead of Hash{Map,Set}.
#![allow(clippy::disallowed_types)]

use std::{
    collections::{HashMap, HashSet},
    env,
    ffi::{CString, OsStr, OsString},
    fmt::Write as FmtWrite,
    fs::{self, DirBuilder},
    io::{BufReader, BufWriter, Write as IOWrite},
    os::{
        fd::{AsRawFd, RawFd},
        linux::fs::MetadataExt,
        unix::fs::{DirBuilderExt, PermissionsExt},
    },
    path::{Path, PathBuf},
    process::{exit, ExitCode},
    rc::Rc,
    str::FromStr,
};

use clap::Parser;
use libcgroups::common::{CgroupManager, ControllerOpt};
use libcontainer::{
    apparmor,
    config::YoukiConfig,
    container::{builder::ContainerBuilder, Container, ContainerStatus},
    error::{ErrInvalidSpec, LibcontainerError, MissingSpecError},
    hooks,
    notify_socket::{NotifyListener, NotifySocket, NOTIFY_FILE},
    process,
    process::{args::ContainerArgs, intel_rdt::delete_resctrl_subdirectory},
    signal::Signal,
    syscall::syscall::SyscallType,
    tty,
    user_ns::UserNamespaceConfig,
    utils,
    utils::{rootless_required, PathBufExt},
    workload::{Executor, ExecutorError, ExecutorValidationError},
};
use liboci_cli::{
    Checkpoint, CommonCmd, Create, Delete, Events, Exec, Features, GlobalOpts, Kill, List, Pause,
    Ps, Resume, Run, StandardCmd, Start, State, Update,
};
use nix::{
    errno::Errno,
    fcntl::OFlag,
    sys::{
        signal,
        signal::kill,
        signalfd::SigSet,
        stat::Mode,
        wait::{waitpid, WaitPidFlag, WaitStatus},
    },
    unistd::{pipe2, read, Gid, Pid, Uid},
};
use oci_spec::runtime::{
    Capabilities as SpecCapabilities, Capability, LinuxBuilder, LinuxCapabilities,
    LinuxCapabilitiesBuilder, LinuxIdMappingBuilder, LinuxNamespace, LinuxNamespaceBuilder,
    LinuxNamespaceType, LinuxPidsBuilder, LinuxResources, LinuxResourcesBuilder,
    LinuxSchedulerPolicy, LinuxSeccompAction, LinuxSeccompBuilder, LinuxSyscallBuilder, Mount,
    Process, ProcessBuilder, Spec,
};
use procfs::process::Namespace;
use serde_json::to_writer_pretty;
use syd::{
    err::{SydError, SydResult},
    fs::path_exists,
    hook::Supervisor,
    log::JsonLinesLogger,
    sandbox::Sandbox,
};
use tabwriter::TabWriter;
use tracing_subscriber::layer::SubscriberExt;

// Set global allocator to mimalloc.
#[global_allocator]
static GLOBAL: mimalloc2_rust::GlobalMiMalloc = mimalloc2_rust::GlobalMiMalloc;

/// A tracing layer that writes messages to the invalid file descriptor -42.
pub struct SydLayer;

impl<S> tracing_subscriber::layer::Layer<S> for SydLayer
where
    S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,
{
    /// Determines if an event should be recorded.
    fn event_enabled(
        &self,
        _event: &tracing::Event,
        _ctx: tracing_subscriber::layer::Context<S>,
    ) -> bool {
        cfg!(feature = "log")
    }

    /// Called when an event occurs.
    fn on_event(&self, _event: &tracing::Event, _ctx: tracing_subscriber::layer::Context<S>) {
        #[cfg(feature = "log")]
        {
            syd::t!("{_event:?}");
        }
    }
}

#[derive(Clone)]
struct SydExecutor {}

impl Executor for SydExecutor {
    fn exec(&self, spec: &Spec) -> Result<(), ExecutorError> {
        // libcontainer sets process name to youki:INIT.
        // Let's avoid the confusion but ignore errors,
        // because this is not really that important...
        let _ = set_name("syd_oci");

        // Log the Spec for debugging (requires features=log).
        syd::t!(
            "Syd-OCI-Spec: {}",
            serde_json::to_string(spec).unwrap_or("?".to_string())
        );

        // `Spec::process` specifies the container process. This
        // property is REQUIRED when start is called.
        // `Process::args` specifies the binary and arguments for the
        // application to execute.
        // SAFETY: We panic if:
        // 1. Process is None.
        // 2. Process::args is None.
        // 3. Process::args is empty.
        #[allow(clippy::disallowed_methods)]
        let proc = spec
            .process()
            .as_ref()
            .expect("oci_spec::runtime::Spec::process");
        #[allow(clippy::disallowed_methods)]
        let argv = proc
            .args()
            .as_ref()
            .expect("oci_spec::runtime::Process::args!None")
            .iter()
            .map(OsString::from)
            .collect::<Vec<_>>();
        #[allow(clippy::disallowed_methods)]
        let (comm, argv) = argv
            .split_first()
            .map(|(arg0, argv)| (arg0, argv.to_vec()))
            .expect("oci_spec::runtime::Process::args!Empty");

        // Populate the environment from the OCI spec.
        if let Some(env) = proc.env() {
            for var in env {
                // Split the environment variable string into key and value.
                if let Some((var, val)) = var.split_once('=') {
                    // Set the environment variable if it's not already set.
                    let var = OsString::from(var);
                    if env::var_os(&var).is_none() {
                        env::set_var(var, OsString::from(val));
                    }
                }
            }
        }

        // Initialize the Syd sandbox.
        let sandbox = Sandbox::try_from(spec)
            .or(Err(ExecutorError::CantHandle("Failed to initialize Syd!")))?;

        // Log the Syd sandbox for debugging (requires features=log).
        syd::t!(
            "Syd-OCI-Sandbox: {}",
            serde_json::to_string(&sandbox).unwrap_or("?".to_string())
        );

        // Ignore all signals except SIG{KILL,STOP,PIPE,CHLD,Core}.
        // This is used to ensure we can deny {rt_,}sigreturn(2) to mitigate SROP.
        syd::ignore_signals().or(Err(ExecutorError::CantHandle("Failed to ignore signals!")))?;

        // Run command under the Syd sandbox and exit with the return code.
        #[allow(clippy::disallowed_methods)]
        Supervisor::run(sandbox, comm, argv, None, None, None)
            .map(i32::from)
            .map(exit)
            .map_err(|err| ExecutorError::Execution(err.into()))?
    }

    fn validate(&self, spec: &Spec) -> Result<(), ExecutorValidationError> {
        let proc = spec
            .process()
            .as_ref()
            .ok_or(ExecutorValidationError::ArgValidationError(
                "spec did not contain process".into(),
            ))?;

        if let Some(args) = proc.args() {
            let envs: Vec<String> = proc.env().as_ref().unwrap_or(&vec![]).clone();
            let path_vars: Vec<&String> = envs.iter().filter(|&e| e.starts_with("PATH=")).collect();
            if path_vars.is_empty() {
                syd::t!("PATH environment variable is not set");
                Err(ExecutorValidationError::ArgValidationError(
                    "PATH environment variable is not set".into(),
                ))?;
            }
            let path_var = path_vars[0].trim_start_matches("PATH=");
            match get_executable_path(&args[0], path_var) {
                None => {
                    syd::t!("executable for container process not found in PATH");
                    Err(ExecutorValidationError::ArgValidationError(format!(
                        "executable '{}' not found in $PATH",
                        args[0]
                    )))?;
                }
                Some(path) => match is_executable(&path) {
                    Ok(true) => {
                        syd::t!("found executable in executor");
                    }
                    Ok(false) => {
                        syd::t!("executable does not have the correct permission set");
                        Err(ExecutorValidationError::ArgValidationError(format!(
                            "executable '{}' at path '{:?}' does not have correct permissions",
                            args[0], path
                        )))?;
                    }
                    Err(err) => {
                        syd::t!("failed to check permissions for executable: {err}");
                        Err(ExecutorValidationError::ArgValidationError(format!(
                            "failed to check permissions for executable '{}' at path '{:?}' : {}",
                            args[0], path, err
                        )))?;
                    }
                },
            }
        }

        Ok(())
    }
}

const NAMESPACE_TYPES: &[&str] = &["ipc", "uts", "net", "pid", "mnt", "cgroup"];
const TENANT_NOTIFY: &str = "not-";
const TENANT_TTY: &str = "tty-";

// Builder that can be used to configure the properties of a process
// that will join an existing container sandbox
struct SydTenantContainerBuilder {
    #[allow(dead_code)]
    base: ContainerBuilder,
    env: HashMap<String, String>,
    cwd: Option<PathBuf>,
    args: Vec<String>,
    no_new_privs: Option<bool>,
    capabilities: Vec<String>,
    process: Option<PathBuf>,
    detached: bool,
    syscall: SyscallType,
    container_id: String,
    pid_file: Option<PathBuf>,
    preserve_fds: i32,
    executor: Box<dyn Executor>,
    root_path: PathBuf,
    console_socket: Option<PathBuf>,
}

impl SydTenantContainerBuilder {
    /// Generates the base configuration for a process that will join
    /// an existing container sandbox from which configuration methods
    /// can be chained
    fn new(opt: GlobalOpts, args: Exec) -> SydResult<Self> {
        let syscall = SyscallType::default();
        let container_id = args.container_id.clone();
        let pid_file = if let Some(ref p) = args.pid_file {
            Some(p.canonicalize_safely()?)
        } else {
            None
        };
        let executor = Box::new(SydExecutor {});
        let mut preserve_fds = args.preserve_fds;
        if opt.log.is_some() {
            preserve_fds += 1; // Preserve Syd's log file descriptor.
        }
        #[allow(clippy::disallowed_methods)]
        let builder = ContainerBuilder::new(container_id.clone(), syscall)
            .with_executor(SydExecutor {})
            .with_root_path(opt.root.clone().unwrap())?
            .with_console_socket(args.console_socket.as_ref())
            .with_pid_file(pid_file.clone())?
            .validate_id()?;
        #[allow(clippy::disallowed_methods)]
        Ok(Self {
            base: builder,
            env: HashMap::new(),
            cwd: None,
            args: Vec::new(),
            no_new_privs: None,
            capabilities: Vec::new(),
            process: None,
            detached: false,
            syscall,
            container_id,
            pid_file,
            preserve_fds,
            executor,
            root_path: opt.root.unwrap(),
            console_socket: args.console_socket,
        })
    }

    /// Sets environment variables for the container
    pub fn with_env(mut self, env: HashMap<String, String>) -> Self {
        self.env = env;
        self
    }

    /// Sets the working directory of the container
    pub fn with_cwd<P: Into<PathBuf>>(mut self, path: Option<P>) -> Self {
        self.cwd = path.map(|p| p.into());
        self
    }

    /// Sets the command the container will be started with
    pub fn with_container_args(mut self, args: Vec<String>) -> Self {
        self.args = args;
        self
    }

    pub fn with_no_new_privs(mut self, no_new_privs: bool) -> Self {
        self.no_new_privs = Some(no_new_privs);
        self
    }

    #[allow(dead_code)]
    pub fn with_capabilities(mut self, capabilities: Vec<String>) -> Self {
        self.capabilities = capabilities;
        self
    }

    pub fn with_process<P: Into<PathBuf>>(mut self, path: Option<P>) -> Self {
        self.process = path.map(|p| p.into());
        self
    }

    pub fn with_detach(mut self, detached: bool) -> Self {
        self.detached = detached;
        self
    }

    /// Joins an existing container
    pub fn build(self) -> Result<Pid, LibcontainerError> {
        let container_dir = self.lookup_container_dir()?;
        let container = self.load_container_state(container_dir.clone())?;
        let mut spec = self.load_init_spec(&container)?;
        self.adapt_spec_for_tenant(&mut spec, &container)?;

        syd::t!("{spec:?}");

        let notify_path = Self::setup_notify_listener(&container_dir)?;
        // convert path of root file system of the container to absolute path.
        #[allow(clippy::disallowed_methods)]
        let rootfs = fs::canonicalize(spec.root().as_ref().ok_or(MissingSpecError::Root)?.path())
            .map_err(LibcontainerError::OtherIO)?;

        // if socket file path is given in commandline options,
        // get file descriptors of console socket.
        let csocketfd = self.setup_tty_socket(&container_dir)?;

        let use_systemd = self.should_use_systemd(&container);
        let user_ns_config = UserNamespaceConfig::new(&spec)?;

        //FIXME: syd's nix != libcontainer's nix
        //let (read_end, write_end) =
        //    pipe2(OFlag::O_CLOEXEC).map_err(LibcontainerError::OtherSyscall)?;
        #[allow(clippy::disallowed_methods)]
        let (read_end, write_end) = pipe2(OFlag::O_CLOEXEC)
            .map_err(|e| LibcontainerError::OtherIO(std::io::Error::from_raw_os_error(e as i32)))?;

        let mut builder_impl = SydContainerBuilderImpl {
            container_type: ContainerType::SydTenantContainer {
                exec_notify_fd: write_end.as_raw_fd(),
            },
            syscall: self.syscall,
            container_id: self.container_id,
            pid_file: self.pid_file,
            console_socket: csocketfd,
            use_systemd,
            spec: Rc::new(spec),
            rootfs,
            user_ns_config,
            notify_path: notify_path.clone(),
            container: None,
            preserve_fds: self.preserve_fds,
            detached: self.detached,
            executor: self.executor,
        };

        let pid = builder_impl.create()?;

        let mut notify_socket = NotifySocket::new(notify_path);
        notify_socket.notify_container_start()?;

        // Explicitly close the write end of the pipe here to notify the
        // `read_end` that the init process is able to move forward. Closing one
        // end of the pipe will immediately signal the other end of the pipe,
        // which we use in the init thread as a form of barrier. `drop` is used
        // here because `OwnedFd` supports it, so we don't have to use `close`
        // here with `RawFd`.
        drop(write_end);

        let mut err_str_buf = Vec::new();

        loop {
            let mut buf = [0; 3];
            //FIXME: syd's nix != libcontainer's nix
            //match unistd::read(read_end, &mut buf).map_err(LibcontainerError::OtherSyscall)? {
            #[allow(clippy::disallowed_methods)]
            match read(read_end.as_raw_fd(), &mut buf).map_err(|e| {
                LibcontainerError::OtherIO(std::io::Error::from_raw_os_error(e as i32))
            })? {
                0 => {
                    if err_str_buf.is_empty() {
                        return Ok(pid);
                    } else {
                        return Err(LibcontainerError::Other(
                            String::from_utf8_lossy(&err_str_buf).to_string(),
                        ));
                    }
                }
                _ => {
                    err_str_buf.extend(buf);
                }
            }
        }
    }

    fn lookup_container_dir(&self) -> Result<PathBuf, LibcontainerError> {
        let container_dir = self.root_path.join(&self.container_id);
        if path_exists(&container_dir).is_err() {
            syd::t!("container dir does not exist");
            return Err(LibcontainerError::NoDirectory);
        }

        Ok(container_dir)
    }

    fn load_init_spec(&self, container: &Container) -> Result<Spec, LibcontainerError> {
        let spec_path = container.bundle().join("config.json");
        let mut spec = syd_spec_load(spec_path)?;

        Self::validate_spec(&spec)?;

        spec.canonicalize_rootfs(container.bundle())?;
        Ok(spec)
    }

    fn validate_spec(spec: &Spec) -> Result<(), LibcontainerError> {
        let version = spec.version();
        if !version.starts_with("1.") {
            syd::t!(
                "runtime spec has incompatible version '{}'. Only 1.X.Y is supported",
                spec.version()
            );
            Err(ErrInvalidSpec::UnsupportedVersion)?;
        }

        if let Some(process) = spec.process() {
            if let Some(io_priority) = process.io_priority() {
                let priority = io_priority.priority();
                let iop_class_res = serde_json::to_string(&io_priority.class());
                match iop_class_res {
                    Ok(_iop_class) => {
                        if !(0..=7).contains(&priority) {
                            syd::t!("io priority '{}' not between 0 and 7 (inclusive), class '{}' not in (IO_PRIO_CLASS_RT,IO_PRIO_CLASS_BE,IO_PRIO_CLASS_IDLE)",
                                priority, _iop_class);
                            Err(ErrInvalidSpec::IoPriority)?;
                        }
                    }
                    Err(_e) => {
                        syd::t!("failed to parse io priority class: {_e}");
                        Err(ErrInvalidSpec::IoPriority)?;
                    }
                }
            }

            if let Some(sc) = process.scheduler() {
                let policy = sc.policy();
                if let Some(nice) = sc.nice() {
                    // https://man7.org/linux/man-pages/man2/sched_setattr.2.html#top_of_page
                    if (*policy == LinuxSchedulerPolicy::SchedBatch
                        || *policy == LinuxSchedulerPolicy::SchedOther)
                        && (*nice < -20 || *nice > 19)
                    {
                        syd::t!("invalid scheduler.nice: '{nice}', must be within -20 to 19");
                        Err(ErrInvalidSpec::Scheduler)?;
                    }
                }
                if let Some(priority) = sc.priority() {
                    if *priority != 0
                        && (*policy != LinuxSchedulerPolicy::SchedFifo
                            && *policy != LinuxSchedulerPolicy::SchedRr)
                    {
                        syd::t!("scheduler.priority can only be specified for SchedFIFO or SchedRR policy");
                        Err(ErrInvalidSpec::Scheduler)?;
                    }
                }
                if *policy != LinuxSchedulerPolicy::SchedDeadline {
                    if let Some(runtime) = sc.runtime() {
                        if *runtime != 0 {
                            syd::t!(
                                "scheduler runtime can only be specified for SchedDeadline policy"
                            );
                            Err(ErrInvalidSpec::Scheduler)?;
                        }
                    }
                    if let Some(deadline) = sc.deadline() {
                        if *deadline != 0 {
                            syd::t!(
                                "scheduler deadline can only be specified for SchedDeadline policy"
                            );
                            Err(ErrInvalidSpec::Scheduler)?;
                        }
                    }
                    if let Some(period) = sc.period() {
                        if *period != 0 {
                            syd::t!(
                                "scheduler period can only be specified for SchedDeadline policy"
                            );
                            Err(ErrInvalidSpec::Scheduler)?;
                        }
                    }
                }
            }
        }

        utils::validate_spec_for_new_user_ns(spec)?;

        Ok(())
    }

    fn load_container_state(&self, container_dir: PathBuf) -> Result<Container, LibcontainerError> {
        let container = Container::load(container_dir)?;
        if !container.can_exec() {
            syd::t!("cannot exec as container");
            return Err(LibcontainerError::IncorrectStatus);
        }

        Ok(container)
    }

    fn adapt_spec_for_tenant(
        &self,
        spec: &mut Spec,
        container: &Container,
    ) -> Result<(), LibcontainerError> {
        let process = if let Some(process) = &self.process {
            self.get_process(process)?
        } else {
            let mut process_builder = ProcessBuilder::default()
                .args(self.get_args()?)
                .env(self.get_environment());
            if let Some(cwd) = self.get_working_dir()? {
                process_builder = process_builder.cwd(cwd);
            }

            if let Some(no_new_priv) = self.get_no_new_privileges() {
                process_builder = process_builder.no_new_privileges(no_new_priv);
            }

            if let Some(caps) = self.get_capabilities(spec)? {
                process_builder = process_builder.capabilities(caps);
            }

            process_builder.build()?
        };

        let container_pid = container.pid().ok_or(LibcontainerError::Other(
            "could not retrieve container init pid".into(),
        ))?;

        //FIXME: syd's procfs != libcontainer's procfs
        // let init_process = procfs::process::Process::new(container_pid.as_raw())?;
        //let ns = self.get_namespaces(init_process.namespaces()?.0)?;
        #[allow(clippy::disallowed_methods)]
        let init_process = procfs::process::Process::new(container_pid.as_raw()).map_err(|_| {
            LibcontainerError::OtherIO(std::io::Error::from_raw_os_error(nix::libc::ESRCH))
        })?;
        #[allow(clippy::disallowed_methods)]
        let ns = self.get_namespaces(
            init_process
                .namespaces()
                .map_err(|_| {
                    LibcontainerError::OtherIO(std::io::Error::from_raw_os_error(nix::libc::ESRCH))
                })?
                .0,
        )?;

        // it should never be the case that linux is not present in spec
        #[allow(clippy::disallowed_methods)]
        let spec_linux = spec.linux().as_ref().unwrap();
        let mut linux_builder = LinuxBuilder::default().namespaces(ns);

        if let Some(ref cgroup_path) = spec_linux.cgroups_path() {
            linux_builder = linux_builder.cgroups_path(cgroup_path.clone());
        }
        let linux = linux_builder.build()?;
        spec.set_process(Some(process)).set_linux(Some(linux));

        Ok(())
    }

    fn get_process(&self, process: &Path) -> Result<Process, LibcontainerError> {
        if path_exists(process).is_err() {
            syd::t!("process.json file does not exist");
            return Err(LibcontainerError::Other(
                "process.json file does not exist".into(),
            ));
        }

        #[allow(clippy::disallowed_methods)]
        let process = utils::open(process).map_err(LibcontainerError::OtherIO)?;
        let reader = BufReader::new(process);
        #[allow(clippy::disallowed_methods)]
        let process_spec =
            serde_json::from_reader(reader).map_err(LibcontainerError::OtherSerialization)?;
        Ok(process_spec)
    }

    fn get_working_dir(&self) -> Result<Option<PathBuf>, LibcontainerError> {
        if let Some(cwd) = &self.cwd {
            if cwd.is_relative() {
                syd::t!("current working directory must be an absolute path");
                return Err(LibcontainerError::Other(
                    "current working directory must be an absolute path".into(),
                ));
            }
            return Ok(Some(cwd.into()));
        }
        Ok(None)
    }

    fn get_args(&self) -> Result<Vec<String>, LibcontainerError> {
        if self.args.is_empty() {
            Err(MissingSpecError::Args)?;
        }

        Ok(self.args.clone())
    }

    fn get_environment(&self) -> Vec<String> {
        self.env.iter().map(|(k, v)| format!("{k}={v}")).collect()
    }

    fn get_no_new_privileges(&self) -> Option<bool> {
        self.no_new_privs
    }

    fn get_capabilities(
        &self,
        spec: &Spec,
    ) -> Result<Option<LinuxCapabilities>, LibcontainerError> {
        if !self.capabilities.is_empty() {
            let mut caps: Vec<syd::caps::Capability> = Vec::with_capacity(self.capabilities.len());
            #[allow(clippy::disallowed_methods)]
            for cap in &self.capabilities {
                caps.push(
                    syd::caps::Capability::from_str(cap)
                        .map_err(|e| LibcontainerError::Other(e.to_string()))?,
                );
            }

            let caps: SpecCapabilities = caps.iter().map(|c| c.spec()).collect();

            if let Some(spec_caps) = spec
                .process()
                .as_ref()
                .ok_or(MissingSpecError::Process)?
                .capabilities()
            {
                let mut capabilities_builder = LinuxCapabilitiesBuilder::default();
                capabilities_builder = match spec_caps.ambient() {
                    Some(ambient) => {
                        let ambient: SpecCapabilities = ambient.union(&caps).copied().collect();
                        capabilities_builder.ambient(ambient)
                    }
                    None => capabilities_builder,
                };
                capabilities_builder = match spec_caps.bounding() {
                    Some(bounding) => {
                        let bounding: SpecCapabilities = bounding.union(&caps).copied().collect();
                        capabilities_builder.bounding(bounding)
                    }
                    None => capabilities_builder,
                };
                capabilities_builder = match spec_caps.effective() {
                    Some(effective) => {
                        let effective: SpecCapabilities = effective.union(&caps).copied().collect();
                        capabilities_builder.effective(effective)
                    }
                    None => capabilities_builder,
                };
                capabilities_builder = match spec_caps.inheritable() {
                    Some(inheritable) => {
                        let inheritable: SpecCapabilities =
                            inheritable.union(&caps).copied().collect();
                        capabilities_builder.inheritable(inheritable)
                    }
                    None => capabilities_builder,
                };
                capabilities_builder = match spec_caps.permitted() {
                    Some(permitted) => {
                        let permitted: SpecCapabilities = permitted.union(&caps).copied().collect();
                        capabilities_builder.permitted(permitted)
                    }
                    None => capabilities_builder,
                };

                let c = capabilities_builder.build()?;
                return Ok(Some(c));
            }

            return Ok(Some(
                LinuxCapabilitiesBuilder::default()
                    .bounding(caps.clone())
                    .effective(caps.clone())
                    .inheritable(caps.clone())
                    .permitted(caps.clone())
                    .ambient(caps)
                    .build()?,
            ));
        }

        Ok(None)
    }

    fn get_namespaces(
        &self,
        init_namespaces: HashMap<OsString, Namespace>,
    ) -> Result<Vec<LinuxNamespace>, LibcontainerError> {
        let mut tenant_namespaces = Vec::with_capacity(init_namespaces.len());

        for &ns_type in NAMESPACE_TYPES {
            if let Some(init_ns) = init_namespaces.get(OsStr::new(ns_type)) {
                let tenant_ns = LinuxNamespaceType::try_from(ns_type)?;
                tenant_namespaces.push(
                    LinuxNamespaceBuilder::default()
                        .typ(tenant_ns)
                        .path(init_ns.path.clone())
                        .build()?,
                )
            }
        }

        Ok(tenant_namespaces)
    }

    fn should_use_systemd(&self, container: &Container) -> bool {
        container.systemd()
    }

    fn setup_notify_listener(container_dir: &Path) -> Result<PathBuf, LibcontainerError> {
        let notify_name = Self::generate_name(container_dir, TENANT_NOTIFY);
        let socket_path = container_dir.join(notify_name);

        Ok(socket_path)
    }

    fn setup_tty_socket(&self, container_dir: &Path) -> Result<Option<RawFd>, LibcontainerError> {
        let tty_name = Self::generate_name(container_dir, TENANT_TTY);
        let csocketfd = if let Some(console_socket) = &self.console_socket {
            Some(tty::setup_console_socket(
                container_dir,
                console_socket,
                &tty_name,
            )?)
        } else {
            None
        };

        Ok(csocketfd)
    }

    fn generate_name(dir: &Path, prefix: &str) -> String {
        loop {
            // SAFETY: Use GRND_RANDOM flag to draw random bytes from the random source.
            let mut rand_buf = [0u8; 2];
            if unsafe {
                nix::libc::getrandom(
                    rand_buf.as_mut_ptr() as *mut nix::libc::c_void,
                    rand_buf.len(),
                    nix::libc::GRND_RANDOM,
                )
            } < 0
            {
                panic!("getrandom: {}", Errno::last());
            }

            // Convert the bytes into an i16
            let rand = i16::from_be_bytes(rand_buf);
            let name = format!("{prefix}{rand:x}");
            if path_exists(&dir.join(&name)).is_err() {
                return name;
            }
        }
    }
}

// Builder to build a Syd container
struct SydInitContainerBuilder {
    #[allow(dead_code)]
    base: ContainerBuilder,
    bundle: PathBuf,
    use_systemd: bool,
    detached: bool,
    console_socket: Option<PathBuf>,
    syscall: SyscallType,
    container_id: String,
    pid_file: Option<PathBuf>,
    preserve_fds: i32,
    executor: Box<dyn Executor>,
    root_path: PathBuf,
}

impl TryFrom<(GlobalOpts, Create)> for SydInitContainerBuilder {
    type Error = SydError;

    /// Generates the base configuration for a new container from which
    /// configuration methods can be chained
    fn try_from(options: (GlobalOpts, Create)) -> SydResult<Self> {
        let (opt, args) = options;
        let syscall = SyscallType::default();
        let container_id = args.container_id.clone();
        let pid_file = if let Some(ref p) = args.pid_file {
            Some(p.canonicalize_safely()?)
        } else {
            None
        };
        let executor = Box::new(SydExecutor {});
        let mut preserve_fds = args.preserve_fds;
        if opt.log.is_some() {
            preserve_fds += 1; // Preserve Syd's log file descriptor.
        }
        #[allow(clippy::disallowed_methods)]
        let builder = ContainerBuilder::new(container_id.clone(), syscall)
            .with_executor(SydExecutor {})
            .with_pid_file(pid_file.clone())?
            .with_console_socket(args.console_socket.as_ref())
            .with_root_path(opt.root.clone().unwrap())?
            .with_preserved_fds(preserve_fds)
            .validate_id()?;
        #[allow(clippy::disallowed_methods)]
        Ok(Self {
            base: builder,
            bundle: args.bundle,
            use_systemd: opt.systemd_cgroup,
            detached: true,
            container_id,
            executor,
            pid_file,
            syscall,
            console_socket: args.console_socket,
            preserve_fds,
            root_path: opt.root.unwrap(),
        })
    }
}

impl TryFrom<(GlobalOpts, Run)> for SydInitContainerBuilder {
    type Error = SydError;

    /// Generates the base configuration for a new container from which
    /// configuration methods can be chained
    fn try_from(options: (GlobalOpts, Run)) -> SydResult<Self> {
        let (opt, args) = options;
        let syscall = SyscallType::default();
        let container_id = args.container_id.clone();
        let pid_file = if let Some(ref p) = args.pid_file {
            Some(p.canonicalize_safely()?)
        } else {
            None
        };
        let executor = Box::new(SydExecutor {});
        let mut preserve_fds = args.preserve_fds;
        if opt.log.is_some() {
            preserve_fds += 1; // Preserve Syd's log file descriptor.
        }
        #[allow(clippy::disallowed_methods)]
        let builder = ContainerBuilder::new(container_id.clone(), syscall)
            .with_executor(SydExecutor {})
            .with_pid_file(pid_file.clone())?
            .with_console_socket(args.console_socket.as_ref())
            .with_root_path(opt.root.clone().unwrap())?
            .with_preserved_fds(preserve_fds)
            .validate_id()?;
        #[allow(clippy::disallowed_methods)]
        Ok(Self {
            base: builder,
            bundle: args.bundle,
            use_systemd: opt.systemd_cgroup,
            detached: true,
            container_id,
            executor,
            pid_file,
            syscall,
            console_socket: args.console_socket,
            preserve_fds,
            root_path: opt.root.unwrap(),
        })
    }
}

impl SydInitContainerBuilder {
    /// Sets if systemd should be used for managing cgroups
    pub fn with_systemd(mut self, should_use: bool) -> Self {
        self.use_systemd = should_use;
        self
    }

    pub fn with_detach(mut self, detached: bool) -> Self {
        self.detached = detached;
        self
    }

    /// Creates a new container
    pub fn build(self) -> Result<Container, LibcontainerError> {
        let spec = self.load_spec()?;
        let container_dir = self.create_container_dir()?;

        let mut container = self.create_container_state(&container_dir)?;
        container
            .set_systemd(self.use_systemd)
            .set_annotations(spec.annotations().clone());

        let notify_path = container_dir.join(NOTIFY_FILE);
        // convert path of root file system of the container to absolute path
        #[allow(clippy::disallowed_methods)]
        let rootfs = fs::canonicalize(spec.root().as_ref().ok_or(MissingSpecError::Root)?.path())
            .map_err(LibcontainerError::OtherIO)?;

        // if socket file path is given in commandline options,
        // get file descriptors of console socket
        let csocketfd = if let Some(console_socket) = &self.console_socket {
            Some(tty::setup_console_socket(
                &container_dir,
                console_socket,
                "tty",
            )?)
        } else {
            None
        };

        syd::t!("parsing user namespace config");
        let user_ns_config = UserNamespaceConfig::new(&spec)?;

        syd::t!("parsing youki config");
        let mut config = YoukiConfig::from_spec(&spec, container.id())?;
        let linux = spec.linux().as_ref().ok_or(MissingSpecError::Linux)?;
        config.cgroup_path = get_cgroup_path(linux.cgroups_path(), &self.container_id);
        #[allow(clippy::disallowed_methods)]
        config.save(&container_dir).map_err(|err| {
            syd::t!("failed to save config: {err}");
            err
        })?;

        let mut builder_impl = SydContainerBuilderImpl {
            container_type: ContainerType::SydInitContainer,
            syscall: self.syscall,
            container_id: self.container_id,
            pid_file: self.pid_file,
            console_socket: csocketfd,
            use_systemd: self.use_systemd,
            spec: Rc::new(spec),
            rootfs,
            user_ns_config,
            notify_path,
            container: Some(container.clone()),
            preserve_fds: self.preserve_fds,
            detached: self.detached,
            executor: self.executor,
        };

        builder_impl.create()?;

        container.refresh_state()?;

        Ok(container)
    }

    fn create_container_dir(&self) -> Result<PathBuf, LibcontainerError> {
        let container_dir = self.root_path.join(&self.container_id);
        syd::t!("container directory will be {container_dir:?}");

        if path_exists(&container_dir).is_ok() {
            syd::t!("container already exists");
            return Err(LibcontainerError::Exist);
        }

        #[allow(clippy::disallowed_methods)]
        std::fs::create_dir_all(&container_dir).map_err(|err| {
            syd::t!("failed to create container directory: {err}");
            LibcontainerError::OtherIO(err)
        })?;

        Ok(container_dir)
    }

    fn load_spec(&self) -> Result<Spec, LibcontainerError> {
        let source_spec_path = self.bundle.join("config.json");
        let mut spec = syd_spec_load(source_spec_path)?;

        // Step 3: Copy user specified config if available.
        let target_syd_path = if let Some(root) = spec.root() {
            root.path().join(".oci.syd-3")
        } else {
            // root is required.
            return Err(ErrInvalidSpec::UnsupportedVersion)?;
        };

        #[allow(clippy::disallowed_methods)]
        let is_rootless_required = rootless_required().map_err(LibcontainerError::OtherIO)?;
        let syd_dir = if !is_rootless_required {
            Some(String::from("/etc/syd/oci"))
        } else if let Ok(path) = env::var("XDG_CONFIG_HOME") {
            Some(format!("{path}/syd/oci"))
        } else if let Ok(path) = env::var("HOME") {
            Some(format!("{path}/.syd/oci"))
        } else {
            None
        };

        if let Some(syd_dir) = syd_dir {
            let mut sources = vec![];
            match (spec.hostname(), spec.domainname()) {
                (Some(hostname), Some(domainname)) => {
                    sources.push(PathBuf::from(format!("{syd_dir}/{domainname}.syd-3")));
                    sources.push(PathBuf::from(format!(
                        "{syd_dir}/{hostname}.{domainname}.syd-3"
                    )));
                    sources.push(PathBuf::from(format!("{syd_dir}/{hostname}.syd-3")));
                }
                (None, Some(domainname)) => {
                    sources.push(PathBuf::from(format!("{syd_dir}/{domainname}.syd-3")));
                }
                (Some(hostname), None) => {
                    sources.push(PathBuf::from(format!("{syd_dir}/{hostname}.syd-3")));
                }
                _ => {}
            };
            sources.push(PathBuf::from(format!("{syd_dir}/default.syd-3")));

            for path in sources {
                #[allow(clippy::disallowed_methods)]
                if path_exists(&path).is_ok() {
                    // Copy the file into the container root.
                    fs::copy(&path, &target_syd_path)
                        .map_err(|err| LibcontainerError::InvalidInput(err.to_string()))?;

                    // Set the file as read-only.
                    // Note, we give read access to group and others to support UserNs correctly.
                    fs::set_permissions(&target_syd_path, fs::Permissions::from_mode(0o444))
                        .map_err(|err| LibcontainerError::InvalidInput(err.to_string()))?;

                    // Stop processing.
                    break;
                }
            }
        }

        Self::validate_spec(&spec)?;

        #[allow(clippy::disallowed_methods)]
        spec.canonicalize_rootfs(&self.bundle).map_err(|err| {
            syd::t!("failed to canonicalize rootfs: {err}");
            err
        })?;

        Ok(spec)
    }

    fn validate_spec(spec: &Spec) -> Result<(), LibcontainerError> {
        let version = spec.version();
        if !version.starts_with("1.") {
            syd::t!(
                "runtime spec has incompatible version '{}'. Only 1.X.Y is supported",
                spec.version()
            );
            Err(ErrInvalidSpec::UnsupportedVersion)?;
        }

        if let Some(process) = spec.process() {
            if let Some(_profile) = process.apparmor_profile() {
                #[allow(clippy::disallowed_methods)]
                let apparmor_is_enabled = apparmor::is_enabled().map_err(|err| {
                    syd::t!("failed to check if apparmor is enabled");
                    LibcontainerError::OtherIO(err)
                })?;
                if !apparmor_is_enabled {
                    syd::t!("apparmor profile exists in the spec, but apparmor is not activated on this system");
                    Err(ErrInvalidSpec::AppArmorNotEnabled)?;
                }
            }

            if let Some(io_priority) = process.io_priority() {
                let priority = io_priority.priority();
                let iop_class_res = serde_json::to_string(&io_priority.class());
                match iop_class_res {
                    Ok(_iop_class) => {
                        if !(0..=7).contains(&priority) {
                            syd::t!("io priority '{}' not between 0 and 7 (inclusive), class '{}' not in (IO_PRIO_CLASS_RT,IO_PRIO_CLASS_BE,IO_PRIO_CLASS_IDLE)",
                                priority, _iop_class);
                            Err(ErrInvalidSpec::IoPriority)?;
                        }
                    }
                    Err(_e) => {
                        syd::t!("failed to parse io priority class: {_e}");
                        Err(ErrInvalidSpec::IoPriority)?;
                    }
                }
            }
        }

        utils::validate_spec_for_new_user_ns(spec)?;

        Ok(())
    }

    fn create_container_state(&self, container_dir: &Path) -> Result<Container, LibcontainerError> {
        let container = Container::new(
            &self.container_id,
            ContainerStatus::Creating,
            None,
            &self.bundle,
            container_dir,
        )?;
        container.save()?;
        Ok(container)
    }
}

struct SydContainerBuilderImpl {
    /// Flag indicating if an init or a tenant container should be created
    pub container_type: ContainerType,
    /// Interface to operating system primitives
    pub syscall: SyscallType,
    /// Flag indicating if systemd should be used for cgroup management
    pub use_systemd: bool,
    /// Id of the container
    pub container_id: String,
    /// OCI compliant runtime spec
    pub spec: Rc<Spec>,
    /// Root filesystem of the container
    pub rootfs: PathBuf,
    /// File which will be used to communicate the pid of the
    /// container process to the higher level runtime
    pub pid_file: Option<PathBuf>,
    /// Socket to communicate the file descriptor of the ptty
    pub console_socket: Option<RawFd>,
    /// Options for new user namespace
    pub user_ns_config: Option<UserNamespaceConfig>,
    /// Path to the Unix Domain Socket to communicate container start
    pub notify_path: PathBuf,
    /// Container state
    pub container: Option<Container>,
    /// File descriptos preserved/passed to the container init process.
    pub preserve_fds: i32,
    /// If the container is to be run in detached mode
    pub detached: bool,
    /// Default executes the specified execution of a generic command
    pub executor: Box<dyn Executor>,
}

impl SydContainerBuilderImpl {
    fn create(&mut self) -> Result<Pid, LibcontainerError> {
        match self.run_container() {
            Ok(pid) => Ok(pid),
            Err(outer) => {
                // Only the init container should be cleaned up in the case of
                // an error.
                if matches!(self.container_type, ContainerType::SydInitContainer) {
                    self.cleanup_container()?;
                }

                Err(outer)
            }
        }
    }

    fn run_container(&mut self) -> Result<Pid, LibcontainerError> {
        let linux = self.spec.linux().as_ref().ok_or(MissingSpecError::Linux)?;
        let cgroups_path = get_cgroup_path(linux.cgroups_path(), &self.container_id);
        let cgroup_config = libcgroups::common::CgroupConfig {
            cgroup_path: cgroups_path,
            systemd_cgroup: self.use_systemd || self.user_ns_config.is_some(),
            container_name: self.container_id.to_owned(),
        };
        let process = self
            .spec
            .process()
            .as_ref()
            .ok_or(MissingSpecError::Process)?;

        if matches!(self.container_type, ContainerType::SydInitContainer) {
            if let Some(hooks) = self.spec.hooks() {
                hooks::run_hooks(
                    hooks.create_runtime().as_ref(),
                    self.container.as_ref(),
                    None,
                )?
            }
        }

        // Need to create the notify socket before we pivot root, since the unix
        // domain socket used here is outside of the rootfs of container. During
        // exec, need to create the socket before we enter into existing mount
        // namespace. We also need to create to socket before entering into the
        // user namespace in the case that the path is located in paths only
        // root can access.
        let notify_listener = NotifyListener::new(&self.notify_path)?;

        // If Out-of-memory score adjustment is set in specification.  set the score
        // value for the current process check
        // https://dev.to/rrampage/surviving-the-linux-oom-killer-2ki9 for some more
        // information.
        //
        // This has to be done before !dumpable because /proc/self/oom_score_adj
        // is not writeable unless you're an privileged user (if !dumpable is
        // set). All children inherit their parent's oom_score_adj value on
        // fork(2) so this will always be propagated properly.
        #[allow(clippy::disallowed_methods)]
        if let Some(oom_score_adj) = process.oom_score_adj() {
            syd::t!("Set OOM score to {oom_score_adj}");
            let mut f = fs::File::create("/proc/self/oom_score_adj").map_err(|err| {
                syd::t!("failed to open /proc/self/oom_score_adj: {err}");
                LibcontainerError::OtherIO(err)
            })?;
            f.write_all(oom_score_adj.to_string().as_bytes())
                .map_err(|err| {
                    syd::t!("failed to write to /proc/self/oom_score_adj: {err}");
                    LibcontainerError::OtherIO(err)
                })?;
        }

        /*
         * syd: With non-dumpable syd's pidfd_getfd will not work, so we
         * can not do this.
        // Make the process non-dumpable, to avoid various race conditions that
        // could cause processes in namespaces we're joining to access host
        // resources (or potentially execute code).
        //
        // However, if the number of namespaces we are joining is 0, we are not
        // going to be switching to a different security context. Thus setting
        // ourselves to be non-dumpable only breaks things (like rootless
        // containers), which is the recommendation from the kernel folks.
        if linux.namespaces().is_some() {
            // SAFETY: Our version of nix does not have prctl::set_dumpable
            let res = unsafe { nix::libc::prctl(nix::libc::PR_SET_DUMPABLE, 0, 0, 0, 0) };
            Errno::result(res).map(drop).map_err(|e| {
                LibcontainerError::Other(format!("error in setting dumpable to false : {e}",))
            })?;
        }
        */

        // This container_args will be passed to the container processes,
        // therefore we will have to move all the variable by value. Since self
        // is a shared reference, we have to clone these variables here.
        let container_args = ContainerArgs {
            container_type: match self.container_type {
                ContainerType::SydInitContainer => process::args::ContainerType::InitContainer,
                ContainerType::SydTenantContainer { exec_notify_fd } => {
                    process::args::ContainerType::TenantContainer { exec_notify_fd }
                }
            },
            syscall: self.syscall,
            spec: Rc::clone(&self.spec),
            rootfs: self.rootfs.to_owned(),
            console_socket: self.console_socket,
            notify_listener,
            preserve_fds: self.preserve_fds,
            container: self.container.to_owned(),
            user_ns_config: self.user_ns_config.to_owned(),
            cgroup_config,
            detached: self.detached,
            executor: self.executor.clone(),
        };

        #[allow(clippy::disallowed_methods)]
        let (init_pid, need_to_clean_up_intel_rdt_dir) =
            process::container_main_process::container_main_process(&container_args).map_err(
                |err| {
                    syd::t!("failed to run container process: {err}");
                    LibcontainerError::MainProcess(err)
                },
            )?;

        // if file to write the pid to is specified, write pid of the child
        #[allow(clippy::disallowed_methods)]
        if let Some(pid_file) = &self.pid_file {
            fs::write(pid_file, format!("{init_pid}")).map_err(|err| {
                syd::t!("failed to write pid to file: {err}");
                LibcontainerError::OtherIO(err)
            })?;
        }

        if let Some(container) = &mut self.container {
            // update status and pid of the container process
            container
                .set_status(ContainerStatus::Created)
                .set_creator(nix::unistd::geteuid().as_raw())
                .set_pid(init_pid.as_raw())
                .set_clean_up_intel_rdt_directory(need_to_clean_up_intel_rdt_dir)
                .save()?;
        }

        // FIXME(alip): syd's nix version != libcontainer nix version
        Ok(Pid::from_raw(init_pid.as_raw()))
    }

    fn cleanup_container(&self) -> Result<(), LibcontainerError> {
        let linux = self.spec.linux().as_ref().ok_or(MissingSpecError::Linux)?;
        let cgroups_path = get_cgroup_path(linux.cgroups_path(), &self.container_id);
        let cmanager =
            libcgroups::common::create_cgroup_manager(libcgroups::common::CgroupConfig {
                cgroup_path: cgroups_path,
                systemd_cgroup: self.use_systemd || self.user_ns_config.is_some(),
                container_name: self.container_id.to_string(),
            })?;

        let mut errors = Vec::new();

        if let Err(e) = cmanager.remove() {
            syd::t!("failed to remove cgroup manager: {e}");
            errors.push(e.to_string());
        }

        if let Some(container) = &self.container {
            if let Some(true) = container.clean_up_intel_rdt_subdirectory() {
                if let Err(e) = delete_resctrl_subdirectory(container.id()) {
                    syd::t!("failed to delete resctrl subdirectory: {e}");
                    errors.push(e.to_string());
                }
            }

            if path_exists(&container.root).is_ok() {
                if let Err(e) = fs::remove_dir_all(&container.root) {
                    syd::t!("failed to delete container root: {e}");
                    errors.push(e.to_string());
                }
            }
        }

        if !errors.is_empty() {
            return Err(LibcontainerError::Other(format!(
                "failed to cleanup container: {}",
                errors.join(";")
            )));
        }

        Ok(())
    }
}

#[derive(Debug, Copy, Clone)]
pub enum ContainerType {
    SydInitContainer,
    SydTenantContainer { exec_notify_fd: RawFd },
}

// Subcommands accepted by Syd, confirming with [OCI runtime-spec](https://github.com/opencontainers/runtime-spec/blob/master/runtime.md)
// Also for a short information, check [runc commandline documentation](https://github.com/opencontainers/runc/blob/master/man/runc.8.md)
#[derive(Parser, Debug)]
enum SubCommand {
    // Standard and common commands handled by the liboci_cli crate
    #[clap(flatten)]
    Standard(Box<StandardCmd>),
    #[clap(flatten)]
    Common(Box<CommonCmd>),
}

#[derive(Parser, Debug)]
#[clap(
    about = "Syd's OCI container runtime",
    author = "Ali Polatel",
    version = env!("CARGO_PKG_VERSION"),
)]
struct Opts {
    #[clap(flatten)]
    opt: GlobalOpts,

    // Standard and common commands handled by the liboci_cli crate
    #[clap(subcommand)]
    sub: SubCommand,
}

fn main() -> SydResult<ExitCode> {
    // Guard against CVE-2019-5736:
    // Copy /proc/self/exe in an anonymous fd (created via memfd_create), seal it and re-execute it.
    // See:
    // - https://github.com/opencontainers/runc/commit/0a8e4117e7f715d5fbeef398405813ce8e88558b
    // - https://github.com/lxc/lxc/commit/6400238d08cdf1ca20d49bafb85f4e224348bf9d
    // Note: syd's procfs protections is another layer of defense against this.
    if env::var_os(syd::config::ENV_QUICK_BOOT).is_none() {
        syd::seal::ensure_sealed()?;
    }

    // Configure mimalloc.
    syd::configure_mimalloc();

    // Parse CLI arguments.
    let mut opts = Opts::parse();

    // Initialize sandbox logging.
    if opts.opt.log.is_some() {
        let level = if opts.opt.debug { "debug" } else { "warn" };
        env::set_var(syd::config::ENV_NO_SYSLOG, "NoThanks");
        JsonLinesLogger::init(opts.opt.log.as_deref(), level)?;
    } else {
        JsonLinesLogger::init(None, "off")?;
    }

    // Initialize youki tracing.
    let fmtlayer = tracing_subscriber::fmt::layer()
        .with_writer(std::io::sink) // Redirect output to a sink (no output)
        .with_span_events(tracing_subscriber::fmt::format::FmtSpan::NONE);
    let subscriber = tracing_subscriber::registry().with(fmtlayer).with(SydLayer);
    tracing::subscriber::set_global_default(subscriber)?;

    // Check root, set a sane default if None.
    make_root(&mut opts.opt)?;

    // Call the relevant subcommand.
    match opts.sub {
        SubCommand::Standard(cmd) => match *cmd {
            StandardCmd::Create(subopts) => cmd_create(opts.opt, subopts),
            StandardCmd::Start(subopts) => cmd_start(opts.opt, subopts),
            StandardCmd::State(subopts) => cmd_state(opts.opt, subopts),
            StandardCmd::Kill(subopts) => cmd_kill(opts.opt, subopts),
            StandardCmd::Delete(subopts) => cmd_delete(opts.opt, subopts),
        },
        SubCommand::Common(cmd) => match *cmd {
            CommonCmd::Features(subopts) => cmd_features(opts.opt, subopts),
            CommonCmd::Ps(subopts) => cmd_ps(opts.opt, subopts),
            CommonCmd::List(subopts) => cmd_list(opts.opt, subopts),
            CommonCmd::Spec(subopts) => cmd_spec(opts.opt, subopts),
            CommonCmd::Pause(subopts) => cmd_pause(opts.opt, subopts),
            CommonCmd::Resume(subopts) => cmd_resume(opts.opt, subopts),
            CommonCmd::Events(subopts) => cmd_events(opts.opt, subopts),
            CommonCmd::Update(subopts) => cmd_update(opts.opt, subopts),
            CommonCmd::Checkpointt(subopts) => cmd_checkpoint(opts.opt, subopts),
            CommonCmd::Exec(subopts) => cmd_exec(opts.opt, subopts),
            CommonCmd::Run(subopts) => cmd_run(opts.opt, subopts),
        },
    }
}

fn cmd_create(opt: GlobalOpts, args: Create) -> SydResult<ExitCode> {
    let systemd_cgroup = opt.systemd_cgroup;
    SydInitContainerBuilder::try_from((opt, args))?
        .with_systemd(systemd_cgroup)
        .with_detach(true)
        .build()?;
    Ok(ExitCode::SUCCESS)
}

fn cmd_start(opt: GlobalOpts, args: Start) -> SydResult<ExitCode> {
    #[allow(clippy::disallowed_methods)]
    let container_root = opt.root.unwrap().join(args.container_id.clone());
    path_exists(&container_root)?;

    let mut container = Container::load(container_root)?;
    container.start()?;

    Ok(ExitCode::SUCCESS)
}

fn cmd_state(opt: GlobalOpts, args: State) -> SydResult<ExitCode> {
    #[allow(clippy::disallowed_methods)]
    let container_root = opt.root.unwrap().join(args.container_id.clone());
    path_exists(&container_root)?;

    let container = Container::load(container_root)?;
    println!("{}", serde_json::to_string_pretty(&container.state)?);

    Ok(ExitCode::SUCCESS)
}

fn cmd_kill(opt: GlobalOpts, args: Kill) -> SydResult<ExitCode> {
    #[allow(clippy::disallowed_methods)]
    let container_root = opt.root.unwrap().join(args.container_id.clone());
    path_exists(&container_root)?;

    let mut container = Container::load(container_root)?;

    let signal: Signal = args.signal.as_str().try_into()?;
    container.kill(signal, args.all)?;

    Ok(ExitCode::SUCCESS)
}

fn cmd_delete(opt: GlobalOpts, args: Delete) -> SydResult<ExitCode> {
    #[allow(clippy::disallowed_methods)]
    let container_root = opt.root.unwrap().join(args.container_id.clone());
    if path_exists(&container_root).is_err() && args.force {
        return Ok(ExitCode::SUCCESS);
    }

    let mut container = Container::load(container_root)?;
    container.delete(args.force)?;

    Ok(ExitCode::SUCCESS)
}

fn cmd_features(_opt: GlobalOpts, _args: Features) -> SydResult<ExitCode> {
    Ok(ExitCode::SUCCESS)
}

fn cmd_ps(opt: GlobalOpts, args: Ps) -> SydResult<ExitCode> {
    #[allow(clippy::disallowed_methods)]
    let container_root = opt.root.unwrap().join(args.container_id.clone());
    path_exists(&container_root)?;

    let container = Container::load(container_root)?;
    let cmanager = libcgroups::common::create_cgroup_manager(libcgroups::common::CgroupConfig {
        cgroup_path: container.spec()?.cgroup_path,
        systemd_cgroup: container.systemd(),
        container_name: container.id().to_string(),
    })?;

    let pids: Vec<i32> = cmanager
        .get_all_pids()?
        .iter()
        .map(|pid| pid.as_raw())
        .collect();

    if args.format == "json" {
        println!("{}", serde_json::to_string(&pids)?);
    } else if args.format == "table" {
        let default_ps_options = vec![String::from("-ef")];
        let ps_options = if args.ps_options.is_empty() {
            &default_ps_options
        } else {
            &args.ps_options
        };
        let output = std::process::Command::new("ps").args(ps_options).output()?;
        if !output.status.success() {
            println!("{}", std::str::from_utf8(&output.stderr)?);
        } else {
            let lines = std::str::from_utf8(&output.stdout)?;
            let lines: Vec<&str> = lines.split('\n').collect();
            let pid_index = get_pid_index(lines[0])?;
            println!("{}", &lines[0]);
            for line in &lines[1..] {
                if line.is_empty() {
                    continue;
                }
                let fields: Vec<&str> = line.split_whitespace().collect();
                let pid: i32 = fields[pid_index].parse()?;
                if pids.contains(&pid) {
                    println!("{line}");
                }
            }
        }
    }

    Ok(ExitCode::SUCCESS)
}

fn cmd_list(opt: GlobalOpts, _args: List) -> SydResult<ExitCode> {
    let mut content = String::new();
    // all containers' data is stored in their respective dir in root directory
    // so we iterate through each and print the various info
    #[allow(clippy::disallowed_methods)]
    for container_dir in fs::read_dir(opt.root.unwrap())? {
        let container_dir = container_dir?.path();
        let state_file = container_dir.join("state.json");
        if path_exists(&state_file).is_err() {
            continue;
        }

        let container = Container::load(container_dir)?;
        let pid = if let Some(pid) = container.pid() {
            pid.to_string()
        } else {
            "".to_owned()
        };

        let user_name = container.creator().unwrap_or_default();

        let created = if let Some(utc) = container.created() {
            utc.to_rfc3339()
        } else {
            "".to_owned()
        };

        let _ = writeln!(
            content,
            "{}\t{}\t{}\t{}\t{}\t{}",
            container.id(),
            pid,
            container.status(),
            container.bundle().display(),
            created,
            user_name.to_string_lossy()
        );
    }

    let mut tab_writer = TabWriter::new(std::io::stdout());
    writeln!(&mut tab_writer, "ID\tPID\tSTATUS\tBUNDLE\tCREATED\tCREATOR")?;
    write!(&mut tab_writer, "{content}")?;
    tab_writer.flush()?;

    Ok(ExitCode::SUCCESS)
}

fn cmd_spec(_opt: GlobalOpts, args: liboci_cli::Spec) -> SydResult<ExitCode> {
    let spec = if args.rootless {
        get_rootless_spec()?
    } else {
        Spec::default()
    };

    // write data to config.json
    let file = fs::File::create("config.json")?;
    let mut writer = BufWriter::new(file);
    to_writer_pretty(&mut writer, &spec)?;
    writer.flush()?;

    Ok(ExitCode::SUCCESS)
}

// Pausing a container indicates suspending all processes in given container
// This uses Freezer cgroup to suspend and resume processes
// For more information see :
// https://man7.org/linux/man-pages/man7/cgroups.7.html
// https://www.kernel.org/doc/Documentation/cgroup-v1/freezer-subsystem.txt
fn cmd_pause(opt: GlobalOpts, args: Pause) -> SydResult<ExitCode> {
    #[allow(clippy::disallowed_methods)]
    let container_root = opt.root.unwrap().join(args.container_id.clone());
    path_exists(&container_root)?;

    let mut container = Container::load(container_root)?;
    container.pause()?;

    Ok(ExitCode::SUCCESS)
}

// Resuming a container indicates resuming all processes in given container from paused state
// This uses Freezer cgroup to suspend and resume processes
// For more information see :
// https://man7.org/linux/man-pages/man7/cgroups.7.html
// https://www.kernel.org/doc/Documentation/cgroup-v1/freezer-subsystem.txt
fn cmd_resume(opt: GlobalOpts, args: Resume) -> SydResult<ExitCode> {
    #[allow(clippy::disallowed_methods)]
    let container_root = opt.root.unwrap().join(args.container_id.clone());
    path_exists(&container_root)?;

    let mut container = Container::load(container_root)?;
    container.resume()?;

    Ok(ExitCode::SUCCESS)
}

fn cmd_events(opt: GlobalOpts, args: Events) -> SydResult<ExitCode> {
    #[allow(clippy::disallowed_methods)]
    let container_root = opt.root.unwrap().join(args.container_id.clone());
    path_exists(&container_root)?;

    let mut container = Container::load(container_root)?;
    container.events(args.interval, args.stats)?;

    Ok(ExitCode::SUCCESS)
}

fn cmd_update(opt: GlobalOpts, args: Update) -> SydResult<ExitCode> {
    #[allow(clippy::disallowed_methods)]
    let container_root = opt.root.unwrap().join(args.container_id.clone());
    path_exists(&container_root)?;

    let container = Container::load(container_root)?;
    let cmanager = libcgroups::common::create_cgroup_manager(libcgroups::common::CgroupConfig {
        cgroup_path: container.spec()?.cgroup_path,
        systemd_cgroup: container.systemd(),
        container_name: container.id().to_string(),
    })?;

    let linux_res: LinuxResources;
    if let Some(resources_path) = args.resources {
        linux_res = if resources_path.to_string_lossy() == "-" {
            serde_json::from_reader(std::io::stdin())?
        } else {
            let file = fs::File::open(resources_path)?;
            let reader = BufReader::new(file);
            serde_json::from_reader(reader)?
        };
    } else {
        let mut builder = LinuxResourcesBuilder::default();
        if let Some(new_pids_limit) = args.pids_limit {
            builder = builder.pids(LinuxPidsBuilder::default().limit(new_pids_limit).build()?);
        }
        linux_res = builder.build()?;
    }

    cmanager.apply(&ControllerOpt {
        resources: &linux_res,
        disable_oom_killer: false,
        oom_score_adj: None,
        freezer_state: None,
    })?;

    Ok(ExitCode::SUCCESS)
}

fn cmd_checkpoint(opt: GlobalOpts, args: Checkpoint) -> SydResult<ExitCode> {
    #[allow(clippy::disallowed_methods)]
    let container_root = opt.root.unwrap().join(args.container_id.clone());
    path_exists(&container_root)?;

    let mut container = Container::load(container_root)?;
    let opts = libcontainer::container::CheckpointOptions {
        ext_unix_sk: args.ext_unix_sk,
        file_locks: args.file_locks,
        image_path: args.image_path,
        leave_running: args.leave_running,
        shell_job: args.shell_job,
        tcp_established: args.tcp_established,
        work_path: args.work_path,
    };
    container.checkpoint(&opts)?;

    Ok(ExitCode::SUCCESS)
}

fn cmd_exec(opt: GlobalOpts, args: Exec) -> SydResult<ExitCode> {
    let pid = {
        let cwd = args.cwd.clone();
        let env = args.env.clone().into_iter().collect();
        let detach = args.detach;
        let no_new_privs = args.no_new_privs;
        let command = args.command.clone();
        let process = args.process.clone();
        let pid = SydTenantContainerBuilder::new(opt, args)?
            .with_detach(detach)
            .with_cwd(cwd)
            .with_env(env)
            .with_process(process)
            .with_no_new_privs(no_new_privs)
            .with_container_args(command)
            .build()?;

        // See https://github.com/containers/youki/pull/1252 for a detailed explanation
        // basically, if there is any error in starting exec, the build above will return error
        // however, if the process does start, and detach is given, we do not wait for it
        // if not detached, then we wait for it using waitpid below
        if detach {
            return Ok(ExitCode::SUCCESS);
        }

        pid
    };

    match waitpid(pid, None)? {
        WaitStatus::Exited(_, status) => Ok(ExitCode::from(status as u8)),
        WaitStatus::Signaled(_, sig, _) => Ok(ExitCode::from(128 + (sig as i32 as u8))),
        _ => Ok(ExitCode::SUCCESS),
    }
}

fn cmd_run(opt: GlobalOpts, args: Run) -> SydResult<ExitCode> {
    let detach = args.detach;
    let systemd_cgroup = opt.systemd_cgroup;
    let mut container = SydInitContainerBuilder::try_from((opt, args))?
        .with_systemd(systemd_cgroup)
        .with_detach(detach)
        .build()?;

    container.start()?;

    if detach {
        return Ok(ExitCode::SUCCESS);
    }

    // Using `debug_assert` here rather than returning an error because this is
    // an invariant. The design when the code path arrives to this point, is that
    // the container state must have recorded the container init pid.
    debug_assert!(
        container.pid().is_some(),
        "expects a container init pid in the container state"
    );
    //FIXME: syd's nix != libcontainer's nix
    //let foreground_result = handle_foreground(container.pid().unwrap());
    #[allow(clippy::disallowed_methods)]
    let foreground_result = handle_foreground(Pid::from_raw(container.pid().unwrap().as_raw()));
    // execute the destruction action after the container finishes running
    container.delete(true)?;
    // return result
    Ok(foreground_result
        .map(|i| ExitCode::from(i as u8))
        .unwrap_or(ExitCode::FAILURE))
}

// Step 1: Add pidfd_getfd to seccomp allowlist.
// Step 2: Add CAP_SYS_PTRACE to Capabilities (but not for rootless containers).
fn syd_spec_load<P: AsRef<Path>>(config: P) -> Result<Spec, LibcontainerError> {
    let mut spec = Spec::load(&config)?;

    if let Some(linux) = spec.linux() {
        if let Some(seccomp) = linux.seccomp() {
            syd::t!(
                "Syd-OCI-Seccomp-Pre: {}",
                serde_json::to_string(&seccomp).unwrap_or("?".to_string())
            );

            let sydallowlist = LinuxSyscallBuilder::default()
                .action(LinuxSeccompAction::ScmpActAllow)
                .names(vec![
                    "pidfd_getfd".to_string(),
                    "process_vm_readv".to_string(),
                    "process_vm_writev".to_string(),
                    "ptrace".to_string(),
                ])
                .build()?;
            let syscalls = if let Some(syscalls) = seccomp.syscalls() {
                let mut syscalls = syscalls.clone();
                syscalls.push(sydallowlist);
                syscalls
            } else {
                vec![sydallowlist]
            };

            let mut builder = LinuxSeccompBuilder::default()
                .default_action(seccomp.default_action())
                .syscalls(syscalls);
            if let Some(default_errno_ret) = seccomp.default_errno_ret() {
                builder = builder.default_errno_ret(default_errno_ret)
            }
            if let Some(flags) = seccomp.flags() {
                builder = builder.flags(flags.clone());
            }
            if let Some(architectures) = seccomp.architectures() {
                builder = builder.architectures(architectures.clone());
            }
            if let Some(listener_path) = seccomp.listener_path() {
                builder = builder.listener_path(listener_path);
            }
            if let Some(listener_metadata) = seccomp.listener_metadata() {
                builder = builder.listener_metadata(listener_metadata);
            }

            let seccomp = builder.build()?;
            syd::t!(
                "Syd-OCI-Seccomp-Post: {}",
                serde_json::to_string(&seccomp).unwrap_or("?".to_string())
            );

            let mut linux = linux.clone();
            linux.set_seccomp(Some(seccomp));
            spec.set_linux(Some(linux));
        }
    }

    if let Some(process) = spec.process() {
        // FIXME: Podman does not like it when we add CAP_SYS_PTRACE to
        // the set and we do not know exactly why and detecting for it
        // seems to be not easily possible (e.g. if we check if CAP_SYS_PTRACE
        // is in permitted set here it'll return true regardless or if we check
        // the inheritable set CAP_SYS_PTRACE is not there for both Podman and
        // Docker).
        if let Some(env) = process.env() {
            if env.contains(&String::from("container=podman")) {
                return Ok(spec);
            }
        }

        let mut p = process.clone();
        if let Some(capabilities) = process.capabilities() {
            let bounding = if let Some(caps) = capabilities.bounding() {
                let mut caps = caps.clone();
                caps.insert(Capability::SysPtrace);
                caps
            } else {
                HashSet::from([Capability::SysPtrace])
            };
            let effective = if let Some(caps) = capabilities.effective() {
                let mut caps = caps.clone();
                caps.insert(Capability::SysPtrace);
                caps
            } else {
                HashSet::from([Capability::SysPtrace])
            };
            let permitted = if let Some(caps) = capabilities.permitted() {
                let mut caps = caps.clone();
                caps.insert(Capability::SysPtrace);
                caps
            } else {
                HashSet::from([Capability::SysPtrace])
            };
            // SAFETY: Inheritable and Ambient are not necessary.
            let caps = LinuxCapabilitiesBuilder::default()
                .bounding(bounding)
                .effective(effective)
                .permitted(permitted)
                .build()?;
            p.set_capabilities(Some(caps));
        } else {
            // SAFETY: Inheritable and Ambient are not necessary.
            let caps = LinuxCapabilitiesBuilder::default()
                .bounding(HashSet::from([Capability::SysPtrace]))
                .effective(HashSet::from([Capability::SysPtrace]))
                .permitted(HashSet::from([Capability::SysPtrace]))
                .build()?;
            p.set_capabilities(Some(caps));
        }
        spec.set_process(Some(p));
    }

    Ok(spec)
}

// handle_foreground will match the `runc` behavior running the foreground mode.
// The Syd main process will wait and reap the container init process. The
// Syd main process also forwards most of the signals to the container init
// process.
fn handle_foreground(init_pid: Pid) -> SydResult<i32> {
    syd::t!("waiting for container init process to exit");
    // We mask all signals here and forward most of the signals to the container
    // init process.
    let signal_set = SigSet::all();
    signal_set.thread_block()?;
    loop {
        match signal_set.wait()? {
            signal::SIGCHLD => {
                // Reap all child until either container init process exits or
                // no more child to be reaped. Once the container init process
                // exits we can then return.
                syd::t!("reaping child processes");
                loop {
                    match waitpid(None, Some(WaitPidFlag::WNOHANG))? {
                        WaitStatus::Exited(pid, status) => {
                            if pid.eq(&init_pid) {
                                return Ok(status);
                            }

                            // Else, some random child process exited, ignoring...
                        }
                        WaitStatus::Signaled(pid, signal, _) => {
                            if pid.eq(&init_pid) {
                                return Ok(signal as i32);
                            }

                            // Else, some random child process exited, ignoring...
                        }
                        WaitStatus::StillAlive => {
                            // No more child to reap.
                            break;
                        }
                        _ => {}
                    }
                }
            }
            signal::SIGURG => {
                // In `runc`, SIGURG is used by go runtime and should not be forwarded to
                // the container process. Here, we just ignore the signal.
            }
            signal::SIGWINCH => {
                // TODO: resize the terminal
            }
            signal => {
                syd::t!("forwarding signal {}", signal as i32);
                // There is nothing we can do if we fail to forward the signal.
                #[allow(clippy::disallowed_methods)]
                let _ = kill(init_pid, Some(signal)).map_err(|_err| {
                    syd::t!("failed to forward signal to container init process: {_err}")
                });
            }
        }
    }
}

fn get_rootless_spec() -> SydResult<Spec> {
    // Remove network and user namespace from the default spec
    let mut namespaces: Vec<LinuxNamespace> =
        libcontainer::oci_spec::runtime::get_default_namespaces()
            .into_iter()
            .filter(|ns| {
                ns.typ() != LinuxNamespaceType::Network && ns.typ() != LinuxNamespaceType::User
            })
            .collect();

    // Add user namespace
    namespaces.push(
        LinuxNamespaceBuilder::default()
            .typ(LinuxNamespaceType::User)
            .build()?,
    );

    let uid = Uid::effective().as_raw();
    let gid = Gid::effective().as_raw();

    let linux = LinuxBuilder::default()
        .namespaces(namespaces)
        .uid_mappings(vec![LinuxIdMappingBuilder::default()
            .host_id(uid)
            .container_id(0_u32)
            .size(1_u32)
            .build()?])
        .gid_mappings(vec![LinuxIdMappingBuilder::default()
            .host_id(gid)
            .container_id(0_u32)
            .size(1_u32)
            .build()?])
        .build()?;

    // Prepare the mounts

    let mut mounts: Vec<Mount> = libcontainer::oci_spec::runtime::get_default_mounts();
    for mount in &mut mounts {
        if mount.destination().eq(Path::new("/sys")) {
            mount
                .set_source(Some(PathBuf::from("/sys")))
                .set_typ(Some(String::from("none")))
                .set_options(Some(vec![
                    "rbind".to_string(),
                    "nosuid".to_string(),
                    "noexec".to_string(),
                    "nodev".to_string(),
                    "ro".to_string(),
                ]));
        } else {
            let options: Vec<String> = mount
                .options()
                .as_ref()
                .unwrap_or(&vec![])
                .iter()
                .filter(|&o| !o.starts_with("gid=") && !o.starts_with("uid="))
                .map(|o| o.to_string())
                .collect();
            mount.set_options(Some(options));
        }
    }

    let mut spec = Spec::default();
    spec.set_linux(Some(linux)).set_mounts(Some(mounts));
    Ok(spec)
}

fn get_pid_index(title: &str) -> SydResult<usize> {
    let titles = title.split_whitespace();

    for (index, name) in titles.enumerate() {
        if name == "PID" {
            return Ok(index);
        }
    }
    Err(Errno::ENOENT.into())
}

fn make_root(opt: &mut GlobalOpts) -> SydResult<()> {
    let uid = Uid::current();

    // Determine root path.
    if opt.root.is_none() {
        let is_rootless_required = rootless_required()?;
        opt.root = Some(if !is_rootless_required {
            PathBuf::from("/run/syd")
        } else if let Ok(path) = env::var("XDG_RUNTIME_DIR") {
            PathBuf::from(format!("{path}/syd"))
        } else {
            PathBuf::from(format!("/run/user/{uid}/syd"))
        });
    };

    let path = match opt.root {
        Some(ref path) => path,
        _ => unreachable!(),
    };

    // Create root directory recursively.
    mkdir_p(path, Mode::S_IRWXU | Mode::S_ISVTX)?;

    // Canonicalize root directory.
    let path = path.canonicalize()?;

    // SAFETY: At this point we may or may not have created the root
    // directory and it may belong to someone else! Check directory
    // metadata to ensure this is not the case.
    assert_eq!(path_uid(&path)?, uid, "UID mismatch on root directory!");

    // All good, set the new root directory in global options.
    opt.root = Some(path);
    Ok(())
}

// Make directory recursively, return canonical path.
fn mkdir_p<P: AsRef<Path>>(dir: P, mode: Mode) -> SydResult<()> {
    Ok(DirBuilder::new()
        .recursive(true)
        .mode(mode.bits())
        .create(&dir)?)
}

// Get the owner of the given path.
fn path_uid<P: AsRef<Path>>(path: P) -> SydResult<Uid> {
    Ok(Uid::from_raw(fs::metadata(&path)?.st_uid()))
}

fn get_executable_path(name: &str, path_var: &str) -> Option<PathBuf> {
    // if path has / in it, we have to assume absolute path, as per runc impl
    if name.contains('/') && path_exists(Path::new(name)).is_ok() {
        return Some(PathBuf::from(name));
    }
    for path in path_var.split(':') {
        let potential_path = PathBuf::from(path).join(name);
        if path_exists(&potential_path).is_ok() {
            return Some(potential_path);
        }
    }
    None
}

fn is_executable(path: &Path) -> std::result::Result<bool, std::io::Error> {
    let metadata = path.metadata()?;
    let permissions = metadata.permissions();
    // we have to check if the path is file and the execute bit
    // is set. In case of directories, the execute bit is also set,
    // so have to check if this is a file or not
    Ok(metadata.is_file() && permissions.mode() & 0o001 != 0)
}

#[allow(clippy::disallowed_methods)]
fn set_name(name: &str) -> std::result::Result<(), Errno> {
    let name = CString::new(name).map_err(|_| Errno::EINVAL)?;

    // SAFETY: syd's nix version does not have prctl::set_name yet.
    let res = unsafe { nix::libc::prctl(nix::libc::PR_SET_NAME, name.as_ptr(), 0, 0, 0) };

    Errno::result(res).map(drop)
}

// If None, it will generate a default path for cgroups.
fn get_cgroup_path(cgroups_path: &Option<PathBuf>, container_id: &str) -> PathBuf {
    match cgroups_path {
        Some(cpath) => cpath.clone(),
        None => PathBuf::from(format!(":syd:{container_id}")),
    }
}
