// Copyright (C) 2026 Ordinary Labs, LLC.
//
// SPDX-License-Identifier: AGPL-3.0-only

mod ops;

use flate2::read::ZlibDecoder;
use futures_util::future::FutureExt;
use hashbrown::{HashMap, HashSet};
use ordinary_config::{
    CompressionAlgorithm, HttpCors, HttpCorsAllowHeaders, HttpCorsAllowMethods,
    HttpCorsAllowOrigin, HttpCorsExposeHeaders, OrdinaryApiLimits,
};
use rayon::iter::IndexedParallelIterator;
use rayon::iter::IntoParallelRefIterator;
use rayon::iter::ParallelIterator;
use std::io::Write;

use axum::Extension;
use axum::Router;
use axum::extract::{ConnectInfo, State};
use axum::http::{HeaderName, Request, StatusCode};
use axum::http::{HeaderValue, Uri};
use axum::response::Response;
use axum::response::{IntoResponse, Redirect};
use axum::routing::{MethodRouter, delete, get, post, put};
use std::collections::BTreeMap;
use std::io::Read;
use std::net::SocketAddr;
use std::path::Path;
use std::process;
use std::str::FromStr;
use std::sync::Arc;
use std::time::Duration;
use tokio::net::TcpStream;
use tokio_rustls::LazyConfigAcceptor;
use tokio_rustls::StartHandshake;
use tokio_rustls::server::TlsStream;
use tower::Layer;

use bytes::Bytes;
use hyper::body::Incoming;
use hyper_util::rt::{TokioExecutor, TokioIo};
use ops::assets::get_asset_to_res;
use ordinary_auth::Auth;
use ordinary_integration::Integration;

use axum::http::header::CONTENT_TYPE;
use ordinary_storage::{ArtifactKind, CacheDependency, Storage};
use tokio::net::TcpListener;

use ordinary_action::{Action, Engine, PrivilegedComponents};
use ordinary_config::{ActionTrigger, HttpMethod, OrdinaryConfig};
use ordinary_template::Template;
use ordinary_types::ContentObject;
use serde::{Deserialize, Serialize};

use tower::ServiceBuilder;
use tower_http::classify::ServerErrorsFailureClass;
use tower_http::compression::CompressionLayer;
use tower_http::decompression::RequestDecompressionLayer;
use tower_http::request_id::{MakeRequestUuid, PropagateRequestIdLayer, SetRequestIdLayer};
use tower_http::trace::TraceLayer;
use tower_service::Service;
use tracing::Span;
use uuid::Uuid;

use rustls_acme::caches::DirCache;
use rustls_acme::{AcmeConfig, is_tls_alpn_challenge};
use tokio::io::AsyncWriteExt;
use tokio_rustls::rustls::crypto::ring;

use anyhow::bail;
use brotli::enc::BrotliEncoderParams;
use flate2::Compression;
use flate2::write::{GzEncoder, ZlibEncoder};
use hickory_client::client::{Client, ClientHandle};
use hickory_client::proto::rr::{DNSClass, Name, Record, RecordType};
use hyper::{HeaderMap, Method, header};
use mime::Mime;
use ordinary_utils::json::JsonValuable;
use ordinary_utils::middleware::{get_etag_hash, modify_etag_for_encoding};
use ordinary_utils::{
    GMT_FORMAT, HeadersDebug, LatencyDisplay, REQUEST_ID_HEADER, WrappedRedactedHashingAlg,
    get_host, redirect_service, response_for_panic, rustls_server_config,
};
use std::fmt::Debug;
use std::pin::pin;
use time::UtcDateTime;
use tokio::sync::watch;
use tokio::sync::watch::{Receiver, Sender};
use tokio::time::timeout;
use tokio_rustls::rustls::server::Acceptor;
use tokio_rustls::{TlsAcceptor, rustls::ServerConfig};
use tower_http::catch_panic::CatchPanicLayer;
use tower_http::cors::{Any, CorsLayer};
use tower_http::set_header::SetResponseHeaderLayer;
use tower_http::timeout::TimeoutLayer;

#[derive(Serialize, Deserialize, Debug)]
struct ClientEvent {
    ts: String,
    lvl: String,
    version: String,
    correlation: String,
    path: String,
    #[serde(skip_serializing_if = "Option::is_none")]
    #[serde(default)]
    query: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    #[serde(default)]
    fragment: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    #[serde(default)]
    flags: Option<BTreeMap<String, String>>,
    #[serde(skip_serializing_if = "Option::is_none")]
    #[serde(default)]
    fields: Option<BTreeMap<String, String>>,
    #[serde(skip_serializing)]
    msg: String,
}

#[derive(PartialEq)]
pub enum ProvisionMode {
    Localhost,
    Staging,
    Production,
}

pub enum SecurityMode<T: AsRef<Path>> {
    Insecure,
    Secure(T, ProvisionMode),
}

pub struct OrdinaryAppServer {
    pub config: Arc<OrdinaryConfig>,
    state: Arc<OrdinaryAppServerState>,
    limits: OrdinaryApiLimits,
    pub service: Router,
    pub log_ips: bool,
    pub log_headers: bool,
}

#[derive(Clone)]
struct OrdinaryAppServerState {
    schema: Option<String>,
    config: Arc<OrdinaryConfig>,

    auth: Arc<Auth>,
    storage: Arc<Storage>,
    templates: Vec<Template>,
    actions: Arc<Vec<Action>>,
    template_route_map: HashMap<String, usize>,
    action_route_map: HashMap<String, usize>,
    registration_actions: Vec<u8>,

    error_template_idx: Option<u8>,
    mfa_totp_template_idx: Option<u8>,

    secure_cookies: bool,
}

impl OrdinaryAppServer {
    pub async fn validate_cnames(
        config: &OrdinaryConfig,
        dns_client: &Arc<tokio::sync::Mutex<Client>>,
    ) -> anyhow::Result<()> {
        let domain = &config.domain;
        let mut dns_client = dns_client.lock().await;

        if let Some(cnames) = &config.cnames {
            for cname in cnames {
                let mut is_valid = false;

                let response = (*dns_client)
                    .query(Name::from_str(cname)?, DNSClass::IN, RecordType::CNAME)
                    .await?;

                let answers: &[Record] = response.answers();

                for record in answers {
                    if let Some(cname) = record.data().as_cname() {
                        let mut cname_string = cname.0.to_string();
                        cname_string.pop();

                        if &cname_string == domain {
                            is_valid = true;
                            break;
                        }
                    }
                }

                if is_valid {
                    tracing::info!(%cname, "valid");
                } else {
                    tracing::error!(%cname, "invalid");
                    bail!("invalid CNAME");
                }
            }
        }

        drop(dns_client);
        Ok(())
    }

    #[allow(
        clippy::too_many_arguments,
        clippy::too_many_lines,
        clippy::fn_params_excessive_bools,
        clippy::needless_pass_by_value
    )]
    pub fn new(
        mut config: OrdinaryConfig,
        limits: OrdinaryApiLimits,
        auth: Arc<Auth>,
        storage: Arc<Storage>,
        secure: bool,
        secure_cookies: bool,
        log_headers: bool,
        log_ips: bool,
        server_span: Option<Span>,
        app_span: Option<Span>,
        privileged_components: Option<PrivilegedComponents>,
    ) -> anyhow::Result<OrdinaryAppServer> {
        let app_span = if let Some(app_span) = app_span {
            app_span
        } else {
            tracing::info_span!("app", domain = %config.domain)
        };

        app_span.in_scope(|| {
            if let Some(assets) = &mut config.assets {
                // todo: pass in default assets cache control from Ordinary API
                assets.init("")?;
            }

            let mut registration_actions = vec![];

            let schema = if config.hide_schema == Some(true) {
                None
            } else {
                Some(serde_json::to_string(&config)?)
            };

            let mut secrets = BTreeMap::new();

            if let Some(secret_configs) = &config.secrets {
                for secret in secret_configs {
                    secrets.insert(secret.name.clone(), secret);
                }
            }

            let mut integrations = Vec::with_capacity(match &config.integrations {
                Some(i) => i.len(),
                None => 0,
            });

            if let Some(mut integration_configs) = config.integrations.clone() {
                integration_configs.sort_by(|a, b| a.idx.cmp(&b.idx));

                for (i, config) in integration_configs.iter().enumerate() {
                    if i != config.idx as usize {
                        tracing::error!("gap in or duplicate integration indexes");
                    }

                    if let Ok(integration) =
                        Integration::new(config.clone(), storage.clone(), &secrets)
                    {
                        integrations.push(integration);
                    }
                }
            }

            let integrations = Arc::new(integrations);

            let engine = Engine::default();

            let action_ct = match &config.actions {
                Some(i) => i.len(),
                None => 0,
            };
            let template_ct = match &config.templates {
                Some(i) => i.len(),
                None => 0,
            };

            let mut actions = Vec::with_capacity(action_ct);
            let mut action_route_map = HashMap::with_capacity(action_ct);

            let mut templates = Vec::with_capacity(template_ct);
            let mut template_route_map = HashMap::with_capacity(template_ct);

            let model_ct = match &config.models {
                Some(i) => i.len(),
                None => 0,
            };
            let content_ct = match &config.content {
                Some(i) => i.definitions.len(),
                None => 0,
            };

            let mut model_map = HashMap::with_capacity(model_ct);
            let mut content_map = HashMap::with_capacity(content_ct);

            if let Some(models) = config.models.clone() {
                for model in models {
                    model_map.insert(model.name.clone(), model);
                }
            }

            if let Some(content) = &config.content {
                for content_def in content.definitions.clone() {
                    content_map.insert(content_def.name.clone(), content_def);
                }
            }

            if let Some(mut action_configs) = config.actions.clone() {
                action_configs.sort_by(|a, b| a.idx.cmp(&b.idx));

                actions = action_configs
                    .par_iter()
                    .enumerate()
                    .filter_map(|(i, action_config)| {
                        if i != action_config.idx as usize {
                            tracing::error!("gap in or duplicate action indexes");
                        }

                        let artifact_span = tracing::info_span!("artifact");

                        let src = artifact_span.in_scope(|| {
                            match storage
                                .artifact
                                .get(action_config.idx, ArtifactKind::Action)
                            {
                                Ok(val) => Some(val),
                                Err(err) => {
                                    tracing::warn!(
                                        "action '{}' not stored - {err}",
                                        action_config.name
                                    );
                                    None
                                }
                            }
                        });

                        Action::new(
                            src,
                            engine.clone(),
                            action_config.clone(),
                            auth.clone(),
                            storage.clone(),
                            integrations.clone(),
                            &model_map,
                            &content_map,
                            &config.actions,
                            privileged_components.clone(),
                        )
                        .ok()
                    })
                    .collect::<Vec<Action>>();

                for action in &actions {
                    for trigger in &action.config.triggered_by {
                        match trigger {
                            ActionTrigger::Registration => {
                                registration_actions.push(action.idx);
                            }
                            ActionTrigger::Json { route, method: _ }
                            | ActionTrigger::Form {
                                route,
                                method: _,
                                redirect: _,
                            } => {
                                action_route_map.insert(route.clone(), action.idx as usize);
                            }
                            _ => {}
                        }
                    }
                }
            }

            registration_actions.shrink_to_fit();

            let mut error_template_idx = None;
            let mut mfa_totp_template_idx = None;

            if let Some(mut template_configs) = config.templates.clone() {
                template_configs.sort_by(|a, b| a.idx.cmp(&b.idx));

                templates = template_configs
                    .par_iter()
                    .enumerate()
                    .filter_map(|(i, template_config)| {
                        if i != template_config.idx as usize {
                            tracing::error!("gap in or duplicate template indexes");
                        }

                        let artifact_span = tracing::info_span!("artifact");

                        let src = artifact_span.in_scope(|| {
                            match storage
                                .artifact
                                .get(template_config.idx, ArtifactKind::Template)
                            {
                                Ok(val) => Some(val),
                                Err(err) => {
                                    tracing::warn!(
                                        name = template_config.name,
                                        %err,
                                        "template not stored",
                                    );
                                    None
                                }
                            }
                        });

                        Template::new(
                            &config.domain,
                            secure,
                            src,
                            template_config.clone(),
                            auth.clone(),
                            &model_map,
                            &content_map,
                            engine.clone(),
                            storage.clone(),
                        )
                        .ok()
                    })
                    .collect::<Vec<Template>>();

                for template in &templates {
                    if let Some(auth) = &config.auth
                        && let Some(totp_template) = &auth.mfa.totp.template
                        && &template.config.name == totp_template
                    {
                        mfa_totp_template_idx = Some(template.idx);
                    }

                    if let Some(error) = &config.error
                        && template.config.name == error.template
                    {
                        error_template_idx = Some(template.idx);
                    }

                    template_route_map.insert(template.config.route.clone(), template.idx as usize);

                    template.start_tasks();
                }
            }

            let redacted_hash = if let Some(logging) = &config.logging
                && let Some(server) = &logging.server
                && let Some(credentials) = &server.credentials
            {
                Arc::new(Some(WrappedRedactedHashingAlg(credentials.clone())))
            } else {
                Arc::new(None)
            };

            let redacted_hash_clone = redacted_hash.clone();

            let config = Arc::new(config);

            let state = Arc::new(OrdinaryAppServerState {
                config: config.clone(),
                schema,
                auth,
                storage,
                templates,
                template_route_map,
                action_route_map,
                registration_actions,
                actions: Arc::new(actions),
                error_template_idx,
                mfa_totp_template_idx,
                secure_cookies,
            });

            let request_id_header = HeaderName::from_static(REQUEST_ID_HEADER);

            let mut router = Router::new();

            router =
                router
                    .route(
                        "/favicon.ico",
                        get(
                            |State(state): State<Arc<OrdinaryAppServerState>>,
                             headers: HeaderMap| async {
                                get_asset_to_res(state, "favicon.ico".into(), headers)
                            },
                        ),
                    )
                    // display the config of the service
                    .route(
                        "/events",
                        post(|body: Bytes| async move {
                            let mut deflater = ZlibDecoder::new(body.as_ref());
                            let mut json = String::new();

                            if let Err(err) = deflater.read_to_string(&mut json) {
                                tracing::error!(%err);
                                return StatusCode::INTERNAL_SERVER_ERROR;
                            }

                            let events: Vec<ClientEvent> = match serde_json::from_str(&json) {
                                Ok(parsed) => parsed,
                                Err(err) => {
                                    tracing::error!(%err);
                                    return StatusCode::INTERNAL_SERVER_ERROR;
                                }
                            };

                            let client_span = tracing::info_span!("client");

                            client_span.in_scope(|| {
                                for event in events {
                                    #[cfg(tracing_unstable)]
                                    {
                                        let lvl = event.lvl.as_str();
                                        let msg = event.msg.as_str();

                                        if let Ok(value) = serde_json::to_value(&event) {
                                            let event = JsonValuable(value);

                                            match lvl {
                                                "trace" => tracing::trace!(
                                                    event = tracing::field::valuable(&event),
                                                    "{msg}"
                                                ),
                                                "debug" => tracing::debug!(
                                                    event = tracing::field::valuable(&event),
                                                    "{msg}"
                                                ),
                                                "info" => {
                                                    tracing::info!(
                                                        event = tracing::field::valuable(&event),
                                                        "{msg}"
                                                    );
                                                }
                                                "warn" => {
                                                    tracing::warn!(
                                                        event = tracing::field::valuable(&event),
                                                        "{msg}"
                                                    );
                                                }
                                                _ => tracing::error!(
                                                    event = tracing::field::valuable(&event),
                                                    "{msg}"
                                                ),
                                            }
                                        }
                                    }

                                    #[cfg(not(tracing_unstable))]
                                    {
                                        match event.lvl.as_ref() {
                                            "trace" => tracing::trace!(event = debug(&event)),
                                            "debug" => tracing::debug!(event = debug(&event)),
                                            "info" => tracing::info!(event = debug(&event)),
                                            "warn" => tracing::warn!(event = debug(&event)),
                                            _ => tracing::error!(event = debug(&event)),
                                        }
                                    }
                                }
                            });

                            StatusCode::OK
                        }),
                    )
                    .route(
                        "/reports/csp",
                        post(|body: String| async move {
                            if let Ok(report) = serde_json::from_str::<serde_json::Value>(&body) {
                                let client_span = tracing::info_span!("client");

                                client_span.in_scope(|| {
                                    #[cfg(tracing_unstable)]
                                    tracing::error!(
                                        report.csp =
                                            tracing::field::valuable(&JsonValuable(report))
                                    );

                                    #[cfg(not(tracing_unstable))]
                                    tracing::error!(report.csp = tracing::field::debug(&report));
                                });

                                StatusCode::OK
                            } else {
                                tracing::error!("failed to parse report");
                                StatusCode::UNSUPPORTED_MEDIA_TYPE
                            }
                        }),
                    );

            if state.config.hide_schema != Some(true) {
                router = router.route(
                    "/.ordinary/schema",
                    get(
                        |State(state): State<Arc<OrdinaryAppServerState>>| async move {
                            if let Some(schema) = &state.schema {
                                (
                                    StatusCode::OK,
                                    [("content-type", "application/json")],
                                    schema.to_owned(),
                                )
                                    .into_response()
                            } else {
                                StatusCode::NOT_FOUND.into_response()
                            }
                        },
                    ),
                );
            }

            if state.config.auth.is_some() {
                router = router // token
                    .route("/accounts/access/get", put(ops::token::fetch_access))
                    // auth
                    .route("/accounts/registration/start", put(ops::registration::start))
                    .route("/accounts/registration/finish", put(ops::registration::finish))
                    .route("/accounts/login/start", put(ops::login::start))
                    .route("/accounts/login/finish", put(ops::login::finish))
                    // for javascript-only
                    .route("/accounts/register", put(ops::registration::hash_only))
                    .route("/accounts/login", put(ops::login::hash_only))
                    // reset password
                    .route(
                        "/accounts/password/reset/login/start",
                        put(ops::password::reset_login_start),
                    )
                    .route(
                        "/accounts/password/reset/login/finish",
                        put(ops::password::reset_login_finish),
                    )
                    .route(
                        "/accounts/password/reset/registration/start",
                        put(ops::password::reset_registration_start),
                    )
                    .route(
                        "/accounts/password/reset/registration/finish",
                        put(ops::password::reset_registration_finish),
                    )
                    // forgot password
                    .route("/accounts/password/forgot/start", put(ops::password::forgot_start))
                    .route("/accounts/password/forgot/finish", put(ops::password::forgot_finish))
                    // reset totp mfa
                    .route("/accounts/mfa/totp/reset/start", put(ops::mfa::reset_totp_start))
                    .route("/accounts/mfa/totp/reset/finish", put(ops::mfa::reset_totp_finish))
                    // lost totp mfa
                    .route("/accounts/mfa/totp/lost/start", put(ops::mfa::lost_totp_start))
                    .route("/accounts/mfa/totp/lost/finish", put(ops::mfa::lost_totp_finish))
                    // delete account
                    .route("/accounts/delete/start", put(ops::account::delete_start))
                    .route("/accounts/delete/finish", put(ops::account::delete_finish))
                    // reset recovery codes
                    .route(
                        "/accounts/recovery-codes/reset/start",
                        put(ops::recovery::reset_codes_start),
                    )
                    .route(
                        "/accounts/recovery-codes/reset/finish",
                        put(ops::recovery::reset_codes_finish),
                    );

                if state.auth.config.cookies_enabled {
                    router = router
                        .route("/accounts/register", post(ops::registration::form))
                        .route("/accounts/login", post(ops::login::form))
                        .route("/accounts/access", get(ops::token::redirect_access))
                        .route(
                            "/accounts/access/get",
                            post(ops::token::fetch_access_cookies),
                        );
                }
            }

            if let Some(action_configs) = &state.config.actions {
                if !action_configs.is_empty() {
                    // todo: handle timeouts
                    router = router.route("/actions/invoke/{idx}", put(ops::actions::invoke));
                }

                for config in action_configs {
                    // todo: default for default should come from Ordinary API
                    let timeout_s = u64::from(
                        config
                            .timeout
                            .unwrap_or(state.config.default_timeout.unwrap_or(10)),
                    );
                    let timeout_layer = TimeoutLayer::with_status_code(
                        StatusCode::REQUEST_TIMEOUT,
                        Duration::from_secs(timeout_s),
                    );

                    for trigger in &config.triggered_by {
                        let (route, mut mr) = match trigger {
                            ActionTrigger::Form {
                                route,
                                method,
                                redirect: _,
                            } => match method {
                                HttpMethod::GET => (route, get(ops::actions::form)),
                                HttpMethod::PUT => (route, put(ops::actions::form)),
                                HttpMethod::POST => (route, post(ops::actions::form)),
                                HttpMethod::DELETE => (route, delete(ops::actions::form)),
                            },
                            ActionTrigger::Json { route, method } => match method {
                                HttpMethod::GET => (route, get(ops::actions::json)),
                                HttpMethod::PUT => (route, put(ops::actions::json)),
                                HttpMethod::POST => (route, post(ops::actions::json)),
                                HttpMethod::DELETE => (route, delete(ops::actions::json)),
                            },
                            _ => {
                                continue;
                            }
                        };

                        mr = mr.route_layer(timeout_layer);
                        mr = Self::apply_cors_to_route(&state.config.cors, &config.cors, mr);

                        router = router.route(route, mr);
                    }
                }
            }

            if let Some(template_configs) = &state.config.templates {
                if !template_configs.is_empty() {
                    // todo: handle timeouts
                    router =
                        router.route("/templates/query/{idx}/{*path}", get(ops::templates::query));
                }

                for config in template_configs {
                    // todo: default for default should come from Ordinary API
                    let timeout_s = u64::from(
                        config
                            .timeout
                            .unwrap_or(state.config.default_timeout.unwrap_or(10)),
                    );

                    let mut route =
                        get(ops::templates::get).route_layer(TimeoutLayer::with_status_code(
                            StatusCode::REQUEST_TIMEOUT,
                            Duration::from_secs(timeout_s),
                        ));

                    route = Self::apply_cors_to_route(&state.config.cors, &config.cors, route);

                    router = router.route(&config.route, route);
                }
            }

            // todo: handle timeouts
            router = router
                .route("/assets/{*path}", get(ops::assets::get))
                .fallback(
                    |State(state): State<Arc<OrdinaryAppServerState>>,
                     uri: Uri,
                     headers: HeaderMap| async move {
                        if let Some(idx) = state.error_template_idx
                            && let Some(err_template) = &state.templates.get(idx as usize)
                        {
                            // todo: base this on accept header

                            let Some(host) = get_host(&headers, &uri) else {
                                tracing::error!("no host");
                                return StatusCode::BAD_REQUEST.into_response();
                            };

                            match err_template.render(
                                host.as_str(),
                                "/404".into(),
                                None,
                                Some(("Not found".into(), 404)),
                                None,
                                &None,
                            ) {
                                Ok(res) => {
                                    return (
                                        StatusCode::NOT_FOUND,
                                        [(CONTENT_TYPE, err_template.config.mime.clone())],
                                        res,
                                    )
                                        .into_response();
                                }
                                Err(err) => tracing::warn!("{err}"),
                            }
                        }

                        StatusCode::NOT_FOUND.into_response()
                    },
                );

            let domain = Arc::new(config.domain.clone());

            let mut service = router
                .with_state(state.clone())
                .layer(
                    ServiceBuilder::new()
                        .layer(CatchPanicLayer::custom(response_for_panic))
                        .layer(RequestDecompressionLayer::new())
                        .layer(CompressionLayer::new()),
                )
                .layer(
                    ServiceBuilder::new()
                        .layer(SetRequestIdLayer::new(
                            request_id_header.clone(),
                            MakeRequestUuid,
                        ))
                        .layer(
                            TraceLayer::new_for_http()
                                .make_span_with(move |req: &Request<_>| {
                                    let request_id = req.headers().get(REQUEST_ID_HEADER);
                                    let host = get_host(req.headers(), req.uri())
                                        .map(tracing::field::display);

                                    let ip = log_ips.then(|| {
                                        req.extensions()
                                            .get::<ConnectInfo<SocketAddr>>()
                                            .map(|addr| tracing::field::display(addr.ip()))
                                    });

                                    let query = req.uri().query().map(tracing::field::display);

                                    if let Some(server_span) = &server_span {
                                        server_span.in_scope(|| match request_id {
                                            Some(rid) => {
                                                tracing::info_span!(
                                                    "app",
                                                    %domain,
                                                    host,
                                                    id = %rid.to_str().unwrap_or(
                                                        Uuid::new_v4().to_string().as_str()
                                                    ),
                                                    ip,
                                                    path = %req.uri().path(),
                                                    query,
                                                )
                                            }
                                            None => {
                                                tracing::info_span!(
                                                    "app",
                                                    %domain,
                                                    host,
                                                    id = %Uuid::new_v4(),
                                                    ip,
                                                    path = %req.uri().path(),
                                                    query,
                                                )
                                            }
                                        })
                                    } else {
                                        match request_id {
                                            Some(rid) => {
                                                tracing::info_span!(
                                                    "app",
                                                    %domain,
                                                    host,
                                                    id = %rid.to_str().unwrap_or(
                                                        Uuid::new_v4().to_string().as_str()
                                                    ),
                                                    ip,
                                                    path = req.uri().path(),
                                                    query,
                                                )
                                            }
                                            None => {
                                                tracing::info_span!(
                                                    "app",
                                                    %domain,
                                                    host,
                                                    id = %Uuid::new_v4(),
                                                    ip,
                                                    path = %req.uri().path(),
                                                    query,
                                                )
                                            }
                                        }
                                    }
                                })
                                .on_request(move |req: &Request<_>, _: &Span| {
                                    let hd = log_headers.then_some(HeadersDebug(
                                        req.headers(),
                                        redacted_hash.clone(),
                                    ));

                                    #[cfg(tracing_unstable)]
                                    let headers =
                                        log_headers.then_some(tracing::field::valuable(&hd));

                                    #[cfg(not(tracing_unstable))]
                                    let headers = log_headers.then_some(tracing::field::debug(&hd));

                                    tracing::info!(
                                        version = ?req.version(),
                                        method = %req.method(),
                                        headers,
                                        "req"
                                    );
                                })
                                .on_response(
                                    move |res: &Response<_>, latency: Duration, _: &Span| {
                                        let hd = log_headers.then_some(HeadersDebug(
                                            res.headers(),
                                            redacted_hash_clone.clone(),
                                        ));

                                        #[cfg(tracing_unstable)]
                                        let headers =
                                            log_headers.then_some(tracing::field::valuable(&hd));

                                        #[cfg(not(tracing_unstable))]
                                        let headers =
                                            log_headers.then_some(tracing::field::debug(&hd));

                                        let status = res.status().as_u16();
                                        let latency = LatencyDisplay(latency.as_nanos() as f64);

                                        if status >= 500 {
                                            tracing::error!(status, headers, %latency, "res");
                                        } else if status >= 400 {
                                            tracing::warn!(status, headers, %latency, "res");
                                        } else {
                                            tracing::info!(status, headers, %latency, "res");
                                        }
                                    },
                                )
                                .on_failure(
                                    |error: ServerErrorsFailureClass, _: Duration, _: &Span| {
                                        tracing::error!(
                                            err = %error,
                                            "fail"
                                        );
                                    },
                                ),
                        )
                        .layer(SetResponseHeaderLayer::overriding(
                            header::ETAG,
                            modify_etag_for_encoding,
                        ))
                        .layer(PropagateRequestIdLayer::new(request_id_header))
                        .layer(SetResponseHeaderLayer::if_not_present(
                            header::SERVER,
                            HeaderValue::from_static("Ordinary"),
                        )),
                );

            if secure {
                service = service.layer(SetResponseHeaderLayer::if_not_present(
                    header::STRICT_TRANSPORT_SECURITY,
                    HeaderValue::from_static("max-age=31536000"),
                ));
            }

            Ok(OrdinaryAppServer {
                config,
                state,
                limits,
                service,
                log_ips,
                log_headers,
            })
        })
    }

    #[allow(clippy::ref_option)]
    fn apply_cors_to_route(
        base_cors: &Option<HttpCors>,
        route_cors: &Option<HttpCors>,
        route: MethodRouter<Arc<OrdinaryAppServerState>>,
    ) -> MethodRouter<Arc<OrdinaryAppServerState>> {
        if let Some(cors_config) = &route_cors {
            let cors_config = if let Some(base_cors) = &base_cors {
                cors_config.overwrite(base_cors)
            } else {
                cors_config.clone()
            };

            let mut cors_layer = CorsLayer::new();

            if let Some(allow_credentials) = cors_config.allow_credentials {
                cors_layer = cors_layer.allow_credentials(allow_credentials);
            }

            if let Some(allow_headers) = cors_config.allow_headers {
                match allow_headers {
                    HttpCorsAllowHeaders::Any => {
                        cors_layer = cors_layer.allow_headers(Any);
                    }
                    HttpCorsAllowHeaders::Headers(headers) => {
                        cors_layer = cors_layer.allow_headers(
                            headers
                                .iter()
                                .filter_map(|v| HeaderName::from_str(v.as_str()).ok())
                                .collect::<Vec<_>>(),
                        );
                    }
                }
            }

            if let Some(max_age) = cors_config.max_age {
                cors_layer = cors_layer.max_age(Duration::from_secs(u64::from(max_age)));
            }

            if let Some(allow_methods) = cors_config.allow_methods {
                match allow_methods {
                    HttpCorsAllowMethods::Any => {
                        cors_layer = cors_layer.allow_methods(Any);
                    }
                    HttpCorsAllowMethods::Methods(methods) => {
                        cors_layer = cors_layer.allow_methods(
                            methods
                                .iter()
                                .filter_map(|v| Method::from_str(v.as_str()).ok())
                                .collect::<Vec<_>>(),
                        );
                    }
                }
            }

            if let Some(allow_origin) = cors_config.allow_origin {
                match allow_origin {
                    HttpCorsAllowOrigin::Any => {
                        cors_layer = cors_layer.allow_origin(Any);
                    }
                    HttpCorsAllowOrigin::Origins(origins) => {
                        cors_layer = cors_layer.allow_origin(
                            origins
                                .iter()
                                .filter_map(|v| HeaderValue::from_str(v.as_str()).ok())
                                .collect::<Vec<_>>(),
                        );
                    }
                }
            }

            if let Some(expose_headers) = cors_config.expose_headers {
                match expose_headers {
                    HttpCorsExposeHeaders::Any => {
                        cors_layer = cors_layer.expose_headers(Any);
                    }
                    HttpCorsExposeHeaders::Headers(headers) => {
                        cors_layer = cors_layer.expose_headers(
                            headers
                                .iter()
                                .filter_map(|v| HeaderName::from_str(v.as_str()).ok())
                                .collect::<Vec<_>>(),
                        );
                    }
                }
            }

            if let Some(allow_private_network) = cors_config.allow_private_network {
                cors_layer = cors_layer.allow_private_network(allow_private_network);
            }

            return route.route_layer(cors_layer);
        }

        route
    }

    pub fn get_invite(
        &self,
        api_domain: &str,
        api_account: &str,
        custom_claims: Option<Bytes>,
    ) -> anyhow::Result<Bytes> {
        self.state
            .auth
            .api_invite_get(api_domain, api_account, custom_claims)
    }

    pub fn list_accounts(&self) -> anyhow::Result<Bytes> {
        self.state.auth.list_accounts()
    }

    pub fn list_items(&self, model_idx: u8, cursor: Option<&[u8]>) -> anyhow::Result<Bytes> {
        self.state.storage.model.list_model(model_idx, cursor)
    }

    pub fn update_content(&self, objects: &Vec<ContentObject>) -> anyhow::Result<()> {
        let storage_span = tracing::info_span!("storage");
        let span = storage_span.in_scope(|| tracing::info_span!("content"));

        span.in_scope(|| self.state.storage.content.update(objects))?;

        let span = storage_span.in_scope(|| tracing::info_span!("cache"));
        span.in_scope(|| {
            self.state
                .storage
                .cache
                .dependency_evict(vec![CacheDependency::Content])
        })?;

        Ok(())
    }

    pub fn put_secret(&self, name: &str, secret: &[u8]) -> anyhow::Result<()> {
        let storage_span = tracing::info_span!("storage");
        let span = storage_span.in_scope(|| tracing::info_span!("secrets"));

        span.in_scope(|| self.state.storage.secrets.put(name, secret))?;

        Ok(())
    }

    pub fn put_asset(&self, path: &str, content: &[u8]) -> anyhow::Result<()> {
        let storage_span = tracing::info_span!("storage");
        let span = storage_span.in_scope(|| tracing::info_span!("asset"));

        span.in_scope(|| {
            if let Some(assets_config) = &self.config.assets {
                let mut no_compress = false;

                let mime = if let Some(ext) = Path::new(path).extension() {
                    let ext_str = ext.to_str();

                    if !self
                        .limits
                        .storage
                        .assets
                        .allowed_extensions
                        .contains(&ext_str.unwrap_or_default().to_owned())
                    {
                        bail!("extension is not allowed");
                    }

                    match ext_str {
                        Some("txt") => mime::TEXT_PLAIN_UTF_8,
                        Some("xml") => mime::TEXT_XML,
                        Some("html") => mime::TEXT_HTML_UTF_8,
                        Some("css") => mime::TEXT_CSS_UTF_8,
                        Some("csv") => mime::TEXT_CSV_UTF_8,
                        Some("js") => mime::TEXT_JAVASCRIPT,
                        Some("png") => {
                            no_compress = true;
                            mime::IMAGE_PNG
                        }
                        Some("apng") => {
                            no_compress = true;
                            Mime::from_str("image/apng")?
                        }
                        Some("gif") => {
                            no_compress = true;
                            mime::IMAGE_GIF
                        }
                        Some("svg") => mime::IMAGE_SVG,
                        Some("jpg" | "jpeg") => {
                            no_compress = true;
                            mime::IMAGE_JPEG
                        }
                        Some("bmp") => {
                            no_compress = true;
                            mime::IMAGE_BMP
                        }
                        Some("tif" | "tiff") => {
                            no_compress = true;
                            Mime::from_str("image/tiff")?
                        }
                        Some("webp") => {
                            no_compress = true;
                            Mime::from_str("image/webp")?
                        }
                        Some("avif") => {
                            no_compress = true;
                            Mime::from_str("image/avif")?
                        }
                        Some("ico") => {
                            no_compress = true;
                            Mime::from_str("image/vnd.microsoft.icon")?
                        }
                        Some("pdf") => {
                            no_compress = true; // todo: make configurable — lots of pdfs leverage internal compression
                            mime::APPLICATION_PDF
                        }
                        Some("json") => mime::APPLICATION_JSON,
                        Some("wasm") => Mime::from_str("application/wasm")?,
                        _ => mime::APPLICATION_OCTET_STREAM,
                    }
                } else {
                    mime::APPLICATION_OCTET_STREAM
                };

                if !no_compress && let Some(precompression) = &assets_config.precompression {
                    for alg in precompression {
                        self.put_asset_with_compression(path, content, Some(alg), &mime)?;
                    }
                }

                self.put_asset_with_compression(path, content, None, &mime)?;

                Ok(())
            } else {
                bail!("no assets configured for this project")
            }
        })
    }

    // todo: compress on client
    #[allow(clippy::too_many_lines)]
    fn put_asset_with_compression(
        &self,
        path: &str,
        content: &[u8],
        compression: Option<&CompressionAlgorithm>,
        mime: &Mime,
    ) -> anyhow::Result<()> {
        let mut asset_builder = flexbuffers::Builder::new(&flexbuffers::BuilderOptions::SHARE_NONE);
        let mut asset_builder_vec = asset_builder.start_vector();

        asset_builder_vec.push(mime.as_ref());

        let mut etag = if let Some(config) = &self.config.assets
            && let Some(http_cache) = &config.http
        {
            get_etag_hash(content, Some(http_cache))
        } else {
            get_etag_hash(content, None)
        };

        if let Some(compression) = compression {
            etag.push(compression.as_char());

            match compression {
                CompressionAlgorithm::Gzip => {
                    let mut encoder = GzEncoder::new(Vec::new(), Compression::default());

                    match encoder.write_all(content) {
                        Ok(()) => match encoder.finish() {
                            Ok(compressed) => {
                                asset_builder_vec.push(flexbuffers::Blob(compressed.as_slice()));
                            }
                            Err(err) => tracing::error!(%err),
                        },
                        Err(err) => tracing::error!(%err),
                    }
                }
                CompressionAlgorithm::Zstd { level } => {
                    match zstd::stream::encode_all(std::io::Cursor::new(content), i32::from(*level))
                    {
                        Ok(compressed) => {
                            asset_builder_vec.push(flexbuffers::Blob(compressed.as_slice()));
                        }
                        Err(err) => tracing::error!(%err),
                    }
                }
                CompressionAlgorithm::Brotli => {
                    let mut compressed = Vec::new();

                    let params = BrotliEncoderParams::default();
                    match brotli::enc::BrotliCompress(
                        &mut content.to_vec().as_slice(),
                        &mut compressed,
                        &params,
                    ) {
                        Ok(_) => {
                            asset_builder_vec.push(flexbuffers::Blob(compressed.as_slice()));
                        }
                        Err(err) => tracing::error!(%err),
                    }
                }
                CompressionAlgorithm::Deflate => {
                    let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());

                    match encoder.write_all(content) {
                        Ok(()) => match encoder.finish() {
                            Ok(compressed) => {
                                asset_builder_vec.push(flexbuffers::Blob(compressed.as_slice()));
                            }
                            Err(err) => tracing::error!(%err),
                        },
                        Err(err) => tracing::error!(%err),
                    }
                }
            }
        } else {
            asset_builder_vec.push(flexbuffers::Blob(content));
        }

        // ETag
        asset_builder_vec.push(etag.as_str());

        // Last-Modified
        asset_builder_vec.push(UtcDateTime::now().format(&GMT_FORMAT)?.as_str());

        asset_builder_vec.end_vector();

        self.state
            .storage
            .asset
            .put(path, asset_builder.view(), compression)?;

        Ok(())
    }

    pub fn set_template(
        &self,
        pos: u8,
        bin: &[u8],
        csp_style: Option<Vec<String>>,
        csp_script: Option<Vec<String>>,
        secure: bool,
    ) -> anyhow::Result<String> {
        let span = tracing::info_span!(
            "template",
            nm = tracing::field::Empty,
            i = tracing::field::Empty
        );

        span.in_scope(|| {
            if let Some(template) = self.state.templates.get(pos as usize) {
                span.record("nm", tracing::field::display(&template.config.name));
                span.record("i", template.config.idx);

                template.set_wasm(bin, &self.config, csp_style, csp_script, secure)?;

                Ok(template.config.name.clone())
            } else {
                bail!("no template with idx: {pos}")
            }
        })
    }

    pub fn set_action(&self, pos: u8, bin: &[u8]) -> anyhow::Result<String> {
        let span = tracing::info_span!(
            "action",
            nm = tracing::field::Empty,
            i = tracing::field::Empty
        );

        span.in_scope(|| {
            if let Some(action) = self.state.actions.get(pos as usize) {
                span.record("nm", tracing::field::display(&action.config.name));
                span.record("i", action.config.idx);

                action.set_wasm(bin)?;

                Ok(action.config.name.clone())
            } else {
                bail!("no action with idx: {pos}")
            }
        })
    }

    pub async fn handle_stream(
        span: &Span,
        tower_service: Router,
        addr: SocketAddr,
        stream: TokioIo<TlsStream<TcpStream>>,
        signal_tx: &Sender<()>,
        close_rx: &Receiver<()>,
    ) {
        let hyper_service = hyper::service::service_fn(move |request: Request<Incoming>| {
            let mut tower_service = Extension(ConnectInfo(addr)).layer(tower_service.clone());
            tower_service.call(request)
        });

        let mut builder = hyper_util::server::conn::auto::Builder::new(TokioExecutor::new());
        builder.http2().enable_connect_protocol();

        let signal_tx = signal_tx.clone();
        let close_rx = close_rx.clone();
        let span = span.clone();

        let mut conn = pin!(builder.serve_connection_with_upgrades(stream, hyper_service));
        let mut signal_closed = pin!(signal_tx.closed().fuse());

        loop {
            tokio::select! {
                result = conn.as_mut() => {
                    match result {
                        Ok(()) => span.in_scope(|| tracing::info!("closed")),
                        Err(err) => span.in_scope(|| tracing::warn!(%err, "connection")),
                    }

                    break;
                }
                () = &mut signal_closed => {
                    span.in_scope(|| tracing::warn!("shutdown"));
                    conn.as_mut().graceful_shutdown();
                }
            }
        }

        drop(close_rx);
    }

    #[allow(clippy::too_many_arguments)]
    pub async fn handle_handshake(
        tls_span: &Span,
        acme_span: &Span,
        start_handshake: StartHandshake<TcpStream>,
        tower_service: Router,
        addr: SocketAddr,
        challenge_rustls_config: Arc<ServerConfig>,
        default_rustls_config: Arc<ServerConfig>,
        signal_tx: &Sender<()>,
        close_rx: &Receiver<()>,
    ) -> anyhow::Result<()> {
        if is_tls_alpn_challenge(&start_handshake.client_hello()) {
            acme_span.in_scope(|| tracing::info!(evt = %"validation", "TLS-ALPN-01"));
            let mut tls = start_handshake.into_stream(challenge_rustls_config).await?;

            tls.shutdown().await?;
        } else {
            tls_span.in_scope(|| tracing::info!("connecting"));
            let as_stream = start_handshake.into_stream(default_rustls_config).await?;

            let stream = TokioIo::new(as_stream);

            Self::handle_stream(tls_span, tower_service, addr, stream, signal_tx, close_rx).await;
        }

        Ok(())
    }

    #[allow(clippy::too_many_lines)]
    pub async fn start<P, F>(
        &self,
        listener: TcpListener,
        redirect_listener: Option<TcpListener>,
        mode: SecurityMode<P>,
        configs: Option<(Arc<ServerConfig>, Arc<ServerConfig>)>,
        signal: fn() -> F,
    ) -> anyhow::Result<()>
    where
        P: AsRef<Path>,
        F: Future<Output = ()> + Send + 'static,
    {
        let span = tracing::info_span!("server", pid = process::id(), domain = &self.config.domain);
        let span_clone = span.clone();

        let redacted_hash = if let Some(logging) = &self.config.logging
            && let Some(server) = &logging.server
            && let Some(credentials) = &server.credentials
        {
            Arc::new(Some(WrappedRedactedHashingAlg(credentials.clone())))
        } else {
            Arc::new(None)
        };

        let mut cnames = HashSet::new();

        if let Some(c) = &self.config.cnames {
            for cname in c {
                cnames.insert(cname.clone());
            }
        }

        let cnames = Arc::new(cnames);
        let cnames_clone = cnames.clone();

        if let Some(listener) = redirect_listener {
            let domain = self.config.domain.clone();

            let log_ips = self.log_ips;
            let log_headers = self.log_headers;

            let request_id_header = HeaderName::from_static(REQUEST_ID_HEADER);

            let local_addr = listener.local_addr()?;

            tokio::spawn(async move {
                span_clone.in_scope(|| tracing::info!("listening on {}", local_addr));

                let handler = async move |uri: Uri,
                                          headers: HeaderMap|
                            -> anyhow::Result<Redirect, StatusCode> {
                    // todo: handle localhost redirecting
                    if let Some(host) = get_host(&headers, &uri)
                        && (host == domain || cnames_clone.contains(&host))
                        && let Some(path_and_query) = uri.path_and_query()
                    {
                        return Ok(Redirect::permanent(&format!(
                            "https://{host}{}",
                            path_and_query.as_str()
                        )));
                    }

                    Err(StatusCode::NOT_FOUND)
                };

                let service = redirect_service(
                    span_clone,
                    redacted_hash,
                    log_ips,
                    log_headers,
                    request_id_header,
                    handler,
                    (),
                );

                if let Err(err) = axum::serve(
                    listener,
                    service.into_make_service_with_connect_info::<SocketAddr>(),
                )
                .with_graceful_shutdown(signal())
                .await
                {
                    tracing::error!(%err, "server closed");
                }
            });
        }

        let local_addr = listener.local_addr()?;
        let log_ips = self.log_ips;

        match mode {
            SecurityMode::Insecure => {
                span.in_scope(|| {
                    tracing::info!(server = %"primary", addr = %local_addr, provision = %"none", "listening");
                });

                let service = self.service.clone();

                axum::serve(
                    listener,
                    service.into_make_service_with_connect_info::<SocketAddr>(),
                )
                .with_graceful_shutdown(signal())
                .await?;

                Ok(())
            }
            SecurityMode::Secure(cert_dir_path, provision) => {
                let span_clone = span.clone();

                let (signal_tx, signal_rx) = watch::channel(());
                let (close_tx, close_rx) = watch::channel(());

                tokio::spawn(async move {
                    signal().await;

                    span_clone.in_scope(|| {
                        tracing::warn!("graceful shutdown");
                    });

                    drop(signal_rx);
                });

                match provision {
                    ProvisionMode::Localhost => {
                        let cert_dir_path = cert_dir_path.as_ref();

                        ordinary_utils::generate_self_signed_localhost_certs(cert_dir_path)?;

                        let rustls_config = rustls_server_config(
                            cert_dir_path.join("key.pem"),
                            cert_dir_path.join("crt.pem"),
                        )?;

                        let tls_acceptor = TlsAcceptor::from(rustls_config);
                        span.in_scope(|| {
                            tracing::info!(server = %"primary", addr = %local_addr, provision = %"localhost", "listening");
                        });

                        loop {
                            let span = span.clone();
                            let (cnx, addr) = tokio::select! {
                                conn = listener.accept() => conn?,
                                () = signal_tx.closed() => {
                                    span.in_scope(|| {
                                       tracing::warn!("not accepting new connections");
                                    });
                                    break;
                                }
                            };

                            let tower_service = self.service.clone();
                            let tls_acceptor = tls_acceptor.clone();

                            let signal_tx = signal_tx.clone();
                            let close_rx = close_rx.clone();

                            tokio::spawn(async move {
                                // todo: determine a better timeout
                                match timeout(Duration::from_secs(3), tls_acceptor.accept(cnx))
                                    .await
                                {
                                    Ok(stream) => {
                                        let stream = match stream {
                                            Ok(stream) => TokioIo::new(stream),
                                            Err(err) => {
                                                span.in_scope(|| {
                                                    if log_ips {
                                                        tracing::warn!(ip = %addr.ip(), port = addr.port(), %err, "accept");
                                                    } else {
                                                        tracing::warn!(%err, "accept");
                                                    }
                                                });

                                                return;
                                            }
                                        };

                                        Self::handle_stream(
                                            &span,
                                            tower_service,
                                            addr,
                                            stream,
                                            &signal_tx,
                                            &close_rx,
                                        )
                                        .await;
                                    }
                                    Err(_) => {
                                        span.in_scope(|| {
                                            if log_ips {
                                                tracing::warn!(ip = %addr.ip(), port = addr.port(), "timeout");
                                            } else {
                                                tracing::warn!("timeout");
                                            }
                                        });
                                    }
                                }
                            });
                        }
                    }
                    ProvisionMode::Staging | ProvisionMode::Production => {
                        let (state, (challenge_rustls_config, default_rustls_config)) =
                            if let Some(configs) = configs {
                                (None, configs)
                            } else {
                                let contacts = self
                                    .config
                                    .contacts
                                    .iter()
                                    .map(|c| format!("mailto:{c}"))
                                    .collect::<Vec<String>>();

                                let cache_path = Path::new(cert_dir_path.as_ref()).join("cache");

                                let state = AcmeConfig::new([&self.config.domain])
                                    .contact(contacts)
                                    .cache_option(Some(DirCache::new(cache_path)))
                                    .directory_lets_encrypt(provision == ProvisionMode::Production)
                                    .state();

                                let challenge_rustls_config = state.challenge_rustls_config();
                                let mut default_rustls_config =
                                    ServerConfig::builder_with_provider(Arc::new(
                                        ring::default_provider(),
                                    ))
                                    .with_safe_default_protocol_versions()?
                                    .with_no_client_auth()
                                    .with_cert_resolver(state.resolver());

                                default_rustls_config.alpn_protocols =
                                    vec![b"h2".to_vec(), b"http/1.1".to_vec()];

                                let default_rustls_config = Arc::new(default_rustls_config);

                                (
                                    Some(state),
                                    (challenge_rustls_config, default_rustls_config),
                                )
                            };

                        let provision_str = match provision {
                            ProvisionMode::Production => "production",
                            ProvisionMode::Staging => "staging",
                            _ => unreachable!(),
                        };

                        let acme_span = span.in_scope(|| {
                            tracing::info_span!(
                                "acme",
                                domain = %self.config.domain,
                                cnames = ?self.config.cnames.clone().unwrap_or_default(),
                                provision = %provision_str
                            )
                        });

                        if let Some(state) = state {
                            ordinary_utils::acme_task(acme_span.clone(), state, signal_tx.clone());
                        }

                        span.in_scope(|| {
                            tracing::info!(
                                provision = %provision_str,
                                "listening on {}",
                                local_addr
                            );
                        });

                        let domain = Arc::new(self.config.domain.clone());

                        loop {
                            let span = span.clone();

                            let (cnx, addr) = tokio::select! {
                                conn = listener.accept() => conn?,
                                () = signal_tx.closed() => {
                                    span.in_scope(|| {
                                       tracing::warn!("not accepting new connections");
                                    });
                                    break;
                                }
                            };

                            let tower_service = self.service.clone();

                            let challenge_rustls_config = challenge_rustls_config.clone();
                            let default_rustls_config = default_rustls_config.clone();
                            let acme_span = acme_span.clone();
                            let cnames = cnames.clone();
                            let signal_tx = signal_tx.clone();
                            let close_rx = close_rx.clone();
                            let domain = domain.clone();

                            tokio::spawn(async move {
                                // todo: determine a better timeout
                                match timeout(
                                    Duration::from_secs(3),
                                    LazyConfigAcceptor::new(Acceptor::default(), cnx),
                                )
                                .await
                                {
                                    Ok(start_handshake) => {
                                        let start_handshake = match start_handshake {
                                            Ok(start_handshake) => start_handshake,
                                            Err(err) => {
                                                if log_ips {
                                                    let tls_span = span.in_scope(|| { tracing::error_span!("tls", ip = %addr.ip(), port = addr.port()) });

                                                    tls_span.in_scope(
                                                        || tracing::error!(%err, "accept"),
                                                    );
                                                } else {
                                                    let tls_span = span
                                                        .in_scope(|| tracing::error_span!("tls"));

                                                    tls_span.in_scope(
                                                        || tracing::error!(%err, "accept"),
                                                    );
                                                }

                                                return;
                                            }
                                        };

                                        if let Some(sni) =
                                            start_handshake.client_hello().server_name()
                                        {
                                            let tls_span = span.in_scope(|| {
                                                if log_ips {
                                                    tracing::info_span!("tls", %sni, ip = %addr.ip(), port = addr.port())
                                                } else {
                                                    tracing::info_span!("tls", %sni)
                                                }
                                            });

                                            if sni == *domain || cnames.contains(sni) {
                                                if let Err(err) = Self::handle_handshake(
                                                    &tls_span,
                                                    &acme_span,
                                                    start_handshake,
                                                    tower_service,
                                                    addr,
                                                    challenge_rustls_config,
                                                    default_rustls_config,
                                                    &signal_tx,
                                                    &close_rx,
                                                )
                                                .await
                                                {
                                                    tls_span.in_scope(|| {
                                                        tracing::error!(%err, "handshake");
                                                    });
                                                }
                                            } else {
                                                tls_span.in_scope(|| tracing::warn!("no match"));
                                            }
                                        } else {
                                            let tls_span = span.in_scope(|| {
                                                if log_ips {
                                                    tracing::info_span!("tls", ip = %addr.ip(), port = addr.port())
                                                } else {
                                                    tracing::info_span!("tls")
                                                }
                                            });

                                            tls_span.in_scope(|| tracing::warn!("no sni"));
                                        }
                                    }
                                    Err(_) => {
                                        if log_ips {
                                            let tls_span = span.in_scope(|| { tracing::error_span!("tls", ip = %addr.ip(), port = addr.port()) });

                                            tls_span.in_scope(|| tracing::error!("timeout"));
                                        } else {
                                            let tls_span =
                                                span.in_scope(|| tracing::error_span!("tls"));

                                            tls_span.in_scope(|| tracing::error!("timeout"));
                                        }
                                    }
                                }
                            });
                        }
                    }
                }

                drop(close_rx);
                drop(listener);

                span.in_scope(|| {
                    tracing::warn!(
                        "waiting for {} task(s) to finish",
                        close_tx.receiver_count()
                    );
                });
                close_tx.closed().await;

                Ok(())
            }
        }
    }
}
