//! Test gpu-trace-perf replay (Mesa CI mode).

use crate::{GpuTracePerfRun, check_html_links};
use bytes::Bytes;
use http_body_util::{BodyExt, Full};
use hyper::body::Incoming;
use hyper::{Method, Request, Response};
use hyper_util::rt::{TokioExecutor, TokioIo};
use hyper_util::server::conn::auto::Builder as ConnBuilder;
use std::collections::{HashMap, HashSet};
use std::convert::Infallible;
use std::io::prelude::*;
use std::path::Path;
use std::sync::{Arc, Mutex};
use tempfile::NamedTempFile;
use tokio::net::TcpListener;

/// Extract the filename and raw file bytes from a `multipart/form-data` body.
///
/// Expects a single `file` part with a `filename` attribute in its
/// `Content-Disposition` header, as produced by the reqwest multipart upload.
fn parse_multipart_file(content_type: &str, body: &[u8]) -> Option<(String, Vec<u8>)> {
    let boundary = content_type
        .split(';')
        .find_map(|p| p.trim().strip_prefix("boundary="))?
        .trim_matches('"')
        .to_string();

    // Locate the start of the first part (after "--<boundary>\r\n").
    let part_start_delim = format!("--{boundary}\r\n").into_bytes();
    let part_start = find_subsequence(body, &part_start_delim)? + part_start_delim.len();

    // Headers end at the first \r\n\r\n.
    let header_end = find_subsequence(&body[part_start..], b"\r\n\r\n")?;
    let headers_str = std::str::from_utf8(&body[part_start..part_start + header_end]).ok()?;

    // File data starts right after the blank line.
    let data_start = part_start + header_end + 4;

    // File data ends before the closing "\r\n--<boundary>" delimiter.
    let end_delim = format!("\r\n--{boundary}").into_bytes();
    let data_end = find_subsequence(&body[data_start..], &end_delim)
        .map(|pos| data_start + pos)
        .unwrap_or(body.len());

    let file_data = body[data_start..data_end].to_vec();

    // Extract filename from the Content-Disposition header.
    let filename = headers_str
        .lines()
        .find(|l| l.to_ascii_lowercase().starts_with("content-disposition:"))?
        .split(';')
        .find_map(|p| p.trim().strip_prefix("filename="))
        .map(|s| s.trim_matches('"').to_string())?;

    Some((filename, file_data))
}

fn find_subsequence(haystack: &[u8], needle: &[u8]) -> Option<usize> {
    haystack.windows(needle.len()).position(|w| w == needle)
}

/// Handles a single HTTP request for the in-memory S3 test server.
async fn handle_s3_request(
    objects: Arc<Mutex<HashMap<String, Vec<u8>>>>,
    required_token: Option<String>,
    jwt_download_paths: Arc<HashSet<String>>,
    upload_forbidden: bool,
    req: Request<Incoming>,
) -> Result<Response<Full<Bytes>>, Infallible> {
    if upload_forbidden && *req.method() == Method::PUT {
        return Ok(Response::builder()
            .status(403)
            .body(Full::new(Bytes::from("Forbidden")))
            .unwrap());
    }
    if let Some(token) = &required_token {
        // Uploads always require auth; downloads only when explicitly flagged.
        let needs_auth = match *req.method() {
            Method::PUT => true,
            _ => jwt_download_paths.contains(req.uri().path()),
        };
        if needs_auth {
            let auth_ok = req
                .headers()
                .get("authorization")
                .and_then(|v| v.to_str().ok())
                .and_then(|s| s.strip_prefix("Bearer "))
                .map(|t| t == token)
                .unwrap_or(false);
            if !auth_ok {
                // Real S3 returns 403 Forbidden for unauthenticated uploads;
                // use 401 for downloads to match the download error-message test.
                let status = if *req.method() == Method::PUT {
                    403
                } else {
                    401
                };
                return Ok(Response::builder()
                    .status(status)
                    .body(Full::new(Bytes::new()))
                    .unwrap());
            }
        }
    }

    let path = req.uri().path().to_string();
    match *req.method() {
        Method::GET => {
            let data = objects.lock().unwrap().get(&path).cloned();
            match data {
                Some(bytes) => Ok(Response::new(Full::new(Bytes::from(bytes)))),
                None => Ok(Response::builder()
                    .status(404)
                    .body(Full::new(Bytes::new()))
                    .unwrap()),
            }
        }
        Method::HEAD => {
            let exists = objects.lock().unwrap().contains_key(&path);
            Ok(Response::builder()
                .status(if exists { 200 } else { 404 })
                .body(Full::new(Bytes::new()))
                .unwrap())
        }
        Method::PUT => {
            let content_type = req
                .headers()
                .get("content-type")
                .and_then(|v| v.to_str().ok())
                .unwrap_or("")
                .to_string();
            let body = req.into_body().collect().await.unwrap().to_bytes();

            // Uploads are multipart/form-data PUT to the directory URL.
            // Parse the filename from the part and store at dir + filename.
            if content_type.starts_with("multipart/form-data") {
                if let Some((filename, data)) = parse_multipart_file(&content_type, &body) {
                    let key = format!("{}/{}", path.trim_end_matches('/'), filename);
                    objects.lock().unwrap().insert(key, data);
                }
            } else {
                objects.lock().unwrap().insert(path, body.to_vec());
            }
            Ok(Response::builder()
                .status(204)
                .body(Full::new(Bytes::new()))
                .unwrap())
        }
        _ => Ok(Response::builder()
            .status(405)
            .body(Full::new(Bytes::new()))
            .unwrap()),
    }
}

/// An in-process HTTP object-store server backed by an in-memory HashMap,
/// used to test trace downloading and snapshot uploading without hitting a
/// real S3 endpoint.  Optionally enforces a Bearer token on all requests.
///
/// Keys are full URL paths (e.g. `/traces-db/glxgears.mock-trace`).
/// Callers are responsible for including the bucket prefix in every key and
/// in every URL they construct from [`url()`].
struct S3TestServer {
    /// The base URL: `http://127.0.0.1:<port>`.
    /// Append `/<bucket>/<key>` to form a full object URL.
    base_url: String,

    objects: Arc<Mutex<HashMap<String, Vec<u8>>>>,

    // Holds the tokio runtime that's running the server.  When S3TestServer is
    // dropped, its accept() will return an error and exit the accept loop,
    // tearing down the server.
    _runtime: tokio::runtime::Runtime,
}

impl S3TestServer {
    /// Start a server, pre-populate it with `files` (full-path key → local path),
    /// and return a ready-to-use server.  No authentication is required.
    ///
    /// Keys must include the bucket prefix, e.g.
    /// `"/traces-db/glxgears.mock-trace"`.
    fn start(files: &[(&str, &Path)]) -> Self {
        Self::start_inner(files, None, false)
    }

    /// Like [`start`], but uploads always require `Authorization: Bearer <token>`,
    /// and downloads require it only for paths listed in `jwt_download_paths`.
    fn start_with_jwt(files: &[(&str, &Path)], jwt_download_paths: &[&str], token: &str) -> Self {
        Self::start_inner(files, Some((jwt_download_paths, token)), false)
    }

    /// Like [`start`], but all PUT requests return 403 Forbidden.
    /// HEAD and GET work normally, so pre-populated objects are visible and
    /// `upload_if_absent` skips the upload when the file already exists.
    fn start_upload_forbidden(files: &[(&str, &Path)]) -> Self {
        Self::start_inner(files, None, true)
    }

    fn start_inner(
        files: &[(&str, &Path)],
        auth: Option<(&[&str], &str)>,
        upload_forbidden: bool,
    ) -> Self {
        // Pre-populate the object store directly.
        let objects: Arc<Mutex<HashMap<String, Vec<u8>>>> = Arc::new(Mutex::new(HashMap::new()));
        for (key, path) in files {
            let content = std::fs::read(path).expect("reading test file");
            objects.lock().unwrap().insert(key.to_string(), content);
        }

        let std_listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
        std_listener.set_nonblocking(true).unwrap();
        let port = std_listener.local_addr().unwrap().port();

        let runtime = tokio::runtime::Builder::new_multi_thread()
            .enable_all()
            .build()
            .unwrap();

        let objs = Arc::clone(&objects);
        let token: Option<String> = auth.map(|(_, t)| t.to_string());
        let jwt_download_paths: Arc<HashSet<String>> = Arc::new(
            auth.map(|(paths, _)| paths.iter().map(|s| s.to_string()).collect())
                .unwrap_or_default(),
        );
        runtime.spawn(async move {
            let listener = TcpListener::from_std(std_listener).unwrap();
            let http = Arc::new(ConnBuilder::new(TokioExecutor::new()));
            loop {
                let (stream, _) = match listener.accept().await {
                    Ok(x) => x,
                    Err(_) => break,
                };
                let objs = Arc::clone(&objs);
                let token = token.clone();
                let jwt = Arc::clone(&jwt_download_paths);
                let http = http.clone();
                tokio::spawn(async move {
                    let svc = hyper::service::service_fn(move |req| {
                        let objs = Arc::clone(&objs);
                        let token = token.clone();
                        let jwt = Arc::clone(&jwt);
                        async move {
                            handle_s3_request(objs, token, jwt, upload_forbidden, req).await
                        }
                    });
                    let _ = http.serve_connection(TokioIo::new(stream), svc).await;
                });
            }
        });

        S3TestServer {
            base_url: format!("http://127.0.0.1:{port}"),
            objects,
            _runtime: runtime,
        }
    }

    /// Returns the URL for the traces bucket (`<base>/traces-db`).
    fn traces_url(&self) -> String {
        format!("{}{TRACES_BUCKET}", self.base_url)
    }

    /// Returns the URL for the results/snapshots bucket (`<base>/results`).
    fn snapshots_url(&self) -> String {
        format!("{}{RESULTS_BUCKET}", self.base_url)
    }

    /// Returns `true` if the given full-path key exists (e.g. `/results/abc.png`).
    fn key_exists(&self, key: &str) -> bool {
        self.objects.lock().unwrap().contains_key(key)
    }

    /// Returns the bytes stored at the given full-path key, or `None`.
    fn get_object(&self, key: &str) -> Option<Vec<u8>> {
        self.objects.lock().unwrap().get(key).cloned()
    }

    /// Stores bytes at the given full-path key.
    fn put_object(&self, key: &str, data: &[u8]) {
        self.objects
            .lock()
            .unwrap()
            .insert(key.to_string(), data.to_vec());
    }
}

/// Bucket path for trace files (mirrors `mesa-tracie-public` in production).
const TRACES_BUCKET: &str = "/traces-db";

/// Bucket path for result snapshots (mirrors `mesa-tracie-results` in production).
const RESULTS_BUCKET: &str = "/results";

/// Full S3 key for a trace file within the traces bucket.
fn trace_key(name: &str) -> String {
    format!("{TRACES_BUCKET}/{name}")
}

/// Full S3 key for a snapshot within the results bucket.
fn snapshot_key(hash: &str) -> String {
    format!("{RESULTS_BUCKET}/{hash}.png")
}

/// Write a minimal traces.toml to a temp file and return it.
fn write_test_toml(download_url: &str, traces: &[(&str, &str)]) -> NamedTempFile {
    write_ci_test_toml(
        download_url,
        &traces
            .iter()
            .map(|(path, checksum)| (*path, *checksum, [].as_slice()))
            .collect::<Vec<_>>(),
    )
}

/// Tags for use in write_ci_test_toml, mirroring the DeviceEntry boolean fields.
#[derive(Debug, Clone, PartialEq)]
enum Tag {
    Skip,
    Singlethread,
}

/// Write a traces.toml with optional per-device tags, serialized via serde.
///
/// `traces` is a slice of `(path, checksum, tags)` tuples.
fn write_ci_test_toml(download_url: &str, traces: &[(&str, &str, &[Tag])]) -> NamedTempFile {
    use gpu_trace_perf::traces_config::*;

    let config = TracesConfig {
        traces_db: TracesDb {
            download_url: download_url.to_string(),
        },
        traces: traces
            .iter()
            .map(|(path, checksum, tags)| {
                let mut devices = std::collections::HashMap::new();
                devices.insert(
                    "test-device".to_string(),
                    DeviceEntry {
                        checksum: checksum.to_string(),
                        skip: tags.contains(&Tag::Skip),
                        singlethread: tags.contains(&Tag::Singlethread),
                        replay_args: vec![],
                    },
                );
                TraceEntry {
                    path: path.to_string(),
                    nonloopable: false,
                    replay_args: vec![],
                    devices,
                }
            })
            .collect(),
    };

    let toml_str = toml::to_string(&config).expect("serializing test TOML");
    let mut f = NamedTempFile::new().unwrap();
    f.write_all(toml_str.as_bytes()).unwrap();
    f
}

/// Run `replay --traces-db src/test_data` with the given TOML and return the
/// actual checksum found in a FAIL line, panicking if no FAIL line is present.
#[cfg(any(
    feature = "testapitrace",
    feature = "testrdc",
    feature = "testgfxr",
    feature = "testangle"
))]
fn extract_actual_checksum_from_fail(stderr: &str) -> String {
    stderr
        .lines()
        .find(|l| l.starts_with("[ERROR] FAIL "))
        .and_then(|l| l.split("actual:").nth(1))
        .map(|s| s.trim().to_string())
        .unwrap_or_else(|| panic!("Could not find actual checksum in stderr:\n{stderr}"))
}

/// Discover the actual checksum for a local trace, then verify a passing replay.
///
/// Runs once with a zero checksum to learn the actual, then again with the
/// correct checksum expecting PASS and exit 0.  Returns the final `CommandOutput`.
#[cfg(any(
    feature = "testapitrace",
    feature = "testrdc",
    feature = "testgfxr",
    feature = "testangle"
))]
fn run_local_replay_and_pass(trace_path: &str) {
    const ZERO_HASH: &str = "0000000000000000000000000000000000000000000000000000000000000000";

    // Step 1: run with zero checksum to discover the actual checksum.
    let toml1 = write_ci_test_toml("http://unused/", &[(trace_path, ZERO_HASH, &[])]);
    let mut run1 = GpuTracePerfRun::new();
    let cache_dir = run1.tempdir();
    run1.setup_ci_replay(toml1.path(), "test-device", Some(cache_dir.clone()));
    run1.arg("--traces-db");
    run1.arg("src/test_data");
    let actual_checksum = extract_actual_checksum_from_fail(&run1.run().stderr);

    // Step 2: run with the correct checksum; expect PASS and exit 0.
    let toml2 = write_ci_test_toml("http://unused/", &[(trace_path, &actual_checksum, &[])]);
    let mut run2 = GpuTracePerfRun::new();
    run2.setup_ci_replay(toml2.path(), "test-device", Some(cache_dir));
    run2.arg("--traces-db");
    run2.arg("src/test_data");
    run2.snapshot_output(&format!("replay_pass_{trace_path}"));
}

static GLXGEARS_MOCK_HASH: &str =
    "2c82ccff2286031d891dac879ae4a075dea283dd80040237957313dc0039a267";
static GLXGEARS_BROKEN_HASH: &str =
    "5411a2e66334f01d5080515b8be426e57ce7199376832262a07483c5f6a8aa58";

fn html_contains(output_dir: &Path, link: &str) -> bool {
    let html = std::fs::read_to_string(output_dir.join("index.html")).expect("reading index.html");
    html.contains(link)
}

/// Download a trace from S3 and replay it.  Verify the trace was fetched
/// (FAIL in stderr because the checksum is intentionally wrong) and that the
/// cache is cleaned up by GC after replay.
#[test]
pub fn ci_download() {
    let server = S3TestServer::start(&[(
        &trace_key("glxgears.mock-trace"),
        Path::new("src/test_data/glxgears.mock-trace"),
    )]);

    let toml = write_test_toml(
        &server.traces_url(),
        &[(
            "glxgears.mock-trace",
            "0000000000000000000000000000000000000000000000000000000000000000",
        )],
    );

    let mut run = GpuTracePerfRun::new();
    let cache_dir = run.tempdir();
    run.setup_ci_replay(toml.path(), "test-device", Some(cache_dir.clone()));

    let output = run.run();

    // Should fail due to the wrong checksum, but not with an ERROR
    // (which would indicate the download or trace-type detection failed).
    assert!(
        output.stderr.contains("FAIL"),
        "Expected FAIL (trace downloaded and replayed), got:\nstdout: {}\nstderr: {}\n",
        output.stdout,
        output.stderr,
    );

    // Post-replay GC should have left the downloaded trace in the cache.
    assert!(std::fs::exists(cache_dir.path().join("glxgears.mock-trace")).unwrap());
}

/// With --traces-db, no download should occur and the cache dir stays empty.
#[test]
pub fn ci_traces_db() {
    let toml = write_test_toml(
        "http://127.0.0.1:0/should-not-be-reached/",
        &[("glxgears.mock-trace", GLXGEARS_MOCK_HASH)],
    );

    let mut run = GpuTracePerfRun::new();
    let cache_dir = run.tempdir();
    run.setup_ci_replay(toml.path(), "test-device", Some(cache_dir.clone()));
    run.arg("--traces-db");
    run.arg("src/test_data");

    let output = run.run();

    // Should fail due to the wrong checksum, but not with an ERROR
    // (an ERROR would mean the local file wasn't found or the trace type
    // was unrecognised, which shouldn't happen in --traces-db mode).
    assert!(
        output.stdout.contains("PASS"),
        "Expected PASS (local trace replayed), got:\nstdout: {}\nstderr: {}\n",
        output.stdout,
        output.stderr,
    );

    // The cache dir should remain empty: no S3 downloads took place.
    let cached: Vec<_> = std::fs::read_dir(cache_dir.path())
        .unwrap()
        .filter_map(|e| e.ok())
        .collect();
    assert!(
        cached.is_empty(),
        "Cache dir should be empty in local traces-db mode (got {cached:?})"
    );
}

/// With --traces-db given an absolute path, relative_test_name must not panic
/// trying to strip the cache dir from the traces-db path.
#[test]
pub fn ci_traces_db_absolute_path() {
    let test_data = std::fs::canonicalize("src/test_data").expect("canonicalize src/test_data");

    let toml = write_test_toml(
        "http://127.0.0.1:0/should-not-be-reached/",
        &[("glxgears.mock-trace", GLXGEARS_MOCK_HASH)],
    );

    let mut run = GpuTracePerfRun::new();
    let cache_dir = run.tempdir();
    run.setup_ci_replay(toml.path(), "test-device", Some(cache_dir));
    run.arg("--traces-db");
    run.path_arg(&test_data);

    let output = run.run();

    assert!(
        output.stdout.contains("PASS"),
        "Expected PASS (local trace replayed with absolute traces-db path), got:\nstdout: {}\nstderr: {}\n",
        output.stdout,
        output.stderr,
    );
}

/// With --traces-db, a trace missing from the local dir falls back to download.
#[test]
pub fn ci_traces_db_miss() {
    let server = S3TestServer::start(&[(
        &trace_key("glxgears.mock-trace"),
        Path::new("src/test_data/glxgears.mock-trace"),
    )]);

    let toml = write_test_toml(
        &server.traces_url(),
        &[("glxgears.mock-trace", GLXGEARS_MOCK_HASH)],
    );

    // Point --traces-db at an empty directory that does not contain the trace.
    let empty_traces_db = tempfile::tempdir().unwrap();
    let mut run = GpuTracePerfRun::new();
    let cache_dir = run.tempdir();
    run.setup_ci_replay(toml.path(), "test-device", Some(cache_dir.clone()));
    run.arg("--traces-db");
    run.path_arg(empty_traces_db.path());

    run.snapshot_output("ci_traces_db_miss");

    // The downloaded trace should now be in the cache dir.
    assert!(
        std::fs::exists(cache_dir.path().join("glxgears.mock-trace")).unwrap(),
        "Downloaded trace should be in cache dir after fallback download"
    );
}

/// A passing replay: run once with a wrong checksum to learn the actual one,
/// then run again with the correct checksum and verify PASS + exit 0.
#[test]
pub fn ci_replay_pass() {
    let server = S3TestServer::start(&[
        (
            &trace_key("glxgears.mock-trace"),
            Path::new("src/test_data/glxgears.mock-trace"),
        ),
        (
            &snapshot_key(GLXGEARS_MOCK_HASH),
            Path::new("src/test_data/glxgears.mock-trace"),
        ),
    ]);
    let toml = write_ci_test_toml(
        &server.traces_url(),
        &[("glxgears.mock-trace", GLXGEARS_MOCK_HASH, &[])],
    );
    let mut run = GpuTracePerfRun::new();
    let output_dir = run.setup_ci_replay(toml.path(), "test-device", None);
    run.arg("--snapshots-url");
    run.arg(&server.snapshots_url());

    run.snapshot_output("ci_replay_pass");

    assert!(
        !output_dir.path().join("checksum-changes.json").exists(),
        "checksum-changes.json should not be written when all traces pass"
    );

    let leftover = find_fullres_pngs(output_dir.path());
    assert!(
        leftover.is_empty(),
        "Full-res PNGs left after successful upload (passing trace): {leftover:?}"
    );

    check_html_links(output_dir.path());
}

/// A failing replay: download trace and expected snapshot from S3, break
/// rendering to produce a wrong checksum → non-zero exit, FAIL in stderr,
/// checksums.patch written, HTML report with diff links valid.
fn ci_replay_fail_helper(expected_png_present: bool) {
    let glxgears_key = trace_key("glxgears.mock-trace");
    let mut s3_files: Vec<(&str, &Path)> = vec![(
        &glxgears_key,
        Path::new("src/test_data/glxgears.mock-trace"),
    )];
    let snapshot_key = snapshot_key(GLXGEARS_MOCK_HASH);
    if expected_png_present {
        s3_files.push((
            &snapshot_key,
            Path::new("src/test_data/glxgears.mock-trace"),
        ));
    }
    let server = S3TestServer::start(&s3_files);
    let toml = write_ci_test_toml(
        &server.traces_url(),
        &[("glxgears.mock-trace", GLXGEARS_MOCK_HASH, &[])],
    );
    let mut run = GpuTracePerfRun::new();
    let output_dir = run.setup_ci_replay(toml.path(), "test-device", None);
    run.arg("--snapshots-url");
    run.arg(&server.snapshots_url());
    run.env("MOCK_BREAK_RENDERING", "1");

    if expected_png_present {
        run.snapshot_output("ci_replay_fail");
    } else {
        run.snapshot_output("ci_replay_fail_no_expected");
    }

    let patch = std::fs::read_to_string(output_dir.path().join("checksums.patch"))
        .expect("reading checksums.patch");

    // The patch must remove the wrong checksum and add the actual one.
    assert!(
        patch.contains(&format!("-checksum = \"{GLXGEARS_MOCK_HASH}\"")),
        "checksums.patch missing '-' line for old checksum:\n{patch}"
    );
    assert!(
        patch.contains(&format!("+checksum = \"{GLXGEARS_BROKEN_HASH}\"")),
        "checksums.patch missing '+' line for new checksum:\n{patch}"
    );

    let leftover = find_fullres_pngs(output_dir.path());
    assert!(
        leftover.is_empty(),
        "Full-res PNGs left after successful upload (failing trace): {leftover:?}"
    );

    check_html_links(output_dir.path());
}

#[test]
pub fn ci_replay_fail_expected_exists() {
    ci_replay_fail_helper(true)
}

#[test]
pub fn ci_replay_fail_no_expected() {
    ci_replay_fail_helper(false)
}

/// A JWT-gated replay: trace download, snapshot upload, and expected-image
/// fetch all require a Bearer token.  Verify that `--jwt` threads the token
/// through every HTTP call end-to-end.
#[test]
pub fn ci_jwt() {
    const TOKEN: &str = "test-jwt-secret";

    let server = S3TestServer::start_with_jwt(
        &[
            (
                &trace_key("glxgears.mock-trace"),
                Path::new("src/test_data/glxgears.mock-trace"),
            ),
            // Pre-populate the expected snapshot so fill_diff_info can fetch it.
            (
                &snapshot_key(GLXGEARS_MOCK_HASH),
                Path::new("src/test_data/glxgears.mock-trace"),
            ),
        ],
        // Only the trace download requires JWT; snapshot GETs are unauthenticated
        // so the HTML link checker can verify them without credentials.
        &[&trace_key("glxgears.mock-trace")],
        TOKEN,
    );

    let toml = write_ci_test_toml(
        &server.traces_url(),
        &[("glxgears.mock-trace", GLXGEARS_MOCK_HASH, &[])],
    );

    let mut jwt_file = NamedTempFile::new().unwrap();
    jwt_file.write_all(TOKEN.as_bytes()).unwrap();

    let mut run = GpuTracePerfRun::new();
    let output_dir = run.setup_ci_replay(toml.path(), "test-device", None);
    run.arg("--jwt");
    run.arg(jwt_file.path().to_str().unwrap());
    run.arg("--snapshots-url");
    run.arg(&server.snapshots_url());
    run.env("MOCK_BREAK_RENDERING", "1");
    run.snapshot_output("ci_jwt");

    // The actual snapshot must have been uploaded via the JWT-authenticated PUT.
    assert!(
        server.key_exists(&snapshot_key(GLXGEARS_BROKEN_HASH)),
        "Expected snapshot uploaded to JWT-gated S3"
    );

    // HTML report should be valid (fill_diff_info used JWT to fetch expected image).
    check_html_links(output_dir.path());
}

/// A flaking replay: same setup as ci_replay_fail but with MOCK_FLAKE_RENDERING.
#[test]
pub fn ci_replay_flake() {
    let server = S3TestServer::start(&[
        (
            &trace_key("glxgears.mock-trace"),
            Path::new("src/test_data/glxgears.mock-trace"),
        ),
        (
            &snapshot_key(GLXGEARS_MOCK_HASH),
            Path::new("src/test_data/glxgears.mock-trace"),
        ),
    ]);
    let toml = write_ci_test_toml(
        &server.traces_url(),
        &[("glxgears.mock-trace", GLXGEARS_MOCK_HASH, &[])],
    );
    let mut run = GpuTracePerfRun::new();
    let output_dir = run.setup_ci_replay(toml.path(), "test-device", None);
    run.arg("--snapshots-url");
    run.arg(&server.snapshots_url());
    run.env("MOCK_FLAKE_RENDERING", "1");
    run.snapshot_output("ci_replay_flake");

    check_html_links(output_dir.path());
}

/// A timed-out replay: MOCK_TIMEOUT causes the snapshot to hang; --timeout 1
/// kills it, and the result should appear as a failure in the HTML report.
#[test]
pub fn ci_replay_timeout() {
    let toml = write_ci_test_toml(
        "http://unused/",
        &[("glxgears.mock-trace", GLXGEARS_MOCK_HASH, &[])],
    );
    let mut run = GpuTracePerfRun::new();
    let output_dir = run.setup_ci_replay(toml.path(), "test-device", None);
    run.arg("--traces-db");
    run.arg("src/test_data");
    run.arg("--timeout");
    run.arg("1");
    run.env("MOCK_TIMEOUT", "1");
    run.snapshot_output("ci_replay_timeout");

    check_html_links(output_dir.path());
}

#[test]
pub fn ci_replay_missing_file() {
    let server = S3TestServer::start(&[(
        &trace_key("glxgears.mock-trace"),
        Path::new("src/test_data/glxgears.mock-trace"),
    )]);
    let toml = write_ci_test_toml(
        &server.traces_url(),
        &[("glxgears-typo.mock-trace", GLXGEARS_MOCK_HASH, &[])],
    );
    let mut run = GpuTracePerfRun::new();
    run.setup_ci_replay(toml.path(), "test-device", None);
    run.snapshot_output("ci_replay_missing_file");
}

/// Without --jwt against a JWT-required server, the download fails with an
/// authentication error.  Verify non-zero exit and a helpful ERROR message.
#[test]
pub fn ci_auth_failure() {
    const TOKEN: &str = "secret-token";

    let server = S3TestServer::start_with_jwt(
        &[(
            &trace_key("glxgears.mock-trace"),
            Path::new("src/test_data/glxgears.mock-trace"),
        )],
        &[&trace_key("glxgears.mock-trace")],
        TOKEN,
    );

    let toml = write_test_toml(
        &server.traces_url(),
        &[("glxgears.mock-trace", GLXGEARS_MOCK_HASH)],
    );

    let mut run = GpuTracePerfRun::new();
    run.setup_ci_replay(toml.path(), "test-device", None);
    // No --jwt argument: the server returns 401.
    let out = run.run();

    assert!(
        !out.status.success(),
        "Expected non-zero exit on auth failure:\nstdout: {}\nstderr: {}\n",
        out.stdout,
        out.stderr,
    );

    // Should be reported as ERROR (download failed), not FAIL (wrong checksum).
    assert!(
        out.stderr.contains("ERROR"),
        "Expected ERROR on auth failure, got:\nstdout: {}\nstderr: {}\n",
        out.stdout,
        out.stderr,
    );

    // The error message should mention HTTP 401 and suggest --jwt.
    assert!(
        out.stderr.contains("401"),
        "Error should mention HTTP 401:\nstdout: {}\nstderr: {}\n",
        out.stdout,
        out.stderr,
    );
    assert!(
        out.stderr.contains("--jwt"),
        "Error should suggest --jwt:\nstdout: {}\nstderr: {}\n",
        out.stdout,
        out.stderr,
    );
}

/// A skipped trace: `skip` tag → exit 0, trace not mentioned in output.
#[test]
pub fn ci_replay_skip() {
    let toml = write_ci_test_toml(
        "http://unused/",
        &[("glxgears.mock-trace", GLXGEARS_MOCK_HASH, &[Tag::Skip])],
    );
    let mut run = GpuTracePerfRun::new();
    run.setup_ci_replay(toml.path(), "test-device", None);
    run.arg("--traces-db");
    run.arg("src/test_data");
    run.snapshot_output("ci_replay_skip");
}

/// A passing apitrace replay.
#[cfg(feature = "testapitrace")]
#[test]
pub fn ci_replay_apitrace() {
    run_local_replay_and_pass("glmark2-pulsar.trace");
}

/// A passing renderdoc replay.
#[cfg(feature = "testrdc")]
#[test]
pub fn ci_replay_renderdoc() {
    run_local_replay_and_pass("vkcube.rdc");
}

/// A singlethread replay: trace tagged `singlethread` must not run concurrently
/// with any other trace.  Verified by parsing the debug log that records when
/// each replay starts and finishes.
#[test]
pub fn ci_replay_singlethread() {
    // Upload the same mock trace under five names so there is real opportunity
    // for parallel execution.  One is tagged singlethread; the rest are normal.
    const SINGLETHREAD: &str = "glxgears-single.mock-trace";
    let trace_path = Path::new("src/test_data/glxgears.mock-trace");
    let server = S3TestServer::start(&[
        (&trace_key("glxgears-a.mock-trace"), trace_path),
        (&trace_key("glxgears-b.mock-trace"), trace_path),
        (&trace_key(SINGLETHREAD), trace_path),
        (&trace_key("glxgears-d.mock-trace"), trace_path),
        (&trace_key("glxgears-e.mock-trace"), trace_path),
    ]);

    let toml = write_ci_test_toml(
        &server.traces_url(),
        &[
            ("glxgears-a.mock-trace", GLXGEARS_MOCK_HASH, &[]),
            ("glxgears-b.mock-trace", GLXGEARS_MOCK_HASH, &[]),
            (SINGLETHREAD, GLXGEARS_MOCK_HASH, &[Tag::Singlethread]),
            ("glxgears-d.mock-trace", GLXGEARS_MOCK_HASH, &[]),
            ("glxgears-e.mock-trace", GLXGEARS_MOCK_HASH, &[]),
        ],
    );

    // -v enables [DEBUG] Starting/Finished replay lines in stdout.
    // -j 4 allows four concurrent replays to exercise the semaphore logic.
    let mut run = GpuTracePerfRun::new();
    run.arg("-v");
    run.setup_ci_replay(toml.path(), "test-device", None);
    run.arg("-j");
    run.arg("4");
    let out = run.run();

    assert!(
        out.status.success(),
        "Expected exit 0:\nstdout: {}\nstderr: {}\n",
        out.stdout,
        out.stderr
    );

    // Parse the "[DEBUG] Starting/Finished replay <path>" lines that
    // replay_one_trace emits after acquiring and before releasing the replay
    // semaphore.  These appear on stdout (DEBUG goes through SimpleLogger →
    // stdout) and use the logical TOML path, not the cache file hash.
    let events: Vec<(bool, &str)> = out
        .stdout
        .lines()
        .filter_map(|line| {
            if let Some(rest) = line.split("[INFO] Running ").nth(1) {
                Some((true, rest.trim()))
            } else if let Some(rest) = line.split("replay: Finished ").nth(1) {
                Some((false, rest.trim()))
            } else {
                None
            }
        })
        .collect();

    assert!(
        !events.is_empty(),
        "No Starting/Finished replay events in stdout; is -v working?\nstdout:\n{}\nstderr:\n{}",
        out.stdout,
        out.stderr
    );

    // Find the index of "Starting" and "Finished" for the singlethread trace.
    let single_start = events
        .iter()
        .position(|(is_start, name)| *is_start && *name == SINGLETHREAD)
        .unwrap_or_else(|| {
            panic!(
                "No 'Starting replay {SINGLETHREAD}' in stdout:\n{}",
                out.stdout
            )
        });
    let single_end = events
        .iter()
        .position(|(is_start, name)| !*is_start && *name == SINGLETHREAD)
        .unwrap_or_else(|| {
            panic!(
                "No 'Finished replay {SINGLETHREAD}' in stdout:\n{}",
                out.stdout
            )
        });

    // Between single_start and single_end, no other trace may start.
    for (idx, (is_start, name)) in events.iter().enumerate() {
        if idx > single_start && idx < single_end && *is_start {
            panic!(
                "Trace '{name}' started while singlethread '{SINGLETHREAD}' was running \
                 (event {idx} between {single_start} and {single_end})!\nstdout:\n{}",
                out.stdout
            );
        }
    }
}

/// Running the same replay twice must not upload the PNG a second time.
/// We pre-populate the S3 key with sentinel bytes; if the second run
/// overwrites it, the assertion fails.
#[test]
pub fn ci_output_upload_skip() {
    let server = S3TestServer::start(&[(
        &trace_key("glxgears.mock-trace"),
        Path::new("src/test_data/glxgears.mock-trace"),
    )]);
    let sentinel = b"sentinel-not-a-png";
    let sk = snapshot_key(GLXGEARS_MOCK_HASH);
    server.put_object(&sk, sentinel);

    let toml_s3 = write_test_toml(
        &server.traces_url(),
        &[("glxgears.mock-trace", GLXGEARS_MOCK_HASH)],
    );
    let mut run = GpuTracePerfRun::new();
    run.setup_ci_replay(toml_s3.path(), "test-device", None);
    run.arg("--snapshots-url");
    run.arg(&server.snapshots_url());
    let out = run.run();
    assert!(
        out.status.success(),
        "Expected exit 0:\nstdout: {}\nstderr: {}",
        out.stdout,
        out.stderr
    );

    // Sentinel must still be intact — no second PUT was made.
    let stored = server
        .get_object(&sk)
        .expect("snapshot key should still exist");
    assert_eq!(
        stored, sentinel,
        "S3 object was overwritten on second run (expected upload to be skipped)\nstdout: {}\nstderr: {}",
        out.stdout, out.stderr,
    );
}

/// Return all full-resolution snapshot PNGs (captured frames) under `dir`.
///
/// Excludes thumbnails (`-preview.png`), downloaded expected images
/// (`-expected.png`) and pixel-diff images (`-diff.png`), which are local
/// artefacts that legitimately remain after upload.
fn find_fullres_pngs(dir: &Path) -> Vec<std::path::PathBuf> {
    walkdir::WalkDir::new(dir)
        .into_iter()
        .filter_map(|e| e.ok())
        .filter(|e| e.file_type().is_file())
        .map(|e| e.into_path())
        .filter(|p| {
            let name = p.file_name().and_then(|n| n.to_str()).unwrap_or("");
            name.ends_with(".png")
                && !name.ends_with("-preview.png")
                && !name.ends_with("-expected.png")
                && !name.ends_with("-diff.png")
        })
        .collect()
}

/// With `--no-upload`, a failing replay does not upload the actual snapshot to S3.
#[test]
pub fn ci_no_upload() {
    let server = S3TestServer::start(&[
        (
            &trace_key("glxgears.mock-trace"),
            Path::new("src/test_data/glxgears.mock-trace"),
        ),
        (
            &snapshot_key(GLXGEARS_MOCK_HASH),
            Path::new("src/test_data/glxgears.mock-trace"),
        ),
    ]);

    let toml = write_ci_test_toml(
        &server.traces_url(),
        &[("glxgears.mock-trace", GLXGEARS_MOCK_HASH, &[])],
    );
    let mut run = GpuTracePerfRun::new();
    let output_dir = run.setup_ci_replay(toml.path(), "test-device", None);
    run.arg("--snapshots-url");
    run.arg(&server.snapshots_url());
    run.arg("--no-upload");
    run.env("MOCK_BREAK_RENDERING", "1");
    run.snapshot_output("ci_no_upload");

    // The actual snapshot must NOT have been uploaded.
    assert!(
        !server.key_exists(&snapshot_key(GLXGEARS_BROKEN_HASH)),
        "Snapshot should not be uploaded with --no-upload",
    );

    check_html_links(output_dir.path());
}

/// A passing replay whose `--snapshots-url` upload is rejected with 403 but
/// `--snapshot-url-must-work` is not set: must exit 0, fall back to
/// `--job-url`, and include the trace in the HTML report.
#[test]
pub fn ci_upload_forbidden() {
    // snapshot server: GET works (for fill_diff_info / link checks); HEAD
    // always returns 404 and PUT always returns 403 so the upload is
    // attempted and rejected, triggering the fallback.
    let snapshot_server = S3TestServer::start_upload_forbidden(&[(
        &trace_key("glxgears.mock-trace"),
        Path::new("src/test_data/glxgears.mock-trace"),
    )]);

    // job server: no auth — this is where the fallback upload lands.
    let job_server = S3TestServer::start(&[]);

    let toml = write_ci_test_toml(
        &snapshot_server.traces_url(),
        &[("glxgears.mock-trace", GLXGEARS_MOCK_HASH, &[])],
    );
    let mut run = GpuTracePerfRun::new();
    let output_dir = run.setup_ci_replay(toml.path(), "test-device", None);
    run.arg("--snapshots-url");
    run.arg(&snapshot_server.snapshots_url());
    run.arg("--job-url");
    run.arg(&job_server.snapshots_url());
    run.snapshot_output("ci_upload_forbidden");

    // The image must have been uploaded to the job server via the fallback.
    assert!(
        job_server.key_exists(&snapshot_key(GLXGEARS_MOCK_HASH)),
        "Fallback image should have been uploaded to job_url"
    );
    // We should have links to the expected image on the job server.
    assert!(html_contains(
        output_dir.path(),
        &format!("{}/{GLXGEARS_MOCK_HASH}.png", job_server.snapshots_url())
    ));

    // All links must resolve.
    check_html_links(output_dir.path());
}

/// When we can't upload, but the file already exists on the snapshot server, we
/// should just link that file, not upload a new copy.
#[test]
pub fn ci_upload_forbidden_file_exists() {
    // snapshot server: GET works (for fill_diff_info / link checks); HEAD
    // always returns 404 and PUT always returns 403 so the upload is
    // attempted and rejected, triggering the fallback.
    let snapshot_server = S3TestServer::start_upload_forbidden(&[
        (
            &trace_key("glxgears.mock-trace"),
            Path::new("src/test_data/glxgears.mock-trace"),
        ),
        (
            &snapshot_key(GLXGEARS_MOCK_HASH),
            Path::new("src/test_data/glxgears.mock-trace"),
        ),
    ]);

    // job server: no auth — this is where the fallback upload lands.
    let job_server = S3TestServer::start(&[]);

    let toml = write_ci_test_toml(
        &snapshot_server.traces_url(),
        &[("glxgears.mock-trace", GLXGEARS_MOCK_HASH, &[])],
    );
    let mut run = GpuTracePerfRun::new();
    let output_dir = run.setup_ci_replay(toml.path(), "test-device", None);
    run.arg("--snapshots-url");
    run.arg(&snapshot_server.snapshots_url());
    run.arg("--job-url");
    run.arg(&job_server.snapshots_url());
    run.snapshot_output("ci_upload_forbidden_file_exists");

    // The image must not have been uploaded to the job URL via the fallback.
    assert!(
        !job_server.key_exists(&snapshot_key(GLXGEARS_MOCK_HASH)),
        "Fallback image should not have been uploaded to job_url"
    );
    // We should have links to the original expected image on the job server.
    assert!(html_contains(
        output_dir.path(),
        &format!(
            "{}/{GLXGEARS_MOCK_HASH}.png",
            snapshot_server.snapshots_url()
        )
    ));

    // All links must resolve.
    check_html_links(output_dir.path());
}

/// Same as `ci_upload_forbidden` but with `--snapshot-url-must-work`: the 403
/// must be a hard failure (FAIL in stderr, non-zero exit) and the trace still
/// appears in the HTML report.
#[test]
pub fn ci_upload_forbidden_must_work() {
    let server = S3TestServer::start_upload_forbidden(&[(
        &trace_key("glxgears.mock-trace"),
        Path::new("src/test_data/glxgears.mock-trace"),
    )]);

    let toml = write_ci_test_toml(
        &server.traces_url(),
        &[("glxgears.mock-trace", GLXGEARS_MOCK_HASH, &[])],
    );
    let mut run = GpuTracePerfRun::new();
    let output_dir = run.setup_ci_replay(toml.path(), "test-device", None);
    run.arg("--snapshots-url");
    run.arg(&server.snapshots_url());
    run.arg("--snapshot-url-must-work");
    run.snapshot_output("ci_upload_forbidden_must_work");

    // The trace row must appear in the HTML
    assert!(html_contains(output_dir.path(), "glxgears.mock-trace"));

    // All links must resolve.
    check_html_links(output_dir.path());
}

/// After a passing replay, `index.html` is written to the output directory
/// and contains the S3 image URLs for both the baseline and the result.
#[test]
pub fn ci_output_html() {
    let server = S3TestServer::start(&[(
        &trace_key("glxgears.mock-trace"),
        Path::new("src/test_data/glxgears.mock-trace"),
    )]);
    let toml_s3 = write_test_toml(
        &server.traces_url(),
        &[("glxgears.mock-trace", GLXGEARS_MOCK_HASH)],
    );
    let mut run = GpuTracePerfRun::new();
    let output_dir = run.setup_ci_replay(toml_s3.path(), "test-device", None);
    run.arg("--snapshots-url");
    run.arg(&server.snapshots_url());
    let out = run.run();

    assert!(
        out.status.success(),
        "Expected exit 0:\nstdout: {}\nstderr: {}",
        out.stdout,
        out.stderr
    );

    // index.html must exist.
    let index_path = output_dir.path().join("index.html");
    assert!(
        index_path.exists(),
        "index.html was not created in output dir"
    );

    let html = std::fs::read_to_string(&index_path).expect("reading index.html");

    // Verify we're using snapshot's index.html template (not ci-replay.html).
    assert!(
        html.contains("Rendering snapshots"),
        "index.html is missing 'Rendering snapshots' heading\n{html}"
    );

    // The S3 image URL for the actual snapshot must appear in the HTML
    // (in data-original2 and href attributes for run_b).
    let actual_image_url = format!("{}/{GLXGEARS_MOCK_HASH}.png", server.snapshots_url());
    assert!(
        html.contains(&actual_image_url),
        "index.html does not contain the actual S3 image URL {actual_image_url}\n{html}"
    );
}

/// checksum-changes.json is written for failing traces and contains the device,
/// trace path, old checksum, and new checksum as structured JSON so external
/// tooling can collate updates across CI jobs.
///
/// ci_replay_pass will do the checking for it not being written when unnecessary.
#[test]
pub fn ci_checksum_changes_json() {
    let server = S3TestServer::start(&[(
        &trace_key("glxgears.mock-trace"),
        Path::new("src/test_data/glxgears.mock-trace"),
    )]);
    let toml = write_ci_test_toml(
        &server.traces_url(),
        &[("glxgears.mock-trace", GLXGEARS_MOCK_HASH, &[])],
    );
    let mut run = GpuTracePerfRun::new();
    let output_dir = run.setup_ci_replay(toml.path(), "test-device", None);
    run.env("MOCK_BREAK_RENDERING", "1");
    let _out = run.run();

    let json_path = output_dir.path().join("checksum-changes.json");
    assert!(
        json_path.exists(),
        "checksum-changes.json should be written when checksums differ"
    );

    let json: serde_json::Value =
        serde_json::from_str(&std::fs::read_to_string(&json_path).unwrap())
            .expect("parsing checksum-changes.json");

    let changes = json
        .as_array()
        .expect("checksum-changes.json is not an array");
    assert_eq!(changes.len(), 1, "expected exactly one change entry");

    let entry = &changes[0];
    assert_eq!(entry["device"], "test-device");
    assert_eq!(entry["trace"], "glxgears.mock-trace");
    assert_eq!(entry["old_checksum"], GLXGEARS_MOCK_HASH);
    assert_eq!(entry["new_checksum"], GLXGEARS_BROKEN_HASH);
}

/// When the config has no traces for the requested device, replay exits 0 and
/// logs an error rather than silently succeeding with no output.
#[test]
pub fn ci_no_traces_for_device() {
    let toml = write_ci_test_toml(
        "http://unused/",
        &[("glxgears.mock-trace", GLXGEARS_MOCK_HASH, &[])],
    );

    let mut run = GpuTracePerfRun::new();
    run.setup_ci_replay(toml.path(), "other-device", None);
    run.snapshot_output("ci_no_traces_for_device");
}

/// A passing gfxreconstruct replay.
#[cfg(feature = "testgfxr")]
#[test]
pub fn ci_replay_gfxr() {
    run_local_replay_and_pass("vkcube-10-frames.gfxr");
}

/// A passing ANGLE trace replay.
#[cfg(feature = "testangle")]
#[test]
pub fn ci_replay_angle() {
    run_local_replay_and_pass("angle_trace_tests/google_maps");
}
