name: CI (slow / overnight)

# Catches the 15+ `#[ignore]`-tagged tests (model-loading, stress, daemon-env)
# that don't run in PR-time CI. Triggered on a daily cron + manual dispatch.
# Failures open a tracking issue rather than blocking merges (overnight runs
# aren't gating).
#
# Background: see #1286.

on:
  schedule:
    # 06:00 UTC daily ≈ 22:00 PT previous day. Once-a-day, not hourly —
    # the suite is heavy and free-tier minutes matter.
    #
    # Re-enabled 2026-05-02 after the #1305 triage closed all 14 surfaced
    # bug classes (slot path, model soft-skip, ref resolution, corrupt
    # cache, BLAKE3 socket, windowing probe, envelope shape, eval DB
    # precheck, format flag, R@1 gate, noise corpus, LLM env-lock,
    # doctest text blocks, ONNX_DIR env-lock). Run #25256531515 was the
    # first fully-green ci-slow.yml execution; both jobs (full-suite and
    # slow-tests-feature) completed cleanly.
    - cron: '0 6 * * *'
  workflow_dispatch:
    inputs:
      include_ignored:
        description: 'Run #[ignore]-tagged tests (model-loading, stress, daemon-env)'
        required: false
        default: 'true'
        type: boolean

permissions:
  contents: read
  issues: write  # auto-file-on-failure step

env:
  CARGO_TERM_COLOR: always
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true

jobs:
  full-suite:
    runs-on: ubuntu-latest
    timeout-minutes: 90
    steps:
      - name: Free up disk space
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/share/boost /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
          sudo docker image prune --all --force || true
          df -h /
      - uses: actions/checkout@v5
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
        with:
          # Distinct shared-key so the slow run doesn't poison the regular
          # CI cache (different feature set, different output binaries).
          shared-key: ci-slow

      - name: Cache HF models
        # Model-loading tests under #[ignore] pull ~3 GB from HF (BGE-large +
        # E5-base + SPLADE + the default EmbeddingGemma); repeated downloads
        # trip HF's 429 rate limit. restore-keys seeds from the PR-time cache.
        uses: actions/cache@v4
        with:
          path: ~/.cache/huggingface
          key: hf-models-slow-v1
          restore-keys: |
            hf-models-

      - name: Build (release)
        run: cargo build --release --verbose

      - name: Warm + verify embedder model
        # The model-loading tests pull the default EmbeddingGemma ONNX,
        # including its big external-weights sidecar `model.onnx_data`. HF
        # downloads are flaky (429 / truncation) and `ensure_model` only WARNS
        # on a failed sidecar fetch — ORT then panics mid-suite with "cannot
        # get file size". Warm via `cqs doctor` (the same `Embedder::new`
        # download path the tests use), retry, and verify the sidecar is
        # present + non-empty, so a bad download fails THIS step loudly instead
        # of poisoning the suite. (#1973 / #1959)
        run: |
          set -uo pipefail
          ok=0
          for attempt in 1 2 3; do
            echo "::group::model warm attempt $attempt"
            cargo run --release --quiet -- doctor || true
            echo "::endgroup::"
            data=$(find "$HOME/.cache/huggingface/hub" -name 'model.onnx_data' -path '*embeddinggemma*' 2>/dev/null | head -1)
            if [ -n "$data" ] && [ -s "$data" ]; then
              ok=1
              echo "model warmed: $data ($(du -h "$data" | cut -f1))"
              break
            fi
            echo "attempt $attempt: model.onnx_data still missing/empty; backing off"
            sleep $((attempt * 20))
          done
          [ "$ok" = 1 ] || { echo "::error::EmbeddingGemma model.onnx_data could not be fetched after 3 attempts"; exit 1; }

      - name: Run full test suite (incl. ignored)
        if: ${{ github.event_name == 'schedule' || inputs.include_ignored != 'false' }}
        # --include-ignored is for #[ignore]d unit/integration tests (GPU,
        # model-download). Doctests run in their own step below WITHOUT it:
        # rustdoc maps ```ignore blocks to #[ignore] doctests, so a combined
        # run force-compiles examples that are ignore-marked precisely
        # because they cannot compile (e.g. pub(crate) item references).
        run: cargo test --release --verbose --lib --bins --tests -- --include-ignored

      - name: Run doctests (normal ignore semantics)
        if: ${{ github.event_name == 'schedule' || inputs.include_ignored != 'false' }}
        run: cargo test --release --verbose --doc

      - name: Run regular suite only (manual dispatch fallback)
        if: ${{ github.event_name == 'workflow_dispatch' && inputs.include_ignored == 'false' }}
        run: cargo test --release --verbose

  slow-tests-feature:
    # Tests gated behind the `slow-tests` cargo feature (subprocess-spawning
    # CLI tests + the heaviest e2e binaries — onboard, eval_subcommand). Most
    # use mock or local-only embedders, but the overlay/embedding tests index a
    # real parent and DO need the default EmbeddingGemma ONNX — so this job
    # warms + verifies the model like full-suite (#1973; the stale "no model
    # download dependency" assumption is what let #1959 panic mid-suite). Could
    # later graduate to PR-time CI if the wall time stays under budget. See
    # Cargo.toml `slow-tests` comment for the gated file inventory. (#1286 Phase 2)
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: Free up disk space
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/share/boost /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
          sudo docker image prune --all --force || true
          df -h /
      - uses: actions/checkout@v5
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
        with:
          shared-key: ci-slow-feature

      - name: Cache HF models
        # The overlay tests need the default EmbeddingGemma ONNX (incl. the
        # ~1GB `model.onnx_data` sidecar); persist it across nights so a good
        # download is reused and a partial one is caught by the warm step.
        uses: actions/cache@v4
        with:
          path: ~/.cache/huggingface
          key: hf-models-slow-feature-v1
          restore-keys: |
            hf-models-

      - name: Warm + verify embedder model
        # See full-suite's warm step — same flaky-download / WARN-on-sidecar
        # rationale. The overlay tests in this suite index a real parent, so a
        # missing `model.onnx_data` panics them at ORT session init. Fail here,
        # not mid-suite. (#1973 / #1959)
        run: |
          set -uo pipefail
          ok=0
          for attempt in 1 2 3; do
            echo "::group::model warm attempt $attempt"
            cargo run --release --features slow-tests --quiet -- doctor || true
            echo "::endgroup::"
            data=$(find "$HOME/.cache/huggingface/hub" -name 'model.onnx_data' -path '*embeddinggemma*' 2>/dev/null | head -1)
            if [ -n "$data" ] && [ -s "$data" ]; then
              ok=1
              echo "model warmed: $data ($(du -h "$data" | cut -f1))"
              break
            fi
            echo "attempt $attempt: model.onnx_data still missing/empty; backing off"
            sleep $((attempt * 20))
          done
          [ "$ok" = 1 ] || { echo "::error::EmbeddingGemma model.onnx_data could not be fetched after 3 attempts"; exit 1; }

      - name: Run slow-tests feature suite
        run: cargo test --release --features slow-tests --verbose

  notify-failure:
    needs: [full-suite, slow-tests-feature]
    if: failure() && github.event_name == 'schedule'
    runs-on: ubuntu-latest
    steps:
      - name: Open issue on failure
        uses: actions/github-script@v7
        with:
          script: |
            const title = `Nightly CI failed on ${context.sha.substr(0, 7)}`;
            const body = [
              `Nightly run [${context.runId}](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}) failed.`,
              ``,
              `Not a PR-blocking failure (overnight cadence). Investigate when convenient.`,
              ``,
              `Triggered by commit ${context.sha} on \`${context.ref}\`.`,
            ].join('\n');
            await github.rest.issues.create({
              owner: context.repo.owner,
              repo: context.repo.repo,
              title,
              body,
              labels: ['ci-flake-or-bug', 'overnight-ci'],
            });
