From 3fba3855138653d23ccd22790db5cf553b0e5152 Mon Sep 17 00:00:00 2001 From: claude Date: Tue, 21 Jul 2026 04:04:30 +0000 Subject: [PATCH] ci: artifact-based coverage and local Firecracker candidate flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each test job now runs once under coverage and uploads a small .coverage.* artifact. The coverage job combines them on ubuntu-latest — no test reruns, no KVM dependency. The infra candidate is built directly on the KVM runner, eliminating the build-infra job and the ~70 s upload + ~83 s combined download. For PRs, no rootfs artifact is transferred at all. Main-branch pushes upload the tested rootfs and matching dropbear so publish-infra publishes the byte-identical artifact. relative_files = True in .coveragerc lets coverage files from different runners combine without path remapping. Closes #446 --- .coveragerc | 4 + .gitea/workflows/test.yml | 211 ++++++++++------------ docs/prds/prd-new-ci-artifact-coverage.md | 110 +++++++++++ scripts/coverage.sh | 36 +++- 4 files changed, 241 insertions(+), 120 deletions(-) create mode 100644 docs/prds/prd-new-ci-artifact-coverage.md diff --git a/.coveragerc b/.coveragerc index 7dc1873..161fde3 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,10 @@ [run] branch = True source = . +# Store paths relative to the project root so .coverage.* files produced on +# different runners (ubuntu-latest vs self-hosted KVM) can be combined by the +# coverage job without a [paths] remapping section. +relative_files = True [report] # Coverage policy: see docs/decisions/0004-coverage-policy.md. diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index 155da5c..fa66b3f 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -9,10 +9,12 @@ # tests/canaries/ — upstream regression canaries; run on a separate # schedule (see canaries.yml), not here # -# Integration tests run once per backend in separate jobs. Each job sets -# BOT_BOTTLE_BACKEND explicitly so the test suite uses the right backend. -# Backends that aren't available on the runner fail the preflight step -# rather than silently skipping inside the test output. +# Each test job runs once under coverage and uploads a small .coverage.* +# artifact. The `coverage` job combines them — no test reruns, no KVM +# dependency on that job. For main-branch pushes only, the tested rootfs +# and matching dropbear are uploaded so `publish-infra` can publish the +# byte-identical artifact that was tested. PRs avoid the ~194 MB rootfs +# transfer entirely. name: test @@ -40,53 +42,6 @@ on: workflow_dispatch: jobs: - stage-firecracker-inputs: - runs-on: [self-hosted, kvm] - # Same guard as the other KVM-runner jobs: don't spin the privileged - # runner for fork PRs (this only copies a non-secret static binary, but - # keep the posture consistent — build-infra/integration/coverage all - # depend on it, so gating here gates the whole Firecracker chain). - if: >- - github.event_name == 'push' || - github.event_name == 'workflow_dispatch' || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name == github.repository) - steps: - - name: Stage the provisioned static dropbear - run: | - mkdir -p firecracker-inputs - cp /var/cache/bot-bottle-fc/dropbear firecracker-inputs/dropbear - - - name: Upload Firecracker build inputs - uses: actions/upload-artifact@v3 - with: - name: firecracker-inputs - path: firecracker-inputs/ - - build-infra: - needs: stage-firecracker-inputs - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Download Firecracker build inputs - uses: actions/download-artifact@v3 - with: - name: firecracker-inputs - path: firecracker-inputs - - - name: Build infra candidate from this checkout - env: - BOT_BOTTLE_FC_DROPBEAR: ${{ github.workspace }}/firecracker-inputs/dropbear - run: python3 -m bot_bottle.backend.firecracker.publish_infra --output infra-candidate - - - name: Upload infra candidate - uses: actions/upload-artifact@v3 - with: - name: infra-candidate - path: infra-candidate/ - unit: runs-on: ubuntu-latest steps: @@ -101,11 +56,17 @@ jobs: - name: Install dev requirements run: python3 -m pip install --break-system-packages -r requirements-dev.txt - - name: Run unit tests - run: python3 -m coverage run -m unittest discover -t . -s tests/unit -v + - name: Run unit tests with coverage + run: python3 -m coverage run --data-file=.coverage.unit -m unittest discover -t . -s tests/unit -v - name: Report unit coverage - run: python3 -m coverage report -m + run: python3 -m coverage report --data-file=.coverage.unit -m + + - name: Upload unit coverage artifact + uses: actions/upload-artifact@v3 + with: + name: coverage-unit + path: .coverage.unit integration-docker: runs-on: ubuntu-latest @@ -115,6 +76,9 @@ jobs: # No actions/setup-python (see the note in the `unit` job); the # container's system Python 3.12 runs the stdlib test suite directly. + - name: Install coverage + run: python3 -m pip install --break-system-packages coverage + - name: Show environment run: | python3 --version @@ -124,10 +88,16 @@ jobs: echo "docker not on PATH — integration tests will skip" fi - - name: Run integration tests (docker) + - name: Run integration tests (docker) with coverage env: BOT_BOTTLE_BACKEND: docker - run: python3 -m unittest discover -t . -s tests/integration -v + run: python3 -m coverage run --data-file=.coverage.docker -m unittest discover -t . -s tests/integration -v + + - name: Upload docker coverage artifact + uses: actions/upload-artifact@v3 + with: + name: coverage-docker + path: .coverage.docker # Integration tests against the Firecracker backend. Runs on a self-hosted # KVM runner (label `kvm`) where /dev/kvm and the TAP/nft pool are available. @@ -137,9 +107,16 @@ jobs: # # Runner prerequisites (provision once; see README "Firecracker on Linux"): # `firecracker` on PATH, `/dev/kvm` accessible, cached kernel + - # static dropbear, and the pool as a persistent systemd unit. + # static dropbear at /var/cache/bot-bottle-fc/dropbear, and the pool as a + # persistent systemd unit. + # + # The infra candidate is built here directly (no artifact download) to + # eliminate the ~70 s ubuntu-latest upload + ~83 s combined download that + # the old build-infra → integration-firecracker + coverage chain incurred. + # For main-branch pushes the tested rootfs and matching dropbear are + # uploaded so publish-infra can publish the byte-identical artifact; PRs + # skip those uploads entirely. integration-firecracker: - needs: build-infra runs-on: [self-hosted, kvm] if: >- github.event_name == 'push' || @@ -159,49 +136,58 @@ jobs: # range overlap; it prints the exact `backend setup` fix. python3 cli.py backend status --backend=firecracker - - name: Download the candidate built from this checkout - uses: actions/download-artifact@v3 - with: - name: infra-candidate - path: infra-candidate + - name: Build infra candidate from this checkout + env: + BOT_BOTTLE_FC_DROPBEAR: /var/cache/bot-bottle-fc/dropbear + run: python3 -m bot_bottle.backend.firecracker.publish_infra --output infra-candidate - name: Replace the persistent infra VM with the candidate run: python3 -c 'from bot_bottle.backend.firecracker import infra_vm; infra_vm.stop()' - # No dev-requirements install: the integration suite runs on stdlib - # `unittest` (pylint/pyright are lint.yml's concern, not this job's), - # and the self-hosted runner's Nix python env has no `pip` module - # (`python3 -m pip` → "No module named pip"). Nothing to install. - - name: Run integration tests (firecracker) + # No dev-requirements install: `coverage` is already provided by the + # self-hosted runner's Nix python env, and that env has no `pip` + # module to install into anyway. + - name: Run integration tests (firecracker) with coverage env: BOT_BOTTLE_BACKEND: firecracker BOT_BOTTLE_INFRA_ARTIFACT_DIR: ${{ github.workspace }}/infra-candidate - run: python3 -m unittest discover -t . -s tests/integration -v + run: python3 -m coverage run --data-file=.coverage.firecracker -m unittest discover -t . -s tests/integration -v - # Combined unit+integration coverage + the diff-coverage gate (the hard - # gate: new/changed lines >= 90%). See docs/decisions/0004-coverage-policy.md. + - name: Upload firecracker coverage artifact + uses: actions/upload-artifact@v3 + with: + name: coverage-firecracker + path: .coverage.firecracker + + # Only upload the large rootfs artifact on main-branch pushes; + # PRs avoid the ~194 MB transfer. publish-infra only runs on main + # and downloads these to publish the byte-identical tested rootfs. + - name: Upload tested rootfs (main branch only) + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: actions/upload-artifact@v3 + with: + name: infra-candidate + path: infra-candidate/ + + - name: Upload dropbear for publish verification (main branch only) + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: actions/upload-artifact@v3 + with: + name: firecracker-inputs + path: /var/cache/bot-bottle-fc/dropbear + + # Combined coverage gate: aggregates .coverage.* artifacts uploaded by each + # test job, then runs the diff-coverage gate (new/changed lines >= 90%). # - # This runs on a self-hosted KVM runner (label `kvm`), NOT ubuntu-latest, - # because the Firecracker backend's subprocess/VM orchestration - # (launch/boot/SSH/isolation-probe) is covered by the integration suite, - # and that suite needs `/dev/kvm` + the provisioned TAP/nft pool — which a - # container-based runner doesn't have. On such a runner the firecracker - # integration test skips and its ~230 orchestration lines read as - # uncovered, so the gate can't pass there. + # Runs on ubuntu-latest — no KVM needed, no test reruns. Coverage files use + # relative_files = True (.coveragerc) so they combine cleanly across runners. # - # Restricted to the same events as integration-firecracker (same-repo PRs, - # push, workflow_dispatch) for the same security reason. - # - # See #414 for the planned follow-up: artifact-based coverage combination - # (run tests once in their respective jobs, combine .coverage files here). - # - # build-infra creates one candidate from the checkout. This job boots that - # same candidate after integration-firecracker has exercised it; the main - # push path publishes the identical bytes only after every required job. + # Restricted to the same events as integration-firecracker: it depends on + # that job's coverage artifact and skips for fork PRs alongside it. coverage: - needs: [build-infra, integration-firecracker] + needs: [unit, integration-docker, integration-firecracker] timeout-minutes: 15 - runs-on: [self-hosted, kvm] + runs-on: ubuntu-latest if: >- github.event_name == 'push' || github.event_name == 'workflow_dispatch' || @@ -213,29 +199,29 @@ jobs: with: fetch-depth: 0 - - name: Preflight — Firecracker host is ready - run: | - command -v firecracker >/dev/null || { - echo "firecracker not on PATH — provision the runner (README: Firecracker on Linux)"; exit 1; } - test -e /dev/kvm || { echo "/dev/kvm missing — KVM not available on this runner"; exit 1; } - # `backend status` exits non-zero unless the TAP pool is up + no - # range overlap; it prints the exact `backend setup` fix. - python3 cli.py backend status --backend=firecracker + - name: Install coverage + run: python3 -m pip install --break-system-packages coverage - - name: Download the candidate already exercised by integration + - name: Download unit coverage artifact uses: actions/download-artifact@v3 with: - name: infra-candidate - path: infra-candidate + name: coverage-unit + path: . + + - name: Download docker coverage artifact + uses: actions/download-artifact@v3 + with: + name: coverage-docker + path: . + + - name: Download firecracker coverage artifact + uses: actions/download-artifact@v3 + with: + name: coverage-firecracker + path: . - # No dev-requirements install: `coverage` is already provided by the - # self-hosted runner's Nix python env, and that env has no `pip` - # module to install into anyway. `scripts/coverage.sh` + - # `diff_coverage.py` need only `coverage` (not pylint/pyright). - name: Combined coverage (unit + integration, incl. firecracker) - env: - BOT_BOTTLE_CI_INFRA_ARTIFACT_DIR: ${{ github.workspace }}/infra-candidate - run: PYTHON=python3 bash scripts/coverage.sh critical + run: PYTHON=python3 bash scripts/coverage.sh aggregate critical - name: Diff-coverage gate (changed lines >= 90%) run: | @@ -243,14 +229,14 @@ jobs: python3 scripts/diff_coverage.py --base origin/main --min 90 publish-infra: - needs: [stage-firecracker-inputs, build-infra, unit, integration-docker, integration-firecracker, coverage] + needs: [unit, integration-docker, integration-firecracker, coverage] runs-on: ubuntu-latest if: github.event_name == 'push' && github.ref == 'refs/heads/main' steps: - name: Checkout the tested revision uses: actions/checkout@v4 - - name: Download the tested candidate + - name: Download the tested rootfs uses: actions/download-artifact@v3 with: name: infra-candidate @@ -258,9 +244,10 @@ jobs: # publish_infra re-derives the version from the checkout to confirm the # bundle matches before uploading, and the version hashes the dropbear - # bytes. Stage the SAME dropbear build-infra used, or the recheck - # computes a ""-dropbear version and rejects the candidate. - - name: Download the staged dropbear (matches build-infra's version) + # bytes. Download the SAME dropbear integration-firecracker used, or + # the recheck computes a ""-dropbear version and rejects the + # candidate. + - name: Download the staged dropbear (matches build's version) uses: actions/download-artifact@v3 with: name: firecracker-inputs diff --git a/docs/prds/prd-new-ci-artifact-coverage.md b/docs/prds/prd-new-ci-artifact-coverage.md new file mode 100644 index 0000000..b14c897 --- /dev/null +++ b/docs/prds/prd-new-ci-artifact-coverage.md @@ -0,0 +1,110 @@ +# PRD prd-new: CI artifact-based coverage and local Firecracker candidate flow + +- **Status:** Active +- **Author:** Claude +- **Created:** 2026-07-21 +- **Issue:** #446 + +## Summary + +Restructure the CI test pipeline to run each test suite exactly once, upload +small `.coverage.*` artifacts, and combine them in a lightweight aggregation +job. Move the infra build onto the KVM runner so the ~194 MB rootfs never +crosses the network for PRs. On main-branch pushes, publish the byte-identical +rootfs that was tested. + +## Motivation + +The prior pipeline had two redundant costs: + +1. **Duplicate artifact transfers.** `build-infra` (ubuntu-latest) built and + uploaded the ~194 MB rootfs; `integration-firecracker` downloaded it; the + `coverage` job downloaded it a second time. Combined download overhead: ~83 + seconds per run, plus the ~70-second upload. + +2. **Duplicate test execution.** `integration-firecracker` ran the Firecracker + integration suite; `coverage` ran the entire unit + integration suite again + on the same KVM runner to collect coverage data. Every line of Firecracker + code was tested twice per CI run. + +## Goals + +- Each test suite (unit, integration-docker, integration-firecracker) executes + exactly once per workflow run. +- PRs incur no large artifact transfers — the rootfs stays on the KVM runner. +- Main-branch pushes publish a byte-for-byte identical rootfs to the one that + passed the integration tests. +- Concurrent workflow runs cannot cross-publish candidates (naturally enforced + by Gitea Actions' per-run artifact scoping). +- Failed or cancelled runs block publication (enforced by the `needs:` chain on + `publish-infra`). + +## Non-goals + +- Changing test semantics or the coverage policy (ADR 0004). +- Removing the KVM runner guard on `integration-firecracker` and `coverage`. +- Changing how `publish_infra.py` builds or uploads the rootfs. + +## Design + +### Job graph + +``` +unit ──────────────────────────────────┐ +integration-docker ────────────────────┤──► coverage ──► publish-infra (main only) +integration-firecracker (KVM) ─────────┘ +``` + +### `unit` + +Unchanged except: `coverage run` writes `--data-file=.coverage.unit`; the file +is uploaded as the `coverage-unit` artifact. + +### `integration-docker` + +Adds a `coverage` install step. `coverage run` writes `--data-file=.coverage.docker`; +the file is uploaded as `coverage-docker`. + +### `integration-firecracker` (KVM runner) + +Replaces the old `stage-firecracker-inputs` → `build-infra` → download chain: + +1. Builds the infra candidate locally with + `BOT_BOTTLE_FC_DROPBEAR=/var/cache/bot-bottle-fc/dropbear`. +2. Boots the candidate and runs integration tests with coverage, writing + `.coverage.firecracker`. +3. Uploads the small `coverage-firecracker` artifact unconditionally. +4. On main-branch pushes only, uploads the rootfs as `infra-candidate` and the + dropbear as `firecracker-inputs` so `publish-infra` can verify and publish + the byte-identical artifact. + +### `coverage` + +Moves from a KVM runner to `ubuntu-latest`. No tests are re-executed: + +1. Downloads `coverage-unit`, `coverage-docker`, and `coverage-firecracker`. +2. Runs `scripts/coverage.sh aggregate critical`, which calls + `coverage combine` then `coverage report`. +3. Runs the diff-coverage gate (`scripts/diff_coverage.py`). + +Coverage files use `relative_files = True` (`.coveragerc`) so they combine +cleanly across runners with different absolute workspace paths. + +### `publish-infra` + +Depends on all four predecessor jobs (unchanged gate). Downloads `infra-candidate` +and `firecracker-inputs` that were uploaded by `integration-firecracker` on +main — the same byte sequence that passed the integration tests. + +### Eliminated jobs + +- `stage-firecracker-inputs`: existed only to copy the dropbear to ubuntu-latest + for `build-infra`. No longer needed. +- `build-infra`: the infra candidate is now built on the KVM runner in + `integration-firecracker`. + +### Script changes + +`scripts/coverage.sh` gains an `aggregate` mode (`coverage.sh aggregate [critical]`) +that combines pre-existing `.coverage.*` files instead of re-running tests. +The existing run mode (`coverage.sh [critical]`) is preserved for local dev. diff --git a/scripts/coverage.sh b/scripts/coverage.sh index 33a3e2f..b202cc7 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -1,15 +1,19 @@ #!/usr/bin/env bash # Combined unit + integration coverage (see docs/decisions/0004-coverage-policy.md). # -# Runs the unit suite, then appends the integration suite (which skips -# cleanly when Docker / the backend CLIs are unavailable), and prints one -# combined report. The integration suite is what scores the subprocess / -# backend orchestration modules, so the number here is the policy's -# yardstick — not the unit-only badge. +# Two modes: # -# Usage: -# scripts/coverage.sh # combined report -# scripts/coverage.sh critical # also report just the critical modules +# scripts/coverage.sh [critical] +# Run mode (default, for local dev): executes the unit suite then the +# integration suite under coverage and prints a combined report. +# +# scripts/coverage.sh aggregate [critical] +# Aggregate mode (used by CI): combines pre-existing .coverage.* files +# produced by individual test jobs and prints a combined report. No tests +# are re-executed; no KVM or Docker dependency. +# +# Pass "critical" as the last argument in either mode to also report just the +# critical modules (ADR 0004 target: 90%). set -euo pipefail cd "$(dirname "$0")/.." @@ -21,6 +25,22 @@ PY="${PYTHON:-python3}" # README "core coverage" badge can't drift; comma-join it for --include. CRITICAL=$(grep -vE '^[[:space:]]*(#|$)' scripts/critical-modules.txt | paste -sd, -) +if [ "${1:-}" = "aggregate" ]; then + # Aggregate mode: combine .coverage.* artifacts already in the workspace. + echo "== combining coverage artifacts ==" >&2 + "$PY" -m coverage combine + + echo "== combined report ==" >&2 + "$PY" -m coverage report -m + + if [ "${2:-}" = "critical" ]; then + echo "== critical modules (ADR 0004 target: 90%) ==" >&2 + "$PY" -m coverage report --include="$CRITICAL" + fi + exit 0 +fi + +# Run mode (default): execute both suites under coverage in this process. rm -f .coverage echo "== unit ==" >&2