PRD: CI artifact-based coverage and local Firecracker candidate flow #447
@@ -1,6 +1,10 @@
|
|||||||
[run]
|
[run]
|
||||||
branch = True
|
branch = True
|
||||||
source = .
|
source = .
|
||||||
|
# Store paths relative to the project root so .coverage.* files produced on
|
||||||
|
# different runners (ubuntu-latest vs self-hosted KVM) can be combined by the
|
||||||
|
# coverage job without a [paths] remapping section.
|
||||||
|
relative_files = True
|
||||||
|
|
||||||
[report]
|
[report]
|
||||||
# Coverage policy: see docs/decisions/0004-coverage-policy.md.
|
# Coverage policy: see docs/decisions/0004-coverage-policy.md.
|
||||||
|
|||||||
+128
-110
@@ -9,10 +9,12 @@
|
|||||||
# tests/canaries/ — upstream regression canaries; run on a separate
|
# tests/canaries/ — upstream regression canaries; run on a separate
|
||||||
# schedule (see canaries.yml), not here
|
# schedule (see canaries.yml), not here
|
||||||
#
|
#
|
||||||
# Integration tests run once per backend in separate jobs. Each job sets
|
# Each test job runs once under coverage and uploads a small .coverage.*
|
||||||
# BOT_BOTTLE_BACKEND explicitly so the test suite uses the right backend.
|
# artifact. The `coverage` job combines them — no test reruns, no KVM
|
||||||
# Backends that aren't available on the runner fail the preflight step
|
# dependency on that job. For main-branch pushes only, the tested rootfs
|
||||||
# rather than silently skipping inside the test output.
|
# and matching dropbear are uploaded so `publish-infra` can publish the
|
||||||
|
# byte-identical artifact that was tested. PRs avoid the ~194 MB rootfs
|
||||||
|
# transfer entirely.
|
||||||
|
|
||||||
name: test
|
name: test
|
||||||
|
|
||||||
@@ -40,53 +42,6 @@ on:
|
|||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
stage-firecracker-inputs:
|
|
||||||
runs-on: [self-hosted, kvm]
|
|
||||||
# Same guard as the other KVM-runner jobs: don't spin the privileged
|
|
||||||
# runner for fork PRs (this only copies a non-secret static binary, but
|
|
||||||
# keep the posture consistent — build-infra/integration/coverage all
|
|
||||||
# depend on it, so gating here gates the whole Firecracker chain).
|
|
||||||
if: >-
|
|
||||||
github.event_name == 'push' ||
|
|
||||||
github.event_name == 'workflow_dispatch' ||
|
|
||||||
(github.event_name == 'pull_request' &&
|
|
||||||
github.event.pull_request.head.repo.full_name == github.repository)
|
|
||||||
steps:
|
|
||||||
- name: Stage the provisioned static dropbear
|
|
||||||
run: |
|
|
||||||
mkdir -p firecracker-inputs
|
|
||||||
cp /var/cache/bot-bottle-fc/dropbear firecracker-inputs/dropbear
|
|
||||||
|
|
||||||
- name: Upload Firecracker build inputs
|
|
||||||
uses: actions/upload-artifact@v3
|
|
||||||
with:
|
|
||||||
name: firecracker-inputs
|
|
||||||
path: firecracker-inputs/
|
|
||||||
|
|
||||||
build-infra:
|
|
||||||
needs: stage-firecracker-inputs
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Download Firecracker build inputs
|
|
||||||
uses: actions/download-artifact@v3
|
|
||||||
with:
|
|
||||||
name: firecracker-inputs
|
|
||||||
path: firecracker-inputs
|
|
||||||
|
|
||||||
- name: Build infra candidate from this checkout
|
|
||||||
env:
|
|
||||||
BOT_BOTTLE_FC_DROPBEAR: ${{ github.workspace }}/firecracker-inputs/dropbear
|
|
||||||
run: python3 -m bot_bottle.backend.firecracker.publish_infra --output infra-candidate
|
|
||||||
|
|
||||||
- name: Upload infra candidate
|
|
||||||
uses: actions/upload-artifact@v3
|
|
||||||
with:
|
|
||||||
name: infra-candidate
|
|
||||||
path: infra-candidate/
|
|
||||||
|
|
||||||
unit:
|
unit:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
@@ -101,12 +56,29 @@ jobs:
|
|||||||
- name: Install dev requirements
|
- name: Install dev requirements
|
||||||
run: python3 -m pip install --break-system-packages -r requirements-dev.txt
|
run: python3 -m pip install --break-system-packages -r requirements-dev.txt
|
||||||
|
|
||||||
- name: Run unit tests
|
- name: Run unit tests with coverage
|
||||||
|
env:
|
||||||
|
COVERAGE_FILE: ${{ github.workspace }}/.coverage.unit
|
||||||
run: python3 -m coverage run -m unittest discover -t . -s tests/unit -v
|
run: python3 -m coverage run -m unittest discover -t . -s tests/unit -v
|
||||||
|
|
||||||
- name: Report unit coverage
|
- name: Report unit coverage
|
||||||
|
env:
|
||||||
|
COVERAGE_FILE: ${{ github.workspace }}/.coverage.unit
|
||||||
run: python3 -m coverage report -m
|
run: python3 -m coverage report -m
|
||||||
|
|
||||||
|
# upload-artifact@v3's glob skips dotfiles, so a bare `.coverage.unit`
|
||||||
|
# silently uploads nothing ("No files were found"). Stage it under a
|
||||||
|
# non-dot name; the coverage job renames it back before `coverage
|
||||||
|
# combine`. `cp` also fails loudly if coverage never wrote the file.
|
||||||
|
- name: Stage unit coverage for upload
|
||||||
|
run: cp .coverage.unit coverage-unit.dat
|
||||||
|
|
||||||
|
- name: Upload unit coverage artifact
|
||||||
|
uses: actions/upload-artifact@v3
|
||||||
|
with:
|
||||||
|
name: coverage-unit
|
||||||
|
path: coverage-unit.dat
|
||||||
|
|
||||||
integration-docker:
|
integration-docker:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
@@ -115,6 +87,9 @@ jobs:
|
|||||||
|
|
||||||
# No actions/setup-python (see the note in the `unit` job); the
|
# No actions/setup-python (see the note in the `unit` job); the
|
||||||
# container's system Python 3.12 runs the stdlib test suite directly.
|
# container's system Python 3.12 runs the stdlib test suite directly.
|
||||||
|
- name: Install coverage
|
||||||
|
run: python3 -m pip install --break-system-packages coverage
|
||||||
|
|
||||||
- name: Show environment
|
- name: Show environment
|
||||||
run: |
|
run: |
|
||||||
python3 --version
|
python3 --version
|
||||||
@@ -124,10 +99,21 @@ jobs:
|
|||||||
echo "docker not on PATH — integration tests will skip"
|
echo "docker not on PATH — integration tests will skip"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Run integration tests (docker)
|
- name: Run integration tests (docker) with coverage
|
||||||
env:
|
env:
|
||||||
BOT_BOTTLE_BACKEND: docker
|
BOT_BOTTLE_BACKEND: docker
|
||||||
run: python3 -m unittest discover -t . -s tests/integration -v
|
COVERAGE_FILE: ${{ github.workspace }}/.coverage.docker
|
||||||
|
run: python3 -m coverage run -m unittest discover -t . -s tests/integration -v
|
||||||
|
|
||||||
|
# Non-dot name so upload-artifact's dotfile-skipping glob picks it up.
|
||||||
|
- name: Stage docker coverage for upload
|
||||||
|
run: cp .coverage.docker coverage-docker.dat
|
||||||
|
|
||||||
|
- name: Upload docker coverage artifact
|
||||||
|
uses: actions/upload-artifact@v3
|
||||||
|
with:
|
||||||
|
name: coverage-docker
|
||||||
|
path: coverage-docker.dat
|
||||||
|
|
||||||
# Integration tests against the Firecracker backend. Runs on a self-hosted
|
# Integration tests against the Firecracker backend. Runs on a self-hosted
|
||||||
# KVM runner (label `kvm`) where /dev/kvm and the TAP/nft pool are available.
|
# KVM runner (label `kvm`) where /dev/kvm and the TAP/nft pool are available.
|
||||||
@@ -137,9 +123,16 @@ jobs:
|
|||||||
#
|
#
|
||||||
# Runner prerequisites (provision once; see README "Firecracker on Linux"):
|
# Runner prerequisites (provision once; see README "Firecracker on Linux"):
|
||||||
# `firecracker` on PATH, `/dev/kvm` accessible, cached kernel +
|
# `firecracker` on PATH, `/dev/kvm` accessible, cached kernel +
|
||||||
# static dropbear, and the pool as a persistent systemd unit.
|
# static dropbear at /var/cache/bot-bottle-fc/dropbear, and the pool as a
|
||||||
|
# persistent systemd unit.
|
||||||
|
#
|
||||||
|
# The infra candidate is built here directly (no artifact download) to
|
||||||
|
# eliminate the ~70 s ubuntu-latest upload + ~83 s combined download that
|
||||||
|
# the old build-infra → integration-firecracker + coverage chain incurred.
|
||||||
|
# For main-branch pushes the tested rootfs and matching dropbear are
|
||||||
|
# uploaded so publish-infra can publish the byte-identical artifact; PRs
|
||||||
|
# skip those uploads entirely.
|
||||||
integration-firecracker:
|
integration-firecracker:
|
||||||
needs: build-infra
|
|
||||||
runs-on: [self-hosted, kvm]
|
runs-on: [self-hosted, kvm]
|
||||||
if: >-
|
if: >-
|
||||||
github.event_name == 'push' ||
|
github.event_name == 'push' ||
|
||||||
@@ -159,49 +152,65 @@ jobs:
|
|||||||
# range overlap; it prints the exact `backend setup` fix.
|
# range overlap; it prints the exact `backend setup` fix.
|
||||||
python3 cli.py backend status --backend=firecracker
|
python3 cli.py backend status --backend=firecracker
|
||||||
|
|
||||||
- name: Download the candidate built from this checkout
|
- name: Build infra candidate from this checkout
|
||||||
uses: actions/download-artifact@v3
|
env:
|
||||||
with:
|
BOT_BOTTLE_FC_DROPBEAR: /var/cache/bot-bottle-fc/dropbear
|
||||||
name: infra-candidate
|
run: python3 -m bot_bottle.backend.firecracker.publish_infra --output infra-candidate --reuse-published
|
||||||
path: infra-candidate
|
|
||||||
|
|
||||||
- name: Replace the persistent infra VM with the candidate
|
- name: Replace the persistent infra VM with the candidate
|
||||||
run: python3 -c 'from bot_bottle.backend.firecracker import infra_vm; infra_vm.stop()'
|
run: python3 -c 'from bot_bottle.backend.firecracker import infra_vm; infra_vm.stop()'
|
||||||
|
|
||||||
# No dev-requirements install: the integration suite runs on stdlib
|
# No dev-requirements install: `coverage` is already provided by the
|
||||||
# `unittest` (pylint/pyright are lint.yml's concern, not this job's),
|
# self-hosted runner's Nix python env, and that env has no `pip`
|
||||||
# and the self-hosted runner's Nix python env has no `pip` module
|
# module to install into anyway.
|
||||||
# (`python3 -m pip` → "No module named pip"). Nothing to install.
|
- name: Run integration tests (firecracker) with coverage
|
||||||
- name: Run integration tests (firecracker)
|
|
||||||
env:
|
env:
|
||||||
BOT_BOTTLE_BACKEND: firecracker
|
BOT_BOTTLE_BACKEND: firecracker
|
||||||
BOT_BOTTLE_INFRA_ARTIFACT_DIR: ${{ github.workspace }}/infra-candidate
|
BOT_BOTTLE_INFRA_ARTIFACT_DIR: ${{ github.workspace }}/infra-candidate
|
||||||
run: python3 -m unittest discover -t . -s tests/integration -v
|
COVERAGE_FILE: ${{ github.workspace }}/.coverage.firecracker
|
||||||
|
run: python3 -m coverage run -m unittest discover -t . -s tests/integration -v
|
||||||
|
|
||||||
# Combined unit+integration coverage + the diff-coverage gate (the hard
|
# Non-dot name so upload-artifact's dotfile-skipping glob picks it up.
|
||||||
# gate: new/changed lines >= 90%). See docs/decisions/0004-coverage-policy.md.
|
- name: Stage firecracker coverage for upload
|
||||||
|
run: cp .coverage.firecracker coverage-firecracker.dat
|
||||||
|
|
||||||
|
- name: Upload firecracker coverage artifact
|
||||||
|
uses: actions/upload-artifact@v3
|
||||||
|
with:
|
||||||
|
name: coverage-firecracker
|
||||||
|
path: coverage-firecracker.dat
|
||||||
|
|
||||||
|
# Only upload the large rootfs artifact on main-branch pushes;
|
||||||
|
# PRs avoid the ~194 MB transfer. publish-infra only runs on main
|
||||||
|
# and downloads these to publish the byte-identical tested rootfs.
|
||||||
|
- name: Upload tested rootfs (main branch only)
|
||||||
|
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||||
|
uses: actions/upload-artifact@v3
|
||||||
|
with:
|
||||||
|
name: infra-candidate
|
||||||
|
path: infra-candidate/
|
||||||
|
|
||||||
|
- name: Upload dropbear for publish verification (main branch only)
|
||||||
|
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||||
|
uses: actions/upload-artifact@v3
|
||||||
|
with:
|
||||||
|
name: firecracker-inputs
|
||||||
|
path: /var/cache/bot-bottle-fc/dropbear
|
||||||
|
|
||||||
|
# Combined coverage gate: aggregates .coverage.* artifacts uploaded by each
|
||||||
|
# test job, then runs the diff-coverage gate (new/changed lines >= 90%).
|
||||||
#
|
#
|
||||||
# This runs on a self-hosted KVM runner (label `kvm`), NOT ubuntu-latest,
|
# Runs on ubuntu-latest — no KVM needed, no test reruns. Coverage files use
|
||||||
# because the Firecracker backend's subprocess/VM orchestration
|
# relative_files = True (.coveragerc) so they combine cleanly across runners.
|
||||||
# (launch/boot/SSH/isolation-probe) is covered by the integration suite,
|
# Each test job sets COVERAGE_FILE to an absolute path so coverage.py writes
|
||||||
# and that suite needs `/dev/kvm` + the provisioned TAP/nft pool — which a
|
# to a known location that upload-artifact can find regardless of runner env.
|
||||||
# container-based runner doesn't have. On such a runner the firecracker
|
|
||||||
# integration test skips and its ~230 orchestration lines read as
|
|
||||||
# uncovered, so the gate can't pass there.
|
|
||||||
#
|
#
|
||||||
# Restricted to the same events as integration-firecracker (same-repo PRs,
|
# Restricted to the same events as integration-firecracker: it depends on
|
||||||
# push, workflow_dispatch) for the same security reason.
|
# that job's coverage artifact and skips for fork PRs alongside it.
|
||||||
#
|
|
||||||
# See #414 for the planned follow-up: artifact-based coverage combination
|
|
||||||
# (run tests once in their respective jobs, combine .coverage files here).
|
|
||||||
#
|
|
||||||
# build-infra creates one candidate from the checkout. This job boots that
|
|
||||||
# same candidate after integration-firecracker has exercised it; the main
|
|
||||||
# push path publishes the identical bytes only after every required job.
|
|
||||||
coverage:
|
coverage:
|
||||||
needs: [build-infra, integration-firecracker]
|
needs: [unit, integration-docker, integration-firecracker]
|
||||||
timeout-minutes: 15
|
timeout-minutes: 15
|
||||||
runs-on: [self-hosted, kvm]
|
runs-on: ubuntu-latest
|
||||||
if: >-
|
if: >-
|
||||||
github.event_name == 'push' ||
|
github.event_name == 'push' ||
|
||||||
github.event_name == 'workflow_dispatch' ||
|
github.event_name == 'workflow_dispatch' ||
|
||||||
@@ -213,29 +222,37 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
|
|
||||||
- name: Preflight — Firecracker host is ready
|
- name: Install coverage
|
||||||
run: |
|
run: python3 -m pip install --break-system-packages coverage
|
||||||
command -v firecracker >/dev/null || {
|
|
||||||
echo "firecracker not on PATH — provision the runner (README: Firecracker on Linux)"; exit 1; }
|
|
||||||
test -e /dev/kvm || { echo "/dev/kvm missing — KVM not available on this runner"; exit 1; }
|
|
||||||
# `backend status` exits non-zero unless the TAP pool is up + no
|
|
||||||
# range overlap; it prints the exact `backend setup` fix.
|
|
||||||
python3 cli.py backend status --backend=firecracker
|
|
||||||
|
|
||||||
- name: Download the candidate already exercised by integration
|
- name: Download unit coverage artifact
|
||||||
uses: actions/download-artifact@v3
|
uses: actions/download-artifact@v3
|
||||||
with:
|
with:
|
||||||
name: infra-candidate
|
name: coverage-unit
|
||||||
path: infra-candidate
|
path: ${{ github.workspace }}
|
||||||
|
|
||||||
|
- name: Download docker coverage artifact
|
||||||
|
uses: actions/download-artifact@v3
|
||||||
|
with:
|
||||||
|
name: coverage-docker
|
||||||
|
path: ${{ github.workspace }}
|
||||||
|
|
||||||
|
- name: Download firecracker coverage artifact
|
||||||
|
uses: actions/download-artifact@v3
|
||||||
|
with:
|
||||||
|
name: coverage-firecracker
|
||||||
|
path: ${{ github.workspace }}
|
||||||
|
|
||||||
|
# Rename the non-dot upload names back to the .coverage.* files that
|
||||||
|
# `coverage combine` discovers (see the staging steps in each test job).
|
||||||
|
- name: Reassemble coverage data files
|
||||||
|
run: |
|
||||||
|
mv coverage-unit.dat .coverage.unit
|
||||||
|
mv coverage-docker.dat .coverage.docker
|
||||||
|
mv coverage-firecracker.dat .coverage.firecracker
|
||||||
|
|
||||||
# No dev-requirements install: `coverage` is already provided by the
|
|
||||||
# self-hosted runner's Nix python env, and that env has no `pip`
|
|
||||||
# module to install into anyway. `scripts/coverage.sh` +
|
|
||||||
# `diff_coverage.py` need only `coverage` (not pylint/pyright).
|
|
||||||
- name: Combined coverage (unit + integration, incl. firecracker)
|
- name: Combined coverage (unit + integration, incl. firecracker)
|
||||||
env:
|
run: PYTHON=python3 bash scripts/coverage.sh aggregate critical
|
||||||
BOT_BOTTLE_CI_INFRA_ARTIFACT_DIR: ${{ github.workspace }}/infra-candidate
|
|
||||||
run: PYTHON=python3 bash scripts/coverage.sh critical
|
|
||||||
|
|
||||||
- name: Diff-coverage gate (changed lines >= 90%)
|
- name: Diff-coverage gate (changed lines >= 90%)
|
||||||
run: |
|
run: |
|
||||||
@@ -243,14 +260,14 @@ jobs:
|
|||||||
python3 scripts/diff_coverage.py --base origin/main --min 90
|
python3 scripts/diff_coverage.py --base origin/main --min 90
|
||||||
|
|
||||||
publish-infra:
|
publish-infra:
|
||||||
needs: [stage-firecracker-inputs, build-infra, unit, integration-docker, integration-firecracker, coverage]
|
needs: [unit, integration-docker, integration-firecracker, coverage]
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout the tested revision
|
- name: Checkout the tested revision
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Download the tested candidate
|
- name: Download the tested rootfs
|
||||||
uses: actions/download-artifact@v3
|
uses: actions/download-artifact@v3
|
||||||
with:
|
with:
|
||||||
name: infra-candidate
|
name: infra-candidate
|
||||||
@@ -258,9 +275,10 @@ jobs:
|
|||||||
|
|
||||||
# publish_infra re-derives the version from the checkout to confirm the
|
# publish_infra re-derives the version from the checkout to confirm the
|
||||||
# bundle matches before uploading, and the version hashes the dropbear
|
# bundle matches before uploading, and the version hashes the dropbear
|
||||||
# bytes. Stage the SAME dropbear build-infra used, or the recheck
|
# bytes. Download the SAME dropbear integration-firecracker used, or
|
||||||
# computes a "<missing>"-dropbear version and rejects the candidate.
|
# the recheck computes a "<missing>"-dropbear version and rejects the
|
||||||
- name: Download the staged dropbear (matches build-infra's version)
|
# candidate.
|
||||||
|
- name: Download the staged dropbear (matches build's version)
|
||||||
uses: actions/download-artifact@v3
|
uses: actions/download-artifact@v3
|
||||||
with:
|
with:
|
||||||
name: firecracker-inputs
|
name: firecracker-inputs
|
||||||
|
|||||||
@@ -131,6 +131,29 @@ def build_artifact(out_dir: Path) -> tuple[str, Path, Path]:
|
|||||||
return version, gz, sha
|
return version, gz, sha
|
||||||
|
|
||||||
|
|
||||||
|
def _try_download_published(out_dir: Path) -> tuple[str, Path, Path] | None:
|
||||||
|
"""If this version's artifact is already in the registry, download the gz
|
||||||
|
and sha to out_dir and return (version, gz_path, sha_path). Returns None
|
||||||
|
when not yet published."""
|
||||||
|
version = infra_artifact.infra_artifact_version(infra_vm._infra_init())
|
||||||
|
sha_url = infra_artifact.artifact_url(version, "rootfs.ext4.gz.sha256")
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(infra_artifact._open(sha_url)):
|
||||||
|
pass
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
if e.code == 404:
|
||||||
|
return None
|
||||||
|
raise SystemExit(f"registry check failed (HTTP {e.code}): {sha_url}")
|
||||||
|
except urllib.error.URLError as e:
|
||||||
|
raise SystemExit(f"registry unreachable: {sha_url} ({e.reason})")
|
||||||
|
print(f"infra rootfs {version} already published — downloading instead of building")
|
||||||
|
gz = out_dir / "rootfs.ext4.gz"
|
||||||
|
sha = out_dir / "rootfs.ext4.gz.sha256"
|
||||||
|
infra_artifact._download(infra_artifact.artifact_url(version, "rootfs.ext4.gz"), gz)
|
||||||
|
infra_artifact._download(infra_artifact.artifact_url(version, "rootfs.ext4.gz.sha256"), sha)
|
||||||
|
return version, gz, sha
|
||||||
|
|
||||||
|
|
||||||
def _publish_bundle(root: Path, token: str) -> str:
|
def _publish_bundle(root: Path, token: str) -> str:
|
||||||
version_file = root / "version.txt"
|
version_file = root / "version.txt"
|
||||||
# Guard the read so a missing version.txt is a clean error, not a raw
|
# Guard the read so a missing version.txt is a clean error, not a raw
|
||||||
@@ -187,6 +210,8 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
help="build a candidate bundle in DIR without publishing")
|
help="build a candidate bundle in DIR without publishing")
|
||||||
mode.add_argument("--publish-dir", type=Path,
|
mode.add_argument("--publish-dir", type=Path,
|
||||||
help="publish an already-built and tested candidate bundle")
|
help="publish an already-built and tested candidate bundle")
|
||||||
|
parser.add_argument("--reuse-published", action="store_true",
|
||||||
|
help="with --output: download from registry if already published instead of building")
|
||||||
args = parser.parse_args(argv)
|
args = parser.parse_args(argv)
|
||||||
|
|
||||||
_, _, token = infra_artifact._config()
|
_, _, token = infra_artifact._config()
|
||||||
@@ -197,6 +222,14 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
|
|
||||||
if args.output is not None:
|
if args.output is not None:
|
||||||
args.output.mkdir(parents=True, exist_ok=True)
|
args.output.mkdir(parents=True, exist_ok=True)
|
||||||
|
reused = None
|
||||||
|
if args.reuse_published:
|
||||||
|
reused = _try_download_published(args.output)
|
||||||
|
if reused is not None:
|
||||||
|
version, _, _ = reused
|
||||||
|
(args.output / "version.txt").write_text(version + "\n", encoding="utf-8")
|
||||||
|
print(f"reused published infra rootfs candidate {version}")
|
||||||
|
return 0
|
||||||
version, _gz, _sha = build_artifact(args.output)
|
version, _gz, _sha = build_artifact(args.output)
|
||||||
(args.output / "version.txt").write_text(version + "\n", encoding="utf-8")
|
(args.output / "version.txt").write_text(version + "\n", encoding="utf-8")
|
||||||
print(f"built infra rootfs candidate {version}")
|
print(f"built infra rootfs candidate {version}")
|
||||||
|
|||||||
@@ -0,0 +1,110 @@
|
|||||||
|
# PRD prd-new: CI artifact-based coverage and local Firecracker candidate flow
|
||||||
|
|
||||||
|
- **Status:** Active
|
||||||
|
- **Author:** Claude
|
||||||
|
- **Created:** 2026-07-21
|
||||||
|
- **Issue:** #446
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Restructure the CI test pipeline to run each test suite exactly once, upload
|
||||||
|
small `.coverage.*` artifacts, and combine them in a lightweight aggregation
|
||||||
|
job. Move the infra build onto the KVM runner so the ~194 MB rootfs never
|
||||||
|
crosses the network for PRs. On main-branch pushes, publish the byte-identical
|
||||||
|
rootfs that was tested.
|
||||||
|
|
||||||
|
## Motivation
|
||||||
|
|
||||||
|
The prior pipeline had two redundant costs:
|
||||||
|
|
||||||
|
1. **Duplicate artifact transfers.** `build-infra` (ubuntu-latest) built and
|
||||||
|
uploaded the ~194 MB rootfs; `integration-firecracker` downloaded it; the
|
||||||
|
`coverage` job downloaded it a second time. Combined download overhead: ~83
|
||||||
|
seconds per run, plus the ~70-second upload.
|
||||||
|
|
||||||
|
2. **Duplicate test execution.** `integration-firecracker` ran the Firecracker
|
||||||
|
integration suite; `coverage` ran the entire unit + integration suite again
|
||||||
|
on the same KVM runner to collect coverage data. Every line of Firecracker
|
||||||
|
code was tested twice per CI run.
|
||||||
|
|
||||||
|
## Goals
|
||||||
|
|
||||||
|
- Each test suite (unit, integration-docker, integration-firecracker) executes
|
||||||
|
exactly once per workflow run.
|
||||||
|
- PRs incur no large artifact transfers — the rootfs stays on the KVM runner.
|
||||||
|
- Main-branch pushes publish a byte-for-byte identical rootfs to the one that
|
||||||
|
passed the integration tests.
|
||||||
|
- Concurrent workflow runs cannot cross-publish candidates (naturally enforced
|
||||||
|
by Gitea Actions' per-run artifact scoping).
|
||||||
|
- Failed or cancelled runs block publication (enforced by the `needs:` chain on
|
||||||
|
`publish-infra`).
|
||||||
|
|
||||||
|
## Non-goals
|
||||||
|
|
||||||
|
- Changing test semantics or the coverage policy (ADR 0004).
|
||||||
|
- Removing the KVM runner guard on `integration-firecracker` and `coverage`.
|
||||||
|
- Changing how `publish_infra.py` builds or uploads the rootfs.
|
||||||
|
|
||||||
|
## Design
|
||||||
|
|
||||||
|
### Job graph
|
||||||
|
|
||||||
|
```
|
||||||
|
unit ──────────────────────────────────┐
|
||||||
|
integration-docker ────────────────────┤──► coverage ──► publish-infra (main only)
|
||||||
|
integration-firecracker (KVM) ─────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
### `unit`
|
||||||
|
|
||||||
|
Unchanged except: `coverage run` writes `--data-file=.coverage.unit`; the file
|
||||||
|
is uploaded as the `coverage-unit` artifact.
|
||||||
|
|
||||||
|
### `integration-docker`
|
||||||
|
|
||||||
|
Adds a `coverage` install step. `coverage run` writes `--data-file=.coverage.docker`;
|
||||||
|
the file is uploaded as `coverage-docker`.
|
||||||
|
|
||||||
|
### `integration-firecracker` (KVM runner)
|
||||||
|
|
||||||
|
Replaces the old `stage-firecracker-inputs` → `build-infra` → download chain:
|
||||||
|
|
||||||
|
1. Builds the infra candidate locally with
|
||||||
|
`BOT_BOTTLE_FC_DROPBEAR=/var/cache/bot-bottle-fc/dropbear`.
|
||||||
|
2. Boots the candidate and runs integration tests with coverage, writing
|
||||||
|
`.coverage.firecracker`.
|
||||||
|
3. Uploads the small `coverage-firecracker` artifact unconditionally.
|
||||||
|
4. On main-branch pushes only, uploads the rootfs as `infra-candidate` and the
|
||||||
|
dropbear as `firecracker-inputs` so `publish-infra` can verify and publish
|
||||||
|
the byte-identical artifact.
|
||||||
|
|
||||||
|
### `coverage`
|
||||||
|
|
||||||
|
Moves from a KVM runner to `ubuntu-latest`. No tests are re-executed:
|
||||||
|
|
||||||
|
1. Downloads `coverage-unit`, `coverage-docker`, and `coverage-firecracker`.
|
||||||
|
2. Runs `scripts/coverage.sh aggregate critical`, which calls
|
||||||
|
`coverage combine` then `coverage report`.
|
||||||
|
3. Runs the diff-coverage gate (`scripts/diff_coverage.py`).
|
||||||
|
|
||||||
|
Coverage files use `relative_files = True` (`.coveragerc`) so they combine
|
||||||
|
cleanly across runners with different absolute workspace paths.
|
||||||
|
|
||||||
|
### `publish-infra`
|
||||||
|
|
||||||
|
Depends on all four predecessor jobs (unchanged gate). Downloads `infra-candidate`
|
||||||
|
and `firecracker-inputs` that were uploaded by `integration-firecracker` on
|
||||||
|
main — the same byte sequence that passed the integration tests.
|
||||||
|
|
||||||
|
### Eliminated jobs
|
||||||
|
|
||||||
|
- `stage-firecracker-inputs`: existed only to copy the dropbear to ubuntu-latest
|
||||||
|
for `build-infra`. No longer needed.
|
||||||
|
- `build-infra`: the infra candidate is now built on the KVM runner in
|
||||||
|
`integration-firecracker`.
|
||||||
|
|
||||||
|
### Script changes
|
||||||
|
|
||||||
|
`scripts/coverage.sh` gains an `aggregate` mode (`coverage.sh aggregate [critical]`)
|
||||||
|
that combines pre-existing `.coverage.*` files instead of re-running tests.
|
||||||
|
The existing run mode (`coverage.sh [critical]`) is preserved for local dev.
|
||||||
+28
-8
@@ -1,15 +1,19 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# Combined unit + integration coverage (see docs/decisions/0004-coverage-policy.md).
|
# Combined unit + integration coverage (see docs/decisions/0004-coverage-policy.md).
|
||||||
#
|
#
|
||||||
# Runs the unit suite, then appends the integration suite (which skips
|
# Two modes:
|
||||||
# cleanly when Docker / the backend CLIs are unavailable), and prints one
|
|
||||||
# combined report. The integration suite is what scores the subprocess /
|
|
||||||
# backend orchestration modules, so the number here is the policy's
|
|
||||||
# yardstick — not the unit-only badge.
|
|
||||||
#
|
#
|
||||||
# Usage:
|
# scripts/coverage.sh [critical]
|
||||||
# scripts/coverage.sh # combined report
|
# Run mode (default, for local dev): executes the unit suite then the
|
||||||
# scripts/coverage.sh critical # also report just the critical modules
|
# integration suite under coverage and prints a combined report.
|
||||||
|
#
|
||||||
|
# scripts/coverage.sh aggregate [critical]
|
||||||
|
# Aggregate mode (used by CI): combines pre-existing .coverage.* files
|
||||||
|
# produced by individual test jobs and prints a combined report. No tests
|
||||||
|
# are re-executed; no KVM or Docker dependency.
|
||||||
|
#
|
||||||
|
# Pass "critical" as the last argument in either mode to also report just the
|
||||||
|
# critical modules (ADR 0004 target: 90%).
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
cd "$(dirname "$0")/.."
|
cd "$(dirname "$0")/.."
|
||||||
@@ -21,6 +25,22 @@ PY="${PYTHON:-python3}"
|
|||||||
# README "core coverage" badge can't drift; comma-join it for --include.
|
# README "core coverage" badge can't drift; comma-join it for --include.
|
||||||
CRITICAL=$(grep -vE '^[[:space:]]*(#|$)' scripts/critical-modules.txt | paste -sd, -)
|
CRITICAL=$(grep -vE '^[[:space:]]*(#|$)' scripts/critical-modules.txt | paste -sd, -)
|
||||||
|
|
||||||
|
if [ "${1:-}" = "aggregate" ]; then
|
||||||
|
# Aggregate mode: combine .coverage.* artifacts already in the workspace.
|
||||||
|
echo "== combining coverage artifacts ==" >&2
|
||||||
|
"$PY" -m coverage combine
|
||||||
|
|
||||||
|
echo "== combined report ==" >&2
|
||||||
|
"$PY" -m coverage report -m
|
||||||
|
|
||||||
|
if [ "${2:-}" = "critical" ]; then
|
||||||
|
echo "== critical modules (ADR 0004 target: 90%) ==" >&2
|
||||||
|
"$PY" -m coverage report --include="$CRITICAL"
|
||||||
|
fi
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Run mode (default): execute both suites under coverage in this process.
|
||||||
rm -f .coverage
|
rm -f .coverage
|
||||||
|
|
||||||
echo "== unit ==" >&2
|
echo "== unit ==" >&2
|
||||||
|
|||||||
@@ -136,6 +136,49 @@ class TestPublishBundle(unittest.TestCase):
|
|||||||
self.assertIn("registry unreachable", str(ctx.exception))
|
self.assertIn("registry unreachable", str(ctx.exception))
|
||||||
|
|
||||||
|
|
||||||
|
class TestTryDownloadPublished(unittest.TestCase):
|
||||||
|
def test_downloads_existing_artifact(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as d, \
|
||||||
|
mock.patch.object(pub.infra_vm, "_infra_init", return_value="init"), \
|
||||||
|
mock.patch.object(
|
||||||
|
pub.infra_artifact, "infra_artifact_version", return_value="v1"
|
||||||
|
), mock.patch.object(
|
||||||
|
pub.urllib.request, "urlopen", return_value=_Resp()
|
||||||
|
), mock.patch.object(pub.infra_artifact, "_download") as download:
|
||||||
|
root = Path(d)
|
||||||
|
result = pub._try_download_published(root)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
("v1", root / "rootfs.ext4.gz", root / "rootfs.ext4.gz.sha256"),
|
||||||
|
result,
|
||||||
|
)
|
||||||
|
self.assertEqual(2, download.call_count)
|
||||||
|
|
||||||
|
def test_missing_artifact_returns_none(self) -> None:
|
||||||
|
missing = urllib.error.HTTPError("u", 404, "missing", Message(), None)
|
||||||
|
with tempfile.TemporaryDirectory() as d, mock.patch.object(
|
||||||
|
pub.urllib.request, "urlopen", side_effect=missing
|
||||||
|
):
|
||||||
|
self.assertIsNone(pub._try_download_published(Path(d)))
|
||||||
|
|
||||||
|
def test_registry_http_failure_is_reported(self) -> None:
|
||||||
|
failure = urllib.error.HTTPError("u", 500, "failed", Message(), None)
|
||||||
|
with tempfile.TemporaryDirectory() as d, mock.patch.object(
|
||||||
|
pub.urllib.request, "urlopen", side_effect=failure
|
||||||
|
):
|
||||||
|
with self.assertRaises(SystemExit) as ctx:
|
||||||
|
pub._try_download_published(Path(d))
|
||||||
|
self.assertIn("registry check failed (HTTP 500)", str(ctx.exception))
|
||||||
|
|
||||||
|
def test_registry_connection_failure_is_reported(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as d, mock.patch.object(
|
||||||
|
pub.urllib.request, "urlopen", side_effect=urllib.error.URLError("offline")
|
||||||
|
):
|
||||||
|
with self.assertRaises(SystemExit) as ctx:
|
||||||
|
pub._try_download_published(Path(d))
|
||||||
|
self.assertIn("registry unreachable", str(ctx.exception))
|
||||||
|
|
||||||
|
|
||||||
class TestMain(unittest.TestCase):
|
class TestMain(unittest.TestCase):
|
||||||
def test_output_builds_candidate_and_records_version(self) -> None:
|
def test_output_builds_candidate_and_records_version(self) -> None:
|
||||||
with tempfile.TemporaryDirectory() as d:
|
with tempfile.TemporaryDirectory() as d:
|
||||||
@@ -147,6 +190,20 @@ class TestMain(unittest.TestCase):
|
|||||||
build.assert_called_once_with(root)
|
build.assert_called_once_with(root)
|
||||||
self.assertEqual("v1\n", (root / "version.txt").read_text())
|
self.assertEqual("v1\n", (root / "version.txt").read_text())
|
||||||
|
|
||||||
|
def test_output_reuses_published_candidate(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
root = Path(d) / "candidate"
|
||||||
|
reused = ("v1", root / "rootfs.ext4.gz", root / "rootfs.ext4.gz.sha256")
|
||||||
|
with mock.patch.object(
|
||||||
|
pub, "_try_download_published", return_value=reused
|
||||||
|
) as reuse, mock.patch.object(pub, "build_artifact") as build:
|
||||||
|
self.assertEqual(
|
||||||
|
0, pub.main(["--output", str(root), "--reuse-published"])
|
||||||
|
)
|
||||||
|
reuse.assert_called_once_with(root)
|
||||||
|
build.assert_not_called()
|
||||||
|
self.assertEqual("v1\n", (root / "version.txt").read_text())
|
||||||
|
|
||||||
def test_publish_dir_publishes_existing_candidate(self) -> None:
|
def test_publish_dir_publishes_existing_candidate(self) -> None:
|
||||||
with tempfile.TemporaryDirectory() as d, \
|
with tempfile.TemporaryDirectory() as d, \
|
||||||
mock.patch.object(pub.infra_artifact, "_config", return_value=("", "", "t")), \
|
mock.patch.object(pub.infra_artifact, "_config", return_value=("", "", "t")), \
|
||||||
|
|||||||
Reference in New Issue
Block a user