From 701f5bf5e3faff082c84140582574cf4a9a28d8a Mon Sep 17 00:00:00 2001 From: codex Date: Sun, 19 Jul 2026 22:22:42 +0000 Subject: [PATCH] ci(infra): test and publish one candidate artifact --- .gitea/workflows/publish-infra.yml | 31 ------- .gitea/workflows/test.yml | 66 ++++++++++++--- README.md | 2 +- .../backend/firecracker/infra_artifact.py | 23 ++++++ .../backend/firecracker/publish_infra.py | 81 ++++++++++++++----- tests/unit/test_infra_artifact.py | 28 +++++++ tests/unit/test_publish_infra.py | 42 ++++++++++ 7 files changed, 210 insertions(+), 63 deletions(-) delete mode 100644 .gitea/workflows/publish-infra.yml diff --git a/.gitea/workflows/publish-infra.yml b/.gitea/workflows/publish-infra.yml deleted file mode 100644 index 401d95d..0000000 --- a/.gitea/workflows/publish-infra.yml +++ /dev/null @@ -1,31 +0,0 @@ -# Build and publish the Firecracker infra rootfs artifact after tests pass on main. -# -# The test suite builds images locally (BOT_BOTTLE_INFRA_BUILD=local). This -# workflow is the separate publish step: it runs AFTER the test workflow -# completes on main pushes, builds the same rootfs, and uploads it to the -# Gitea generic-package registry so operators can pull it without Docker. -# -# Auth: BOT_BOTTLE_INFRA_ARTIFACT_TOKEN must be a Gitea token with -# write:package on the target owner (set in repo secrets). - -name: publish-infra - -on: - workflow_run: - workflows: [test] - types: [completed] - branches: [main] - -jobs: - publish: - runs-on: [self-hosted, kvm] - # Only publish when the test workflow succeeded — never on a failing main. - if: github.event.workflow_run.conclusion == 'success' - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Build and publish infra rootfs artifact - env: - BOT_BOTTLE_INFRA_ARTIFACT_TOKEN: ${{ secrets.BOT_BOTTLE_INFRA_ARTIFACT_TOKEN }} - run: python3 -m bot_bottle.backend.firecracker.publish_infra diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index 6fc341b..5ac0137 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -40,6 +40,21 @@ on: workflow_dispatch: jobs: + build-infra: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build infra candidate from this checkout + run: python3 -m bot_bottle.backend.firecracker.publish_infra --output infra-candidate + + - name: Upload infra candidate + uses: actions/upload-artifact@v3 + with: + name: infra-candidate + path: infra-candidate/ + unit: runs-on: ubuntu-latest steps: @@ -89,9 +104,10 @@ jobs: # PRs don't execute untrusted code on the privileged runner. # # Runner prerequisites (provision once; see README "Firecracker on Linux"): - # `firecracker` on PATH, `/dev/kvm` accessible, Docker, cached kernel + + # `firecracker` on PATH, `/dev/kvm` accessible, cached kernel + # static dropbear, and the pool as a persistent systemd unit. integration-firecracker: + needs: build-infra runs-on: [self-hosted, kvm] if: >- github.event_name == 'push' || @@ -111,6 +127,15 @@ jobs: # range overlap; it prints the exact `backend setup` fix. python3 cli.py backend status --backend=firecracker + - name: Download the candidate built from this checkout + uses: actions/download-artifact@v3 + with: + name: infra-candidate + path: infra-candidate + + - name: Replace the persistent infra VM with the candidate + run: python3 -c 'from bot_bottle.backend.firecracker import infra_vm; infra_vm.stop()' + # No dev-requirements install: the integration suite runs on stdlib # `unittest` (pylint/pyright are lint.yml's concern, not this job's), # and the self-hosted runner's Nix python env has no `pip` module @@ -118,10 +143,7 @@ jobs: - name: Run integration tests (firecracker) env: BOT_BOTTLE_BACKEND: firecracker - # Build images locally from the checked-out source — never pull a - # pre-published artifact during tests. Publishing happens post-merge - # in publish-infra.yml. - BOT_BOTTLE_INFRA_BUILD: local + BOT_BOTTLE_INFRA_ARTIFACT_DIR: ${{ github.workspace }}/infra-candidate run: python3 -m unittest discover -t . -s tests/integration -v # Combined unit+integration coverage + the diff-coverage gate (the hard @@ -141,10 +163,11 @@ jobs: # See #414 for the planned follow-up: artifact-based coverage combination # (run tests once in their respective jobs, combine .coverage files here). # - # BOT_BOTTLE_INFRA_BUILD=local: build images from source here, never pull a - # pre-published artifact. Registry publishing runs post-merge in - # publish-infra.yml, gated on tests passing. + # build-infra creates one candidate from the checkout. This job boots that + # same candidate after integration-firecracker has exercised it; the main + # push path publishes the identical bytes only after every required job. coverage: + needs: [build-infra, integration-firecracker] timeout-minutes: 15 runs-on: [self-hosted, kvm] if: >- @@ -167,16 +190,41 @@ jobs: # range overlap; it prints the exact `backend setup` fix. python3 cli.py backend status --backend=firecracker + - name: Download the candidate already exercised by integration + uses: actions/download-artifact@v3 + with: + name: infra-candidate + path: infra-candidate + # No dev-requirements install: `coverage` is already provided by the # self-hosted runner's Nix python env, and that env has no `pip` # module to install into anyway. `scripts/coverage.sh` + # `diff_coverage.py` need only `coverage` (not pylint/pyright). - name: Combined coverage (unit + integration, incl. firecracker) env: - BOT_BOTTLE_INFRA_BUILD: local + BOT_BOTTLE_INFRA_ARTIFACT_DIR: ${{ github.workspace }}/infra-candidate run: PYTHON=python3 bash scripts/coverage.sh critical - name: Diff-coverage gate (changed lines >= 90%) run: | git fetch --no-tags origin main:refs/remotes/origin/main python3 scripts/diff_coverage.py --base origin/main --min 90 + + publish-infra: + needs: [build-infra, unit, integration-docker, integration-firecracker, coverage] + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + steps: + - name: Checkout the tested revision + uses: actions/checkout@v4 + + - name: Download the tested candidate + uses: actions/download-artifact@v3 + with: + name: infra-candidate + path: infra-candidate + + - name: Publish the tested candidate + env: + BOT_BOTTLE_INFRA_ARTIFACT_TOKEN: ${{ secrets.BOT_BOTTLE_INFRA_ARTIFACT_TOKEN }} + run: python3 -m bot_bottle.backend.firecracker.publish_infra --publish-dir infra-candidate diff --git a/README.md b/README.md index ba68c6e..557bd44 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ BOT_BOTTLE_BACKEND=firecracker ./cli.py start > **NixOS:** enable `virtualisation.docker`, ensure the KVM module is loaded (`boot.kernelModules = [ "kvm-intel" ];` or `kvm-amd`), and add your user to the `kvm` and `docker` groups. For the network pool, consume the flake module — `imports = [ inputs.bot-bottle.nixosModules.firecracker-netpool ]; services.bot-bottle-firecracker = { enable = true; owner = "you"; };` — then `nixos-rebuild switch` (imperative nft/TAP rules don't survive a rebuild; channel users can `imports = [ /nix/firecracker-netpool.nix ]`). `firecracker` isn't in nixpkgs by default as a user binary — install the release binary (pin the version) and put it on `PATH`. -> **CI:** the coverage gate (`.gitea/workflows/test.yml` → `coverage` job) runs on a self-hosted runner labelled `kvm`, because the Firecracker backend's VM/SSH orchestration is exercised only by the integration suite, which needs `/dev/kvm` + the provisioned pool (a container runner would skip it and read as uncovered). Provision that runner exactly like a normal Firecracker host — `firecracker` on `PATH`, `/dev/kvm`, Docker, the cached guest kernel + static dropbear, and the pool installed as the persistent systemd unit — then register it with the `kvm` label. The unit/lint jobs still run on `ubuntu-latest`. +> **CI:** the coverage gate (`.gitea/workflows/test.yml` → `coverage` job) runs on a self-hosted runner labelled `kvm`, because the Firecracker backend's VM/SSH orchestration is exercised only by the integration suite, which needs `/dev/kvm` + the provisioned pool (a container runner would skip it and read as uncovered). Provision that runner exactly like a normal Firecracker host — `firecracker` on `PATH`, `/dev/kvm`, the cached guest kernel + static dropbear, and the pool installed as the persistent systemd unit — then register it with the `kvm` label. A Docker-capable hosted job builds the candidate once; KVM tests boot those exact bytes, and a successful main run publishes them. The unit/lint jobs still run on `ubuntu-latest`. ```sh ./cli.py start # builds the image on first run, drops you into claude diff --git a/bot_bottle/backend/firecracker/infra_artifact.py b/bot_bottle/backend/firecracker/infra_artifact.py index ba91cb8..8d76846 100644 --- a/bot_bottle/backend/firecracker/infra_artifact.py +++ b/bot_bottle/backend/firecracker/infra_artifact.py @@ -113,6 +113,7 @@ def artifact_url(version: str, filename: str) -> str: _GZ_NAME = "rootfs.ext4.gz" _SHA_NAME = "rootfs.ext4.gz.sha256" +_CANDIDATE_DIR_ENV = "BOT_BOTTLE_INFRA_ARTIFACT_DIR" def _cache_root(version: str) -> Path: @@ -162,6 +163,28 @@ def ensure_artifact_gz(version: str) -> Path: """The verified, cached `rootfs.ext4.gz` for `version` — downloading it (and its `.sha256`) once, then reusing it. Fail-closed on a checksum mismatch: the partial is removed and we die rather than boot an unverified rootfs.""" + candidate_dir = os.environ.get(_CANDIDATE_DIR_ENV, "").strip() + if candidate_dir: + root = Path(candidate_dir) + declared = (root / "version.txt").read_text(encoding="utf-8").strip() + if declared != version: + die( + f"infra candidate version mismatch: expected {version}, " + f"bundle contains {declared or ''}" + ) + gz = root / _GZ_NAME + sha = root / _SHA_NAME + if not gz.is_file() or not sha.is_file(): + die(f"infra candidate bundle is incomplete: {root}") + expected = sha.read_text().split()[0].strip().lower() + actual = _sha256_file(gz) + if actual != expected: + die( + f"infra candidate checksum mismatch for {version}:\n" + f" expected {expected}\n actual {actual}" + ) + return gz + root = _cache_root(version) root.mkdir(parents=True, exist_ok=True) gz = root / _GZ_NAME diff --git a/bot_bottle/backend/firecracker/publish_infra.py b/bot_bottle/backend/firecracker/publish_infra.py index eb11946..495b1d1 100644 --- a/bot_bottle/backend/firecracker/publish_infra.py +++ b/bot_bottle/backend/firecracker/publish_infra.py @@ -11,7 +11,8 @@ The `` is `infra_artifact.infra_artifact_version(...)`, the content hash of the rootfs inputs, so a launch host at the same code checkout resolves the exact artifact this produced. - python3 -m bot_bottle.backend.firecracker.publish_infra [--dry-run] [--force] + python3 -m bot_bottle.backend.firecracker.publish_infra --output DIR + python3 -m bot_bottle.backend.firecracker.publish_infra --publish-dir DIR Auth: a token with `write:package` on the target owner, from `BOT_BOTTLE_INFRA_ARTIFACT_TOKEN`. @@ -24,7 +25,6 @@ import gzip import hashlib import shutil import sys -import tempfile import urllib.error import urllib.request from pathlib import Path @@ -131,36 +131,73 @@ def build_artifact(out_dir: Path) -> tuple[str, Path, Path]: return version, gz, sha +def _publish_bundle(root: Path, token: str) -> str: + version = (root / "version.txt").read_text(encoding="utf-8").strip() + expected = infra_artifact.infra_artifact_version(infra_vm._infra_init()) + if version != expected: + raise SystemExit( + f"artifact bundle version {version!r} does not match checkout {expected!r}" + ) + gz = root / "rootfs.ext4.gz" + sha = root / "rootfs.ext4.gz.sha256" + if not gz.is_file() or not sha.is_file(): + raise SystemExit(f"incomplete artifact bundle: {root}") + expected_sha = sha.read_text().split()[0].strip().lower() + if _sha256(gz) != expected_sha: + raise SystemExit("artifact bundle checksum mismatch") + + gz_url = infra_artifact.artifact_url(version, gz.name) + sha_url = infra_artifact.artifact_url(version, sha.name) + about_url = infra_artifact.artifact_url(version, _ABOUT_NAME) + + # Publishing is idempotent. If this exact complete artifact is already + # present, a test-only main commit is a no-op. Otherwise clear any partial + # upload left by an interrupted prior attempt and upload the complete set. + try: + with urllib.request.urlopen(infra_artifact._open(sha_url)) as resp: + remote_sha = resp.read().decode("utf-8").split()[0].strip().lower() + except urllib.error.HTTPError as e: + if e.code != 404: + raise SystemExit(f"checking existing artifact failed (HTTP {e.code})") + remote_sha = "" + except urllib.error.URLError as e: + raise SystemExit(f"registry unreachable: {sha_url} ({e.reason})") + if remote_sha == expected_sha: + print(f"infra rootfs {version} already published") + return version + + for url in (gz_url, sha_url, about_url): + _delete(url, token) + _put(gz_url, gz, token) + _put(sha_url, sha.read_bytes(), token) + _put(about_url, _ABOUT_TEXT.encode(), token) + return version + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( prog="publish_infra", description="Build + publish the infra rootfs artifact.") - parser.add_argument("--dry-run", action="store_true", - help="build the artifact but do not upload") - parser.add_argument("--force", action="store_true", - help="overwrite an already-published artifact of this version") + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--output", type=Path, + help="build a candidate bundle in DIR without publishing") + mode.add_argument("--publish-dir", type=Path, + help="publish an already-built and tested candidate bundle") args = parser.parse_args(argv) _, _, token = infra_artifact._config() - if not args.dry_run and not token: + if args.publish_dir is not None and not token: raise SystemExit( "no publish token: set BOT_BOTTLE_INFRA_ARTIFACT_TOKEN to a token " "with write:package") - with tempfile.TemporaryDirectory(prefix="bb-publish-infra.") as tmp: - version, gz, sha = build_artifact(Path(tmp)) - gz_url = infra_artifact.artifact_url(version, gz.name) - sha_url = infra_artifact.artifact_url(version, sha.name) - about_url = infra_artifact.artifact_url(version, _ABOUT_NAME) - if args.dry_run: - print(f"dry-run: would upload -> {gz_url}") - return 0 - if args.force: - _delete(gz_url, token) - _delete(sha_url, token) - _delete(about_url, token) - _put(gz_url, gz, token) # streamed from disk (hundreds of MB) - _put(sha_url, sha.read_bytes(), token) # tiny, in-memory is fine - _put(about_url, _ABOUT_TEXT.encode(), token) # package description + if args.output is not None: + args.output.mkdir(parents=True, exist_ok=True) + version, _gz, _sha = build_artifact(args.output) + (args.output / "version.txt").write_text(version + "\n", encoding="utf-8") + print(f"built infra rootfs candidate {version}") + return 0 + + version = _publish_bundle(args.publish_dir, token) print(f"published infra rootfs {version}") return 0 diff --git a/tests/unit/test_infra_artifact.py b/tests/unit/test_infra_artifact.py index 805758a..4699354 100644 --- a/tests/unit/test_infra_artifact.py +++ b/tests/unit/test_infra_artifact.py @@ -129,6 +129,34 @@ class TestVersionInputs(unittest.TestCase): class TestEnsureArtifact(_CacheMixin): + def test_uses_verified_ci_candidate_without_network(self) -> None: + version = "deadbeef00000000" + gz = _gz(b"candidate ext4") + with tempfile.TemporaryDirectory() as d: + root = Path(d) + (root / "version.txt").write_text(version + "\n") + (root / "rootfs.ext4.gz").write_bytes(gz) + digest = hashlib.sha256(gz).hexdigest() + (root / "rootfs.ext4.gz.sha256").write_text( + f"{digest} rootfs.ext4.gz\n") + with mock.patch.dict(os.environ, { + "BOT_BOTTLE_INFRA_ARTIFACT_DIR": str(root), + }), mock.patch.object(ia.urllib.request, "urlopen") as net: + path = ia.ensure_artifact_gz(version) + self.assertEqual(root / "rootfs.ext4.gz", path) + net.assert_not_called() + + def test_rejects_candidate_for_another_version(self) -> None: + with tempfile.TemporaryDirectory() as d: + root = Path(d) + (root / "version.txt").write_text("wrong\n") + with mock.patch.dict(os.environ, { + "BOT_BOTTLE_INFRA_ARTIFACT_DIR": str(root), + }): + with self.assertRaises(Die) as ctx: + ia.ensure_artifact_gz("expected") + self.assertIn("version mismatch", str(ctx.exception.message)) + def test_downloads_verifies_and_caches(self) -> None: version = "deadbeef00000000" gz = _gz(b"fake ext4 bytes") diff --git a/tests/unit/test_publish_infra.py b/tests/unit/test_publish_infra.py index f1228ae..b257c08 100644 --- a/tests/unit/test_publish_infra.py +++ b/tests/unit/test_publish_infra.py @@ -6,8 +6,10 @@ read it into memory. Network is mocked; no Docker, no real build. from __future__ import annotations +import hashlib import tempfile import unittest +import urllib.error import urllib.request from pathlib import Path from unittest import mock @@ -58,5 +60,45 @@ class TestPut(unittest.TestCase): self.assertEqual(b"abc123 rootfs\n", captured[0].data) +class TestPublishBundle(unittest.TestCase): + def _bundle(self, root: Path, version: str) -> None: + payload = b"candidate" + (root / "version.txt").write_text(version + "\n") + (root / "rootfs.ext4.gz").write_bytes(payload) + digest = hashlib.sha256(payload).hexdigest() + (root / "rootfs.ext4.gz.sha256").write_text( + f"{digest} rootfs.ext4.gz\n") + + def test_existing_identical_artifact_is_success(self) -> None: + with tempfile.TemporaryDirectory() as d: + root = Path(d) + self._bundle(root, "v1") + sha = (root / "rootfs.ext4.gz.sha256").read_bytes() + response = mock.MagicMock() + response.__enter__.return_value.read.return_value = sha + with mock.patch.object( + pub.infra_artifact, "infra_artifact_version", return_value="v1" + ), mock.patch.object( + pub.urllib.request, "urlopen", return_value=response + ), mock.patch.object(pub, "_put") as put: + self.assertEqual("v1", pub._publish_bundle(root, "token")) + put.assert_not_called() + + def test_partial_artifact_is_replaced(self) -> None: + with tempfile.TemporaryDirectory() as d: + root = Path(d) + self._bundle(root, "v1") + missing = urllib.error.HTTPError("u", 404, "missing", {}, None) + with mock.patch.object( + pub.infra_artifact, "infra_artifact_version", return_value="v1" + ), mock.patch.object( + pub.urllib.request, "urlopen", side_effect=missing + ), mock.patch.object(pub, "_delete") as delete, \ + mock.patch.object(pub, "_put") as put: + pub._publish_bundle(root, "token") + self.assertEqual(3, delete.call_count) + self.assertEqual(3, put.call_count) + + if __name__ == "__main__": unittest.main()