ci(infra): test and publish one candidate artifact

.gitea/workflows/publish-infra.yml

@@ -1,31 +0,0 @@
-# Build and publish the Firecracker infra rootfs artifact after tests pass on main.
-#
-# The test suite builds images locally (BOT_BOTTLE_INFRA_BUILD=local). This
-# workflow is the separate publish step: it runs AFTER the test workflow
-# completes on main pushes, builds the same rootfs, and uploads it to the
-# Gitea generic-package registry so operators can pull it without Docker.
-#
-# Auth: BOT_BOTTLE_INFRA_ARTIFACT_TOKEN must be a Gitea token with
-# write:package on the target owner (set in repo secrets).
-
-name: publish-infra
-
-on:
-  workflow_run:
-    workflows: [test]
-    types: [completed]
-    branches: [main]
-
-jobs:
-  publish:
-    runs-on: [self-hosted, kvm]
-    # Only publish when the test workflow succeeded — never on a failing main.
-    if: github.event.workflow_run.conclusion == 'success'
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Build and publish infra rootfs artifact
-        env:
-          BOT_BOTTLE_INFRA_ARTIFACT_TOKEN: ${{ secrets.BOT_BOTTLE_INFRA_ARTIFACT_TOKEN }}
-        run: python3 -m bot_bottle.backend.firecracker.publish_infra

.gitea/workflows/test.yml

@@ -40,6 +40,21 @@ on:
   workflow_dispatch:
 
 jobs:
+  build-infra:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Build infra candidate from this checkout
+        run: python3 -m bot_bottle.backend.firecracker.publish_infra --output infra-candidate
+
+      - name: Upload infra candidate
+        uses: actions/upload-artifact@v3
+        with:
+          name: infra-candidate
+          path: infra-candidate/
+
   unit:
     runs-on: ubuntu-latest
     steps:
@@ -89,9 +104,10 @@ jobs:
   # PRs don't execute untrusted code on the privileged runner.
   #
   # Runner prerequisites (provision once; see README "Firecracker on Linux"):
-  # `firecracker` on PATH, `/dev/kvm` accessible, Docker, cached kernel +
+  # `firecracker` on PATH, `/dev/kvm` accessible, cached kernel +
   # static dropbear, and the pool as a persistent systemd unit.
   integration-firecracker:
+    needs: build-infra
     runs-on: [self-hosted, kvm]
     if: >-
       github.event_name == 'push' ||
@@ -111,6 +127,15 @@ jobs:
           # range overlap; it prints the exact `backend setup` fix.
           python3 cli.py backend status --backend=firecracker
 
+      - name: Download the candidate built from this checkout
+        uses: actions/download-artifact@v3
+        with:
+          name: infra-candidate
+          path: infra-candidate
+
+      - name: Replace the persistent infra VM with the candidate
+        run: python3 -c 'from bot_bottle.backend.firecracker import infra_vm; infra_vm.stop()'
+
       # No dev-requirements install: the integration suite runs on stdlib
       # `unittest` (pylint/pyright are lint.yml's concern, not this job's),
       # and the self-hosted runner's Nix python env has no `pip` module
@@ -118,10 +143,7 @@ jobs:
       - name: Run integration tests (firecracker)
         env:
           BOT_BOTTLE_BACKEND: firecracker
-          # Build images locally from the checked-out source — never pull a
-          # pre-published artifact during tests. Publishing happens post-merge
-          # in publish-infra.yml.
-          BOT_BOTTLE_INFRA_BUILD: local
+          BOT_BOTTLE_INFRA_ARTIFACT_DIR: ${{ github.workspace }}/infra-candidate
         run: python3 -m unittest discover -t . -s tests/integration -v
 
   # Combined unit+integration coverage + the diff-coverage gate (the hard
@@ -141,10 +163,11 @@ jobs:
   # See #414 for the planned follow-up: artifact-based coverage combination
   # (run tests once in their respective jobs, combine .coverage files here).
   #
-  # BOT_BOTTLE_INFRA_BUILD=local: build images from source here, never pull a
-  # pre-published artifact. Registry publishing runs post-merge in
-  # publish-infra.yml, gated on tests passing.
+  # build-infra creates one candidate from the checkout. This job boots that
+  # same candidate after integration-firecracker has exercised it; the main
+  # push path publishes the identical bytes only after every required job.
   coverage:
+    needs: [build-infra, integration-firecracker]
     timeout-minutes: 15
     runs-on: [self-hosted, kvm]
     if: >-
@@ -167,16 +190,41 @@ jobs:
           # range overlap; it prints the exact `backend setup` fix.
           python3 cli.py backend status --backend=firecracker
 
+      - name: Download the candidate already exercised by integration
+        uses: actions/download-artifact@v3
+        with:
+          name: infra-candidate
+          path: infra-candidate
+
       # No dev-requirements install: `coverage` is already provided by the
       # self-hosted runner's Nix python env, and that env has no `pip`
       # module to install into anyway. `scripts/coverage.sh` +
       # `diff_coverage.py` need only `coverage` (not pylint/pyright).
       - name: Combined coverage (unit + integration, incl. firecracker)
         env:
-          BOT_BOTTLE_INFRA_BUILD: local
+          BOT_BOTTLE_INFRA_ARTIFACT_DIR: ${{ github.workspace }}/infra-candidate
         run: PYTHON=python3 bash scripts/coverage.sh critical
 
       - name: Diff-coverage gate (changed lines >= 90%)
         run: |
           git fetch --no-tags origin main:refs/remotes/origin/main
           python3 scripts/diff_coverage.py --base origin/main --min 90
+
+  publish-infra:
+    needs: [build-infra, unit, integration-docker, integration-firecracker, coverage]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    steps:
+      - name: Checkout the tested revision
+        uses: actions/checkout@v4
+
+      - name: Download the tested candidate
+        uses: actions/download-artifact@v3
+        with:
+          name: infra-candidate
+          path: infra-candidate
+
+      - name: Publish the tested candidate
+        env:
+          BOT_BOTTLE_INFRA_ARTIFACT_TOKEN: ${{ secrets.BOT_BOTTLE_INFRA_ARTIFACT_TOKEN }}
+        run: python3 -m bot_bottle.backend.firecracker.publish_infra --publish-dir infra-candidate

README.md

@@ -90,7 +90,7 @@ BOT_BOTTLE_BACKEND=firecracker ./cli.py start <agent>
 
 > **NixOS:** enable `virtualisation.docker`, ensure the KVM module is loaded (`boot.kernelModules = [ "kvm-intel" ];` or `kvm-amd`), and add your user to the `kvm` and `docker` groups. For the network pool, consume the flake module — `imports = [ inputs.bot-bottle.nixosModules.firecracker-netpool ]; services.bot-bottle-firecracker = { enable = true; owner = "you"; };` — then `nixos-rebuild switch` (imperative nft/TAP rules don't survive a rebuild; channel users can `imports = [ <bot-bottle>/nix/firecracker-netpool.nix ]`). `firecracker` isn't in nixpkgs by default as a user binary — install the release binary (pin the version) and put it on `PATH`.
 
-> **CI:** the coverage gate (`.gitea/workflows/test.yml` → `coverage` job) runs on a self-hosted runner labelled `kvm`, because the Firecracker backend's VM/SSH orchestration is exercised only by the integration suite, which needs `/dev/kvm` + the provisioned pool (a container runner would skip it and read as uncovered). Provision that runner exactly like a normal Firecracker host — `firecracker` on `PATH`, `/dev/kvm`, Docker, the cached guest kernel + static dropbear, and the pool installed as the persistent systemd unit — then register it with the `kvm` label. The unit/lint jobs still run on `ubuntu-latest`.
+> **CI:** the coverage gate (`.gitea/workflows/test.yml` → `coverage` job) runs on a self-hosted runner labelled `kvm`, because the Firecracker backend's VM/SSH orchestration is exercised only by the integration suite, which needs `/dev/kvm` + the provisioned pool (a container runner would skip it and read as uncovered). Provision that runner exactly like a normal Firecracker host — `firecracker` on `PATH`, `/dev/kvm`, the cached guest kernel + static dropbear, and the pool installed as the persistent systemd unit — then register it with the `kvm` label. A Docker-capable hosted job builds the candidate once; KVM tests boot those exact bytes, and a successful main run publishes them. The unit/lint jobs still run on `ubuntu-latest`.
 
 ```sh
 ./cli.py start <agent>   # builds the image on first run, drops you into claude

bot_bottle/backend/firecracker/infra_artifact.py

@@ -113,6 +113,7 @@ def artifact_url(version: str, filename: str) -> str:
 
 _GZ_NAME = "rootfs.ext4.gz"
 _SHA_NAME = "rootfs.ext4.gz.sha256"
+_CANDIDATE_DIR_ENV = "BOT_BOTTLE_INFRA_ARTIFACT_DIR"
 
 
 def _cache_root(version: str) -> Path:
@@ -162,6 +163,28 @@ def ensure_artifact_gz(version: str) -> Path:
     """The verified, cached `rootfs.ext4.gz` for `version` — downloading it (and
     its `.sha256`) once, then reusing it. Fail-closed on a checksum mismatch:
     the partial is removed and we die rather than boot an unverified rootfs."""
+    candidate_dir = os.environ.get(_CANDIDATE_DIR_ENV, "").strip()
+    if candidate_dir:
+        root = Path(candidate_dir)
+        declared = (root / "version.txt").read_text(encoding="utf-8").strip()
+        if declared != version:
+            die(
+                f"infra candidate version mismatch: expected {version}, "
+                f"bundle contains {declared or '<empty>'}"
+            )
+        gz = root / _GZ_NAME
+        sha = root / _SHA_NAME
+        if not gz.is_file() or not sha.is_file():
+            die(f"infra candidate bundle is incomplete: {root}")
+        expected = sha.read_text().split()[0].strip().lower()
+        actual = _sha256_file(gz)
+        if actual != expected:
+            die(
+                f"infra candidate checksum mismatch for {version}:\n"
+                f"    expected {expected}\n    actual   {actual}"
+            )
+        return gz
+
     root = _cache_root(version)
     root.mkdir(parents=True, exist_ok=True)
     gz = root / _GZ_NAME

bot_bottle/backend/firecracker/publish_infra.py

@@ -11,7 +11,8 @@ The `<version>` is `infra_artifact.infra_artifact_version(...)`, the content
 hash of the rootfs inputs, so a launch host at the same code checkout resolves
 the exact artifact this produced.
 
-  python3 -m bot_bottle.backend.firecracker.publish_infra [--dry-run] [--force]
+  python3 -m bot_bottle.backend.firecracker.publish_infra --output DIR
+  python3 -m bot_bottle.backend.firecracker.publish_infra --publish-dir DIR
 
 Auth: a token with `write:package` on the target owner, from
 `BOT_BOTTLE_INFRA_ARTIFACT_TOKEN`.
@@ -24,7 +25,6 @@ import gzip
 import hashlib
 import shutil
 import sys
-import tempfile
 import urllib.error
 import urllib.request
 from pathlib import Path
@@ -131,36 +131,73 @@ def build_artifact(out_dir: Path) -> tuple[str, Path, Path]:
     return version, gz, sha
 
 
+def _publish_bundle(root: Path, token: str) -> str:
+    version = (root / "version.txt").read_text(encoding="utf-8").strip()
+    expected = infra_artifact.infra_artifact_version(infra_vm._infra_init())
+    if version != expected:
+        raise SystemExit(
+            f"artifact bundle version {version!r} does not match checkout {expected!r}"
+        )
+    gz = root / "rootfs.ext4.gz"
+    sha = root / "rootfs.ext4.gz.sha256"
+    if not gz.is_file() or not sha.is_file():
+        raise SystemExit(f"incomplete artifact bundle: {root}")
+    expected_sha = sha.read_text().split()[0].strip().lower()
+    if _sha256(gz) != expected_sha:
+        raise SystemExit("artifact bundle checksum mismatch")
+
+    gz_url = infra_artifact.artifact_url(version, gz.name)
+    sha_url = infra_artifact.artifact_url(version, sha.name)
+    about_url = infra_artifact.artifact_url(version, _ABOUT_NAME)
+
+    # Publishing is idempotent. If this exact complete artifact is already
+    # present, a test-only main commit is a no-op. Otherwise clear any partial
+    # upload left by an interrupted prior attempt and upload the complete set.
+    try:
+        with urllib.request.urlopen(infra_artifact._open(sha_url)) as resp:
+            remote_sha = resp.read().decode("utf-8").split()[0].strip().lower()
+    except urllib.error.HTTPError as e:
+        if e.code != 404:
+            raise SystemExit(f"checking existing artifact failed (HTTP {e.code})")
+        remote_sha = ""
+    except urllib.error.URLError as e:
+        raise SystemExit(f"registry unreachable: {sha_url} ({e.reason})")
+    if remote_sha == expected_sha:
+        print(f"infra rootfs {version} already published")
+        return version
+
+    for url in (gz_url, sha_url, about_url):
+        _delete(url, token)
+    _put(gz_url, gz, token)
+    _put(sha_url, sha.read_bytes(), token)
+    _put(about_url, _ABOUT_TEXT.encode(), token)
+    return version
+
+
 def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(
         prog="publish_infra", description="Build + publish the infra rootfs artifact.")
-    parser.add_argument("--dry-run", action="store_true",
-                        help="build the artifact but do not upload")
-    parser.add_argument("--force", action="store_true",
-                        help="overwrite an already-published artifact of this version")
+    mode = parser.add_mutually_exclusive_group(required=True)
+    mode.add_argument("--output", type=Path,
+                      help="build a candidate bundle in DIR without publishing")
+    mode.add_argument("--publish-dir", type=Path,
+                      help="publish an already-built and tested candidate bundle")
     args = parser.parse_args(argv)
 
     _, _, token = infra_artifact._config()
-    if not args.dry_run and not token:
+    if args.publish_dir is not None and not token:
         raise SystemExit(
             "no publish token: set BOT_BOTTLE_INFRA_ARTIFACT_TOKEN to a token "
             "with write:package")
 
-    with tempfile.TemporaryDirectory(prefix="bb-publish-infra.") as tmp:
-        version, gz, sha = build_artifact(Path(tmp))
-        gz_url = infra_artifact.artifact_url(version, gz.name)
-        sha_url = infra_artifact.artifact_url(version, sha.name)
-        about_url = infra_artifact.artifact_url(version, _ABOUT_NAME)
-        if args.dry_run:
-            print(f"dry-run: would upload -> {gz_url}")
-            return 0
-        if args.force:
-            _delete(gz_url, token)
-            _delete(sha_url, token)
-            _delete(about_url, token)
-        _put(gz_url, gz, token)  # streamed from disk (hundreds of MB)
-        _put(sha_url, sha.read_bytes(), token)  # tiny, in-memory is fine
-        _put(about_url, _ABOUT_TEXT.encode(), token)  # package description
+    if args.output is not None:
+        args.output.mkdir(parents=True, exist_ok=True)
+        version, _gz, _sha = build_artifact(args.output)
+        (args.output / "version.txt").write_text(version + "\n", encoding="utf-8")
+        print(f"built infra rootfs candidate {version}")
+        return 0
+
+    version = _publish_bundle(args.publish_dir, token)
     print(f"published infra rootfs {version}")
     return 0
 

tests/unit/test_infra_artifact.py

@@ -129,6 +129,34 @@ class TestVersionInputs(unittest.TestCase):
 
 
 class TestEnsureArtifact(_CacheMixin):
+    def test_uses_verified_ci_candidate_without_network(self) -> None:
+        version = "deadbeef00000000"
+        gz = _gz(b"candidate ext4")
+        with tempfile.TemporaryDirectory() as d:
+            root = Path(d)
+            (root / "version.txt").write_text(version + "\n")
+            (root / "rootfs.ext4.gz").write_bytes(gz)
+            digest = hashlib.sha256(gz).hexdigest()
+            (root / "rootfs.ext4.gz.sha256").write_text(
+                f"{digest}  rootfs.ext4.gz\n")
+            with mock.patch.dict(os.environ, {
+                "BOT_BOTTLE_INFRA_ARTIFACT_DIR": str(root),
+            }), mock.patch.object(ia.urllib.request, "urlopen") as net:
+                path = ia.ensure_artifact_gz(version)
+            self.assertEqual(root / "rootfs.ext4.gz", path)
+            net.assert_not_called()
+
+    def test_rejects_candidate_for_another_version(self) -> None:
+        with tempfile.TemporaryDirectory() as d:
+            root = Path(d)
+            (root / "version.txt").write_text("wrong\n")
+            with mock.patch.dict(os.environ, {
+                "BOT_BOTTLE_INFRA_ARTIFACT_DIR": str(root),
+            }):
+                with self.assertRaises(Die) as ctx:
+                    ia.ensure_artifact_gz("expected")
+            self.assertIn("version mismatch", str(ctx.exception.message))
+
     def test_downloads_verifies_and_caches(self) -> None:
         version = "deadbeef00000000"
         gz = _gz(b"fake ext4 bytes")

tests/unit/test_publish_infra.py

@@ -6,8 +6,10 @@ read it into memory. Network is mocked; no Docker, no real build.
 
 from __future__ import annotations
 
+import hashlib
 import tempfile
 import unittest
+import urllib.error
 import urllib.request
 from pathlib import Path
 from unittest import mock
@@ -58,5 +60,45 @@ class TestPut(unittest.TestCase):
         self.assertEqual(b"abc123  rootfs\n", captured[0].data)
 
 
+class TestPublishBundle(unittest.TestCase):
+    def _bundle(self, root: Path, version: str) -> None:
+        payload = b"candidate"
+        (root / "version.txt").write_text(version + "\n")
+        (root / "rootfs.ext4.gz").write_bytes(payload)
+        digest = hashlib.sha256(payload).hexdigest()
+        (root / "rootfs.ext4.gz.sha256").write_text(
+            f"{digest}  rootfs.ext4.gz\n")
+
+    def test_existing_identical_artifact_is_success(self) -> None:
+        with tempfile.TemporaryDirectory() as d:
+            root = Path(d)
+            self._bundle(root, "v1")
+            sha = (root / "rootfs.ext4.gz.sha256").read_bytes()
+            response = mock.MagicMock()
+            response.__enter__.return_value.read.return_value = sha
+            with mock.patch.object(
+                pub.infra_artifact, "infra_artifact_version", return_value="v1"
+            ), mock.patch.object(
+                pub.urllib.request, "urlopen", return_value=response
+            ), mock.patch.object(pub, "_put") as put:
+                self.assertEqual("v1", pub._publish_bundle(root, "token"))
+            put.assert_not_called()
+
+    def test_partial_artifact_is_replaced(self) -> None:
+        with tempfile.TemporaryDirectory() as d:
+            root = Path(d)
+            self._bundle(root, "v1")
+            missing = urllib.error.HTTPError("u", 404, "missing", {}, None)
+            with mock.patch.object(
+                pub.infra_artifact, "infra_artifact_version", return_value="v1"
+            ), mock.patch.object(
+                pub.urllib.request, "urlopen", side_effect=missing
+            ), mock.patch.object(pub, "_delete") as delete, \
+                    mock.patch.object(pub, "_put") as put:
+                pub._publish_bundle(root, "token")
+            self.assertEqual(3, delete.call_count)
+            self.assertEqual(3, put.call_count)
+
+
 if __name__ == "__main__":
     unittest.main()