.gitea/workflows/test.yml

@@ -4,16 +4,15 @@
 # dependencies are required to execute it. Tests are split by directory:
 #
 #   tests/unit/         — pure unit tests; always run
-#   tests/integration/  — need a reachable Docker daemon; skip cleanly
+#   tests/integration/  — need a reachable backend; skip cleanly when
-#                         (via tests/_docker.py:skip_unless_docker) when
+#                         the backend isn't available on the runner
-#                         Docker isn't available on the runner
 #   tests/canaries/     — upstream regression canaries; run on a separate
 #                         schedule (see canaries.yml), not here
 #
-# This workflow assumes the Gitea Actions runner exposes the host Docker
+# Integration tests run once per backend in separate jobs. Each job sets
-# socket to the job container so `docker` commands inside the job can
+# BOT_BOTTLE_BACKEND explicitly so the test suite uses the right backend.
-# reach the daemon. If that's not yet configured on the runner the
+# Backends that aren't available on the runner fail the preflight step
-# integration tests will skip rather than fail.
+# rather than silently skipping inside the test output.
 
 name: test
 
@@ -23,11 +22,71 @@ on:
       - main
     paths:
       - '**.py'
+      - '.gitea/workflows/**.yml'
+      - 'scripts/**'
+      - 'README.md'
+      # Dockerfiles and pyproject.toml are baked into the infra rootfs; a
+      # change here alters what the integration/coverage jobs build locally.
+      - 'Dockerfile*'
+      - 'pyproject.toml'
   pull_request:
     paths:
       - '**.py'
+      - '.gitea/workflows/**.yml'
+      - 'scripts/**'
+      - 'README.md'
+      - 'Dockerfile*'
+      - 'pyproject.toml'
+  workflow_dispatch:
 
 jobs:
+  stage-firecracker-inputs:
+    runs-on: [self-hosted, kvm]
+    # Same guard as the other KVM-runner jobs: don't spin the privileged
+    # runner for fork PRs (this only copies a non-secret static binary, but
+    # keep the posture consistent — build-infra/integration/coverage all
+    # depend on it, so gating here gates the whole Firecracker chain).
+    if: >-
+      github.event_name == 'push' ||
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'pull_request' &&
+       github.event.pull_request.head.repo.full_name == github.repository)
+    steps:
+      - name: Stage the provisioned static dropbear
+        run: |
+          mkdir -p firecracker-inputs
+          cp /var/cache/bot-bottle-fc/dropbear firecracker-inputs/dropbear
+
+      - name: Upload Firecracker build inputs
+        uses: actions/upload-artifact@v3
+        with:
+          name: firecracker-inputs
+          path: firecracker-inputs/
+
+  build-infra:
+    needs: stage-firecracker-inputs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Download Firecracker build inputs
+        uses: actions/download-artifact@v3
+        with:
+          name: firecracker-inputs
+          path: firecracker-inputs
+
+      - name: Build infra candidate from this checkout
+        env:
+          BOT_BOTTLE_FC_DROPBEAR: ${{ github.workspace }}/firecracker-inputs/dropbear
+        run: python3 -m bot_bottle.backend.firecracker.publish_infra --output infra-candidate
+
+      - name: Upload infra candidate
+        uses: actions/upload-artifact@v3
+        with:
+          name: infra-candidate
+          path: infra-candidate/
+
   unit:
     runs-on: ubuntu-latest
     steps:
@@ -48,7 +107,7 @@ jobs:
       - name: Report unit coverage
         run: python3 -m coverage report -m
 
-  integration:
+  integration-docker:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -65,33 +124,150 @@ jobs:
             echo "docker not on PATH — integration tests will skip"
           fi
 
-      - name: Run integration tests
+      - name: Run integration tests (docker)
+        env:
+          BOT_BOTTLE_BACKEND: docker
         run: python3 -m unittest discover -t . -s tests/integration -v
 
-  # Combined unit+integration coverage report (informational). See
+  # Integration tests against the Firecracker backend. Runs on a self-hosted
-  # docs/decisions/0004-coverage-policy.md.
+  # KVM runner (label `kvm`) where /dev/kvm and the TAP/nft pool are available.
   #
-  # The hard diff-coverage gate (changed lines >= 90%) is DEFERRED: the
+  # Restricted to same-repo PRs, push to main, and workflow_dispatch — fork
-  # Firecracker backend's VM/SSH orchestration is covered by the integration
+  # PRs don't execute untrusted code on the privileged runner.
-  # suite, which needs /dev/kvm + the provisioned TAP/nft pool — a
+  #
-  # container-based runner skips it and those lines read uncovered, so the
+  # Runner prerequisites (provision once; see README "Firecracker on Linux"):
-  # gate can't pass here. Re-enabling it on a self-hosted KVM runner is
+  # `firecracker` on PATH, `/dev/kvm` accessible, cached kernel +
-  # tracked separately (see PRD 0069 / #348 and the ci-runner branch).
+  # static dropbear, and the pool as a persistent systemd unit.
+  integration-firecracker:
+    needs: build-infra
+    runs-on: [self-hosted, kvm]
+    if: >-
+      github.event_name == 'push' ||
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'pull_request' &&
+       github.event.pull_request.head.repo.full_name == github.repository)
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Preflight — Firecracker host is ready
+        run: |
+          command -v firecracker >/dev/null || {
+            echo "firecracker not on PATH — provision the runner (README: Firecracker on Linux)"; exit 1; }
+          test -e /dev/kvm || { echo "/dev/kvm missing — KVM not available on this runner"; exit 1; }
+          # `backend status` exits non-zero unless the TAP pool is up + no
+          # range overlap; it prints the exact `backend setup` fix.
+          python3 cli.py backend status --backend=firecracker
+
+      - name: Download the candidate built from this checkout
+        uses: actions/download-artifact@v3
+        with:
+          name: infra-candidate
+          path: infra-candidate
+
+      - name: Replace the persistent infra VM with the candidate
+        run: python3 -c 'from bot_bottle.backend.firecracker import infra_vm; infra_vm.stop()'
+
+      # No dev-requirements install: the integration suite runs on stdlib
+      # `unittest` (pylint/pyright are lint.yml's concern, not this job's),
+      # and the self-hosted runner's Nix python env has no `pip` module
+      # (`python3 -m pip` → "No module named pip"). Nothing to install.
+      - name: Run integration tests (firecracker)
+        env:
+          BOT_BOTTLE_BACKEND: firecracker
+          BOT_BOTTLE_INFRA_ARTIFACT_DIR: ${{ github.workspace }}/infra-candidate
+        run: python3 -m unittest discover -t . -s tests/integration -v
+
+  # Combined unit+integration coverage + the diff-coverage gate (the hard
+  # gate: new/changed lines >= 90%). See docs/decisions/0004-coverage-policy.md.
+  #
+  # This runs on a self-hosted KVM runner (label `kvm`), NOT ubuntu-latest,
+  # because the Firecracker backend's subprocess/VM orchestration
+  # (launch/boot/SSH/isolation-probe) is covered by the integration suite,
+  # and that suite needs `/dev/kvm` + the provisioned TAP/nft pool — which a
+  # container-based runner doesn't have. On such a runner the firecracker
+  # integration test skips and its ~230 orchestration lines read as
+  # uncovered, so the gate can't pass there.
+  #
+  # Restricted to the same events as integration-firecracker (same-repo PRs,
+  # push, workflow_dispatch) for the same security reason.
+  #
+  # See #414 for the planned follow-up: artifact-based coverage combination
+  # (run tests once in their respective jobs, combine .coverage files here).
+  #
+  # build-infra creates one candidate from the checkout. This job boots that
+  # same candidate after integration-firecracker has exercised it; the main
+  # push path publishes the identical bytes only after every required job.
   coverage:
-    runs-on: ubuntu-latest
+    needs: [build-infra, integration-firecracker]
+    timeout-minutes: 15
+    runs-on: [self-hosted, kvm]
+    if: >-
+      github.event_name == 'push' ||
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'pull_request' &&
+       github.event.pull_request.head.repo.full_name == github.repository)
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      # No actions/setup-python: the runner image already ships Python 3.12,
+      - name: Preflight — Firecracker host is ready
-      # and older act_runner engines mishandle setup-python's PATH (coverage
+        run: |
-      # lands in one interpreter, `python3` resolves to another). Install
+          command -v firecracker >/dev/null || {
-      # straight into the ephemeral job container's system Python —
+            echo "firecracker not on PATH — provision the runner (README: Firecracker on Linux)"; exit 1; }
-      # --break-system-packages is safe because the container is disposable.
+          test -e /dev/kvm || { echo "/dev/kvm missing — KVM not available on this runner"; exit 1; }
-      - name: Install dev requirements
+          # `backend status` exits non-zero unless the TAP pool is up + no
-        run: python3 -m pip install --break-system-packages -r requirements-dev.txt
+          # range overlap; it prints the exact `backend setup` fix.
+          python3 cli.py backend status --backend=firecracker
 
-      - name: Combined coverage report (unit + integration)
+      - name: Download the candidate already exercised by integration
+        uses: actions/download-artifact@v3
+        with:
+          name: infra-candidate
+          path: infra-candidate
+
+      # No dev-requirements install: `coverage` is already provided by the
+      # self-hosted runner's Nix python env, and that env has no `pip`
+      # module to install into anyway. `scripts/coverage.sh` +
+      # `diff_coverage.py` need only `coverage` (not pylint/pyright).
+      - name: Combined coverage (unit + integration, incl. firecracker)
+        env:
+          BOT_BOTTLE_CI_INFRA_ARTIFACT_DIR: ${{ github.workspace }}/infra-candidate
         run: PYTHON=python3 bash scripts/coverage.sh critical
+
+      - name: Diff-coverage gate (changed lines >= 90%)
+        run: |
+          git fetch --no-tags origin main:refs/remotes/origin/main
+          python3 scripts/diff_coverage.py --base origin/main --min 90
+
+  publish-infra:
+    needs: [stage-firecracker-inputs, build-infra, unit, integration-docker, integration-firecracker, coverage]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    steps:
+      - name: Checkout the tested revision
+        uses: actions/checkout@v4
+
+      - name: Download the tested candidate
+        uses: actions/download-artifact@v3
+        with:
+          name: infra-candidate
+          path: infra-candidate
+
+      # publish_infra re-derives the version from the checkout to confirm the
+      # bundle matches before uploading, and the version hashes the dropbear
+      # bytes. Stage the SAME dropbear build-infra used, or the recheck
+      # computes a "<missing>"-dropbear version and rejects the candidate.
+      - name: Download the staged dropbear (matches build-infra's version)
+        uses: actions/download-artifact@v3
+        with:
+          name: firecracker-inputs
+          path: firecracker-inputs
+
+      - name: Publish the tested candidate
+        env:
+          BOT_BOTTLE_INFRA_ARTIFACT_TOKEN: ${{ secrets.BOT_BOTTLE_INFRA_ARTIFACT_TOKEN }}
+          BOT_BOTTLE_FC_DROPBEAR: ${{ github.workspace }}/firecracker-inputs/dropbear
+        run: python3 -m bot_bottle.backend.firecracker.publish_infra --publish-dir infra-candidate

.gitea/workflows/tracker-policy-pr.yml

@@ -2,7 +2,7 @@ name: tracker-policy-pr
 
 on:
   pull_request:
-    types: [opened, edited, reopened, synchronized, labeled, unlabeled]
+    types: [opened, edited, reopened, synchronize, labeled, unlabeled]
 
 jobs:
   check-pr:

Dockerfile.gateway

@@ -98,6 +98,9 @@ RUN pip install --no-cache-dir /src/
 
 # mitmdump -s requires a file path, not a module. Write a one-line shim that
 # re-exports `addons` from the installed package; mitmdump finds it there.
+# WORKDIR here also creates /app so the shim + COPYs below can write into it
+# (nothing created /app before this point).
+WORKDIR /app
 RUN printf 'from bot_bottle.egress_addon import addons\n' > /app/egress_addon.py
 COPY bot_bottle/egress_entrypoint.sh /app/egress-entrypoint.sh
 RUN chmod +x /app/egress-entrypoint.sh
@@ -117,8 +120,6 @@ RUN mkdir -p \
 # subset the bottle uses.
 EXPOSE 8888 9099 9418 9420 9100
 
-WORKDIR /app
-
 # PID 1 is the supervisor. It owns signal handling and exit-code
 # propagation; no `exec` chain in the entrypoint itself.
 ENTRYPOINT ["python3", "-m", "bot_bottle.gateway_init"]

README.md

@@ -90,6 +90,8 @@ BOT_BOTTLE_BACKEND=firecracker ./cli.py start <agent>
 
 > **NixOS:** enable `virtualisation.docker`, ensure the KVM module is loaded (`boot.kernelModules = [ "kvm-intel" ];` or `kvm-amd`), and add your user to the `kvm` and `docker` groups. For the network pool, consume the flake module — `imports = [ inputs.bot-bottle.nixosModules.firecracker-netpool ]; services.bot-bottle-firecracker = { enable = true; owner = "you"; };` — then `nixos-rebuild switch` (imperative nft/TAP rules don't survive a rebuild; channel users can `imports = [ <bot-bottle>/nix/firecracker-netpool.nix ]`). `firecracker` isn't in nixpkgs by default as a user binary — install the release binary (pin the version) and put it on `PATH`.
 
+> **CI:** the coverage gate (`.gitea/workflows/test.yml` → `coverage` job) runs on a self-hosted runner labelled `kvm`, because the Firecracker backend's VM/SSH orchestration is exercised only by the integration suite, which needs `/dev/kvm` + the provisioned pool (a container runner would skip it and read as uncovered). Provision that runner exactly like a normal Firecracker host — `firecracker` on `PATH`, `/dev/kvm`, the cached guest kernel + static dropbear, and the pool installed as the persistent systemd unit — then register it with the `kvm` label. A Docker-capable hosted job builds the candidate once; KVM tests boot those exact bytes, and a successful main run publishes them. The unit/lint jobs still run on `ubuntu-latest`.
+
 ```sh
 ./cli.py start <agent>   # builds the image on first run, drops you into claude
 ```

bot_bottle/audit_store.py

@@ -42,7 +42,7 @@ class AuditStore(DbStore):
         super().__init__(db_path or host_db_path(), migrations)
 
     def write_audit_entry(self, entry: AuditEntry) -> Path:
-        with self._connect() as conn:
+        with self._connection() as conn:
             conn.execute(
                 """
                 INSERT INTO supervise_audit_entries (
@@ -66,7 +66,7 @@ class AuditStore(DbStore):
     def read_audit_entries(self, component: str, slug: str) -> list[AuditEntry]:
         if not self.db_path.is_file():
             return []
-        with self._connect() as conn:
+        with self._connection() as conn:
             rows = conn.execute(
                 """
                 SELECT * FROM supervise_audit_entries

bot_bottle/backend/docker/setup.py

@@ -27,10 +27,14 @@ def _docker_on_path() -> bool:
 def _daemon_reachable() -> bool:
     if not _docker_on_path():
         return False
+    try:
         return subprocess.run(
             ["docker", "info"],
-        stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
+            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+            check=False, timeout=5,
         ).returncode == 0
+    except subprocess.TimeoutExpired:
+        return False
 
 
 def _print_install_pointer() -> None:

bot_bottle/backend/firecracker/image_builder.py

@@ -38,25 +38,39 @@ _BUILD_TIMEOUT_SECONDS = 900.0
 
 
 def _dockerfile_hash(dockerfile: Path) -> str:
-    """Cache key: the Dockerfile's content. The shipped agent Dockerfiles
+    """The Dockerfile's content hash. The shipped agent Dockerfiles COPY
-    COPY nothing from the build context (see .dockerignore), so their content
+    nothing from the build context (see .dockerignore), so their content fully
-    fully determines the image; a Dockerfile that adds COPY will want the
+    determines the built image; a Dockerfile that adds COPY will want the
     context folded in here too."""
     return hashlib.sha256(dockerfile.read_bytes()).hexdigest()[:16]
 
 
+def _rootfs_digest(dockerfile: Path) -> str:
+    """Cache key for the built AND boot-injected agent rootfs. Two inputs
+    determine the on-disk rootfs: the Dockerfile (the image) and the guest init
+    injected into it (`util._GUEST_INIT`). Folding the init in means a fix to
+    it — e.g. making /tmp world-writable — busts the cache instead of silently
+    reusing a stale rootfs built with the old init."""
+    h = hashlib.sha256()
+    h.update(_dockerfile_hash(dockerfile).encode())
+    h.update(b"\0")
+    h.update(util._GUEST_INIT.encode())
+    return h.hexdigest()[:16]
+
+
 def build_agent_rootfs_dir(
     dockerfile: Path, *, image_tag: str, smoke_test: tuple[str, ...] = (),
 ) -> Path:
     """Build `dockerfile` in the infra VM (buildah, no host docker), export its
     rootfs, inject the guest boot bits, and return the cached base dir — the
-    same shape `util.build_rootfs_ext4` consumes. Cached by Dockerfile content,
+    same shape `util.build_rootfs_ext4` consumes. Cached by Dockerfile content
-    so a repeat launch skips the rebuild.
+    + injected guest init, so a repeat launch skips the rebuild but an init or
+    Dockerfile change rebuilds.
 
     `smoke_test` (the provider's declared argv, e.g. `("claude","--version")`)
     is run in the freshly built image before export, catching an npm
     silent-failure image at build time rather than at first agent use."""
-    digest = _dockerfile_hash(dockerfile)
+    digest = _rootfs_digest(dockerfile)
     base = util.cache_dir() / "rootfs" / f"agent-{digest}"
     if (base / ".bb-ready").is_file():
         info(f"using cached agent rootfs {base.name}")

bot_bottle/backend/firecracker/infra_artifact.py

@@ -85,6 +85,11 @@ def infra_artifact_version(init_script: str, *, repo_root: Path = _REPO_ROOT) ->
         h.update(name.encode())
         h.update(b"\0")
         h.update((repo_root / name).read_bytes())
+    h.update(b"pyproject.toml\0")
+    h.update((repo_root / "pyproject.toml").read_bytes())
+    h.update(b"dropbear\0")
+    dropbear = util.dropbear_path()
+    h.update(dropbear.read_bytes() if dropbear.is_file() else b"<missing>")
     h.update(b"init\0")
     h.update(init_script.encode())
     return h.hexdigest()[:16]
@@ -111,6 +116,7 @@ def artifact_url(version: str, filename: str) -> str:
 
 _GZ_NAME = "rootfs.ext4.gz"
 _SHA_NAME = "rootfs.ext4.gz.sha256"
+_CANDIDATE_DIR_ENV = "BOT_BOTTLE_INFRA_ARTIFACT_DIR"
 
 
 def _cache_root(version: str) -> Path:
@@ -160,6 +166,33 @@ def ensure_artifact_gz(version: str) -> Path:
     """The verified, cached `rootfs.ext4.gz` for `version` — downloading it (and
     its `.sha256`) once, then reusing it. Fail-closed on a checksum mismatch:
     the partial is removed and we die rather than boot an unverified rootfs."""
+    candidate_dir = os.environ.get(_CANDIDATE_DIR_ENV, "").strip()
+    if candidate_dir:
+        root = Path(candidate_dir)
+        version_file = root / "version.txt"
+        # Guard the read so a missing version.txt is a clean error, not a raw
+        # FileNotFoundError.
+        if not version_file.is_file():
+            die(f"infra candidate bundle is incomplete: {root}")
+        declared = version_file.read_text(encoding="utf-8").strip()
+        if declared != version:
+            die(
+                f"infra candidate version mismatch: expected {version}, "
+                f"bundle contains {declared or '<empty>'}"
+            )
+        gz = root / _GZ_NAME
+        sha = root / _SHA_NAME
+        if not gz.is_file() or not sha.is_file():
+            die(f"infra candidate bundle is incomplete: {root}")
+        expected = sha.read_text().split()[0].strip().lower()
+        actual = _sha256_file(gz)
+        if actual != expected:
+            die(
+                f"infra candidate checksum mismatch for {version}:\n"
+                f"    expected {expected}\n    actual   {actual}"
+            )
+        return gz
+
     root = _cache_root(version)
     root.mkdir(parents=True, exist_ok=True)
     gz = root / _GZ_NAME

bot_bottle/backend/firecracker/infra_vm.py

@@ -161,20 +161,23 @@ def ensure_running() -> InfraVm:
     slot = netpool.orch_slot()
     url = f"http://{slot.guest_ip}:{CONTROL_PLANE_PORT}"
     key = _infra_dir() / "id_ed25519"
-    if key.exists() and _health_ok(url):
+    want = _expected_version()
+    if _adoptable(key, url, want):
         info(f"adopting running infra VM at {url}")
         return InfraVm(guest_ip=slot.guest_ip, private_key=key)
 
     with _singleton_lock():
         # Re-check under the lock: another launcher may have booted it while
         # we waited for the lock (double-checked, so we adopt not re-boot).
-        if key.exists() and _health_ok(url):
+        if _adoptable(key, url, want):
             info(f"adopting running infra VM at {url}")
             return InfraVm(guest_ip=slot.guest_ip, private_key=key)
-        stop()  # clear a stale/hung VM holding the link before booting fresh
+        # Clear a stale/hung/OUTDATED VM holding the link before booting fresh.
+        stop()
         ensure_built()
         infra = boot()
         wait_for_health(infra)
+        _record_booted_version(want)
         return infra
 
 
@@ -193,9 +196,15 @@ def _singleton_lock() -> Generator[None, None, None]:
 
 
 def stop() -> None:
-    """Stop the infra VM singleton (idempotent — absent is success)."""
+    """Stop the infra VM singleton (idempotent — absent is success). Reaps the
+    recorded VMM AND any orphaned firecracker still bound to the infra config —
+    the PID file drifts after crashes / out-of-band kills, and a survivor would
+    hold the orchestrator TAP so the next boot dies with "tap … Resource busy".
+    Drops the version marker so a stopped VM is never treated as adoptable."""
     _kill_pidfile()
+    _kill_infra_firecrackers()
     _pid_file().unlink(missing_ok=True)
+    _version_file().unlink(missing_ok=True)
 
 
 def boot() -> InfraVm:
@@ -238,6 +247,36 @@ def _pid_file() -> Path:
     return _infra_dir() / "vm.pid"
 
 
+def _version_file() -> Path:
+    """Records the infra-artifact version the *running* VM booted from, so a
+    later launcher can tell whether the singleton it found is the current code.
+    Without it, a healthy VM built from an older image gets adopted forever and
+    the new code never boots — every infra change would need an out-of-band
+    kill to dislodge the stale VM (and races whatever launched next)."""
+    return _infra_dir() / "booted-version"
+
+
+def _expected_version() -> str:
+    return infra_artifact.infra_artifact_version(_infra_init())
+
+
+def _adoptable(key: Path, url: str, want: str) -> bool:
+    """Adopt a running infra VM only if it booted from the CURRENT version and
+    its control plane is healthy. A missing/mismatched marker means a prior
+    launcher booted an older infra image — reboot rather than reuse stale code."""
+    if not key.exists():
+        return False
+    try:
+        booted = _version_file().read_text(encoding="utf-8").strip()
+    except OSError:
+        return False
+    return booted == want and _health_ok(url)
+
+
+def _record_booted_version(version: str) -> None:
+    _version_file().write_text(version + "\n", encoding="utf-8")
+
+
 # The registry "volume": a host-side ext4 file attached to the infra VM as a
 # second virtio-block device (guest /dev/vdb), mounted at the control plane's
 # DB dir. It outlives the ephemeral rootfs, so the bottle registry survives an
@@ -309,6 +348,28 @@ def _kill_pidfile() -> None:
         pass
 
 
+def _kill_infra_firecrackers(proc_root: Path = Path("/proc")) -> None:
+    """SIGKILL any firecracker VMM whose `--config-file` is this host's infra
+    config, independent of the PID file — reaps orphans it lost track of so the
+    orchestrator TAP is free to rebind. Scoped to the infra config path, so the
+    interactive pool's agent/infra VMs (other config paths) are untouched."""
+    cfg = str(_infra_dir() / "config.json")
+    for entry in proc_root.iterdir():
+        if not entry.name.isdigit():
+            continue
+        try:
+            if (entry / "comm").read_text().strip() != "firecracker":
+                continue
+            args = (entry / "cmdline").read_bytes().split(b"\0")
+        except OSError:
+            continue  # process vanished / not ours
+        if any(a.decode("utf-8", "replace") == cfg for a in args):
+            try:
+                os.kill(int(entry.name), signal.SIGKILL)
+            except (OSError, ValueError):
+                pass
+
+
 def _health_ok(url: str) -> bool:
     try:
         with urllib.request.urlopen(f"{url}/health", timeout=1.0) as resp:
@@ -433,7 +494,7 @@ BOT_BOTTLE_ROOT=/var/lib/bot-bottle python3 -m bot_bottle.orchestrator \\
 BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise \\
 BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{CONTROL_PLANE_PORT} \\
 SUPERVISE_DB_PATH=/var/lib/bot-bottle/db/bot-bottle.db \\
-    python3 /app/gateway_init.py &
+    python3 -m bot_bottle.gateway_init &
 
 # Reap as PID 1; children are backgrounded, so `wait` blocks.
 while : ; do wait ; done

bot_bottle/backend/firecracker/publish_infra.py

@@ -11,7 +11,8 @@ The `<version>` is `infra_artifact.infra_artifact_version(...)`, the content
 hash of the rootfs inputs, so a launch host at the same code checkout resolves
 the exact artifact this produced.
 
-  python3 -m bot_bottle.backend.firecracker.publish_infra [--dry-run] [--force]
+  python3 -m bot_bottle.backend.firecracker.publish_infra --output DIR
+  python3 -m bot_bottle.backend.firecracker.publish_infra --publish-dir DIR
 
 Auth: a token with `write:package` on the target owner, from
 `BOT_BOTTLE_INFRA_ARTIFACT_TOKEN`.
@@ -24,7 +25,6 @@ import gzip
 import hashlib
 import shutil
 import sys
-import tempfile
 import urllib.error
 import urllib.request
 from pathlib import Path
@@ -131,36 +131,79 @@ def build_artifact(out_dir: Path) -> tuple[str, Path, Path]:
     return version, gz, sha
 
 
+def _publish_bundle(root: Path, token: str) -> str:
+    version_file = root / "version.txt"
+    # Guard the read so a missing version.txt is a clean error, not a raw
+    # FileNotFoundError.
+    if not version_file.is_file():
+        raise SystemExit(f"incomplete artifact bundle: {root}")
+    version = version_file.read_text(encoding="utf-8").strip()
+    expected = infra_artifact.infra_artifact_version(infra_vm._infra_init())
+    if version != expected:
+        raise SystemExit(
+            f"artifact bundle version {version!r} does not match checkout {expected!r}"
+        )
+    gz = root / "rootfs.ext4.gz"
+    sha = root / "rootfs.ext4.gz.sha256"
+    if not gz.is_file() or not sha.is_file():
+        raise SystemExit(f"incomplete artifact bundle: {root}")
+    expected_sha = sha.read_text().split()[0].strip().lower()
+    if _sha256(gz) != expected_sha:
+        raise SystemExit("artifact bundle checksum mismatch")
+
+    gz_url = infra_artifact.artifact_url(version, gz.name)
+    sha_url = infra_artifact.artifact_url(version, sha.name)
+    about_url = infra_artifact.artifact_url(version, _ABOUT_NAME)
+
+    # Publishing is idempotent. If this exact complete artifact is already
+    # present, a test-only main commit is a no-op. Otherwise clear any partial
+    # upload left by an interrupted prior attempt and upload the complete set.
+    try:
+        with urllib.request.urlopen(infra_artifact._open(sha_url)) as resp:
+            remote_sha = resp.read().decode("utf-8").split()[0].strip().lower()
+    except urllib.error.HTTPError as e:
+        if e.code != 404:
+            raise SystemExit(f"checking existing artifact failed (HTTP {e.code})")
+        remote_sha = ""
+    except urllib.error.URLError as e:
+        raise SystemExit(f"registry unreachable: {sha_url} ({e.reason})")
+    if remote_sha == expected_sha:
+        print(f"infra rootfs {version} already published")
+        return version
+
+    for url in (gz_url, sha_url, about_url):
+        _delete(url, token)
+    _put(gz_url, gz, token)
+    _put(sha_url, sha.read_bytes(), token)
+    _put(about_url, _ABOUT_TEXT.encode(), token)
+    return version
+
+
 def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(
         prog="publish_infra", description="Build + publish the infra rootfs artifact.")
-    parser.add_argument("--dry-run", action="store_true",
+    mode = parser.add_mutually_exclusive_group(required=True)
-                        help="build the artifact but do not upload")
+    mode.add_argument("--output", type=Path,
-    parser.add_argument("--force", action="store_true",
+                      help="build a candidate bundle in DIR without publishing")
-                        help="overwrite an already-published artifact of this version")
+    mode.add_argument("--publish-dir", type=Path,
+                      help="publish an already-built and tested candidate bundle")
     args = parser.parse_args(argv)
 
     _, _, token = infra_artifact._config()
-    if not args.dry_run and not token:
+    if args.publish_dir is not None and not token:
         raise SystemExit(
             "no publish token: set BOT_BOTTLE_INFRA_ARTIFACT_TOKEN to a token "
             "with write:package")
 
-    with tempfile.TemporaryDirectory(prefix="bb-publish-infra.") as tmp:
+    if args.output is not None:
-        version, gz, sha = build_artifact(Path(tmp))
+        args.output.mkdir(parents=True, exist_ok=True)
-        gz_url = infra_artifact.artifact_url(version, gz.name)
+        version, _gz, _sha = build_artifact(args.output)
-        sha_url = infra_artifact.artifact_url(version, sha.name)
+        (args.output / "version.txt").write_text(version + "\n", encoding="utf-8")
-        about_url = infra_artifact.artifact_url(version, _ABOUT_NAME)
+        print(f"built infra rootfs candidate {version}")
-        if args.dry_run:
-            print(f"dry-run: would upload -> {gz_url}")
         return 0
-        if args.force:
+
-            _delete(gz_url, token)
+    assert args.publish_dir is not None
-            _delete(sha_url, token)
+    version = _publish_bundle(args.publish_dir, token)
-            _delete(about_url, token)
-        _put(gz_url, gz, token)  # streamed from disk (hundreds of MB)
-        _put(sha_url, sha.read_bytes(), token)  # tiny, in-memory is fine
-        _put(about_url, _ABOUT_TEXT.encode(), token)  # package description
     print(f"published infra rootfs {version}")
     return 0
 

bot_bottle/backend/firecracker/util.py

@@ -368,6 +368,11 @@ mount -t devtmpfs dev /dev 2>/dev/null
 mkdir -p /dev/pts && mount -t devpts devpts /dev/pts 2>/dev/null
 mount -o remount,rw / 2>/dev/null
 
+# /tmp must be world-writable + sticky. The rootless rootfs build can land
+# it 0755/root-owned, leaving the agent (uid 1000 node) unable to create
+# scratch dirs there — git worktrees, build temp, `git init /tmp/...`, etc.
+mkdir -p /tmp && chmod 1777 /tmp
+
 # Install the per-bottle SSH pubkey from the kernel cmdline.
 KEY=$(sed -n 's/.*bb_pubkey=\([^ ]*\).*/\1/p' /proc/cmdline | base64 -d 2>/dev/null)
 if [ -n "$KEY" ]; then

bot_bottle/db_store.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import sqlite3
+from contextlib import contextmanager
 from pathlib import Path
 
 try:
@@ -28,12 +29,21 @@ class DbStore:
         conn.row_factory = sqlite3.Row
         return conn
 
+    @contextmanager
+    def _connection(self):
+        conn = self._connect()
+        try:
+            with conn:
+                yield conn
+        finally:
+            conn.close()
+
     def is_migrated(self) -> bool:
         """Return True if the DB is fully up-to-date, False if migration is needed."""
         if not self.db_path.exists():
             return False
         try:
-            with self._connect() as conn:
+            with self._connection() as conn:
                 row = conn.execute(
                     "SELECT version FROM schema_versions WHERE module = ?",
                     (self._migrations.schema_key,),
@@ -45,7 +55,7 @@ class DbStore:
 
     def migrate(self) -> None:
         """Apply any pending migrations and set permissions on the DB file."""
-        with self._connect() as conn:
+        with self._connection() as conn:
             self._migrations.apply(conn)
         self._chmod()
 

bot_bottle/orchestrator/registry.py

@@ -167,7 +167,7 @@ class RegistryStore(DbStore):
             metadata=metadata,
             policy=policy,
         )
-        with self._connect() as conn:
+        with self._connection() as conn:
             conn.execute(
                 "DELETE FROM orchestrator_bottles "
                 "WHERE source_ip = ? AND state = 'active' AND bottle_id != ?",
@@ -193,7 +193,7 @@ class RegistryStore(DbStore):
     def set_policy(self, bottle_id: str, policy: str) -> bool:
         """Update a bottle's policy in place (live reload). Returns True if
         the bottle exists."""
-        with self._connect() as conn:
+        with self._connection() as conn:
             cur = conn.execute(
                 "UPDATE orchestrator_bottles SET policy = ? WHERE bottle_id = ?",
                 (policy, bottle_id),
@@ -203,7 +203,7 @@ class RegistryStore(DbStore):
 
     def deregister(self, bottle_id: str) -> bool:
         """Remove a bottle. Returns True if a row was deleted."""
-        with self._connect() as conn:
+        with self._connection() as conn:
             cur = conn.execute(
                 "DELETE FROM orchestrator_bottles WHERE bottle_id = ?", (bottle_id,)
             )
@@ -211,7 +211,7 @@ class RegistryStore(DbStore):
 
     def get(self, bottle_id: str) -> BottleRecord | None:
         """Return the bottle by id, or None if absent."""
-        with self._connect() as conn:
+        with self._connection() as conn:
             row = conn.execute(
                 "SELECT * FROM orchestrator_bottles WHERE bottle_id = ?", (bottle_id,)
             ).fetchone()
@@ -219,7 +219,7 @@ class RegistryStore(DbStore):
 
     def all(self) -> list[BottleRecord]:
         """Every registered bottle, oldest first."""
-        with self._connect() as conn:
+        with self._connection() as conn:
             rows = conn.execute(
                 "SELECT * FROM orchestrator_bottles ORDER BY created_at"
             ).fetchall()
@@ -232,7 +232,7 @@ class RegistryStore(DbStore):
         source IP is unspoofable (Firecracker `/31` + nft) and the control
         plane is reachable only by the trusted gateway; pair with the
         identity token (`attribute`) elsewhere."""
-        with self._connect() as conn:
+        with self._connection() as conn:
             rows = conn.execute(
                 "SELECT * FROM orchestrator_bottles "
                 "WHERE source_ip = ? AND state = 'active'",

bot_bottle/queue_store.py

@@ -66,7 +66,7 @@ class QueueStore(DbStore):
         super().__init__(resolved, migrations)
 
     def write_proposal(self, proposal: Proposal) -> Path:
-        with self._connect() as conn:
+        with self._connection() as conn:
             conn.execute(
                 """
                 INSERT OR REPLACE INTO supervise_proposals (
@@ -89,7 +89,7 @@ class QueueStore(DbStore):
         return self.db_path
 
     def read_proposal(self, proposal_id: str) -> Proposal:
-        with self._connect() as conn:
+        with self._connection() as conn:
             row = conn.execute(
                 """
                 SELECT * FROM supervise_proposals
@@ -104,7 +104,7 @@ class QueueStore(DbStore):
     def list_pending_proposals(self) -> list[Proposal]:
         if not self.db_path.is_file():
             return []
-        with self._connect() as conn:
+        with self._connection() as conn:
             rows = conn.execute(
                 """
                 SELECT p.* FROM supervise_proposals p
@@ -125,7 +125,7 @@ class QueueStore(DbStore):
     def list_all_pending_proposals(self) -> list[Proposal]:
         if not self.db_path.is_file():
             return []
-        with self._connect() as conn:
+        with self._connection() as conn:
             rows = conn.execute(
                 """
                 SELECT p.* FROM supervise_proposals p
@@ -142,7 +142,7 @@ class QueueStore(DbStore):
         return [self._row_to_proposal(row) for row in rows]
 
     def write_response(self, response: Response) -> Path:
-        with self._connect() as conn:
+        with self._connection() as conn:
             conn.execute(
                 """
                 INSERT OR REPLACE INTO supervise_responses (
@@ -161,7 +161,7 @@ class QueueStore(DbStore):
         return self.db_path
 
     def read_response(self, proposal_id: str) -> Response:
-        with self._connect() as conn:
+        with self._connection() as conn:
             row = conn.execute(
                 """
                 SELECT * FROM supervise_responses
@@ -176,7 +176,7 @@ class QueueStore(DbStore):
     def archive_proposal(self, proposal_id: str) -> None:
         if not self.db_path.is_file():
             return
-        with self._connect() as conn:
+        with self._connection() as conn:
             conn.execute(
                 """
                 UPDATE supervise_proposals SET archived = 1

pyproject.toml

@@ -1,6 +1,6 @@
 [build-system]
 requires = ["setuptools>=68"]
-build-backend = "setuptools.backends.legacy:build"
+build-backend = "setuptools.build_meta"
 
 [project]
 name = "bot-bottle"

scripts/coverage.sh

@@ -26,7 +26,9 @@ rm -f .coverage
 echo "== unit ==" >&2
 "$PY" -m coverage run -m unittest discover -t . -s tests/unit
 
-echo "== integration (skips without Docker) ==" >&2
+echo "== integration (firecracker; skips docker tests) ==" >&2
+BOT_BOTTLE_BACKEND=firecracker SKIP_DOCKER_TESTS=1 \
+    BOT_BOTTLE_INFRA_ARTIFACT_DIR="${BOT_BOTTLE_CI_INFRA_ARTIFACT_DIR:-}" \
     "$PY" -m coverage run --append -m unittest discover -t . -s tests/integration
 
 echo "== combined report ==" >&2

tests/_docker.py

@@ -2,24 +2,44 @@
 
 from __future__ import annotations
 
+import os
 import shutil
 import subprocess
 import unittest
 
 
 def docker_available() -> bool:
+    if os.environ.get("SKIP_DOCKER_TESTS"):
+        return False
     if shutil.which("docker") is None:
         return False
+    try:
         return (
             subprocess.run(
                 ["docker", "info"],
                 stdout=subprocess.DEVNULL,
                 stderr=subprocess.DEVNULL,
                 check=False,
+                timeout=5,
             ).returncode
             == 0
         )
+    except subprocess.TimeoutExpired:
+        return False
 
 
 def skip_unless_docker(reason: str = "docker unreachable"):
     return unittest.skipUnless(docker_available(), reason)
+
+
+def skip_unless_docker_or_firecracker(
+    reason: str = "neither Docker nor Firecracker selected",
+):
+    """Skip a backend-agnostic test unless one supported backend can run.
+
+    Firecracker does not require the host Docker daemon. The KVM coverage job
+    deliberately sets ``SKIP_DOCKER_TESTS`` to exclude Docker-only integration
+    classes while still exercising this path.
+    """
+    firecracker_selected = os.environ.get("BOT_BOTTLE_BACKEND") == "firecracker"
+    return unittest.skipUnless(firecracker_selected or docker_available(), reason)

tests/integration/test_sandbox_escape.py

@@ -31,7 +31,7 @@ from pathlib import Path
 from bot_bottle.backend import BottleSpec, get_bottle_backend
 from bot_bottle.bottle_state import cleanup_state
 from bot_bottle.manifest import ManifestIndex
-from tests._docker import skip_unless_docker
+from tests._docker import skip_unless_docker_or_firecracker
 
 
 # Secrets planted in the bottle env as literals (agents substitute via
@@ -67,13 +67,14 @@ _DUMMY_HOST_KEY = (
 )
 
 
-@skip_unless_docker()
+@skip_unless_docker_or_firecracker()
 @unittest.skipIf(
-    os.environ.get("GITEA_ACTIONS") == "true",
+    os.environ.get("GITEA_ACTIONS") == "true"
-    "skipped under act_runner: egress_tls_init uses a host bind mount "
+    and os.environ.get("BOT_BOTTLE_BACKEND") != "firecracker",
-    "the runner container can't see, and the network topology hides "
+    "skipped under act_runner unless BOT_BOTTLE_BACKEND=firecracker: "
-    "sibling-gateway visibility — same constraint as the other "
+    "egress_tls_init uses a host bind mount the runner container can't "
-    "bottle-bringup integration tests",
+    "see, and the network topology hides sibling-gateway visibility — "
+    "these constraints don't apply on the self-hosted KVM runner",
 )
 class TestSandboxEscape(unittest.TestCase):
     """End-to-end attacks against a real bottle. The bottle stays
@@ -90,11 +91,9 @@ class TestSandboxEscape(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls) -> None:
-        # Docker is always required (the agent + companion containers run under it,
+        # Pin Docker when BOT_BOTTLE_BACKEND is unset to preserve the
-        # and VM backends still use it for the gateway); the
+        # Docker-backed CI path. Firecracker uses its persistent infra VM for
-        # class-level @skip_unless_docker already covers that. Pin
+        # the shared gateway and therefore does not require host Docker.
-        # Docker when BOT_BOTTLE_BACKEND is unset to preserve the
-        # Docker-backed CI path.
         cls._backend_name = os.environ.get("BOT_BOTTLE_BACKEND", "docker")
 
         # Throwaway static key for the git-gate fixture. It need not

tests/unit/test_backend_setup.py

@@ -215,6 +215,18 @@ class TestDockerSetupStatus(unittest.TestCase):
         with patch.object(dk.shutil, "which", return_value=None):
             self.assertFalse(dk._daemon_reachable())
 
+    def test_daemon_reachable_true_when_daemon_responds(self):
+        with patch.object(dk.shutil, "which", return_value="/usr/bin/docker"), \
+             patch.object(dk.subprocess, "run",
+                          return_value=subprocess.CompletedProcess([], 0)):
+            self.assertTrue(dk._daemon_reachable())
+
+    def test_daemon_reachable_false_on_timeout(self):
+        with patch.object(dk.shutil, "which", return_value="/usr/bin/docker"), \
+             patch.object(dk.subprocess, "run",
+                          side_effect=subprocess.TimeoutExpired(["docker", "info"], 5)):
+            self.assertFalse(dk._daemon_reachable())
+
     def test_status_reports_missing_docker(self):
         with patch.object(dk.shutil, "which", return_value=None):
             rc, out = _cap(dk.status)

tests/unit/test_cli_start_selector.py

@@ -57,6 +57,14 @@ class TestCmdStartSelector(unittest.TestCase):
         self._bottle_picker_mock = self._bottle_picker_patch.start()
         self._bottle_picker_mock.return_value = ["claude"]  # default: one bottle selected
 
+        # name_color_modal opens /dev/tty and blocks on keyboard input on
+        # self-hosted runners that have a real controlling terminal. Stub it
+        # out like the other tui pickers so tests don't wait for a keypress.
+        self._modal_patch = patch.object(
+            tui_mod, "name_color_modal", return_value=("researcher", ""),
+        )
+        self._modal_patch.start()
+
         self._env_patch = patch.dict(os.environ, {}, clear=False)
         self._env_patch.start()
         os.environ.pop("BOT_BOTTLE_BACKEND", None)
@@ -66,6 +74,7 @@ class TestCmdStartSelector(unittest.TestCase):
         self._launch_patch.stop()
         self._agent_picker_patch.stop()
         self._bottle_picker_patch.stop()
+        self._modal_patch.stop()
         self._env_patch.stop()
 
     # ------------------------------------------------------------------

tests/unit/test_db_store.py

@@ -0,0 +1,58 @@
+"""Unit: DbStore._connection() context manager and is_migrated()."""
+
+from __future__ import annotations
+
+import sqlite3
+import tempfile
+import unittest
+from pathlib import Path
+
+from bot_bottle.db_store import DbStore
+from bot_bottle.migrations import TableMigrations
+
+
+def _store(tmp: Path) -> DbStore:
+    migrations = TableMigrations("test", ["CREATE TABLE items (id INTEGER PRIMARY KEY)"])
+    return DbStore(tmp / "test.db", migrations)
+
+
+class TestDbStoreIsMigrated(unittest.TestCase):
+    def test_returns_false_when_db_absent(self):
+        with tempfile.TemporaryDirectory() as d:
+            store = _store(Path(d))
+            self.assertFalse(store.is_migrated())
+
+    def test_returns_false_when_schema_versions_missing(self):
+        # DB file exists but has no schema_versions table → OperationalError → False.
+        with tempfile.TemporaryDirectory() as d:
+            store = _store(Path(d))
+            conn = sqlite3.connect(store.db_path)
+            conn.close()
+            self.assertFalse(store.is_migrated())
+
+    def test_returns_true_after_migrate(self):
+        with tempfile.TemporaryDirectory() as d:
+            store = _store(Path(d))
+            store.migrate()
+            self.assertTrue(store.is_migrated())
+
+    def test_returns_false_when_behind(self):
+        with tempfile.TemporaryDirectory() as d:
+            migrations = TableMigrations(
+                "test",
+                [
+                    "CREATE TABLE items (id INTEGER PRIMARY KEY)",
+                    "ALTER TABLE items ADD COLUMN name TEXT",
+                ],
+            )
+            store = DbStore(Path(d) / "test.db", migrations)
+            # Apply only the first migration manually.
+            conn = sqlite3.connect(store.db_path)
+            with conn:
+                TableMigrations("test", [migrations.migrations[0]]).apply(conn)
+            conn.close()
+            self.assertFalse(store.is_migrated())
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/unit/test_docker_test_helpers.py

@@ -0,0 +1,35 @@
+"""Tests for integration-test backend selection helpers."""
+
+from __future__ import annotations
+
+import os
+import unittest
+from unittest.mock import patch
+
+from tests._docker import skip_unless_docker_or_firecracker
+
+
+class TestSkipUnlessDockerOrFirecracker(unittest.TestCase):
+    def test_firecracker_runs_when_docker_tests_are_disabled(self):
+        with patch.dict(
+            os.environ,
+            {"BOT_BOTTLE_BACKEND": "firecracker", "SKIP_DOCKER_TESTS": "1"},
+            clear=True,
+        ):
+            decorated = skip_unless_docker_or_firecracker()(type("Case", (), {}))
+
+        self.assertFalse(getattr(decorated, "__unittest_skip__", False))
+
+    def test_non_firecracker_still_skips_when_docker_tests_are_disabled(self):
+        with patch.dict(
+            os.environ,
+            {"BOT_BOTTLE_BACKEND": "docker", "SKIP_DOCKER_TESTS": "1"},
+            clear=True,
+        ):
+            decorated = skip_unless_docker_or_firecracker()(type("Case", (), {}))
+
+        self.assertTrue(getattr(decorated, "__unittest_skip__", False))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/unit/test_firecracker_backend.py

@@ -35,9 +35,12 @@ class TestNetpoolSlots(unittest.TestCase):
     def test_slot_ip_math_31_pairs(self):
         with patch.dict(os.environ, {"BOT_BOTTLE_FC_IP_BASE": "100.64.0.0"}):
             s0, s1 = netpool.slot(0), netpool.slot(1)
-        self.assertEqual(("bbfc0", "100.64.0.0", "100.64.0.1"),
+        # Iface names track netpool's (env-driven) prefix — the KVM CI runner
+        # overrides it for its isolated pool, so don't hardcode "bbfc".
+        pfx = netpool.IFACE_PREFIX
+        self.assertEqual((f"{pfx}0", "100.64.0.0", "100.64.0.1"),
                          (s0.iface, s0.host_ip, s0.guest_ip))
-        self.assertEqual(("bbfc1", "100.64.0.2", "100.64.0.3"),
+        self.assertEqual((f"{pfx}1", "100.64.0.2", "100.64.0.3"),
                          (s1.iface, s1.host_ip, s1.guest_ip))
 
     def test_guest_cidr_is_31(self):
@@ -180,11 +183,14 @@ class TestNetpoolOverlap(unittest.TestCase):
         self.assertEqual("tailscale0", conflicts[0].dev)
 
     def test_ignores_own_taps_and_default(self):
+        # The "own tap" route uses netpool's (env-driven) iface name, so the
+        # test still exercises the self-ignore path on the KVM CI runner, whose
+        # BOT_BOTTLE_FC_IFACE_PREFIX differs from the default.
         with patch.dict(os.environ, {"BOT_BOTTLE_FC_IP_BASE": "10.243.0.0",
                                      "BOT_BOTTLE_FC_POOL_SIZE": "8"}), \
              self._routes([
                  {"dst": "default", "dev": "enp4s0"},
-                 {"dst": "10.243.0.0/31", "dev": "bbfc0"},
+                 {"dst": "10.243.0.0/31", "dev": netpool.slot(0).iface},
                  {"dst": "192.168.1.0/24", "dev": "enp4s0"},
              ]):
             self.assertEqual([], netpool.overlapping_routes())
@@ -203,7 +209,7 @@ class TestNetpoolAllocation(unittest.TestCase):
             # over (and here, exhaust the pool).
             slot, lock = netpool.allocate("first")
             self.addCleanup(lock.close)
-            self.assertEqual("bbfc0", slot.iface)
+            self.assertEqual(netpool.slot(0).iface, slot.iface)
             with patch.object(netpool, "die",
                               side_effect=SystemExit("exhausted")):
                 with self.assertRaises(SystemExit):
@@ -323,9 +329,14 @@ class TestNetpoolDefaultsSingleSource(unittest.TestCase):
         with patch.dict(os.environ, {}, clear=True):
             self.assertEqual(int(d["BOT_BOTTLE_FC_POOL_SIZE"]), netpool.pool_size())
             self.assertEqual(d["BOT_BOTTLE_FC_IP_BASE"], netpool.ip_base())
-        # Module constants resolve through the same shared file.
+        # The module's *defaults* come from the same shared file. Assert the
-        self.assertEqual(d["BOT_BOTTLE_FC_IFACE_PREFIX"], netpool.IFACE_PREFIX)
+        # parsed defaults (not IFACE_PREFIX/NFT_TABLE, which layer a live env
-        self.assertEqual(d["BOT_BOTTLE_FC_NFT_TABLE"], netpool.NFT_TABLE)
+        # override on top — the KVM CI runner sets those for its isolated pool,
+        # which would otherwise mask this single-source check).
+        self.assertEqual(d["BOT_BOTTLE_FC_IFACE_PREFIX"],
+                         netpool._DEFAULTS["BOT_BOTTLE_FC_IFACE_PREFIX"])
+        self.assertEqual(d["BOT_BOTTLE_FC_NFT_TABLE"],
+                         netpool._DEFAULTS["BOT_BOTTLE_FC_NFT_TABLE"])
 
     def test_env_var_overrides_the_shared_default(self):
         with patch.dict(os.environ, {"BOT_BOTTLE_FC_IP_BASE": "10.99.0.0"}):

tests/unit/test_firecracker_helpers.py

@@ -63,9 +63,12 @@ class TestNetpoolProbes(unittest.TestCase):
         self.assertEqual(2, ok.call_count)
 
     def test_missing_taps(self):
+        # Derive the expected iface from netpool's (env-driven) config rather
+        # than hardcoding "bbfc1": the KVM CI runner sets BOT_BOTTLE_FC_* for
+        # its isolated pool, so the prefix there is not the default.
         with patch.dict("os.environ", {"BOT_BOTTLE_FC_POOL_SIZE": "2"}), \
              patch.object(netpool, "tap_present", side_effect=[True, False]):
-            self.assertEqual(["bbfc1"], netpool.missing_taps())
+            self.assertEqual([netpool.slot(1).iface], netpool.missing_taps())
 
     def test_orch_slot_is_top_of_ip_base_16(self):
         # Dedicated orchestrator link: /31 at the top of the IP_BASE /16,

tests/unit/test_firecracker_image_builder.py

@@ -24,7 +24,7 @@ class TestBuildAgentRootfsDir(unittest.TestCase):
         self.addCleanup(self._tmp.cleanup)
 
     def test_cache_hit_skips_rebuild(self):
-        digest = image_builder._dockerfile_hash(self.dockerfile)
+        digest = image_builder._rootfs_digest(self.dockerfile)
         base = self.cache / "rootfs" / f"agent-{digest}"
         base.mkdir(parents=True)
         (base / ".bb-ready").write_text("ok\n")
@@ -55,6 +55,17 @@ class TestBuildAgentRootfsDir(unittest.TestCase):
             image_builder._dockerfile_hash(other),
         )
 
+    def test_rootfs_digest_tracks_dockerfile_and_init(self):
+        # Same Dockerfile, different injected init -> different rootfs key, so
+        # an init fix (e.g. /tmp perms) rebuilds instead of reusing a stale
+        # rootfs; different Dockerfiles also differ.
+        base = image_builder._rootfs_digest(self.dockerfile)
+        with patch.object(image_builder.util, "_GUEST_INIT", "#!/bin/sh\n# changed\n"):
+            self.assertNotEqual(base, image_builder._rootfs_digest(self.dockerfile))
+        other = self.cache / "Dockerfile2"
+        other.write_text("FROM python:3.12-slim\n")
+        self.assertNotEqual(base, image_builder._rootfs_digest(other))
+
 
 class TestSmokeTest(unittest.TestCase):
     def test_empty_argv_is_noop(self):

tests/unit/test_firecracker_infra_vm.py

@@ -38,7 +38,9 @@ class TestBuildInfraRootfs(unittest.TestCase):
         # and exports PATH so gateway_init's subprocess daemons find python3.
         init = build.call_args.kwargs["init_script"]
         self.assertIn("bot_bottle.orchestrator", init)
-        self.assertIn("gateway_init.py", init)
+        # Gateway launches via the installed package (there is no
+        # /app/gateway_init.py file since the daemons moved into bot_bottle).
+        self.assertIn("bot_bottle.gateway_init", init)
         self.assertIn("export PATH=", init)
         # Persistent registry volume mounted at the DB dir before the CP starts.
         self.assertIn("/dev/vdb", init)
@@ -98,7 +100,11 @@ class TestRegistryVolume(unittest.TestCase):
 class TestEnsureBuilt(unittest.TestCase):
     def test_default_pulls_artifact_without_docker(self):
         # PRD 0069 Stage 2: the launch host pulls the prebuilt rootfs; no Docker.
-        with patch.object(infra_vm.docker_mod, "build_image") as build, \
+        # Pin BOT_BOTTLE_INFRA_BUILD off: the coverage CI job exports it =local
+        # for the integration suite, and that ambient value would otherwise send
+        # this default-path test down the local Docker-build branch.
+        with patch.dict(os.environ, {"BOT_BOTTLE_INFRA_BUILD": ""}), \
+                patch.object(infra_vm.docker_mod, "build_image") as build, \
                 patch.object(infra_vm.infra_artifact, "ensure_artifact_gz") as pull:
             infra_vm.ensure_built()
         build.assert_not_called()
@@ -138,20 +144,51 @@ class TestWaitForHealth(unittest.TestCase):
 
 
 class TestEnsureRunningSingleton(unittest.TestCase):
-    def test_adopts_when_healthy(self):
+    def test_adopts_when_healthy_and_version_matches(self):
-        # A healthy control plane + existing key -> adopt (no boot), vm=None.
+        # Healthy control plane + existing key + matching version marker
-        with patch.object(infra_vm, "_health_ok", return_value=True), \
+        # -> adopt (no boot), vm=None.
-             patch.object(infra_vm, "_infra_dir") as d, \
+        import tempfile
+        with tempfile.TemporaryDirectory() as td:
+            d = Path(td)
+            (d / "id_ed25519").write_text("k")
+            (d / "booted-version").write_text("v-current\n")
+            with patch.object(infra_vm, "_infra_dir", return_value=d), \
+                 patch.object(infra_vm, "_expected_version", return_value="v-current"), \
+                 patch.object(infra_vm, "_health_ok", return_value=True), \
                  patch.object(infra_vm, "boot") as boot:
-            keydir = MagicMock()
-            (keydir / "id_ed25519").exists.return_value = True
-            d.return_value = keydir
                 infra = infra_vm.ensure_running()
         boot.assert_not_called()
         self.assertIsNone(infra.vm)
 
+    def test_reboots_when_version_stale(self):
+        # Healthy control plane but the running VM booted an OLDER image
+        # (marker mismatch) -> reboot rather than adopt stale code.
+        import tempfile
+        with tempfile.TemporaryDirectory() as td:
+            d = Path(td)
+            (d / "id_ed25519").write_text("k")
+            (d / "booted-version").write_text("v-old\n")
+            with patch.object(infra_vm, "_infra_dir", return_value=d), \
+                 patch.object(infra_vm, "_expected_version", return_value="v-current"), \
+                 patch.object(infra_vm, "_health_ok", return_value=True), \
+                 patch.object(infra_vm, "stop") as stop, \
+                 patch.object(infra_vm, "ensure_built"), \
+                 patch.object(infra_vm, "wait_for_health"), \
+                 patch.object(infra_vm, "boot") as boot:
+                boot.return_value = infra_vm.InfraVm(
+                    guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock())
+                infra_vm.ensure_running()
+            stop.assert_called_once()      # dislodge the outdated VM
+            boot.assert_called_once()
+            # The fresh boot records the current version for the next launcher.
+            self.assertEqual("v-current\n", (d / "booted-version").read_text())
+
     def test_boots_when_unhealthy(self):
-        with patch.object(infra_vm, "_health_ok", return_value=False), \
+        import tempfile
+        with tempfile.TemporaryDirectory() as td:
+            with patch.object(infra_vm, "_infra_dir", return_value=Path(td)), \
+                 patch.object(infra_vm, "_expected_version", return_value="v-current"), \
+                 patch.object(infra_vm, "_health_ok", return_value=False), \
                  patch.object(infra_vm, "stop") as stop, \
                  patch.object(infra_vm, "ensure_built") as built, \
                  patch.object(infra_vm, "boot") as boot, \
@@ -185,5 +222,74 @@ class TestKillPidfile(unittest.TestCase):
             kill.assert_not_called()
 
 
+class TestAdoptable(unittest.TestCase):
+    def _dir(self, td: str, *, key: bool = True, version: str | None = None) -> Path:
+        d = Path(td)
+        if key:
+            (d / "id_ed25519").write_text("k")
+        if version is not None:
+            (d / "booted-version").write_text(version + "\n")
+        return d
+
+    def test_true_when_key_version_and_health(self):
+        import tempfile
+        with tempfile.TemporaryDirectory() as td:
+            d = self._dir(td, version="v1")
+            with patch.object(infra_vm, "_infra_dir", return_value=d), \
+                 patch.object(infra_vm, "_health_ok", return_value=True):
+                self.assertTrue(infra_vm._adoptable(d / "id_ed25519", "u", "v1"))
+
+    def test_false_when_key_missing(self):
+        import tempfile
+        with tempfile.TemporaryDirectory() as td:
+            d = self._dir(td, key=False, version="v1")
+            with patch.object(infra_vm, "_infra_dir", return_value=d):
+                self.assertFalse(infra_vm._adoptable(d / "id_ed25519", "u", "v1"))
+
+    def test_false_when_no_version_marker(self):
+        import tempfile
+        with tempfile.TemporaryDirectory() as td:
+            d = self._dir(td)  # key present, no booted-version
+            with patch.object(infra_vm, "_infra_dir", return_value=d):
+                self.assertFalse(infra_vm._adoptable(d / "id_ed25519", "u", "v1"))
+
+    def test_false_when_version_mismatch(self):
+        import tempfile
+        with tempfile.TemporaryDirectory() as td:
+            d = self._dir(td, version="v-old")
+            with patch.object(infra_vm, "_infra_dir", return_value=d), \
+                 patch.object(infra_vm, "_health_ok", return_value=True):
+                self.assertFalse(infra_vm._adoptable(d / "id_ed25519", "u", "v1"))
+
+
+class TestKillInfraFirecrackers(unittest.TestCase):
+    def _fake_proc(self, root: Path, pid: int, comm: str, cmdline: list[str]) -> None:
+        p = root / str(pid)
+        p.mkdir()
+        (p / "comm").write_text(comm + "\n")
+        (p / "cmdline").write_bytes(b"\0".join(a.encode() for a in cmdline) + b"\0")
+
+    def test_kills_only_matching_infra_firecracker(self):
+        import tempfile
+        with tempfile.TemporaryDirectory() as td, \
+             tempfile.TemporaryDirectory() as proc:
+            infra_dir = Path(td)
+            cfg = str(infra_dir / "config.json")
+            root = Path(proc)
+            # target: firecracker bound to the infra config -> killed
+            self._fake_proc(root, 111, "firecracker",
+                            ["firecracker", "--no-api", "--config-file", cfg])
+            # a firecracker for a different (interactive) VM -> spared
+            self._fake_proc(root, 222, "firecracker",
+                            ["firecracker", "--config-file", "/home/u/other.json"])
+            # a non-firecracker process on the same config path -> spared
+            self._fake_proc(root, 333, "python3", ["python3", cfg])
+            (root / "not-a-pid").mkdir()
+            with patch.object(infra_vm, "_infra_dir", return_value=infra_dir), \
+                 patch.object(infra_vm.os, "kill") as kill:
+                infra_vm._kill_infra_firecrackers(proc_root=root)
+            kill.assert_called_once_with(111, infra_vm.signal.SIGKILL)
+
+
 if __name__ == "__main__":
     unittest.main()

tests/unit/test_infra_artifact.py

@@ -50,7 +50,12 @@ class _CacheMixin(unittest.TestCase):
         self._env = mock.patch.dict(
             os.environ,
             {"BOT_BOTTLE_FC_CACHE": self._tmp.name,
-             "BOT_BOTTLE_INFRA_ARTIFACT_TOKEN": ""},
+             "BOT_BOTTLE_INFRA_ARTIFACT_TOKEN": "",
+             # Pin the candidate-dir override off: the coverage CI job exports a
+             # candidate dir for the integration suite, and an ambient value
+             # would send these registry-pull tests down the local-bundle path.
+             # Cases that exercise the candidate path set it explicitly.
+             "BOT_BOTTLE_INFRA_ARTIFACT_DIR": ""},
             clear=False,
         )
         self._env.start()
@@ -93,6 +98,31 @@ class TestVersionInputs(unittest.TestCase):
         (pkg / "netpool.defaults.env").write_text("FOO=1\n")
         for name in ("Dockerfile.orchestrator", "Dockerfile.gateway", "Dockerfile.infra"):
             (root / name).write_text(f"FROM scratch  # {name}\n")
+        (root / "pyproject.toml").write_text("[project]\nname = 'bot-bottle'\n")
+
+    def test_pyproject_toml_change_bumps_version(self) -> None:
+        with tempfile.TemporaryDirectory() as d:
+            root = Path(d)
+            self._fake_repo(root)
+            before = ia.infra_artifact_version("init", repo_root=root)
+            (root / "pyproject.toml").write_text(
+                "[project]\nname = 'bot-bottle'\ndependencies = ['httpx']\n")
+            after = ia.infra_artifact_version("init", repo_root=root)
+        self.assertNotEqual(before, after)
+
+    def test_dropbear_change_bumps_version(self) -> None:
+        with tempfile.TemporaryDirectory() as d:
+            root = Path(d)
+            self._fake_repo(root)
+            dropbear = root / "dropbear"
+            dropbear.write_bytes(b"dropbear-v1")
+            with mock.patch.dict(os.environ, {
+                "BOT_BOTTLE_FC_DROPBEAR": str(dropbear),
+            }):
+                before = ia.infra_artifact_version("init", repo_root=root)
+                dropbear.write_bytes(b"dropbear-v2")
+                after = ia.infra_artifact_version("init", repo_root=root)
+        self.assertNotEqual(before, after)
 
     def test_non_python_file_change_bumps_version(self) -> None:
         with tempfile.TemporaryDirectory() as d:
@@ -118,6 +148,58 @@ class TestVersionInputs(unittest.TestCase):
 
 
 class TestEnsureArtifact(_CacheMixin):
+    def test_uses_verified_ci_candidate_without_network(self) -> None:
+        version = "deadbeef00000000"
+        gz = _gz(b"candidate ext4")
+        with tempfile.TemporaryDirectory() as d:
+            root = Path(d)
+            (root / "version.txt").write_text(version + "\n")
+            (root / "rootfs.ext4.gz").write_bytes(gz)
+            digest = hashlib.sha256(gz).hexdigest()
+            (root / "rootfs.ext4.gz.sha256").write_text(
+                f"{digest}  rootfs.ext4.gz\n")
+            with mock.patch.dict(os.environ, {
+                "BOT_BOTTLE_INFRA_ARTIFACT_DIR": str(root),
+            }), mock.patch.object(ia.urllib.request, "urlopen") as net:
+                path = ia.ensure_artifact_gz(version)
+            self.assertEqual(root / "rootfs.ext4.gz", path)
+            net.assert_not_called()
+
+    def test_rejects_candidate_for_another_version(self) -> None:
+        with tempfile.TemporaryDirectory() as d:
+            root = Path(d)
+            (root / "version.txt").write_text("wrong\n")
+            with mock.patch.dict(os.environ, {
+                "BOT_BOTTLE_INFRA_ARTIFACT_DIR": str(root),
+            }):
+                with self.assertRaises(Die) as ctx:
+                    ia.ensure_artifact_gz("expected")
+            self.assertIn("version mismatch", str(ctx.exception.message))
+
+    def test_rejects_incomplete_candidate(self) -> None:
+        with tempfile.TemporaryDirectory() as d:
+            root = Path(d)
+            (root / "version.txt").write_text("v1\n")
+            with mock.patch.dict(os.environ, {
+                "BOT_BOTTLE_INFRA_ARTIFACT_DIR": str(root),
+            }):
+                with self.assertRaises(Die) as ctx:
+                    ia.ensure_artifact_gz("v1")
+            self.assertIn("incomplete", str(ctx.exception.message))
+
+    def test_rejects_candidate_checksum_mismatch(self) -> None:
+        with tempfile.TemporaryDirectory() as d:
+            root = Path(d)
+            (root / "version.txt").write_text("v1\n")
+            (root / "rootfs.ext4.gz").write_bytes(b"bad")
+            (root / "rootfs.ext4.gz.sha256").write_text("0" * 64 + "  rootfs.ext4.gz\n")
+            with mock.patch.dict(os.environ, {
+                "BOT_BOTTLE_INFRA_ARTIFACT_DIR": str(root),
+            }):
+                with self.assertRaises(Die) as ctx:
+                    ia.ensure_artifact_gz("v1")
+            self.assertIn("checksum mismatch", str(ctx.exception.message))
+
     def test_downloads_verifies_and_caches(self) -> None:
         version = "deadbeef00000000"
         gz = _gz(b"fake ext4 bytes")

tests/unit/test_publish_infra.py

@@ -6,8 +6,11 @@ read it into memory. Network is mocked; no Docker, no real build.
 
 from __future__ import annotations
 
+import hashlib
+from email.message import Message
 import tempfile
 import unittest
+import urllib.error
 import urllib.request
 from pathlib import Path
 from unittest import mock
@@ -58,5 +61,99 @@ class TestPut(unittest.TestCase):
         self.assertEqual(b"abc123  rootfs\n", captured[0].data)
 
 
+class TestPublishBundle(unittest.TestCase):
+    def _bundle(self, root: Path, version: str) -> None:
+        payload = b"candidate"
+        (root / "version.txt").write_text(version + "\n")
+        (root / "rootfs.ext4.gz").write_bytes(payload)
+        digest = hashlib.sha256(payload).hexdigest()
+        (root / "rootfs.ext4.gz.sha256").write_text(
+            f"{digest}  rootfs.ext4.gz\n")
+
+    def test_existing_identical_artifact_is_success(self) -> None:
+        with tempfile.TemporaryDirectory() as d:
+            root = Path(d)
+            self._bundle(root, "v1")
+            sha = (root / "rootfs.ext4.gz.sha256").read_bytes()
+            response = mock.MagicMock()
+            response.__enter__.return_value.read.return_value = sha
+            with mock.patch.object(
+                pub.infra_artifact, "infra_artifact_version", return_value="v1"
+            ), mock.patch.object(
+                pub.urllib.request, "urlopen", return_value=response
+            ), mock.patch.object(pub, "_put") as put:
+                self.assertEqual("v1", pub._publish_bundle(root, "token"))
+            put.assert_not_called()
+
+    def test_partial_artifact_is_replaced(self) -> None:
+        with tempfile.TemporaryDirectory() as d:
+            root = Path(d)
+            self._bundle(root, "v1")
+            missing = urllib.error.HTTPError("u", 404, "missing", Message(), None)
+            with mock.patch.object(
+                pub.infra_artifact, "infra_artifact_version", return_value="v1"
+            ), mock.patch.object(
+                pub.urllib.request, "urlopen", side_effect=missing
+            ), mock.patch.object(pub, "_delete") as delete, \
+                    mock.patch.object(pub, "_put") as put:
+                pub._publish_bundle(root, "token")
+            self.assertEqual(3, delete.call_count)
+            self.assertEqual(3, put.call_count)
+
+    def test_rejects_bundle_for_different_checkout(self) -> None:
+        with tempfile.TemporaryDirectory() as d:
+            root = Path(d)
+            self._bundle(root, "old")
+            with mock.patch.object(
+                pub.infra_artifact, "infra_artifact_version", return_value="new"
+            ):
+                with self.assertRaises(SystemExit) as ctx:
+                    pub._publish_bundle(root, "token")
+            self.assertIn("does not match checkout", str(ctx.exception))
+
+    def test_rejects_bad_bundle_checksum(self) -> None:
+        with tempfile.TemporaryDirectory() as d:
+            root = Path(d)
+            self._bundle(root, "v1")
+            (root / "rootfs.ext4.gz").write_bytes(b"tampered")
+            with mock.patch.object(
+                pub.infra_artifact, "infra_artifact_version", return_value="v1"
+            ):
+                with self.assertRaises(SystemExit) as ctx:
+                    pub._publish_bundle(root, "token")
+            self.assertIn("checksum mismatch", str(ctx.exception))
+
+    def test_registry_lookup_failure_is_reported(self) -> None:
+        with tempfile.TemporaryDirectory() as d:
+            root = Path(d)
+            self._bundle(root, "v1")
+            failure = urllib.error.URLError("offline")
+            with mock.patch.object(
+                pub.infra_artifact, "infra_artifact_version", return_value="v1"
+            ), mock.patch.object(pub.urllib.request, "urlopen", side_effect=failure):
+                with self.assertRaises(SystemExit) as ctx:
+                    pub._publish_bundle(root, "token")
+            self.assertIn("registry unreachable", str(ctx.exception))
+
+
+class TestMain(unittest.TestCase):
+    def test_output_builds_candidate_and_records_version(self) -> None:
+        with tempfile.TemporaryDirectory() as d:
+            root = Path(d) / "candidate"
+            with mock.patch.object(
+                pub, "build_artifact", return_value=("v1", root / "g", root / "s")
+            ) as build:
+                self.assertEqual(0, pub.main(["--output", str(root)]))
+            build.assert_called_once_with(root)
+            self.assertEqual("v1\n", (root / "version.txt").read_text())
+
+    def test_publish_dir_publishes_existing_candidate(self) -> None:
+        with tempfile.TemporaryDirectory() as d, \
+                mock.patch.object(pub.infra_artifact, "_config", return_value=("", "", "t")), \
+                mock.patch.object(pub, "_publish_bundle", return_value="v1") as publish:
+            self.assertEqual(0, pub.main(["--publish-dir", d]))
+            publish.assert_called_once_with(Path(d), "t")
+
+
 if __name__ == "__main__":
     unittest.main()