ci(coverage): install dev requirements on the KVM runner

The self-hosted KVM runner is a persistent machine, so
--break-system-packages is inappropriate. Use --user instead so
coverage (and pyright/pylint for future jobs) land in ~/.local
and survive between runs without touching the system Python.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

.gitea/workflows/canaries.yml

@@ -22,10 +22,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.12"
-
+      # No actions/setup-python: canaries are stdlib unittest on the image's
+      # system Python 3.12 (older act_runner mishandles setup-python's PATH).
       - name: Run canaries
         run: python3 -m unittest discover -t . -s tests/canaries -v

.gitea/workflows/prd-number.yml

@@ -37,11 +37,8 @@ jobs:
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.12"
-
+      # No actions/setup-python: the inline script is stdlib-only on the
+      # image's system Python 3.12 (older act_runner mishandles its PATH).
       - name: Configure git
         run: |
           git config user.name "github-actions[bot]"

.gitea/workflows/test.yml

@@ -68,30 +68,48 @@ jobs:
       - name: Run integration tests
         run: python3 -m unittest discover -t . -s tests/integration -v
 
-  # Combined unit+integration coverage report (informational). See
-  # docs/decisions/0004-coverage-policy.md.
+  # Combined unit+integration coverage + the diff-coverage gate (the hard
+  # gate: new/changed lines >= 90%). See docs/decisions/0004-coverage-policy.md.
   #
-  # The hard diff-coverage gate (changed lines >= 90%) is DEFERRED: the
-  # Firecracker backend's VM/SSH orchestration is covered by the integration
-  # suite, which needs /dev/kvm + the provisioned TAP/nft pool — a
-  # container-based runner skips it and those lines read uncovered, so the
-  # gate can't pass here. Re-enabling it on a self-hosted KVM runner is
-  # tracked separately (see PRD 0069 / #348 and the ci-runner branch).
+  # This runs on a self-hosted KVM runner (label `kvm`), NOT ubuntu-latest,
+  # because the Firecracker backend's subprocess/VM orchestration
+  # (launch/boot/SSH/isolation-probe) is covered by the integration suite,
+  # and that suite needs `/dev/kvm` + the provisioned TAP/nft pool — which a
+  # container-based runner doesn't have. On such a runner the firecracker
+  # integration test skips and its ~230 orchestration lines read as
+  # uncovered, so the gate can't pass there.
+  #
+  # Runner prerequisites (provision once on the host; see the README
+  # "Firecracker on Linux" section): the `firecracker` binary on PATH,
+  # `/dev/kvm` accessible to the runner user, Docker, the cached guest
+  # kernel + static dropbear (BOT_BOTTLE_FC_KERNEL / BOT_BOTTLE_FC_DROPBEAR),
+  # and the network pool installed as the persistent systemd unit
+  # (`./cli.py backend setup --backend=firecracker`). The preflight step
+  # below fails fast with instructions if anything is missing.
   coverage:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, kvm]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      # No actions/setup-python: the runner image already ships Python 3.12,
-      # and older act_runner engines mishandle setup-python's PATH (coverage
-      # lands in one interpreter, `python3` resolves to another). Install
-      # straight into the ephemeral job container's system Python —
-      # --break-system-packages is safe because the container is disposable.
-      - name: Install dev requirements
-        run: python3 -m pip install --break-system-packages -r requirements-dev.txt
+      - name: Preflight — Firecracker host is ready
+        run: |
+          command -v firecracker >/dev/null || {
+            echo "firecracker not on PATH — provision the runner (README: Firecracker on Linux)"; exit 1; }
+          test -e /dev/kvm || { echo "/dev/kvm missing — KVM not available on this runner"; exit 1; }
+          # `backend status` exits non-zero unless the TAP pool is up + no
+          # range overlap; it prints the exact `backend setup` fix.
+          python3 cli.py backend status --backend=firecracker
 
-      - name: Combined coverage report (unit + integration)
+      - name: Install dev requirements
+        run: python3 -m pip install --user -r requirements-dev.txt
+
+      - name: Combined coverage (unit + integration, incl. firecracker)
         run: PYTHON=python3 bash scripts/coverage.sh critical
+
+      - name: Diff-coverage gate (changed lines >= 90%)
+        run: |
+          git fetch --no-tags origin main:refs/remotes/origin/main
+          python3 scripts/diff_coverage.py --base origin/main --min 90

.gitea/workflows/update-badges.yml

@@ -20,21 +20,18 @@ jobs:
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.12'
-
+      # No actions/setup-python: the runner image ships Python 3.12 and older
+      # act_runner engines mishandle setup-python's PATH. Install into the
+      # ephemeral job container's system Python (--break-system-packages is
+      # safe because the container is disposable).
       - name: Install dev dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements-dev.txt
+        run: python3 -m pip install --break-system-packages -r requirements-dev.txt
 
       - name: Run coverage and extract percentage
         id: coverage
         run: |
-          python -m coverage run -m unittest discover -t . -s tests/unit > /dev/null 2>&1 || true
-          PERCENT=$(python -m coverage report 2>/dev/null | grep '^TOTAL' | grep -oP '\d+(?=%)' | tail -1)
+          python3 -m coverage run -m unittest discover -t . -s tests/unit > /dev/null 2>&1 || true
+          PERCENT=$(python3 -m coverage report 2>/dev/null | grep '^TOTAL' | grep -oP '\d+(?=%)' | tail -1)
           echo "percent=$PERCENT" >> $GITHUB_OUTPUT
           echo "Coverage: $PERCENT%"
 
@@ -45,7 +42,7 @@ jobs:
           # the single source of truth in scripts/critical-modules.txt; every
           # core module is unit-tested, so the unit-only run is accurate for it.
           INCLUDE=$(grep -vE '^[[:space:]]*(#|$)' scripts/critical-modules.txt | paste -sd, -)
-          PERCENT=$(python -m coverage report --include="$INCLUDE" 2>/dev/null | grep '^TOTAL' | grep -oP '\d+(?=%)' | tail -1)
+          PERCENT=$(python3 -m coverage report --include="$INCLUDE" 2>/dev/null | grep '^TOTAL' | grep -oP '\d+(?=%)' | tail -1)
           echo "percent=$PERCENT" >> $GITHUB_OUTPUT
           echo "Core coverage: $PERCENT%"
 

README.md

@@ -5,8 +5,8 @@
 # bot-bottle
 
 [![test](https://gitea.dideric.is/didericis/bot-bottle/actions/workflows/test.yml/badge.svg?branch=main)](https://gitea.dideric.is/didericis/bot-bottle/actions?workflow=test.yml)
-[![coverage](https://img.shields.io/badge/coverage-82%25-brightgreen)](https://coverage.readthedocs.io/)
-[![core coverage](https://img.shields.io/badge/core%20coverage-95%25-brightgreen)](https://gitea.dideric.is/didericis/bot-bottle/src/branch/main/docs/decisions/0004-coverage-policy.md)
+[![coverage](https://img.shields.io/badge/coverage-81%25-brightgreen)](https://coverage.readthedocs.io/)
+[![core coverage](https://img.shields.io/badge/core%20coverage-94%25-brightgreen)](https://gitea.dideric.is/didericis/bot-bottle/src/branch/main/docs/decisions/0004-coverage-policy.md)
 
 **Problem:** Developer wants to run a coding agent without supervision, but they don't want a prompt injected or misbehaving agent wrecking their environment or exfiltrating sensitive data.
 
@@ -90,6 +90,8 @@ BOT_BOTTLE_BACKEND=firecracker ./cli.py start <agent>
 
 > **NixOS:** enable `virtualisation.docker`, ensure the KVM module is loaded (`boot.kernelModules = [ "kvm-intel" ];` or `kvm-amd`), and add your user to the `kvm` and `docker` groups. For the network pool, consume the flake module — `imports = [ inputs.bot-bottle.nixosModules.firecracker-netpool ]; services.bot-bottle-firecracker = { enable = true; owner = "you"; };` — then `nixos-rebuild switch` (imperative nft/TAP rules don't survive a rebuild; channel users can `imports = [ <bot-bottle>/nix/firecracker-netpool.nix ]`). `firecracker` isn't in nixpkgs by default as a user binary — install the release binary (pin the version) and put it on `PATH`.
 
+> **CI:** the coverage gate (`.gitea/workflows/test.yml` → `coverage` job) runs on a self-hosted runner labelled `kvm`, because the Firecracker backend's VM/SSH orchestration is exercised only by the integration suite, which needs `/dev/kvm` + the provisioned pool (a container runner would skip it and read as uncovered). Provision that runner exactly like a normal Firecracker host — `firecracker` on `PATH`, `/dev/kvm`, Docker, the cached guest kernel + static dropbear, and the pool installed as the persistent systemd unit — then register it with the `kvm` label. The unit/lint jobs still run on `ubuntu-latest`.
+
 ```sh
 ./cli.py start <agent>   # builds the image on first run, drops you into claude
 ```

bot_bottle/backend/__init__.py

@@ -40,7 +40,7 @@ from abc import ABC, abstractmethod
 from contextlib import AbstractContextManager
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Generic, Sequence, TypeVar
+from typing import TYPE_CHECKING, Any, Generic, Sequence, TypeVar
 
 from ..agent_provider import AgentProvisionPlan, get_provider, build_agent_provision_plan
 from ..egress import EgressPlan
@@ -54,6 +54,9 @@ from ..workspace import WorkspacePlan, workspace_plan
 from .print_util import print_multi, visible_agent_env_names
 from .util import host_skill_dir
 
+if TYPE_CHECKING:
+    from .freeze import CommitCancelled, Freezer, get_freezer
+
 
 @dataclass(frozen=True)
 class BottleSpec:
@@ -584,28 +587,63 @@ class BottleBackend(ABC, Generic[PlanT, CleanupT]):
         Not called by the launch path or the test suite."""
 
 
-# Import concrete backend classes AFTER the base types are defined, so
-# each backend module can pull BottleSpec / BottlePlan / BottleBackend
-# via `from . import ...` without hitting a partially-initialized module.
-from .docker import DockerBottleBackend  # noqa: E402  # pylint: disable=wrong-import-position
-from .firecracker import FirecrackerBottleBackend  # noqa: E402  # pylint: disable=wrong-import-position
-from .macos_container import MacosContainerBottleBackend  # noqa: E402  # pylint: disable=wrong-import-position
-
-# Freezer is imported after the backend classes for the same reason:
-# Freezer.commit_slug constructs ActiveAgent, which must be fully
-# defined first.
-from .freeze import CommitCancelled, Freezer, get_freezer  # noqa: E402  # pylint: disable=wrong-import-position
+# _backends is None until the first call to _get_backends(), at which
+# point all three concrete backend classes are imported and instantiated.
+# Keeping the imports out of module scope means that importing any
+# backend sub-module (e.g. `backend.docker.util`) no longer drags the
+# firecracker and macos-container implementations into memory.
+#
+# Tests may replace _backends with a {name: fake} dict via patch.object;
+# _get_backends() returns the current module-level value as-is when it
+# is not None, so test fakes take effect without triggering real imports.
+_backends: dict[str, BottleBackend[Any, Any]] | None = None
 
 
-# The dict is heterogeneous: each value is a BottleBackend specialized
-# over its own plan type. Concrete plan types are erased here because
-# the registry is selected at runtime and the CLI only needs the
-# unparameterized methods (prepare → plan → launch(plan), cleanup, etc.).
-_BACKENDS: dict[str, BottleBackend[Any, Any]] = {
-    "docker": DockerBottleBackend(),
-    "firecracker": FirecrackerBottleBackend(),
-    "macos-container": MacosContainerBottleBackend(),
-}
+def _get_backends() -> dict[str, BottleBackend[Any, Any]]:
+    """Return the registry of all backend instances, loading lazily on first call."""
+    global _backends  # pylint: disable=global-statement
+    if _backends is None:
+        from .docker import DockerBottleBackend
+        from .firecracker import FirecrackerBottleBackend
+        from .macos_container import MacosContainerBottleBackend
+        _backends = {
+            "docker": DockerBottleBackend(),
+            "firecracker": FirecrackerBottleBackend(),
+            "macos-container": MacosContainerBottleBackend(),
+        }
+    return _backends
+
+
+def __getattr__(name: str) -> Any:
+    """Lazily surface concrete backend classes and freeze symbols at the
+    package level so existing `from bot_bottle.backend import X` and
+    `patch.object(backend_mod, X, ...)` call-sites keep working without
+    forcing an import of every backend at module-init time."""
+    if name == "DockerBottleBackend":
+        from .docker import DockerBottleBackend
+        globals()[name] = DockerBottleBackend
+        return DockerBottleBackend
+    if name == "FirecrackerBottleBackend":
+        from .firecracker import FirecrackerBottleBackend
+        globals()[name] = FirecrackerBottleBackend
+        return FirecrackerBottleBackend
+    if name == "MacosContainerBottleBackend":
+        from .macos_container import MacosContainerBottleBackend
+        globals()[name] = MacosContainerBottleBackend
+        return MacosContainerBottleBackend
+    if name == "CommitCancelled":
+        from .freeze import CommitCancelled
+        globals()[name] = CommitCancelled
+        return CommitCancelled
+    if name == "Freezer":
+        from .freeze import Freezer
+        globals()[name] = Freezer
+        return Freezer
+    if name == "get_freezer":
+        from .freeze import get_freezer
+        globals()[name] = get_freezer
+        return get_freezer
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
 
 
 def get_bottle_backend(
@@ -623,10 +661,11 @@ def get_bottle_backend(
     Dies with a pointer at the known backends if the chosen name
     isn't implemented."""
     resolved = name or os.environ.get("BOT_BOTTLE_BACKEND") or _default_backend_name()
-    if resolved not in _BACKENDS:
-        known = ", ".join(sorted(_BACKENDS))
+    backends = _get_backends()
+    if resolved not in backends:
+        known = ", ".join(sorted(backends))
         die(f"unknown backend {resolved!r}; known backends: {known}")
-    return _BACKENDS[resolved]
+    return backends[resolved]
 
 
 def _default_backend_name() -> str:
@@ -636,16 +675,17 @@ def _default_backend_name() -> str:
     # `firecracker` binary isn't installed yet: selecting it here routes
     # start through firecracker's preflight, which prints an install
     # pointer, instead of silently falling back to docker.
+    from .firecracker import FirecrackerBottleBackend
     if FirecrackerBottleBackend.is_host_capable():
         return "firecracker"
     return "docker"
 
 
 def known_backend_names() -> tuple[str, ...]:
-    """Sorted tuple of all backend keys in `_BACKENDS`. Used by
+    """Sorted tuple of all backend keys in `_get_backends()`. Used by
     argparse (`--backend` choices) and the dashboard's backend
     picker."""
-    return tuple(sorted(_BACKENDS))
+    return tuple(sorted(_get_backends()))
 
 
 def has_backend(name: str) -> bool:
@@ -657,9 +697,10 @@ def has_backend(name: str) -> bool:
 
     Returns False for unknown names so callers can pass
     arbitrary input without separate validation."""
-    if name not in _BACKENDS:
+    backends = _get_backends()
+    if name not in backends:
         return False
-    return _BACKENDS[name].is_available()
+    return backends[name].is_available()
 
 
 def enumerate_active_agents() -> list[ActiveAgent]:
@@ -675,10 +716,11 @@ def enumerate_active_agents() -> list[ActiveAgent]:
     deterministic tiebreaker. Agents with missing metadata
     (`started_at == ""`) sort first."""
     out: list[ActiveAgent] = []
-    for name in known_backend_names():
-        if not has_backend(name):
+    backends = _get_backends()
+    for name in sorted(backends):
+        if not backends[name].is_available():
             continue
-        out.extend(_BACKENDS[name].enumerate_active())
+        out.extend(backends[name].enumerate_active())
     out.sort(key=lambda a: (a.started_at, a.slug))
     return out
 

bot_bottle/backend/docker/util.py

@@ -8,7 +8,7 @@ import os
 import re
 import shutil
 import subprocess
-from typing import Iterable, Iterator
+from typing import Iterator
 
 from ...docker_cmd import run_docker
 from ...log import die, info
@@ -32,12 +32,7 @@ def container_name_candidates(base: str) -> Iterator[str]:
 def runsc_available() -> bool:
     """Return True if the Docker daemon has the gVisor (`runsc`) runtime
     registered. Called once per prepare; the result lives on the plan."""
-    r = subprocess.run(
-        ["docker", "info", "--format", "{{json .Runtimes}}"],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
+    r = run_docker(["docker", "info", "--format", "{{json .Runtimes}}"])
     return r.returncode == 0 and "runsc" in r.stdout
 
 
@@ -51,20 +46,15 @@ def require_docker() -> None:
 
 
 def image_exists(ref: str) -> bool:
-    return _silent_run(["docker", "image", "inspect", ref]) == 0
+    return run_docker(["docker", "image", "inspect", ref]).returncode == 0
 
 
 def container_exists(name: str) -> bool:
     """Returns True if a container (running or stopped) with the given
     name exists. Uses `docker ps -a -q -f name=^<name>$` so substring
     matches don't false-positive."""
-    result = subprocess.run(
-        ["docker", "ps", "-a", "-q", "-f", f"name=^{name}$"],
-        capture_output=True,
-        text=True,
-        check=True,
-    )
-    return bool(result.stdout.strip())
+    result = run_docker(["docker", "ps", "-a", "-q", "-f", f"name=^{name}$"])
+    return result.returncode == 0 and bool(result.stdout.strip())
 
 
 def force_remove_container(name: str) -> None:
@@ -72,12 +62,7 @@ def force_remove_container(name: str) -> None:
     doesn't — and the rm itself is best-effort (errors swallowed) so
     this is safe to register as a teardown callback."""
     if container_exists(name):
-        subprocess.run(
-            ["docker", "rm", "-f", name],
-            stdout=subprocess.DEVNULL,
-            stderr=subprocess.DEVNULL,
-            check=False,
-        )
+        run_docker(["docker", "rm", "-f", name])
 
 
 def docker_exec_root(container: str, argv: list[str]) -> None:
@@ -205,22 +190,10 @@ def verify_agent_image(image: str, argv: tuple[str, ...]) -> None:
 def commit_container(container_name: str, image_tag: str) -> None:
     """Run `docker commit <container_name> <image_tag>` to snapshot the
     running container's filesystem state as a local Docker image."""
-    result = subprocess.run(
-        ["docker", "commit", container_name, image_tag],
-        capture_output=True, text=True, check=False,
-    )
+    result = run_docker(["docker", "commit", container_name, image_tag])
     if result.returncode != 0:
         die(
             f"docker commit {container_name!r} → {image_tag!r} failed: "
             f"{(result.stderr or '').strip() or '<no stderr>'}"
         )
     info(f"committed {container_name!r} → {image_tag!r}")
-
-
-def _silent_run(cmd: Iterable[str]) -> int:
-    return subprocess.run(
-        list(cmd),
-        stdout=subprocess.DEVNULL,
-        stderr=subprocess.DEVNULL,
-        check=False,
-    ).returncode

tests/unit/test_backend_selection.py

@@ -40,7 +40,7 @@ class TestGetBottleBackend(unittest.TestCase):
                 return True
 
         with patch.dict(os.environ, {}, clear=True), \
-             patch.object(backend_mod, "_BACKENDS", {
+             patch.object(backend_mod, "_backends", {
                  "macos-container": _FakeBackend(),
                  "docker": _FakeBackend(),
              }):
@@ -61,7 +61,7 @@ class TestGetBottleBackend(unittest.TestCase):
         with patch.dict(os.environ, {}, clear=True), \
              patch.object(backend_mod.FirecrackerBottleBackend,
                           "is_host_capable", classmethod(lambda cls: False)), \
-             patch.object(backend_mod, "_BACKENDS", {
+             patch.object(backend_mod, "_backends", {
                  "macos-container": _FakeBackend("macos-container", False),
                  "docker": _FakeBackend("docker", True),
              }):
@@ -83,7 +83,7 @@ class TestGetBottleBackend(unittest.TestCase):
         with patch.dict(os.environ, {}, clear=True), \
              patch.object(backend_mod.FirecrackerBottleBackend,
                           "is_host_capable", classmethod(lambda cls: True)), \
-             patch.object(backend_mod, "_BACKENDS", {
+             patch.object(backend_mod, "_backends", {
                  "macos-container": _FakeBackend("macos-container", False),
                  "firecracker": _FakeBackend("firecracker", False),
                  "docker": _FakeBackend("docker", True),
@@ -133,7 +133,7 @@ class TestEnumerateActiveAgents(unittest.TestCase):
                 return self._items
 
         with patch.object(
-            backend_mod, "_BACKENDS",
+            backend_mod, "_backends",
             {"docker": _FakeBackend([a]), "firecracker": _FakeBackend([b])},
         ):
             self.assertEqual([a, b], enumerate_active_agents())
@@ -167,7 +167,7 @@ class TestEnumerateActiveAgents(unittest.TestCase):
                 return self._items
 
         with patch.object(
-            backend_mod, "_BACKENDS",
+            backend_mod, "_backends",
             {
                 "docker": _FakeBackend([newer, tie_b]),
                 "firecracker": _FakeBackend([missing_metadata, tie_a]),
@@ -187,7 +187,7 @@ class TestEnumerateActiveAgents(unittest.TestCase):
                 return []
 
         with patch.object(
-            backend_mod, "_BACKENDS",
+            backend_mod, "_backends",
             {"docker": _FakeBackend(), "firecracker": _FakeBackend()},
         ):
             self.assertEqual([], enumerate_active_agents())
@@ -218,7 +218,7 @@ class TestEnumerateActiveAgents(unittest.TestCase):
                 return self._items
 
         with patch.object(
-            backend_mod, "_BACKENDS",
+            backend_mod, "_backends",
             {
                 "docker": _FakeBackend([present], available=True),
                 "firecracker": _FakeBackend([hidden], available=False),
@@ -234,7 +234,7 @@ class TestHasBackend(unittest.TestCase):
                 return False
 
         with patch.object(
-            backend_mod, "_BACKENDS", {"docker": _FakeBackend()},
+            backend_mod, "_backends", {"docker": _FakeBackend()},
         ):
             from bot_bottle.backend import has_backend
             self.assertFalse(has_backend("docker"))

tests/unit/test_docker_util_image.py

@@ -29,7 +29,7 @@ def _fail(stderr: str = "boom") -> subprocess.CompletedProcess:  # type: ignore
 class TestCommitContainer(unittest.TestCase):
     def test_runs_docker_commit(self):
         with patch.object(
-            docker_mod.subprocess, "run", return_value=_ok(),
+            docker_mod, "run_docker", return_value=_ok(),
         ) as run, patch.object(docker_mod, "info"):
             docker_mod.commit_container(
                 "bot-bottle-dev-abc12",
@@ -47,7 +47,7 @@ class TestCommitContainer(unittest.TestCase):
 
     def test_dies_on_docker_commit_failure(self):
         with patch.object(
-            docker_mod.subprocess, "run", return_value=_fail("No such container"),
+            docker_mod, "run_docker", return_value=_fail("No such container"),
         ), patch.object(
             docker_mod, "die", side_effect=SystemExit("die"),
         ) as die:
@@ -58,7 +58,7 @@ class TestCommitContainer(unittest.TestCase):
 
     def test_die_message_includes_image_tag(self):
         with patch.object(
-            docker_mod.subprocess, "run", return_value=_fail("boom"),
+            docker_mod, "run_docker", return_value=_fail("boom"),
         ), patch.object(
             docker_mod, "die", side_effect=SystemExit("die"),
         ) as die: