feat(firecracker): build agent images in a builder VM, not host docker (Stage 3)

Replace the host `docker build` + `docker export` behind the Firecracker
agent rootfs with an in-VM buildah build. `image_builder.build_agent_rootfs_dir`
boots a throwaway builder VM (the orchestrator image, which carries
buildah) on the NAT'd orchestrator link, sends the Dockerfile over SSH,
`buildah build`s it, smoke-tests the result with `buildah run` (the
image's own PATH, so it catches an npm silent-failure stub), and streams
the rootfs tar back into the content-addressed cache dir — the same base
dir `util.build_rootfs_ext4` already turns into a bootable ext4 with
`mke2fs -d`. No host Docker daemon, no root-equivalent `docker` group;
an untrusted Dockerfile runs in a confined microVM, not on the host.

- new firecracker/image_builder.py (boot → build → smoke → stream).
- launch.py: `_build_agent_image` → `_build_agent_base`, returning the
  base dir from the builder VM. The committed-snapshot (freeze/migrate)
  path still exports via host docker until it too is ported.
- util: `_inject_guest_boot` → public `inject_guest_boot` (shared with
  the builder). unit tests for the cache decision + smoke-test paths.

Verified on a KVM host end-to-end: builds the real claude Dockerfile
(node:22-slim + npm claude-code) in-VM in ~60s, the produced agent VM
boots and `claude --version` returns 2.1.172; the content cache skips
the rebuild on a repeat launch.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck

bot_bottle/backend/firecracker/image_builder.py

@@ -0,0 +1,200 @@
+"""Docker-free agent-image builds for the Firecracker backend (PRD 0069 Stage 3).
+
+Instead of `docker build` on the host, an agent's Dockerfile is built inside a
+throwaway Firecracker *builder VM* running buildah (rootless, daemonless): no
+host Docker daemon, no root-equivalent `docker` group, and the untrusted
+Dockerfile executes in a confined microVM rather than on the host.
+
+Flow (on a cache miss, keyed by Dockerfile content):
+  1. boot the builder VM (the orchestrator image, which carries buildah) on
+     the NAT'd orchestrator link (`netpool.orch_slot()`), so the Dockerfile's
+     `FROM` pulls + apt/npm reach the internet;
+  2. send the Dockerfile over SSH and `buildah build` it;
+  3. `buildah mount` the built image and stream its rootfs tar back to the
+     host, extracting it into the cache dir;
+  4. inject the guest boot bits + mark the cache ready — the same base-dir
+     shape `firecracker.util` turns into a bootable ext4 with `mke2fs -d`.
+
+The builder shares the orchestrator's TAP; once the orchestrator is a
+persistent VM (Stage B) these builds move onto its control plane instead of a
+throwaway boot.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import shutil
+import subprocess
+from pathlib import Path
+
+from ...log import die, info
+from . import firecracker_vm, netpool, util
+
+# The builder runs the orchestrator image — it is the component that carries
+# buildah (see Dockerfile.orchestrator). Built from the host docker image via
+# the bootstrap export path until we pull a pre-built image instead.
+_ORCHESTRATOR_IMAGE = "bot-bottle-orchestrator:latest"
+
+# The builder VM makes DIRECT registry/apt connections (not through the
+# gateway proxy the way agents do), and the kernel `ip=` cmdline sets no
+# resolver — so give it one. Public for now; build-time egress through the
+# gateway will remove the need for it.
+_BUILDER_RESOLVER = "1.1.1.1"
+
+# vfs + chroot: buildah works as root in the bare microVM (no fuse-overlayfs /
+# overlay module / subuid maps). Must match Dockerfile.orchestrator's env.
+# `--isolation` is a build/run-only flag; `from`/`mount` take just the store.
+_BUILD_FLAGS = "--isolation chroot --storage-driver vfs"
+_STORE_FLAG = "--storage-driver vfs"
+
+_SSH_READY_TIMEOUT = 40.0
+_BUILD_TIMEOUT_SECONDS = 900.0
+_LOCAL_TAG = "bot-bottle-agent-build"
+
+
+def _dockerfile_hash(dockerfile: Path) -> str:
+    """Cache key: the Dockerfile's content. The shipped agent Dockerfiles
+    COPY nothing from the build context (see .dockerignore), so their content
+    fully determines the image; a Dockerfile that adds COPY will want the
+    context folded in here too."""
+    return hashlib.sha256(dockerfile.read_bytes()).hexdigest()[:16]
+
+
+def build_agent_rootfs_dir(
+    dockerfile: Path, *, image_tag: str, smoke_test: tuple[str, ...] = (),
+) -> Path:
+    """Build `dockerfile` in a Firecracker builder VM (buildah, no host
+    docker), export its rootfs, inject the guest boot bits, and return the
+    cached base dir — the same shape `util.build_rootfs_ext4` consumes.
+    Cached by Dockerfile content, so a repeat launch skips the rebuild.
+
+    `smoke_test` (the provider's declared argv, e.g. `("claude","--version")`)
+    is run in the freshly built image before export, catching an npm
+    silent-failure image at build time rather than at first agent use."""
+    digest = _dockerfile_hash(dockerfile)
+    base = util.cache_dir() / "rootfs" / f"agent-{digest}"
+    ready = base / ".bb-ready"
+    if ready.is_file():
+        info(f"using cached agent rootfs {base.name}")
+        return base
+
+    if base.exists():
+        shutil.rmtree(base, ignore_errors=True)
+    base.mkdir(parents=True)
+
+    info(f"building agent image {image_tag!r} in a firecracker builder VM")
+    _build_in_vm(dockerfile, base, smoke_test)
+    util.inject_guest_boot(base)
+    ready.write_text("ok\n")
+    return base
+
+
+def _build_in_vm(dockerfile: Path, base: Path, smoke_test: tuple[str, ...]) -> None:
+    """Boot the builder VM, build the Dockerfile with buildah, and stream the
+    resulting rootfs into `base`. Tears the VM down on every exit path."""
+    slot = netpool.orch_slot()
+    if not netpool.tap_present(slot.iface):
+        die(f"orchestrator/builder link {slot.iface} not present.\n"
+            f"    ./cli.py backend setup --backend=firecracker")
+
+    # The builder runs the orchestrator rootfs (bootstrap: exported from the
+    # host docker image; a pulled image replaces this later).
+    orch_base = util.build_base_rootfs_dir(_ORCHESTRATOR_IMAGE)
+    run_dir = util.cache_dir() / "run" / "builder"
+    run_dir.mkdir(parents=True, exist_ok=True)
+    rootfs = run_dir / "rootfs.ext4"
+    # vfs copies every layer, so the builder disk needs generous headroom.
+    util.build_rootfs_ext4(orch_base, rootfs, slack_mib=8192)
+    private_key, pubkey = util.generate_keypair(run_dir)
+
+    vm = firecracker_vm.boot(
+        name="bot-bottle-builder", rootfs=rootfs, tap=slot.iface,
+        guest_ip=slot.guest_ip, host_ip=slot.host_ip, pubkey=pubkey,
+        run_dir=run_dir, mem_mib=4096,
+    )
+    try:
+        firecracker_vm.wait_for_ssh(vm, private_key, timeout=_SSH_READY_TIMEOUT)
+        _ssh(private_key, slot.guest_ip,
+             f"printf 'nameserver {_BUILDER_RESOLVER}\\n' > /etc/resolv.conf "
+             f"&& mkdir -p /build/ctx")
+        _send_dockerfile(private_key, slot.guest_ip, dockerfile)
+        _buildah_build(private_key, slot.guest_ip)
+        _smoke_test(private_key, slot.guest_ip, smoke_test)
+        _stream_rootfs(private_key, slot.guest_ip, base)
+    finally:
+        vm.terminate()
+
+
+def _ssh(private_key: Path, guest_ip: str, script: str,
+         *, timeout: float = 60.0) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        util.ssh_base_argv(private_key, guest_ip) + [script],
+        capture_output=True, text=True, timeout=timeout, check=False,
+    )
+
+
+def _send_dockerfile(private_key: Path, guest_ip: str, dockerfile: Path) -> None:
+    proc = subprocess.run(
+        util.ssh_base_argv(private_key, guest_ip) + ["cat > /build/Dockerfile"],
+        input=dockerfile.read_bytes(), capture_output=True, timeout=30, check=False,
+    )
+    if proc.returncode != 0:
+        die(f"sending Dockerfile to the builder VM failed: "
+            f"{proc.stderr.decode(errors='replace').strip()}")
+
+
+def _buildah_build(private_key: Path, guest_ip: str) -> None:
+    build = _ssh(
+        private_key, guest_ip,
+        f"buildah build {_BUILD_FLAGS} -t {_LOCAL_TAG} "
+        f"-f /build/Dockerfile /build/ctx",
+        timeout=_BUILD_TIMEOUT_SECONDS,
+    )
+    if build.returncode != 0:
+        tail = "\n".join((build.stdout + build.stderr).strip().splitlines()[-20:])
+        die(f"buildah build in the builder VM failed:\n{tail}")
+
+
+def _smoke_test(private_key: Path, guest_ip: str, argv: tuple[str, ...]) -> None:
+    """Run the provider's smoke argv inside the freshly built image
+    (`buildah run`, which uses the image's own PATH), failing the build
+    loudly if the CLI is a broken stub. No-op without a declared test."""
+    if not argv:
+        return
+    cmd = (
+        f"set -e; ctr=$(buildah from {_STORE_FLAG} {_LOCAL_TAG}); "
+        f"buildah run {_BUILD_FLAGS} \"$ctr\" -- {' '.join(argv)}; rc=$?; "
+        f"buildah rm \"$ctr\" >/dev/null 2>&1 || true; exit $rc"
+    )
+    result = _ssh(private_key, guest_ip, cmd, timeout=120)
+    if result.returncode != 0:
+        detail = (result.stdout + result.stderr).strip().splitlines()[-10:]
+        die(f"agent image failed its post-build smoke test "
+            f"({' '.join(argv)}):\n" + "\n".join(detail))
+
+
+def _stream_rootfs(private_key: Path, guest_ip: str, base: Path) -> None:
+    """`buildah mount` the built image in the VM and pipe its rootfs tar
+    straight into `base` on the host (extracted as the non-root host user, so
+    uid 0 isn't preserved — the guest init restores /root ownership, same as
+    the docker-export path)."""
+    export = (
+        f"set -e; ctr=$(buildah from {_STORE_FLAG} {_LOCAL_TAG}); "
+        f"mnt=$(buildah mount {_STORE_FLAG} \"$ctr\"); "
+        f"tar -C \"$mnt\" -cf - ."
+    )
+    ssh_proc = subprocess.Popen(
+        util.ssh_base_argv(private_key, guest_ip) + [export],
+        stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+    )
+    assert ssh_proc.stdout is not None
+    untar = subprocess.run(
+        ["tar", "-x", "-C", str(base)], stdin=ssh_proc.stdout, check=False,
+    )
+    ssh_proc.stdout.close()
+    ssh_err = (ssh_proc.stderr.read().decode(errors="replace")
+               if ssh_proc.stderr else "")
+    rc = ssh_proc.wait()
+    if rc != 0 or untar.returncode != 0:
+        die(f"exporting the built rootfs from the builder VM failed: "
+            f"{ssh_err.strip() or '<no stderr>'}")

bot_bottle/backend/firecracker/launch.py

@@ -48,7 +48,7 @@ from ...supervise import SUPERVISE_PORT
 from ..docker import util as docker_mod
 from ..docker.egress import EGRESS_PORT
 from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
-from . import firecracker_vm, isolation_probe, netpool, util
+from . import firecracker_vm, image_builder, isolation_probe, netpool, util
 from .bottle import FirecrackerBottle
 from .bottle_plan import FirecrackerBottlePlan
 from .consolidated_launch import (
@@ -57,7 +57,6 @@ from .consolidated_launch import (
 )
 
 
-_REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent)
 _GIT_HTTP_PORT = 9420
 
 
@@ -85,10 +84,10 @@ def launch(
             raise teardown_exc
 
     try:
-        # Step 1: agent image. The sidecar bundle image is built by the
-        # orchestrator service (ensure_running → ensure_built); we only
-        # build the agent image here. Use a committed snapshot when available.
-        plan = _build_agent_image(plan)
+        # Step 1: agent rootfs. Built from the Dockerfile inside a Firecracker
+        # builder VM (buildah, no host docker); a committed snapshot is reused
+        # when present. Returns the base dir the per-bottle ext4 is made from.
+        plan, agent_base = _build_agent_base(plan)
 
         # Step 2: mint the git-gate dynamic (gitea) deploy keys, if any.
         git_gate_plan = plan.git_gate_plan
@@ -154,11 +153,10 @@ def launch(
         )
 
         # Step 6: build the per-bottle rootfs + SSH key, then boot.
-        base_dir = util.build_base_rootfs_dir(plan.image)
         run_dir = util.cache_dir() / "run" / plan.slug
         run_dir.mkdir(parents=True, exist_ok=True)
         rootfs = run_dir / "rootfs.ext4"
-        util.build_rootfs_ext4(base_dir, rootfs)
+        util.build_rootfs_ext4(agent_base, rootfs)
         private_key, pubkey = util.generate_keypair(run_dir)
 
         vm = firecracker_vm.boot(
@@ -199,19 +197,27 @@ def launch(
         teardown()
 
 
-def _build_agent_image(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan:
+def _build_agent_base(
+    plan: FirecrackerBottlePlan,
+) -> tuple[FirecrackerBottlePlan, Path]:
+    """Produce the agent's base rootfs dir. Primary path: build the Dockerfile
+    inside a Firecracker builder VM (buildah, no host docker), smoke-testing
+    the image before export. A committed snapshot (freeze/migrate) is still
+    exported via the host docker path until that is ported too."""
     committed = read_committed_image(plan.slug)
     if committed and docker_mod.image_exists(committed):
         info(f"using committed image {committed!r}")
-        return dataclasses.replace(
+        plan = dataclasses.replace(
             plan,
             agent_provision=dataclasses.replace(plan.agent_provision, image=committed),
         )
-    docker_mod.build_image(plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path)
-    docker_mod.verify_agent_image(
-        plan.image, runtime_for(plan.agent_provider_template).smoke_test,
+        return plan, util.build_base_rootfs_dir(committed)
+    base = image_builder.build_agent_rootfs_dir(
+        Path(plan.dockerfile_path),
+        image_tag=plan.image,
+        smoke_test=runtime_for(plan.agent_provider_template).smoke_test,
     )
-    return plan
+    return plan, base
 
 
 # --- agent guest env -------------------------------------------------

bot_bottle/backend/firecracker/util.py

@@ -200,12 +200,12 @@ def build_base_rootfs_dir(image_ref: str) -> Path:
             stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
         )
 
-    _inject_guest_boot(base)
+    inject_guest_boot(base)
     ready.write_text("ok\n")
     return base
 
 
-def _inject_guest_boot(rootfs: Path) -> None:
+def inject_guest_boot(rootfs: Path) -> None:
     """Drop the static dropbear and the PID-1 init into the rootfs."""
     shutil.copy2(dropbear_path(), rootfs / "bb-dropbear")
     os.chmod(rootfs / "bb-dropbear", 0o755)

tests/unit/test_firecracker_image_builder.py

@@ -0,0 +1,74 @@
+"""Unit tests for the docker-free Firecracker agent-image builder.
+
+The VM boot / SSH / buildah plumbing (`_build_in_vm`) is integration-tested
+on a KVM host; here we cover the cache decision, the boot-bit injection, and
+the smoke-test no-op — the logic that must hold without a VM.
+"""
+
+from __future__ import annotations
+
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from bot_bottle.backend.firecracker import image_builder
+
+
+class TestBuildAgentRootfsDir(unittest.TestCase):
+    def setUp(self):
+        self._tmp = tempfile.TemporaryDirectory()
+        self.cache = Path(self._tmp.name)
+        self.dockerfile = self.cache / "Dockerfile"
+        self.dockerfile.write_text("FROM node:22-slim\n")
+        self.addCleanup(self._tmp.cleanup)
+
+    def test_cache_hit_skips_rebuild(self):
+        digest = image_builder._dockerfile_hash(self.dockerfile)
+        base = self.cache / "rootfs" / f"agent-{digest}"
+        base.mkdir(parents=True)
+        (base / ".bb-ready").write_text("ok\n")
+        with patch.object(image_builder.util, "cache_dir", return_value=self.cache), \
+             patch.object(image_builder, "_build_in_vm") as build:
+            out = image_builder.build_agent_rootfs_dir(
+                self.dockerfile, image_tag="t:latest")
+        build.assert_not_called()
+        self.assertEqual(base, out)
+
+    def test_cache_miss_builds_injects_and_marks_ready(self):
+        with patch.object(image_builder.util, "cache_dir", return_value=self.cache), \
+             patch.object(image_builder, "_build_in_vm") as build, \
+             patch.object(image_builder.util, "inject_guest_boot") as inject:
+            out = image_builder.build_agent_rootfs_dir(
+                self.dockerfile, image_tag="t:latest", smoke_test=("claude", "--version"))
+        build.assert_called_once()
+        # smoke_test threads through to the VM build.
+        self.assertEqual(("claude", "--version"), build.call_args.args[2])
+        inject.assert_called_once_with(out)
+        self.assertTrue((out / ".bb-ready").is_file())
+
+    def test_content_addressed_cache_key(self):
+        other = self.cache / "Dockerfile2"
+        other.write_text("FROM python:3.12-slim\n")
+        self.assertNotEqual(
+            image_builder._dockerfile_hash(self.dockerfile),
+            image_builder._dockerfile_hash(other),
+        )
+
+
+class TestSmokeTest(unittest.TestCase):
+    def test_empty_argv_is_noop(self):
+        with patch.object(image_builder, "_ssh") as ssh:
+            image_builder._smoke_test(Path("/k"), "10.0.0.1", ())
+        ssh.assert_not_called()
+
+    def test_failed_smoke_dies(self):
+        import subprocess
+        result = subprocess.CompletedProcess([], 1, stdout="broken", stderr="")
+        with patch.object(image_builder, "_ssh", return_value=result), \
+             self.assertRaises(SystemExit):
+            image_builder._smoke_test(Path("/k"), "10.0.0.1", ("claude", "--version"))
+
+
+if __name__ == "__main__":
+    unittest.main()