From 73ee081c65fa9808017169c0b1e2c8e436894416 Mon Sep 17 00:00:00 2001 From: didericis Date: Wed, 15 Jul 2026 13:08:32 -0400 Subject: [PATCH] feat(firecracker): build agent images in a builder VM, not host docker (Stage 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the host `docker build` + `docker export` behind the Firecracker agent rootfs with an in-VM buildah build. `image_builder.build_agent_rootfs_dir` boots a throwaway builder VM (the orchestrator image, which carries buildah) on the NAT'd orchestrator link, sends the Dockerfile over SSH, `buildah build`s it, smoke-tests the result with `buildah run` (the image's own PATH, so it catches an npm silent-failure stub), and streams the rootfs tar back into the content-addressed cache dir — the same base dir `util.build_rootfs_ext4` already turns into a bootable ext4 with `mke2fs -d`. No host Docker daemon, no root-equivalent `docker` group; an untrusted Dockerfile runs in a confined microVM, not on the host. - new firecracker/image_builder.py (boot → build → smoke → stream). - launch.py: `_build_agent_image` → `_build_agent_base`, returning the base dir from the builder VM. The committed-snapshot (freeze/migrate) path still exports via host docker until it too is ported. - util: `_inject_guest_boot` → public `inject_guest_boot` (shared with the builder). unit tests for the cache decision + smoke-test paths. Verified on a KVM host end-to-end: builds the real claude Dockerfile (node:22-slim + npm claude-code) in-VM in ~60s, the produced agent VM boots and `claude --version` returns 2.1.172; the content cache skips the rebuild on a repeat launch. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck --- .../backend/firecracker/image_builder.py | 200 ++++++++++++++++++ bot_bottle/backend/firecracker/launch.py | 34 +-- bot_bottle/backend/firecracker/util.py | 4 +- tests/unit/test_firecracker_image_builder.py | 74 +++++++ 4 files changed, 296 insertions(+), 16 deletions(-) create mode 100644 bot_bottle/backend/firecracker/image_builder.py create mode 100644 tests/unit/test_firecracker_image_builder.py diff --git a/bot_bottle/backend/firecracker/image_builder.py b/bot_bottle/backend/firecracker/image_builder.py new file mode 100644 index 0000000..0682de9 --- /dev/null +++ b/bot_bottle/backend/firecracker/image_builder.py @@ -0,0 +1,200 @@ +"""Docker-free agent-image builds for the Firecracker backend (PRD 0069 Stage 3). + +Instead of `docker build` on the host, an agent's Dockerfile is built inside a +throwaway Firecracker *builder VM* running buildah (rootless, daemonless): no +host Docker daemon, no root-equivalent `docker` group, and the untrusted +Dockerfile executes in a confined microVM rather than on the host. + +Flow (on a cache miss, keyed by Dockerfile content): + 1. boot the builder VM (the orchestrator image, which carries buildah) on + the NAT'd orchestrator link (`netpool.orch_slot()`), so the Dockerfile's + `FROM` pulls + apt/npm reach the internet; + 2. send the Dockerfile over SSH and `buildah build` it; + 3. `buildah mount` the built image and stream its rootfs tar back to the + host, extracting it into the cache dir; + 4. inject the guest boot bits + mark the cache ready — the same base-dir + shape `firecracker.util` turns into a bootable ext4 with `mke2fs -d`. + +The builder shares the orchestrator's TAP; once the orchestrator is a +persistent VM (Stage B) these builds move onto its control plane instead of a +throwaway boot. +""" + +from __future__ import annotations + +import hashlib +import shutil +import subprocess +from pathlib import Path + +from ...log import die, info +from . import firecracker_vm, netpool, util + +# The builder runs the orchestrator image — it is the component that carries +# buildah (see Dockerfile.orchestrator). Built from the host docker image via +# the bootstrap export path until we pull a pre-built image instead. +_ORCHESTRATOR_IMAGE = "bot-bottle-orchestrator:latest" + +# The builder VM makes DIRECT registry/apt connections (not through the +# gateway proxy the way agents do), and the kernel `ip=` cmdline sets no +# resolver — so give it one. Public for now; build-time egress through the +# gateway will remove the need for it. +_BUILDER_RESOLVER = "1.1.1.1" + +# vfs + chroot: buildah works as root in the bare microVM (no fuse-overlayfs / +# overlay module / subuid maps). Must match Dockerfile.orchestrator's env. +# `--isolation` is a build/run-only flag; `from`/`mount` take just the store. +_BUILD_FLAGS = "--isolation chroot --storage-driver vfs" +_STORE_FLAG = "--storage-driver vfs" + +_SSH_READY_TIMEOUT = 40.0 +_BUILD_TIMEOUT_SECONDS = 900.0 +_LOCAL_TAG = "bot-bottle-agent-build" + + +def _dockerfile_hash(dockerfile: Path) -> str: + """Cache key: the Dockerfile's content. The shipped agent Dockerfiles + COPY nothing from the build context (see .dockerignore), so their content + fully determines the image; a Dockerfile that adds COPY will want the + context folded in here too.""" + return hashlib.sha256(dockerfile.read_bytes()).hexdigest()[:16] + + +def build_agent_rootfs_dir( + dockerfile: Path, *, image_tag: str, smoke_test: tuple[str, ...] = (), +) -> Path: + """Build `dockerfile` in a Firecracker builder VM (buildah, no host + docker), export its rootfs, inject the guest boot bits, and return the + cached base dir — the same shape `util.build_rootfs_ext4` consumes. + Cached by Dockerfile content, so a repeat launch skips the rebuild. + + `smoke_test` (the provider's declared argv, e.g. `("claude","--version")`) + is run in the freshly built image before export, catching an npm + silent-failure image at build time rather than at first agent use.""" + digest = _dockerfile_hash(dockerfile) + base = util.cache_dir() / "rootfs" / f"agent-{digest}" + ready = base / ".bb-ready" + if ready.is_file(): + info(f"using cached agent rootfs {base.name}") + return base + + if base.exists(): + shutil.rmtree(base, ignore_errors=True) + base.mkdir(parents=True) + + info(f"building agent image {image_tag!r} in a firecracker builder VM") + _build_in_vm(dockerfile, base, smoke_test) + util.inject_guest_boot(base) + ready.write_text("ok\n") + return base + + +def _build_in_vm(dockerfile: Path, base: Path, smoke_test: tuple[str, ...]) -> None: + """Boot the builder VM, build the Dockerfile with buildah, and stream the + resulting rootfs into `base`. Tears the VM down on every exit path.""" + slot = netpool.orch_slot() + if not netpool.tap_present(slot.iface): + die(f"orchestrator/builder link {slot.iface} not present.\n" + f" ./cli.py backend setup --backend=firecracker") + + # The builder runs the orchestrator rootfs (bootstrap: exported from the + # host docker image; a pulled image replaces this later). + orch_base = util.build_base_rootfs_dir(_ORCHESTRATOR_IMAGE) + run_dir = util.cache_dir() / "run" / "builder" + run_dir.mkdir(parents=True, exist_ok=True) + rootfs = run_dir / "rootfs.ext4" + # vfs copies every layer, so the builder disk needs generous headroom. + util.build_rootfs_ext4(orch_base, rootfs, slack_mib=8192) + private_key, pubkey = util.generate_keypair(run_dir) + + vm = firecracker_vm.boot( + name="bot-bottle-builder", rootfs=rootfs, tap=slot.iface, + guest_ip=slot.guest_ip, host_ip=slot.host_ip, pubkey=pubkey, + run_dir=run_dir, mem_mib=4096, + ) + try: + firecracker_vm.wait_for_ssh(vm, private_key, timeout=_SSH_READY_TIMEOUT) + _ssh(private_key, slot.guest_ip, + f"printf 'nameserver {_BUILDER_RESOLVER}\\n' > /etc/resolv.conf " + f"&& mkdir -p /build/ctx") + _send_dockerfile(private_key, slot.guest_ip, dockerfile) + _buildah_build(private_key, slot.guest_ip) + _smoke_test(private_key, slot.guest_ip, smoke_test) + _stream_rootfs(private_key, slot.guest_ip, base) + finally: + vm.terminate() + + +def _ssh(private_key: Path, guest_ip: str, script: str, + *, timeout: float = 60.0) -> subprocess.CompletedProcess[str]: + return subprocess.run( + util.ssh_base_argv(private_key, guest_ip) + [script], + capture_output=True, text=True, timeout=timeout, check=False, + ) + + +def _send_dockerfile(private_key: Path, guest_ip: str, dockerfile: Path) -> None: + proc = subprocess.run( + util.ssh_base_argv(private_key, guest_ip) + ["cat > /build/Dockerfile"], + input=dockerfile.read_bytes(), capture_output=True, timeout=30, check=False, + ) + if proc.returncode != 0: + die(f"sending Dockerfile to the builder VM failed: " + f"{proc.stderr.decode(errors='replace').strip()}") + + +def _buildah_build(private_key: Path, guest_ip: str) -> None: + build = _ssh( + private_key, guest_ip, + f"buildah build {_BUILD_FLAGS} -t {_LOCAL_TAG} " + f"-f /build/Dockerfile /build/ctx", + timeout=_BUILD_TIMEOUT_SECONDS, + ) + if build.returncode != 0: + tail = "\n".join((build.stdout + build.stderr).strip().splitlines()[-20:]) + die(f"buildah build in the builder VM failed:\n{tail}") + + +def _smoke_test(private_key: Path, guest_ip: str, argv: tuple[str, ...]) -> None: + """Run the provider's smoke argv inside the freshly built image + (`buildah run`, which uses the image's own PATH), failing the build + loudly if the CLI is a broken stub. No-op without a declared test.""" + if not argv: + return + cmd = ( + f"set -e; ctr=$(buildah from {_STORE_FLAG} {_LOCAL_TAG}); " + f"buildah run {_BUILD_FLAGS} \"$ctr\" -- {' '.join(argv)}; rc=$?; " + f"buildah rm \"$ctr\" >/dev/null 2>&1 || true; exit $rc" + ) + result = _ssh(private_key, guest_ip, cmd, timeout=120) + if result.returncode != 0: + detail = (result.stdout + result.stderr).strip().splitlines()[-10:] + die(f"agent image failed its post-build smoke test " + f"({' '.join(argv)}):\n" + "\n".join(detail)) + + +def _stream_rootfs(private_key: Path, guest_ip: str, base: Path) -> None: + """`buildah mount` the built image in the VM and pipe its rootfs tar + straight into `base` on the host (extracted as the non-root host user, so + uid 0 isn't preserved — the guest init restores /root ownership, same as + the docker-export path).""" + export = ( + f"set -e; ctr=$(buildah from {_STORE_FLAG} {_LOCAL_TAG}); " + f"mnt=$(buildah mount {_STORE_FLAG} \"$ctr\"); " + f"tar -C \"$mnt\" -cf - ." + ) + ssh_proc = subprocess.Popen( + util.ssh_base_argv(private_key, guest_ip) + [export], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + assert ssh_proc.stdout is not None + untar = subprocess.run( + ["tar", "-x", "-C", str(base)], stdin=ssh_proc.stdout, check=False, + ) + ssh_proc.stdout.close() + ssh_err = (ssh_proc.stderr.read().decode(errors="replace") + if ssh_proc.stderr else "") + rc = ssh_proc.wait() + if rc != 0 or untar.returncode != 0: + die(f"exporting the built rootfs from the builder VM failed: " + f"{ssh_err.strip() or ''}") diff --git a/bot_bottle/backend/firecracker/launch.py b/bot_bottle/backend/firecracker/launch.py index 246441e..0ebb916 100644 --- a/bot_bottle/backend/firecracker/launch.py +++ b/bot_bottle/backend/firecracker/launch.py @@ -48,7 +48,7 @@ from ...supervise import SUPERVISE_PORT from ..docker import util as docker_mod from ..docker.egress import EGRESS_PORT from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH -from . import firecracker_vm, isolation_probe, netpool, util +from . import firecracker_vm, image_builder, isolation_probe, netpool, util from .bottle import FirecrackerBottle from .bottle_plan import FirecrackerBottlePlan from .consolidated_launch import ( @@ -57,7 +57,6 @@ from .consolidated_launch import ( ) -_REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent) _GIT_HTTP_PORT = 9420 @@ -85,10 +84,10 @@ def launch( raise teardown_exc try: - # Step 1: agent image. The sidecar bundle image is built by the - # orchestrator service (ensure_running → ensure_built); we only - # build the agent image here. Use a committed snapshot when available. - plan = _build_agent_image(plan) + # Step 1: agent rootfs. Built from the Dockerfile inside a Firecracker + # builder VM (buildah, no host docker); a committed snapshot is reused + # when present. Returns the base dir the per-bottle ext4 is made from. + plan, agent_base = _build_agent_base(plan) # Step 2: mint the git-gate dynamic (gitea) deploy keys, if any. git_gate_plan = plan.git_gate_plan @@ -154,11 +153,10 @@ def launch( ) # Step 6: build the per-bottle rootfs + SSH key, then boot. - base_dir = util.build_base_rootfs_dir(plan.image) run_dir = util.cache_dir() / "run" / plan.slug run_dir.mkdir(parents=True, exist_ok=True) rootfs = run_dir / "rootfs.ext4" - util.build_rootfs_ext4(base_dir, rootfs) + util.build_rootfs_ext4(agent_base, rootfs) private_key, pubkey = util.generate_keypair(run_dir) vm = firecracker_vm.boot( @@ -199,19 +197,27 @@ def launch( teardown() -def _build_agent_image(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan: +def _build_agent_base( + plan: FirecrackerBottlePlan, +) -> tuple[FirecrackerBottlePlan, Path]: + """Produce the agent's base rootfs dir. Primary path: build the Dockerfile + inside a Firecracker builder VM (buildah, no host docker), smoke-testing + the image before export. A committed snapshot (freeze/migrate) is still + exported via the host docker path until that is ported too.""" committed = read_committed_image(plan.slug) if committed and docker_mod.image_exists(committed): info(f"using committed image {committed!r}") - return dataclasses.replace( + plan = dataclasses.replace( plan, agent_provision=dataclasses.replace(plan.agent_provision, image=committed), ) - docker_mod.build_image(plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path) - docker_mod.verify_agent_image( - plan.image, runtime_for(plan.agent_provider_template).smoke_test, + return plan, util.build_base_rootfs_dir(committed) + base = image_builder.build_agent_rootfs_dir( + Path(plan.dockerfile_path), + image_tag=plan.image, + smoke_test=runtime_for(plan.agent_provider_template).smoke_test, ) - return plan + return plan, base # --- agent guest env ------------------------------------------------- diff --git a/bot_bottle/backend/firecracker/util.py b/bot_bottle/backend/firecracker/util.py index aa29e8a..9455edb 100644 --- a/bot_bottle/backend/firecracker/util.py +++ b/bot_bottle/backend/firecracker/util.py @@ -200,12 +200,12 @@ def build_base_rootfs_dir(image_ref: str) -> Path: stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, ) - _inject_guest_boot(base) + inject_guest_boot(base) ready.write_text("ok\n") return base -def _inject_guest_boot(rootfs: Path) -> None: +def inject_guest_boot(rootfs: Path) -> None: """Drop the static dropbear and the PID-1 init into the rootfs.""" shutil.copy2(dropbear_path(), rootfs / "bb-dropbear") os.chmod(rootfs / "bb-dropbear", 0o755) diff --git a/tests/unit/test_firecracker_image_builder.py b/tests/unit/test_firecracker_image_builder.py new file mode 100644 index 0000000..1ac2224 --- /dev/null +++ b/tests/unit/test_firecracker_image_builder.py @@ -0,0 +1,74 @@ +"""Unit tests for the docker-free Firecracker agent-image builder. + +The VM boot / SSH / buildah plumbing (`_build_in_vm`) is integration-tested +on a KVM host; here we cover the cache decision, the boot-bit injection, and +the smoke-test no-op — the logic that must hold without a VM. +""" + +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from bot_bottle.backend.firecracker import image_builder + + +class TestBuildAgentRootfsDir(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.cache = Path(self._tmp.name) + self.dockerfile = self.cache / "Dockerfile" + self.dockerfile.write_text("FROM node:22-slim\n") + self.addCleanup(self._tmp.cleanup) + + def test_cache_hit_skips_rebuild(self): + digest = image_builder._dockerfile_hash(self.dockerfile) + base = self.cache / "rootfs" / f"agent-{digest}" + base.mkdir(parents=True) + (base / ".bb-ready").write_text("ok\n") + with patch.object(image_builder.util, "cache_dir", return_value=self.cache), \ + patch.object(image_builder, "_build_in_vm") as build: + out = image_builder.build_agent_rootfs_dir( + self.dockerfile, image_tag="t:latest") + build.assert_not_called() + self.assertEqual(base, out) + + def test_cache_miss_builds_injects_and_marks_ready(self): + with patch.object(image_builder.util, "cache_dir", return_value=self.cache), \ + patch.object(image_builder, "_build_in_vm") as build, \ + patch.object(image_builder.util, "inject_guest_boot") as inject: + out = image_builder.build_agent_rootfs_dir( + self.dockerfile, image_tag="t:latest", smoke_test=("claude", "--version")) + build.assert_called_once() + # smoke_test threads through to the VM build. + self.assertEqual(("claude", "--version"), build.call_args.args[2]) + inject.assert_called_once_with(out) + self.assertTrue((out / ".bb-ready").is_file()) + + def test_content_addressed_cache_key(self): + other = self.cache / "Dockerfile2" + other.write_text("FROM python:3.12-slim\n") + self.assertNotEqual( + image_builder._dockerfile_hash(self.dockerfile), + image_builder._dockerfile_hash(other), + ) + + +class TestSmokeTest(unittest.TestCase): + def test_empty_argv_is_noop(self): + with patch.object(image_builder, "_ssh") as ssh: + image_builder._smoke_test(Path("/k"), "10.0.0.1", ()) + ssh.assert_not_called() + + def test_failed_smoke_dies(self): + import subprocess + result = subprocess.CompletedProcess([], 1, stdout="broken", stderr="") + with patch.object(image_builder, "_ssh", return_value=result), \ + self.assertRaises(SystemExit): + image_builder._smoke_test(Path("/k"), "10.0.0.1", ("claude", "--version")) + + +if __name__ == "__main__": + unittest.main()