diff --git a/Dockerfile.gateway b/Dockerfile.gateway index e5456ea..4b3ec9b 100644 --- a/Dockerfile.gateway +++ b/Dockerfile.gateway @@ -39,28 +39,32 @@ # with Dockerfile.git-gate's prior base (now deleted at chunk 3). FROM zricethezav/gitleaks@sha256:c00b6bd0aeb3071cbcb79009cb16a60dd9e0a7c60e2be9ab65d25e6bc8abbb7f AS gitleaks-src -# Stage 2: assembly. mitmproxy/mitmproxy is debian-slim-based with -# Python + mitmdump pre-installed — heavier than the others, so -# this stage starts there and pulls the standalone binaries in. -FROM mitmproxy/mitmproxy:11.1.3 - -# Run as root inside the bundle. The bundle is the isolation -# boundary; per-daemon user separation inside it is not load-bearing -# and complicates the supervisor's spawn path. -USER root +# Stage 2: assembly. Based on `python:3.12-slim` (Debian trixie) rather +# than the `mitmproxy/mitmproxy` image (Debian bookworm) so the whole +# stack — gateway here, and the firecracker infra image that builds +# FROM this — lands on trixie, whose buildah (1.39) can build agent +# Dockerfiles that use heredocs. mitmproxy is pip-installed to the same +# effect as the upstream image. (bookworm's buildah is 1.28, which can't +# parse `RUN ... < str: @@ -63,10 +44,10 @@ def _dockerfile_hash(dockerfile: Path) -> str: def build_agent_rootfs_dir( dockerfile: Path, *, image_tag: str, smoke_test: tuple[str, ...] = (), ) -> Path: - """Build `dockerfile` in a Firecracker builder VM (buildah, no host - docker), export its rootfs, inject the guest boot bits, and return the - cached base dir — the same shape `util.build_rootfs_ext4` consumes. - Cached by Dockerfile content, so a repeat launch skips the rebuild. + """Build `dockerfile` in the infra VM (buildah, no host docker), export its + rootfs, inject the guest boot bits, and return the cached base dir — the + same shape `util.build_rootfs_ext4` consumes. Cached by Dockerfile content, + so a repeat launch skips the rebuild. `smoke_test` (the provider's declared argv, e.g. `("claude","--version")`) is run in the freshly built image before export, catching an npm @@ -82,47 +63,38 @@ def build_agent_rootfs_dir( shutil.rmtree(base, ignore_errors=True) base.mkdir(parents=True) - info(f"building agent image {image_tag!r} in a firecracker builder VM") - _build_in_vm(dockerfile, base, smoke_test) + info(f"building agent image {image_tag!r} in the infra VM") + _build_in_infra(dockerfile, base, smoke_test, digest) util.inject_guest_boot(base) ready.write_text("ok\n") return base -def _build_in_vm(dockerfile: Path, base: Path, smoke_test: tuple[str, ...]) -> None: - """Boot the builder VM, build the Dockerfile with buildah, and stream the - resulting rootfs into `base`. Tears the VM down on every exit path.""" - slot = netpool.orch_slot() - if not netpool.tap_present(slot.iface): - die(f"orchestrator/builder link {slot.iface} not present.\n" - f" ./cli.py backend setup --backend=firecracker") - - # The builder runs the orchestrator rootfs (bootstrap: exported from the - # host docker image; a pulled image replaces this later). - orch_base = util.build_base_rootfs_dir(_ORCHESTRATOR_IMAGE) - run_dir = util.cache_dir() / "run" / "builder" - run_dir.mkdir(parents=True, exist_ok=True) - rootfs = run_dir / "rootfs.ext4" - # vfs copies every layer, so the builder disk needs generous headroom. - util.build_rootfs_ext4(orch_base, rootfs, slack_mib=8192) - private_key, pubkey = util.generate_keypair(run_dir) - - vm = firecracker_vm.boot( - name="bot-bottle-builder", rootfs=rootfs, tap=slot.iface, - guest_ip=slot.guest_ip, host_ip=slot.host_ip, pubkey=pubkey, - run_dir=run_dir, mem_mib=4096, - ) +def _build_in_infra( + dockerfile: Path, base: Path, smoke_test: tuple[str, ...], digest: str, +) -> None: + """Ensure the infra VM is up, `buildah build` the Dockerfile in it, smoke + test the image, and stream its rootfs into `base`. The infra VM persists; + only the per-build container/image/context are cleaned up.""" + infra = infra_vm.ensure_running() + key, ip = infra.private_key, infra.guest_ip + tag = f"bot-bottle-agent-build-{digest}" + ctx = f"/tmp/agent-build-{digest}" try: - firecracker_vm.wait_for_ssh(vm, private_key, timeout=_SSH_READY_TIMEOUT) - _ssh(private_key, slot.guest_ip, - f"printf 'nameserver {_BUILDER_RESOLVER}\\n' > /etc/resolv.conf " - f"&& mkdir -p /build/ctx") - _send_dockerfile(private_key, slot.guest_ip, dockerfile) - _buildah_build(private_key, slot.guest_ip) - _smoke_test(private_key, slot.guest_ip, smoke_test) - _stream_rootfs(private_key, slot.guest_ip, base) + prep = _ssh(key, ip, f"rm -rf {ctx}; mkdir -p {ctx}/ctx") + if prep.returncode != 0: + die(f"preparing build dir in the infra VM failed: {prep.stderr.strip()}") + _send_dockerfile(key, ip, dockerfile, ctx) + _buildah_build(key, ip, ctx, tag) + _smoke_test(key, ip, tag, smoke_test) + _stream_rootfs(key, ip, tag, base) finally: - vm.terminate() + # Keep the ephemeral infra rootfs from accumulating build state. Builds + # are serial per launch, so removing all working containers is safe. + _ssh(key, ip, + f"buildah rm {_STORE_FLAG} -a >/dev/null 2>&1; " + f"buildah rmi {_STORE_FLAG} {tag} >/dev/null 2>&1; rm -rf {ctx}", + timeout=60) def _ssh(private_key: Path, guest_ip: str, script: str, @@ -133,36 +105,35 @@ def _ssh(private_key: Path, guest_ip: str, script: str, ) -def _send_dockerfile(private_key: Path, guest_ip: str, dockerfile: Path) -> None: +def _send_dockerfile(private_key: Path, guest_ip: str, dockerfile: Path, ctx: str) -> None: proc = subprocess.run( - util.ssh_base_argv(private_key, guest_ip) + ["cat > /build/Dockerfile"], + util.ssh_base_argv(private_key, guest_ip) + [f"cat > {ctx}/Dockerfile"], input=dockerfile.read_bytes(), capture_output=True, timeout=30, check=False, ) if proc.returncode != 0: - die(f"sending Dockerfile to the builder VM failed: " + die(f"sending Dockerfile to the infra VM failed: " f"{proc.stderr.decode(errors='replace').strip()}") -def _buildah_build(private_key: Path, guest_ip: str) -> None: +def _buildah_build(private_key: Path, guest_ip: str, ctx: str, tag: str) -> None: build = _ssh( private_key, guest_ip, - f"buildah build {_BUILD_FLAGS} -t {_LOCAL_TAG} " - f"-f /build/Dockerfile /build/ctx", + f"buildah build {_BUILD_FLAGS} -t {tag} -f {ctx}/Dockerfile {ctx}/ctx", timeout=_BUILD_TIMEOUT_SECONDS, ) if build.returncode != 0: tail = "\n".join((build.stdout + build.stderr).strip().splitlines()[-20:]) - die(f"buildah build in the builder VM failed:\n{tail}") + die(f"buildah build in the infra VM failed:\n{tail}") -def _smoke_test(private_key: Path, guest_ip: str, argv: tuple[str, ...]) -> None: +def _smoke_test(private_key: Path, guest_ip: str, tag: str, argv: tuple[str, ...]) -> None: """Run the provider's smoke argv inside the freshly built image (`buildah run`, which uses the image's own PATH), failing the build loudly if the CLI is a broken stub. No-op without a declared test.""" if not argv: return cmd = ( - f"set -e; ctr=$(buildah from {_STORE_FLAG} {_LOCAL_TAG}); " + f"set -e; ctr=$(buildah from {_STORE_FLAG} {tag}); " f"buildah run {_BUILD_FLAGS} \"$ctr\" -- {' '.join(argv)}; rc=$?; " f"buildah rm \"$ctr\" >/dev/null 2>&1 || true; exit $rc" ) @@ -173,13 +144,12 @@ def _smoke_test(private_key: Path, guest_ip: str, argv: tuple[str, ...]) -> None f"({' '.join(argv)}):\n" + "\n".join(detail)) -def _stream_rootfs(private_key: Path, guest_ip: str, base: Path) -> None: - """`buildah mount` the built image in the VM and pipe its rootfs tar +def _stream_rootfs(private_key: Path, guest_ip: str, tag: str, base: Path) -> None: + """`buildah mount` the built image in the infra VM and pipe its rootfs tar straight into `base` on the host (extracted as the non-root host user, so - uid 0 isn't preserved — the guest init restores /root ownership, same as - the docker-export path).""" + uid 0 isn't preserved — the guest init restores /root ownership).""" export = ( - f"set -e; ctr=$(buildah from {_STORE_FLAG} {_LOCAL_TAG}); " + f"set -e; ctr=$(buildah from {_STORE_FLAG} {tag}); " f"mnt=$(buildah mount {_STORE_FLAG} \"$ctr\"); " f"tar -C \"$mnt\" -cf - ." ) @@ -196,5 +166,5 @@ def _stream_rootfs(private_key: Path, guest_ip: str, base: Path) -> None: if ssh_proc.stderr else "") rc = ssh_proc.wait() if rc != 0 or untar.returncode != 0: - die(f"exporting the built rootfs from the builder VM failed: " + die(f"exporting the built rootfs from the infra VM failed: " f"{ssh_err.strip() or ''}") diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py index d7416e4..0fdf6f0 100644 --- a/bot_bottle/backend/firecracker/infra_vm.py +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -39,6 +39,7 @@ from . import firecracker_vm, netpool, util # a pull-from-registry mode lands later. _INFRA_IMAGE = "bot-bottle-infra:latest" _GATEWAY_IMAGE = "bot-bottle-gateway:latest" +_ORCHESTRATOR_IMAGE = "bot-bottle-orchestrator:latest" _REPO_ROOT = Path(__file__).resolve().parents[3] CONTROL_PLANE_PORT = 8099 @@ -105,10 +106,12 @@ class InfraVm: def ensure_built() -> None: - """Build the infra image from source (bootstrap via host docker): the - gateway data-plane image, then the infra image that layers the - control-plane source onto it. A pull-from-registry mode replaces this - later.""" + """Build the infra image from source (bootstrap via host docker). The + infra image `COPY --from`s the orchestrator image and is `FROM` the + gateway image, so both must exist first. A pull-from-registry mode + replaces this later.""" + docker_mod.build_image( + _ORCHESTRATOR_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.orchestrator") docker_mod.build_image( _GATEWAY_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.gateway") docker_mod.build_image( diff --git a/tests/unit/test_firecracker_image_builder.py b/tests/unit/test_firecracker_image_builder.py index 1ac2224..92c949c 100644 --- a/tests/unit/test_firecracker_image_builder.py +++ b/tests/unit/test_firecracker_image_builder.py @@ -1,6 +1,6 @@ """Unit tests for the docker-free Firecracker agent-image builder. -The VM boot / SSH / buildah plumbing (`_build_in_vm`) is integration-tested +The VM boot / SSH / buildah plumbing (`_build_in_infra`) is integration-tested on a KVM host; here we cover the cache decision, the boot-bit injection, and the smoke-test no-op — the logic that must hold without a VM. """ @@ -29,7 +29,7 @@ class TestBuildAgentRootfsDir(unittest.TestCase): base.mkdir(parents=True) (base / ".bb-ready").write_text("ok\n") with patch.object(image_builder.util, "cache_dir", return_value=self.cache), \ - patch.object(image_builder, "_build_in_vm") as build: + patch.object(image_builder, "_build_in_infra") as build: out = image_builder.build_agent_rootfs_dir( self.dockerfile, image_tag="t:latest") build.assert_not_called() @@ -37,7 +37,7 @@ class TestBuildAgentRootfsDir(unittest.TestCase): def test_cache_miss_builds_injects_and_marks_ready(self): with patch.object(image_builder.util, "cache_dir", return_value=self.cache), \ - patch.object(image_builder, "_build_in_vm") as build, \ + patch.object(image_builder, "_build_in_infra") as build, \ patch.object(image_builder.util, "inject_guest_boot") as inject: out = image_builder.build_agent_rootfs_dir( self.dockerfile, image_tag="t:latest", smoke_test=("claude", "--version")) @@ -59,7 +59,7 @@ class TestBuildAgentRootfsDir(unittest.TestCase): class TestSmokeTest(unittest.TestCase): def test_empty_argv_is_noop(self): with patch.object(image_builder, "_ssh") as ssh: - image_builder._smoke_test(Path("/k"), "10.0.0.1", ()) + image_builder._smoke_test(Path("/k"), "10.0.0.1", "tag", ()) ssh.assert_not_called() def test_failed_smoke_dies(self): @@ -67,7 +67,7 @@ class TestSmokeTest(unittest.TestCase): result = subprocess.CompletedProcess([], 1, stdout="broken", stderr="") with patch.object(image_builder, "_ssh", return_value=result), \ self.assertRaises(SystemExit): - image_builder._smoke_test(Path("/k"), "10.0.0.1", ("claude", "--version")) + image_builder._smoke_test(Path("/k"), "10.0.0.1", "tag", ("claude", "--version")) if __name__ == "__main__": diff --git a/tests/unit/test_firecracker_infra_vm.py b/tests/unit/test_firecracker_infra_vm.py index e858654..8b3e876 100644 --- a/tests/unit/test_firecracker_infra_vm.py +++ b/tests/unit/test_firecracker_infra_vm.py @@ -95,11 +95,14 @@ class TestRegistryVolume(unittest.TestCase): class TestEnsureBuilt(unittest.TestCase): - def test_builds_gateway_then_infra(self): + def test_builds_deps_before_infra(self): with patch.object(infra_vm.docker_mod, "build_image") as build: infra_vm.ensure_built() tags = [c.args[0] for c in build.call_args_list] - self.assertEqual([infra_vm._GATEWAY_IMAGE, infra_vm._INFRA_IMAGE], tags) + # infra is FROM gateway and COPY --from orchestrator, so both first. + self.assertEqual(infra_vm._INFRA_IMAGE, tags[-1]) + self.assertIn(infra_vm._ORCHESTRATOR_IMAGE, tags[:-1]) + self.assertIn(infra_vm._GATEWAY_IMAGE, tags[:-1]) class TestWaitForHealth(unittest.TestCase):