diff --git a/Dockerfile.orchestrator b/Dockerfile.orchestrator index e81ae3b..0bb8d25 100644 --- a/Dockerfile.orchestrator +++ b/Dockerfile.orchestrator @@ -49,6 +49,14 @@ ENV STORAGE_DRIVER=vfs \ # No third-party *Python* deps — the control plane stays stdlib only. WORKDIR /app +# Bake the (stdlib-only) control-plane source so the image is +# self-contained — it runs from a built image, no runtime bind-mount. +# This is what the Firecracker infra VM boots (a guest can't bind-mount +# host source); the docker backend may still bind-mount /app for dev +# live-reload, which simply overlays this baked copy. `.dockerignore` +# keeps .git/docs/*.md out of the context. +COPY bot_bottle /app/bot_bottle + # Documentation only; lifecycle.py overrides the entrypoint to # `python3 -m bot_bottle.orchestrator` with the runtime flags. ENTRYPOINT ["python3", "-m", "bot_bottle.orchestrator"] diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py new file mode 100644 index 0000000..1568a1a --- /dev/null +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -0,0 +1,145 @@ +"""The per-host infra VM for the Firecracker backend (PRD 0070 Stage B). + +A single persistent microVM that runs the orchestrator **control plane** (and, +in a following step, the gateway **data plane**) — the trusted per-host service +the docker backend runs as containers. It boots on the NAT'd orchestrator link +(`netpool.orch_slot()`): the host CLI reaches its control plane over HTTP at the +guest IP, and agent VMs reach its gateway ports over VM-to-VM routing. + +Build-from-source (the default while the design churns): the rootfs is exported +from the locally built orchestrator image, which bakes the stdlib-only +control-plane source. A pull-from-registry mode (Gitea's OCI registry) becomes +the default later. + +SSH is left enabled for debugging; the control plane is the load-bearing +surface. +""" + +from __future__ import annotations + +import time +import urllib.error +import urllib.request +from dataclasses import dataclass +from pathlib import Path + +from ...log import die, info +from . import firecracker_vm, netpool, util + +_ORCHESTRATOR_IMAGE = "bot-bottle-orchestrator:latest" +CONTROL_PLANE_PORT = 8099 + +# The infra VM makes direct upstream connections (gateway egress, and buildah +# during builds), and the kernel `ip=` cmdline sets no resolver. Public for +# now; routing DNS through a filtered path is a later refinement. +_INFRA_RESOLVER = "1.1.1.1" + +_HEALTH_TIMEOUT_SECONDS = 45.0 +_HEALTH_POLL_SECONDS = 0.5 + + +@dataclass +class InfraVm: + """A booted infra VM: its VMM handle, guest IP, and debug SSH key.""" + + vm: firecracker_vm.VmHandle + guest_ip: str + private_key: Path + + @property + def control_plane_url(self) -> str: + return f"http://{self.guest_ip}:{CONTROL_PLANE_PORT}" + + def terminate(self) -> None: + self.vm.terminate() + + +def build_infra_rootfs_dir() -> Path: + """The infra VM's base rootfs: the orchestrator image (with its baked, + stdlib-only source) prepared with the control-plane init as PID 1.""" + return util.build_base_rootfs_dir( + _ORCHESTRATOR_IMAGE, variant="-infra", init_script=_infra_init(), + ) + + +def boot() -> InfraVm: + """Boot the infra VM on the orchestrator link. Caller waits for health + (`wait_for_health`) and tears it down.""" + slot = netpool.orch_slot() + if not netpool.tap_present(slot.iface): + die(f"orchestrator link {slot.iface} not present.\n" + f" ./cli.py backend setup --backend=firecracker") + + base = build_infra_rootfs_dir() + run_dir = util.cache_dir() / "run" / "infra" + run_dir.mkdir(parents=True, exist_ok=True) + rootfs = run_dir / "rootfs.ext4" + util.build_rootfs_ext4(base, rootfs, slack_mib=8192) + private_key, pubkey = util.generate_keypair(run_dir) + + info(f"booting infra VM on {slot.iface} (guest {slot.guest_ip})") + vm = firecracker_vm.boot( + name="bot-bottle-infra", rootfs=rootfs, tap=slot.iface, + guest_ip=slot.guest_ip, host_ip=slot.host_ip, pubkey=pubkey, + run_dir=run_dir, mem_mib=4096, + ) + return InfraVm(vm=vm, guest_ip=slot.guest_ip, private_key=private_key) + + +def wait_for_health( + infra: InfraVm, *, timeout: float = _HEALTH_TIMEOUT_SECONDS, +) -> None: + """Poll the control plane's /health until it answers 200 or the deadline + passes. Dies (with the console tail) if the VMM exits early.""" + url = f"{infra.control_plane_url}/health" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if not infra.vm.is_alive(): + die(f"infra VM exited during boot (rc={infra.vm.process.returncode}).\n" + f"{firecracker_vm._console_tail(infra.vm.console_log)}") + try: + with urllib.request.urlopen(url, timeout=1.0) as resp: + if resp.status == 200: + info(f"infra control plane healthy at {infra.control_plane_url}") + return + except (urllib.error.URLError, TimeoutError, OSError): + pass + time.sleep(_HEALTH_POLL_SECONDS) + die(f"infra control plane at {url} did not become healthy within " + f"{timeout:.0f}s.\n{firecracker_vm._console_tail(infra.vm.console_log)}") + + +def _infra_init() -> str: + """PID-1 init for the infra VM: mount the pseudo-filesystems, wire a + resolver, start dropbear (debug SSH), then launch the control plane. The + gateway daemons are added here in the next step.""" + return f"""#!/bin/sh +# bot-bottle Firecracker infra VM init (PID 1). +mount -t proc proc /proc 2>/dev/null +mount -t sysfs sys /sys 2>/dev/null +mount -t devtmpfs dev /dev 2>/dev/null +mkdir -p /dev/pts && mount -t devpts devpts /dev/pts 2>/dev/null +mount -o remount,rw / 2>/dev/null + +# Direct upstream resolver (control-plane / gateway egress + buildah). +printf 'nameserver {_INFRA_RESOLVER}\\n' > /etc/resolv.conf 2>/dev/null + +# Debug SSH: install the per-boot pubkey from the kernel cmdline. +KEY=$(sed -n 's/.*bb_pubkey=\\([^ ]*\\).*/\\1/p' /proc/cmdline | base64 -d 2>/dev/null) +if [ -n "$KEY" ]; then + mkdir -p /root/.ssh + printf '%s\\n' "$KEY" > /root/.ssh/authorized_keys + chmod 700 /root/.ssh && chmod 600 /root/.ssh/authorized_keys +fi +chown -R 0:0 /root 2>/dev/null || true +mkdir -p /etc/dropbear /run /var/lib/bot-bottle +/bb-dropbear -R -E -p 22 & + +# Control plane. Source is baked at /app; the package is stdlib-only. +cd /app +BOT_BOTTLE_ROOT=/var/lib/bot-bottle python3 -m bot_bottle.orchestrator \\ + --host 0.0.0.0 --port {CONTROL_PLANE_PORT} --broker stub & + +# Reap as PID 1; children are backgrounded, so `wait` blocks. +while : ; do wait ; done +""" diff --git a/bot_bottle/backend/firecracker/util.py b/bot_bottle/backend/firecracker/util.py index 9455edb..50bcd9e 100644 --- a/bot_bottle/backend/firecracker/util.py +++ b/bot_bottle/backend/firecracker/util.py @@ -159,15 +159,22 @@ def docker_image_id(ref: str) -> str: return result.stdout.strip().replace("sha256:", "")[:16] -def build_base_rootfs_dir(image_ref: str) -> Path: - """Export the agent image's filesystem and inject the guest init + - static dropbear. Cached by image digest — the per-bottle bits +def build_base_rootfs_dir( + image_ref: str, *, variant: str = "", init_script: str | None = None, +) -> Path: + """Export the image's filesystem and inject the guest init + static + dropbear. Cached by image digest — the per-bottle bits (authorized_keys, IP) are passed at boot via the kernel cmdline, so this tree carries nothing bottle-specific and is safely shared. + `variant` suffixes the cache key so the same image can be prepared + with a different `init_script` (e.g. the infra VM boots the same + orchestrator image as the builder but runs the control plane as + PID 1, not the SSH-only agent init) without a cache collision. + Returns the prepared directory (read as the `mke2fs -d` source).""" digest = docker_image_id(image_ref) - base = cache_dir() / "rootfs" / digest + base = cache_dir() / "rootfs" / f"{digest}{variant}" ready = base / ".bb-ready" if ready.is_file(): return base @@ -200,17 +207,19 @@ def build_base_rootfs_dir(image_ref: str) -> Path: stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, ) - inject_guest_boot(base) + inject_guest_boot(base, init_script=init_script) ready.write_text("ok\n") return base -def inject_guest_boot(rootfs: Path) -> None: - """Drop the static dropbear and the PID-1 init into the rootfs.""" +def inject_guest_boot(rootfs: Path, init_script: str | None = None) -> None: + """Drop the static dropbear and the PID-1 init into the rootfs. + `init_script` defaults to the SSH-only agent init; the infra VM + passes its own (control plane + gateway) init.""" shutil.copy2(dropbear_path(), rootfs / "bb-dropbear") os.chmod(rootfs / "bb-dropbear", 0o755) init = rootfs / "bb-init" - init.write_text(_GUEST_INIT) + init.write_text(init_script or _GUEST_INIT) os.chmod(init, 0o755) diff --git a/tests/unit/test_firecracker_infra_vm.py b/tests/unit/test_firecracker_infra_vm.py new file mode 100644 index 0000000..ab21758 --- /dev/null +++ b/tests/unit/test_firecracker_infra_vm.py @@ -0,0 +1,61 @@ +"""Unit tests for the Firecracker infra VM (control-plane VM boot). + +The KVM boot / HTTP reachability is integration-tested on a KVM host; here +we cover the URL shape, the rootfs-variant wiring, and the health-poll +decisions that must hold without a VM. +""" + +from __future__ import annotations + +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +from bot_bottle.backend.firecracker import infra_vm + + +class TestControlPlaneUrl(unittest.TestCase): + def test_url_uses_guest_ip_and_port(self): + infra = infra_vm.InfraVm( + vm=MagicMock(), guest_ip="10.243.255.1", private_key=Path("/k")) + self.assertEqual( + f"http://10.243.255.1:{infra_vm.CONTROL_PLANE_PORT}", + infra.control_plane_url, + ) + + +class TestBuildInfraRootfs(unittest.TestCase): + def test_uses_infra_variant_and_init(self): + with patch.object(infra_vm.util, "build_base_rootfs_dir") as build: + build.return_value = Path("/cache/rootfs/x-infra") + infra_vm.build_infra_rootfs_dir() + build.assert_called_once() + self.assertEqual(infra_vm._ORCHESTRATOR_IMAGE, build.call_args.args[0]) + self.assertEqual("-infra", build.call_args.kwargs["variant"]) + # The init runs the control plane, not the SSH-only agent init. + self.assertIn("bot_bottle.orchestrator", build.call_args.kwargs["init_script"]) + + +class TestWaitForHealth(unittest.TestCase): + def _infra(self, alive: bool = True) -> infra_vm.InfraVm: + vm = MagicMock() + vm.is_alive.return_value = alive + return infra_vm.InfraVm(vm=vm, guest_ip="10.0.0.1", private_key=Path("/k")) + + def test_returns_on_200(self): + infra = self._infra() + cm = MagicMock() + cm.__enter__.return_value.status = 200 + with patch.object(infra_vm.urllib.request, "urlopen", return_value=cm): + infra_vm.wait_for_health(infra, timeout=5) # must not raise + + def test_dies_when_vm_exits(self): + infra = self._infra(alive=False) + infra.vm.process.returncode = 1 + with patch.object(infra_vm.firecracker_vm, "_console_tail", return_value=""), \ + self.assertRaises(SystemExit): + infra_vm.wait_for_health(infra, timeout=5) + + +if __name__ == "__main__": + unittest.main()