feat(firecracker): run the orchestrator control plane as a VM (Stage B, 1/n)

First slice of Stage B: the orchestrator control plane runs as a
persistent Firecracker infra VM instead of a Docker container. The host
CLI reaches it over HTTP at the orchestrator link's guest IP; agent VMs
will reach its gateway ports (added next) over VM-to-VM routing.

- Dockerfile.orchestrator bakes the stdlib-only control-plane source
  (COPY bot_bottle) so the image is self-contained and runs from a built
  image with no runtime bind-mount — a guest VM can't bind-mount host
  source. (Build-from-source stays the default; a pull-from-registry mode
  lands later. The docker backend's dev bind-mount still overlays this.)
- util.build_base_rootfs_dir / inject_guest_boot take a `variant` +
  `init_script`, so the same orchestrator image is prepared two ways
  without a cache collision: the builder VM keeps the SSH-only agent init;
  the infra VM gets a control-plane PID-1 init.
- new firecracker/infra_vm.py: boot the infra VM on the orchestrator link,
  run `python -m bot_bottle.orchestrator` as PID 1, and poll /health.

Verified on a KVM host: infra VM boots, control plane answers
`GET /health -> 200 {"status":"ok"}` from the host over the TAP link.
Next: fold gateway_init (egress/git-gate/supervise) into the same VM,
then agent->gateway routing.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck

Dockerfile.orchestrator

@@ -49,6 +49,14 @@ ENV STORAGE_DRIVER=vfs \
 # No third-party *Python* deps — the control plane stays stdlib only.
 WORKDIR /app
 
+# Bake the (stdlib-only) control-plane source so the image is
+# self-contained — it runs from a built image, no runtime bind-mount.
+# This is what the Firecracker infra VM boots (a guest can't bind-mount
+# host source); the docker backend may still bind-mount /app for dev
+# live-reload, which simply overlays this baked copy. `.dockerignore`
+# keeps .git/docs/*.md out of the context.
+COPY bot_bottle /app/bot_bottle
+
 # Documentation only; lifecycle.py overrides the entrypoint to
 # `python3 -m bot_bottle.orchestrator` with the runtime flags.
 ENTRYPOINT ["python3", "-m", "bot_bottle.orchestrator"]

bot_bottle/backend/firecracker/infra_vm.py

@@ -0,0 +1,145 @@
+"""The per-host infra VM for the Firecracker backend (PRD 0070 Stage B).
+
+A single persistent microVM that runs the orchestrator **control plane** (and,
+in a following step, the gateway **data plane**) — the trusted per-host service
+the docker backend runs as containers. It boots on the NAT'd orchestrator link
+(`netpool.orch_slot()`): the host CLI reaches its control plane over HTTP at the
+guest IP, and agent VMs reach its gateway ports over VM-to-VM routing.
+
+Build-from-source (the default while the design churns): the rootfs is exported
+from the locally built orchestrator image, which bakes the stdlib-only
+control-plane source. A pull-from-registry mode (Gitea's OCI registry) becomes
+the default later.
+
+SSH is left enabled for debugging; the control plane is the load-bearing
+surface.
+"""
+
+from __future__ import annotations
+
+import time
+import urllib.error
+import urllib.request
+from dataclasses import dataclass
+from pathlib import Path
+
+from ...log import die, info
+from . import firecracker_vm, netpool, util
+
+_ORCHESTRATOR_IMAGE = "bot-bottle-orchestrator:latest"
+CONTROL_PLANE_PORT = 8099
+
+# The infra VM makes direct upstream connections (gateway egress, and buildah
+# during builds), and the kernel `ip=` cmdline sets no resolver. Public for
+# now; routing DNS through a filtered path is a later refinement.
+_INFRA_RESOLVER = "1.1.1.1"
+
+_HEALTH_TIMEOUT_SECONDS = 45.0
+_HEALTH_POLL_SECONDS = 0.5
+
+
+@dataclass
+class InfraVm:
+    """A booted infra VM: its VMM handle, guest IP, and debug SSH key."""
+
+    vm: firecracker_vm.VmHandle
+    guest_ip: str
+    private_key: Path
+
+    @property
+    def control_plane_url(self) -> str:
+        return f"http://{self.guest_ip}:{CONTROL_PLANE_PORT}"
+
+    def terminate(self) -> None:
+        self.vm.terminate()
+
+
+def build_infra_rootfs_dir() -> Path:
+    """The infra VM's base rootfs: the orchestrator image (with its baked,
+    stdlib-only source) prepared with the control-plane init as PID 1."""
+    return util.build_base_rootfs_dir(
+        _ORCHESTRATOR_IMAGE, variant="-infra", init_script=_infra_init(),
+    )
+
+
+def boot() -> InfraVm:
+    """Boot the infra VM on the orchestrator link. Caller waits for health
+    (`wait_for_health`) and tears it down."""
+    slot = netpool.orch_slot()
+    if not netpool.tap_present(slot.iface):
+        die(f"orchestrator link {slot.iface} not present.\n"
+            f"    ./cli.py backend setup --backend=firecracker")
+
+    base = build_infra_rootfs_dir()
+    run_dir = util.cache_dir() / "run" / "infra"
+    run_dir.mkdir(parents=True, exist_ok=True)
+    rootfs = run_dir / "rootfs.ext4"
+    util.build_rootfs_ext4(base, rootfs, slack_mib=8192)
+    private_key, pubkey = util.generate_keypair(run_dir)
+
+    info(f"booting infra VM on {slot.iface} (guest {slot.guest_ip})")
+    vm = firecracker_vm.boot(
+        name="bot-bottle-infra", rootfs=rootfs, tap=slot.iface,
+        guest_ip=slot.guest_ip, host_ip=slot.host_ip, pubkey=pubkey,
+        run_dir=run_dir, mem_mib=4096,
+    )
+    return InfraVm(vm=vm, guest_ip=slot.guest_ip, private_key=private_key)
+
+
+def wait_for_health(
+    infra: InfraVm, *, timeout: float = _HEALTH_TIMEOUT_SECONDS,
+) -> None:
+    """Poll the control plane's /health until it answers 200 or the deadline
+    passes. Dies (with the console tail) if the VMM exits early."""
+    url = f"{infra.control_plane_url}/health"
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        if not infra.vm.is_alive():
+            die(f"infra VM exited during boot (rc={infra.vm.process.returncode}).\n"
+                f"{firecracker_vm._console_tail(infra.vm.console_log)}")
+        try:
+            with urllib.request.urlopen(url, timeout=1.0) as resp:
+                if resp.status == 200:
+                    info(f"infra control plane healthy at {infra.control_plane_url}")
+                    return
+        except (urllib.error.URLError, TimeoutError, OSError):
+            pass
+        time.sleep(_HEALTH_POLL_SECONDS)
+    die(f"infra control plane at {url} did not become healthy within "
+        f"{timeout:.0f}s.\n{firecracker_vm._console_tail(infra.vm.console_log)}")
+
+
+def _infra_init() -> str:
+    """PID-1 init for the infra VM: mount the pseudo-filesystems, wire a
+    resolver, start dropbear (debug SSH), then launch the control plane. The
+    gateway daemons are added here in the next step."""
+    return f"""#!/bin/sh
+# bot-bottle Firecracker infra VM init (PID 1).
+mount -t proc proc /proc 2>/dev/null
+mount -t sysfs sys /sys 2>/dev/null
+mount -t devtmpfs dev /dev 2>/dev/null
+mkdir -p /dev/pts && mount -t devpts devpts /dev/pts 2>/dev/null
+mount -o remount,rw / 2>/dev/null
+
+# Direct upstream resolver (control-plane / gateway egress + buildah).
+printf 'nameserver {_INFRA_RESOLVER}\\n' > /etc/resolv.conf 2>/dev/null
+
+# Debug SSH: install the per-boot pubkey from the kernel cmdline.
+KEY=$(sed -n 's/.*bb_pubkey=\\([^ ]*\\).*/\\1/p' /proc/cmdline | base64 -d 2>/dev/null)
+if [ -n "$KEY" ]; then
+    mkdir -p /root/.ssh
+    printf '%s\\n' "$KEY" > /root/.ssh/authorized_keys
+    chmod 700 /root/.ssh && chmod 600 /root/.ssh/authorized_keys
+fi
+chown -R 0:0 /root 2>/dev/null || true
+mkdir -p /etc/dropbear /run /var/lib/bot-bottle
+/bb-dropbear -R -E -p 22 &
+
+# Control plane. Source is baked at /app; the package is stdlib-only.
+cd /app
+BOT_BOTTLE_ROOT=/var/lib/bot-bottle python3 -m bot_bottle.orchestrator \\
+    --host 0.0.0.0 --port {CONTROL_PLANE_PORT} --broker stub &
+
+# Reap as PID 1; children are backgrounded, so `wait` blocks.
+while : ; do wait ; done
+"""

bot_bottle/backend/firecracker/util.py

@@ -159,15 +159,22 @@ def docker_image_id(ref: str) -> str:
     return result.stdout.strip().replace("sha256:", "")[:16]
 
 
-def build_base_rootfs_dir(image_ref: str) -> Path:
-    """Export the agent image's filesystem and inject the guest init +
-    static dropbear. Cached by image digest — the per-bottle bits
+def build_base_rootfs_dir(
+    image_ref: str, *, variant: str = "", init_script: str | None = None,
+) -> Path:
+    """Export the image's filesystem and inject the guest init + static
+    dropbear. Cached by image digest — the per-bottle bits
     (authorized_keys, IP) are passed at boot via the kernel cmdline, so
     this tree carries nothing bottle-specific and is safely shared.
 
+    `variant` suffixes the cache key so the same image can be prepared
+    with a different `init_script` (e.g. the infra VM boots the same
+    orchestrator image as the builder but runs the control plane as
+    PID 1, not the SSH-only agent init) without a cache collision.
+
     Returns the prepared directory (read as the `mke2fs -d` source)."""
     digest = docker_image_id(image_ref)
-    base = cache_dir() / "rootfs" / digest
+    base = cache_dir() / "rootfs" / f"{digest}{variant}"
     ready = base / ".bb-ready"
     if ready.is_file():
         return base
@@ -200,17 +207,19 @@ def build_base_rootfs_dir(image_ref: str) -> Path:
             stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
         )
 
-    inject_guest_boot(base)
+    inject_guest_boot(base, init_script=init_script)
     ready.write_text("ok\n")
     return base
 
 
-def inject_guest_boot(rootfs: Path) -> None:
-    """Drop the static dropbear and the PID-1 init into the rootfs."""
+def inject_guest_boot(rootfs: Path, init_script: str | None = None) -> None:
+    """Drop the static dropbear and the PID-1 init into the rootfs.
+    `init_script` defaults to the SSH-only agent init; the infra VM
+    passes its own (control plane + gateway) init."""
     shutil.copy2(dropbear_path(), rootfs / "bb-dropbear")
     os.chmod(rootfs / "bb-dropbear", 0o755)
     init = rootfs / "bb-init"
-    init.write_text(_GUEST_INIT)
+    init.write_text(init_script or _GUEST_INIT)
     os.chmod(init, 0o755)
 
 

tests/unit/test_firecracker_infra_vm.py

@@ -0,0 +1,61 @@
+"""Unit tests for the Firecracker infra VM (control-plane VM boot).
+
+The KVM boot / HTTP reachability is integration-tested on a KVM host; here
+we cover the URL shape, the rootfs-variant wiring, and the health-poll
+decisions that must hold without a VM.
+"""
+
+from __future__ import annotations
+
+import unittest
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+from bot_bottle.backend.firecracker import infra_vm
+
+
+class TestControlPlaneUrl(unittest.TestCase):
+    def test_url_uses_guest_ip_and_port(self):
+        infra = infra_vm.InfraVm(
+            vm=MagicMock(), guest_ip="10.243.255.1", private_key=Path("/k"))
+        self.assertEqual(
+            f"http://10.243.255.1:{infra_vm.CONTROL_PLANE_PORT}",
+            infra.control_plane_url,
+        )
+
+
+class TestBuildInfraRootfs(unittest.TestCase):
+    def test_uses_infra_variant_and_init(self):
+        with patch.object(infra_vm.util, "build_base_rootfs_dir") as build:
+            build.return_value = Path("/cache/rootfs/x-infra")
+            infra_vm.build_infra_rootfs_dir()
+        build.assert_called_once()
+        self.assertEqual(infra_vm._ORCHESTRATOR_IMAGE, build.call_args.args[0])
+        self.assertEqual("-infra", build.call_args.kwargs["variant"])
+        # The init runs the control plane, not the SSH-only agent init.
+        self.assertIn("bot_bottle.orchestrator", build.call_args.kwargs["init_script"])
+
+
+class TestWaitForHealth(unittest.TestCase):
+    def _infra(self, alive: bool = True) -> infra_vm.InfraVm:
+        vm = MagicMock()
+        vm.is_alive.return_value = alive
+        return infra_vm.InfraVm(vm=vm, guest_ip="10.0.0.1", private_key=Path("/k"))
+
+    def test_returns_on_200(self):
+        infra = self._infra()
+        cm = MagicMock()
+        cm.__enter__.return_value.status = 200
+        with patch.object(infra_vm.urllib.request, "urlopen", return_value=cm):
+            infra_vm.wait_for_health(infra, timeout=5)  # must not raise
+
+    def test_dies_when_vm_exits(self):
+        infra = self._infra(alive=False)
+        infra.vm.process.returncode = 1
+        with patch.object(infra_vm.firecracker_vm, "_console_tail", return_value=""), \
+             self.assertRaises(SystemExit):
+            infra_vm.wait_for_health(infra, timeout=5)
+
+
+if __name__ == "__main__":
+    unittest.main()