diff --git a/bot_bottle/backend/firecracker/gateway.py b/bot_bottle/backend/firecracker/gateway.py index 97d9bb6c..30859d53 100644 --- a/bot_bottle/backend/firecracker/gateway.py +++ b/bot_bottle/backend/firecracker/gateway.py @@ -9,7 +9,7 @@ only the token the host hands it via `connect_to_orchestrator`. What deliberately stays in `infra_vm` (not moved here): the plane-agnostic VM substrate the orchestrator VM shares — booting a VM from the shared rootfs -(`_boot_vm`), the stable SSH keypair, the secret-push retry loop, and the PID +(`boot_vm`), the stable SSH keypair, the secret-push retry loop, and the PID lifecycle — plus the pair coordinator (`ensure_running`: orchestrator-first health gate, singleton lock, adoption/version marker). The guest-side gateway daemon startup lives in the shared `bb_role=gateway` init branch baked into the @@ -96,12 +96,12 @@ class FirecrackerGateway(Gateway): ) # Boot on the gateway link from the shared rootfs (bb_role=gateway), then # push the token the init waits for before starting the data plane. - vm = infra_vm._boot_vm( + vm = infra_vm.boot_vm( name=GATEWAY_NAME, slot=netpool.gw_slot(), run_dir=infra_vm._gw_dir(), role="gateway", mem_mib=_GW_MEM_MIB, extra_boot_args=f"bb_orch={orchestrator_guest_ip}", ) - infra_vm._push_secret( + infra_vm.push_secret( vm, self._gateway_token, _GUEST_GATEWAY_JWT_PATH, "the gateway JWT to the gateway VM (its data plane will not start)", ) diff --git a/bot_bottle/backend/firecracker/image_builder.py b/bot_bottle/backend/firecracker/image_builder.py index 55b4e122..3f250ae6 100644 --- a/bot_bottle/backend/firecracker/image_builder.py +++ b/bot_bottle/backend/firecracker/image_builder.py @@ -126,7 +126,7 @@ def _build_in_infra( and stream its rootfs into `base`. The infra VMs persist; only the per-build container/image/context are cleaned up.""" orchestrator = infra_vm.ensure_running().orchestrator - key, ip = orchestrator.private_key, orchestrator.guest_ip + key, ip = orchestrator.ssh_target() tag = f"bot-bottle-agent-build-{digest}" ctx = f"/tmp/agent-build-{digest}" smoke_ctr, export_ctr = f"{tag}-smoke", f"{tag}-export" diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py index 420f1ca0..bca52c81 100644 --- a/bot_bottle/backend/firecracker/infra_vm.py +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -43,13 +43,12 @@ from pathlib import Path from typing import TYPE_CHECKING, Generator from ...log import die, info -from ...orchestrator_auth import ROLE_GATEWAY, mint -from ...paths import host_orchestrator_token from ..docker import util as docker_mod from . import firecracker_vm, infra_artifact, netpool, util if TYPE_CHECKING: from .gateway import FirecrackerGateway + from .orchestrator import FirecrackerOrchestrator # The orchestrator VM's signing-key path on its persistent /dev/vdb volume # (mounted at BOT_BOTTLE_ROOT=/var/lib/bot-bottle). The launcher seeds this @@ -76,19 +75,12 @@ ORCHESTRATOR_PORT = 8099 EGRESS_PORT = 9099 SUPERVISE_PORT = 9100 GIT_HTTP_PORT = 9420 -# Memory ceiling (fixed at boot, demand-paged — no balloon reclaim). The -# orchestrator keeps the build headroom (buildah's 2-4 GB working set during -# in-VM agent builds); the gateway's slimmer ceiling is set by the gateway -# service — PRD 0070 "Memory: fixed ceilings". -_ORCH_MEM_MIB = 4096 # The infra VMs make direct upstream connections (gateway egress, and buildah # during builds), and the kernel `ip=` cmdline sets no resolver. Public for # now; routing DNS through a filtered path is a later refinement. _INFRA_RESOLVER = "1.1.1.1" -_HEALTH_TIMEOUT_SECONDS = 45.0 -_HEALTH_POLL_SECONDS = 0.5 _CA_TIMEOUT_SECONDS = 30.0 # How long the launcher retries pushing a secret while the guest's SSH comes # up. Below the init's own wait window, so a failed push dies here first. @@ -107,24 +99,20 @@ class InfraVm: private_key: Path vm: firecracker_vm.VmHandle | None = None - @property - def orchestrator_url(self) -> str: - return f"http://{self.guest_ip}:{ORCHESTRATOR_PORT}" - @dataclass class InfraEndpoint: - """How to reach the running per-host pair. `orchestrator` is the control - plane (host CLI + registration + in-VM builds); `gateway` is the data-plane - `Gateway` service (agent egress / git-http / supervise, CA fetch, git-gate - provisioning). Mirrors the docker/macOS `InfraEndpoint` shape.""" + """How to reach the running per-host pair. `orchestrator` is the control-plane + `Orchestrator` service (host CLI + registration + in-VM builds); `gateway` is + the data-plane `Gateway` service (agent egress / git-http / supervise, CA + fetch, git-gate provisioning). Mirrors the docker/macOS `InfraEndpoint`.""" - orchestrator: InfraVm + orchestrator: "FirecrackerOrchestrator" gateway: "FirecrackerGateway" @property def orchestrator_url(self) -> str: - return self.orchestrator.orchestrator_url + return self.orchestrator.url() def gateway_ca_pem(self, *, timeout: float = _CA_TIMEOUT_SECONDS) -> str: return self.gateway.ca_cert_pem(timeout=timeout) @@ -186,8 +174,14 @@ def ensure_running() -> InfraEndpoint: Concurrency-safe: the cold stop/build/boot path is serialized by a host flock, so two simultaneous first launches don't both boot on the same links/PIDs. The healthy fast-path takes no lock.""" + # Local imports: the orchestrator + gateway services import this module for + # the shared VM substrate, so importing them at module scope would cycle. + from .orchestrator import FirecrackerOrchestrator + from .gateway import FirecrackerGateway + key = _infra_dir() / "id_ed25519" - url = f"http://{netpool.orch_slot().guest_ip}:{ORCHESTRATOR_PORT}" + orchestrator = FirecrackerOrchestrator() + url = orchestrator.url() want = _expected_version() if _adoptable(key, url, want): info(f"adopting running infra VMs (orchestrator at {url})") @@ -202,29 +196,26 @@ def ensure_running() -> InfraEndpoint: # Clear stale/hung/OUTDATED VMs holding either link before booting fresh. stop() ensure_built() - orchestrator = boot_orchestrator() - wait_for_health(orchestrator) - # The orchestrator holds the signing key; mint the role-scoped `gateway` - # JWT on the host and hand it to the gateway service, which never sees - # the key (#469). The gateway owns its own boot — see FirecrackerGateway. - # Local import: the gateway service imports this module for the shared VM - # substrate, so importing it at module scope would cycle. - from .gateway import FirecrackerGateway - gateway_token = mint(ROLE_GATEWAY, host_orchestrator_token()) + # Orchestrator first — the gateway daemons reach the control plane at + # startup. Each service owns its own boot; the orchestrator (which holds + # the signing key) mints the role-scoped `gateway` JWT for the gateway, + # which never sees the key (#469). + orchestrator.ensure_running() gateway = FirecrackerGateway() - gateway.connect_to_orchestrator(orchestrator.orchestrator_url, gateway_token) + gateway.connect_to_orchestrator( + orchestrator.gateway_url(), orchestrator.mint_gateway_token()) _record_booted_version(want) return InfraEndpoint(orchestrator=orchestrator, gateway=gateway) -def _adopt(key: Path) -> InfraEndpoint: - """Build handles to the already-running pair from the stable key + the - fixed links (no live VM handle — teardown / CA fetch go through the PID +def _adopt(key: Path) -> "InfraEndpoint": + """Build the service handles for the already-running pair from the stable key + + the fixed links (no live VM handle — teardown / CA fetch go through the PID files + stable key).""" + from .orchestrator import FirecrackerOrchestrator from .gateway import FirecrackerGateway return InfraEndpoint( - orchestrator=InfraVm( - guest_ip=netpool.orch_slot().guest_ip, private_key=key), + orchestrator=FirecrackerOrchestrator(), gateway=FirecrackerGateway( InfraVm(guest_ip=netpool.gw_slot().guest_ip, private_key=key)), ) @@ -259,24 +250,7 @@ def stop() -> None: _version_file().unlink(missing_ok=True) -def boot_orchestrator() -> InfraVm: - """Boot the orchestrator (control-plane) VM (detached, so it outlives the - launcher) on the orchestrator link, and seed its signing key. Prefer - `ensure_running`.""" - slot = netpool.orch_slot() - orch = _boot_vm( - name="bot-bottle-orchestrator", slot=slot, run_dir=_orch_dir(), - role="orchestrator", mem_mib=_ORCH_MEM_MIB, - data_drive=_ensure_registry_volume(), - ) - # Push the host-canonical signing key over SSH (the init waits for it - # before starting the control plane). The host token file stays the single - # source of truth, so a co-running Docker/macOS control plane keeps working. - _push_signing_key(orch) - return orch - - -def _boot_vm( +def boot_vm( *, name: str, slot: netpool.Slot, @@ -376,50 +350,7 @@ def _record_booted_version(version: str) -> None: _version_file().write_text(version + "\n", encoding="utf-8") -# The registry "volume": a host-side ext4 file attached to the orchestrator VM -# as a second virtio-block device (guest /dev/vdb), mounted at the control -# plane's DB dir. It outlives the ephemeral rootfs, so the bottle registry -# survives an orchestrator-VM restart — the firecracker analogue of a docker -# volume. It is a plain ext4 file: `sudo mount -o loop ` on the host -# (with the VM stopped) to inspect bot-bottle.db directly. The gateway VM has -# no such volume — the data plane never opens the DB (#469). -_REGISTRY_SIZE = "512M" - - -def registry_volume_path() -> Path: - return _infra_dir() / "registry.ext4" - - -def _ensure_registry_volume() -> Path: - """Create the empty ext4 registry volume on first use; reuse it after.""" - vol = registry_volume_path() - if vol.exists(): - return vol - info(f"creating infra registry volume {vol} ({_REGISTRY_SIZE})") - proc = subprocess.run( - ["mke2fs", "-q", "-t", "ext4", "-F", str(vol), _REGISTRY_SIZE], - capture_output=True, text=True, check=False, - ) - if proc.returncode != 0: - vol.unlink(missing_ok=True) - die(f"creating registry volume failed: {proc.stderr.strip()}") - return vol - - -def _push_signing_key(infra: InfraVm) -> None: - """Push the host-canonical signing key into the freshly booted orchestrator - VM over SSH (atomic write), so its control plane verifies tokens with the - same key the host CLI signs from. Retries until the guest's SSH is up (it - comes up before the init waits for this file); dies if it never lands, - since the VM then refuses to start its control plane rather than run OPEN.""" - _push_secret( - infra, host_orchestrator_token(), _GUEST_SIGNING_KEY_PATH, - "the control-plane signing key to the orchestrator VM " - "(its control plane will not start)", - ) - - -def _push_secret(infra: InfraVm, secret: str, dest: str, what: str) -> None: +def push_secret(infra: InfraVm, secret: str, dest: str, what: str) -> None: """Pipe `secret` to an atomic write of `dest` in the guest over SSH, retrying while the guest's SSH comes up; die (naming `what`) if it never succeeds. Bare-pipe input keeps the value off argv.""" @@ -525,31 +456,6 @@ def _health_ok(url: str) -> bool: return False -def wait_for_health( - infra: InfraVm, *, timeout: float = _HEALTH_TIMEOUT_SECONDS, -) -> None: - """Poll the orchestrator's /health until it answers 200 or the deadline - passes. Dies (with the console tail) if the VMM exits early.""" - url = f"{infra.orchestrator_url}/health" - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - if infra.vm is not None and not infra.vm.is_alive(): - die(f"orchestrator VM exited during boot (rc={infra.vm.process.returncode}).\n" - f"{firecracker_vm._console_tail(infra.vm.console_log)}") - try: - with urllib.request.urlopen(url, timeout=1.0) as resp: - if resp.status == 200: - info(f"orchestrator control plane healthy at {infra.orchestrator_url}") - return - except (urllib.error.URLError, TimeoutError, OSError): - pass - time.sleep(_HEALTH_POLL_SECONDS) - tail = (firecracker_vm._console_tail(infra.vm.console_log) - if infra.vm is not None else "") - die(f"orchestrator control plane at {url} did not become healthy within " - f"{timeout:.0f}s.\n{tail}") - - def _infra_init() -> str: """PID-1 init for the infra VMs. Shared setup (pseudo-filesystems, PATH, resolver, debug SSH), then a `bb_role` cmdline branch selecting the plane: diff --git a/bot_bottle/backend/firecracker/orchestrator.py b/bot_bottle/backend/firecracker/orchestrator.py new file mode 100644 index 00000000..86dfa5aa --- /dev/null +++ b/bot_bottle/backend/firecracker/orchestrator.py @@ -0,0 +1,159 @@ +"""The Firecracker orchestrator (control plane) as a microVM (PRD 0070). + +`FirecrackerOrchestrator` is the Firecracker implementation of the backend-neutral +`Orchestrator` service, and owns the control plane's host-side logic directly: +booting the orchestrator microVM on its NAT'd link with the persistent registry +volume (/dev/vdb — the sole opener of `bot-bottle.db`), seeding the host-canonical +signing key over SSH, and waiting for `/health`. The host CLI reaches it at its +guest IP, and so does the gateway (`bb_orch` cmdline), so `url()` == `gateway_url()`. + +What deliberately stays in `infra_vm` (not moved here): the plane-agnostic VM +substrate the gateway VM also shares — booting a VM from the shared rootfs +(`boot_vm`), the stable SSH keypair, the secret-push retry, the PID lifecycle — +plus the pair coordinator (`ensure_running`: adopt-or-boot-both under a singleton +lock with a shared version marker) and the single shared `bb_role`-branched init +baked into the one published rootfs. Those are the shared seam. +""" + +from __future__ import annotations + +import subprocess +import time +from pathlib import Path + +from ...log import die, info +from ...paths import host_orchestrator_token +from ...orchestrator.lifecycle import ( + DEFAULT_STARTUP_TIMEOUT_SECONDS, + Orchestrator, +) +from . import firecracker_vm, infra_vm, netpool +from .infra_vm import ORCHESTRATOR_PORT + +# The orchestrator microVM's name (the `bb_role=orchestrator` boot tag / run +# dir). Fixed per host — one control-plane VM. +ORCHESTRATOR_NAME = "bot-bottle-orchestrator" + +# Memory ceiling (fixed at boot, demand-paged). The orchestrator keeps the build +# headroom (buildah's 2-4 GB working set during in-VM agent builds) — PRD 0070 +# "Memory: fixed ceilings". +_ORCH_MEM_MIB = 4096 + +_HEALTH_POLL_SECONDS = 0.5 + +# The registry "volume": a host-side ext4 file attached to the orchestrator VM as +# a second virtio-block device (guest /dev/vdb), mounted at the control plane's DB +# dir. It outlives the ephemeral rootfs, so the bottle registry survives an +# orchestrator-VM restart — the firecracker analogue of a docker volume. A plain +# ext4 file: `sudo mount -o loop ` on the host (VM stopped) to inspect +# bot-bottle.db. The gateway VM has no such volume — the data plane never opens +# the DB (#469). +_REGISTRY_SIZE = "512M" + + +def registry_volume_path() -> Path: + return infra_vm._infra_dir() / "registry.ext4" + + +class FirecrackerOrchestrator(Orchestrator): + """The control plane as a Firecracker microVM on the orchestrator link. + + The shared rootfs is built/downloaded by `infra_vm.ensure_built` (the ABC's + `ensure_built` no-op here); `ensure_running` boots the VM, seeds the signing + key, and blocks until `/health` answers.""" + + name = ORCHESTRATOR_NAME + + def __init__(self, vm: infra_vm.InfraVm | None = None) -> None: + # The live VM handle when this process booted it; None when adapting an + # already-running orchestrator (health/URL go through the fixed link). + self._vm = vm + + def url(self) -> str: + """The orchestrator VM's guest IP URL — where the host CLI reaches the + control plane.""" + return f"http://{netpool.orch_slot().guest_ip}:{ORCHESTRATOR_PORT}" + + def gateway_url(self) -> str: + """Same guest IP the host uses — the gateway resolves the orchestrator at + `bb_orch=` off its cmdline.""" + return self.url() + + def ssh_target(self) -> tuple[Path, str]: + """(private key, guest IP) for SSHing into the orchestrator VM. In-VM + agent-image builds (buildah) run here — the build host drives them over + SSH with the stable infra key.""" + return infra_vm._infra_dir() / "id_ed25519", netpool.orch_slot().guest_ip + + def is_running(self) -> bool: + return infra_vm._pidfile_alive(infra_vm._orch_dir()) + + def stop(self) -> None: + """Stop only the orchestrator VM (idempotent). A dead orchestrator fails + `infra_vm._adoptable`'s health check, so no version marker to clear.""" + infra_vm._kill_pidfile(infra_vm._orch_dir()) + infra_vm._pid_file(infra_vm._orch_dir()).unlink(missing_ok=True) + + def ensure_running( + self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS, + ) -> None: + """Boot the orchestrator VM on its link with the persistent registry + volume, seed the signing key over SSH, and block until `/health` answers. + Idempotent — a live, healthy control plane is left alone (the pair + coordinator otherwise `stop()`s it before calling, so this boots fresh).""" + if self.is_running() and self.is_healthy(): + return + vm = infra_vm.boot_vm( + name=ORCHESTRATOR_NAME, slot=netpool.orch_slot(), + run_dir=infra_vm._orch_dir(), role="orchestrator", mem_mib=_ORCH_MEM_MIB, + data_drive=self._ensure_registry_volume(), + ) + # Push the host-canonical signing key (the init waits for it before + # starting the control plane). The host token file stays the single + # source of truth, so a co-running docker/macOS control plane keeps + # working; the guest verifies tokens with the same key the CLI signs from. + infra_vm.push_secret( + vm, host_orchestrator_token(), infra_vm._GUEST_SIGNING_KEY_PATH, + "the control-plane signing key to the orchestrator VM " + "(its control plane will not start)", + ) + self._vm = vm + self._wait_for_health(vm, timeout=startup_timeout) + + def _wait_for_health( + self, vm: infra_vm.InfraVm, *, timeout: float, + ) -> None: + """Poll `/health` until it answers 200 or the deadline passes. Dies (with + the console tail) if the VMM exits early.""" + url = f"{self.url()}/health" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if vm.vm is not None and not vm.vm.is_alive(): + die(f"orchestrator VM exited during boot (rc={vm.vm.process.returncode}).\n" + f"{firecracker_vm._console_tail(vm.vm.console_log)}") + if self.is_healthy(): + info(f"orchestrator control plane healthy at {self.url()}") + return + time.sleep(_HEALTH_POLL_SECONDS) + tail = (firecracker_vm._console_tail(vm.vm.console_log) + if vm.vm is not None else "") + die(f"orchestrator control plane at {url} did not become healthy within " + f"{timeout:.0f}s.\n{tail}") + + def _ensure_registry_volume(self) -> Path: + """Create the empty ext4 registry volume on first use; reuse it after.""" + vol = registry_volume_path() + if vol.exists(): + return vol + info(f"creating infra registry volume {vol} ({_REGISTRY_SIZE})") + proc = subprocess.run( + ["mke2fs", "-q", "-t", "ext4", "-F", str(vol), _REGISTRY_SIZE], + capture_output=True, text=True, check=False, + ) + if proc.returncode != 0: + vol.unlink(missing_ok=True) + die(f"creating registry volume failed: {proc.stderr.strip()}") + return vol + + +__all__ = ["FirecrackerOrchestrator", "ORCHESTRATOR_NAME", "registry_volume_path"] diff --git a/tests/unit/test_firecracker_gateway.py b/tests/unit/test_firecracker_gateway.py index 1a52ad0d..2c1b1bfb 100644 --- a/tests/unit/test_firecracker_gateway.py +++ b/tests/unit/test_firecracker_gateway.py @@ -1,7 +1,7 @@ """Unit: the Firecracker gateway data plane as a microVM (PRD 0070). The gateway service owns its host-side logic directly: booting the gateway VM -(via the shared `infra_vm._boot_vm` substrate), seeding the pre-minted token, +(via the shared `infra_vm.boot_vm` substrate), seeding the pre-minted token, fetching the CA over SSH, and the SSH exec/cp provisioning transport. """ @@ -37,21 +37,21 @@ class TestFirecrackerGatewayConnect(unittest.TestCase): def test_refuses_without_orchestrator_url(self) -> None: gw = FirecrackerGateway() - with patch.object(infra_vm, "_boot_vm") as boot: + with patch.object(infra_vm, "boot_vm") as boot: with self.assertRaises(GatewayError): gw.connect_to_orchestrator("", _TOKEN) boot.assert_not_called() def test_refuses_without_gateway_token(self) -> None: gw = FirecrackerGateway() - with patch.object(infra_vm, "_boot_vm") as boot: + with patch.object(infra_vm, "boot_vm") as boot: with self.assertRaises(GatewayError): gw.connect_to_orchestrator(_ORCH_URL, "") boot.assert_not_called() def test_refuses_a_url_without_a_host(self) -> None: gw = FirecrackerGateway() - with patch.object(infra_vm, "_boot_vm") as boot: + with patch.object(infra_vm, "boot_vm") as boot: with self.assertRaises(GatewayError): gw.connect_to_orchestrator("http://:8099", _TOKEN) boot.assert_not_called() @@ -59,8 +59,8 @@ class TestFirecrackerGatewayConnect(unittest.TestCase): def test_boots_the_gateway_vm_and_seeds_the_token(self) -> None: gw = FirecrackerGateway() booted = infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k")) - with patch.object(infra_vm, "_boot_vm", return_value=booted) as boot, \ - patch.object(infra_vm, "_push_secret") as push: + with patch.object(infra_vm, "boot_vm", return_value=booted) as boot, \ + patch.object(infra_vm, "push_secret") as push: gw.connect_to_orchestrator(_ORCH_URL, _TOKEN) # Booted on the gateway link with the gateway role; the orchestrator's # guest IP (parsed off the URL) rides the cmdline as bb_orch. diff --git a/tests/unit/test_firecracker_infra_vm.py b/tests/unit/test_firecracker_infra_vm.py index b9296261..ba918070 100644 --- a/tests/unit/test_firecracker_infra_vm.py +++ b/tests/unit/test_firecracker_infra_vm.py @@ -15,65 +15,51 @@ from unittest.mock import MagicMock, patch from bot_bottle.backend.firecracker import infra_vm -class TestPushSecrets(unittest.TestCase): - """`_push_signing_key` (orchestrator) seeds the guest's secret over SSH via - `_push_secret`. The host token file stays the single source of truth, never - clobbered per-backend (#469). The gateway's own JWT push lives with the - gateway service (test_firecracker_gateway).""" +class TestPushSecret(unittest.TestCase): + """`push_secret` is the shared substrate both services seed a guest secret + over SSH with (atomic write, retry while SSH comes up). The per-plane pushes + live with each service (orchestrator: signing key; gateway: JWT).""" def _infra(self) -> infra_vm.InfraVm: return infra_vm.InfraVm(guest_ip="10.0.0.1", private_key=Path("/k")) - def test_signing_key_written_atomically(self): + def test_written_atomically(self): proc = MagicMock(returncode=0, stderr="") - with patch.object(infra_vm, "host_orchestrator_token", return_value="host-key"), \ - patch.object(infra_vm.subprocess, "run", return_value=proc) as run: - infra_vm._push_signing_key(self._infra()) - # Piped the key to an atomic write of the guest token path. - self.assertEqual("host-key", run.call_args.kwargs["input"]) + with patch.object(infra_vm.subprocess, "run", return_value=proc) as run: + infra_vm.push_secret(self._infra(), "secret", "/dest", "the thing") + self.assertEqual("secret", run.call_args.kwargs["input"]) remote_cmd = run.call_args.args[0][-1] - self.assertIn(f"cat > {infra_vm._GUEST_SIGNING_KEY_PATH}.tmp", remote_cmd) - self.assertIn(f"mv {infra_vm._GUEST_SIGNING_KEY_PATH}.tmp " - f"{infra_vm._GUEST_SIGNING_KEY_PATH}", remote_cmd) + self.assertIn("cat > /dest.tmp", remote_cmd) + self.assertIn("mv /dest.tmp /dest", remote_cmd) def test_dies_when_push_never_succeeds(self): proc = MagicMock(returncode=255, stderr="ssh: connect refused") - with patch.object(infra_vm, "host_orchestrator_token", return_value="host-key"), \ - patch.object(infra_vm, "_SECRET_PUSH_TIMEOUT_SECONDS", 0.05), \ + with patch.object(infra_vm, "_SECRET_PUSH_TIMEOUT_SECONDS", 0.05), \ patch.object(infra_vm, "_SECRET_PUSH_POLL_SECONDS", 0.0), \ patch.object(infra_vm.subprocess, "run", return_value=proc), \ patch.object(infra_vm, "die", side_effect=SystemExit) as die: with self.assertRaises(SystemExit): - infra_vm._push_signing_key(self._infra()) + infra_vm.push_secret(self._infra(), "secret", "/dest", "the thing") die.assert_called_once() -class TestOrchestratorUrl(unittest.TestCase): - def test_url_uses_guest_ip_and_port(self): - infra = infra_vm.InfraVm( - vm=MagicMock(), guest_ip="10.243.255.1", private_key=Path("/k")) - self.assertEqual( - f"http://10.243.255.1:{infra_vm.ORCHESTRATOR_PORT}", - infra.orchestrator_url, - ) - - class TestInfraEndpoint(unittest.TestCase): - """The endpoint mirrors docker/macOS: it exposes the orchestrator's URL and - delegates the CA fetch to the gateway service.""" + """The endpoint mirrors docker/macOS: it exposes the orchestrator service's + URL and delegates the CA fetch to the gateway service.""" def _endpoint(self) -> infra_vm.InfraEndpoint: from bot_bottle.backend.firecracker.gateway import FirecrackerGateway + from bot_bottle.backend.firecracker.orchestrator import FirecrackerOrchestrator return infra_vm.InfraEndpoint( - orchestrator=infra_vm.InfraVm(guest_ip="10.243.255.1", private_key=Path("/k")), + orchestrator=FirecrackerOrchestrator(), gateway=FirecrackerGateway( infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k"))), ) - def test_orchestrator_url_is_the_orchestrator_vm(self): + def test_orchestrator_url_is_the_orchestrator_service(self): ep = self._endpoint() - self.assertEqual( - f"http://10.243.255.1:{infra_vm.ORCHESTRATOR_PORT}", ep.orchestrator_url) + ip = infra_vm.netpool.orch_slot().guest_ip + self.assertEqual(f"http://{ip}:{infra_vm.ORCHESTRATOR_PORT}", ep.orchestrator_url) def test_gateway_ca_pem_delegates_to_the_gateway_service(self): ep = self._endpoint() @@ -122,64 +108,6 @@ class TestBuildInfraRootfs(unittest.TestCase): self.assertIn("BOT_BOTTLE_ORCHESTRATOR_URL=http://$ORCH:", init) -class TestBootRole(unittest.TestCase): - """`boot_orchestrator` boots the shared rootfs with the `bb_role=orchestrator` - cmdline, the orchestrator memory ceiling + registry volume, and seeds the - signing key. (The gateway's boot lives in the gateway service — - test_firecracker_gateway.)""" - - def _boot_env(self): - vm = MagicMock() - vm.process.pid = 4242 - return vm - - def test_orchestrator_role_mem_and_registry(self): - import tempfile - vm = self._boot_env() - with tempfile.TemporaryDirectory() as td, \ - patch.object(infra_vm.netpool, "tap_present", return_value=True), \ - patch.object(infra_vm.infra_artifact, "local_build_requested", return_value=False), \ - patch.object(infra_vm.infra_artifact, "materialize_ext4"), \ - patch.object(infra_vm.infra_artifact, "infra_artifact_version", return_value="v"), \ - patch.object(infra_vm, "_stable_keypair", return_value=(Path("/k"), "pub")), \ - patch.object(infra_vm, "_ensure_registry_volume", return_value=Path("/reg")), \ - patch.object(infra_vm, "_push_signing_key") as push, \ - patch.object(infra_vm.firecracker_vm, "boot", return_value=vm) as boot, \ - patch.object(infra_vm, "_orch_dir", return_value=Path(td)): - infra_vm.boot_orchestrator() - kw = boot.call_args.kwargs - self.assertIn("bb_role=orchestrator", kw["extra_boot_args"]) - self.assertEqual(infra_vm._ORCH_MEM_MIB, kw["mem_mib"]) - self.assertEqual(Path("/reg"), kw["data_drive"]) # DB volume on the CP - push.assert_called_once() # signing key seeded - - -class TestRegistryVolume(unittest.TestCase): - def test_reuses_existing_volume(self): - import tempfile - with tempfile.TemporaryDirectory() as td: - vol = Path(td) / "registry.ext4" - vol.write_bytes(b"") # already present - with patch.object(infra_vm, "registry_volume_path", return_value=vol), \ - patch.object(infra_vm.subprocess, "run") as run: - out = infra_vm._ensure_registry_volume() - run.assert_not_called() # no mke2fs when it exists - self.assertEqual(vol, out) - - def test_creates_volume_when_missing(self): - import tempfile - from subprocess import CompletedProcess - with tempfile.TemporaryDirectory() as td: - vol = Path(td) / "registry.ext4" - with patch.object(infra_vm, "registry_volume_path", return_value=vol), \ - patch.object(infra_vm.subprocess, "run", - return_value=CompletedProcess([], 0)) as run: - infra_vm._ensure_registry_volume() - argv = run.call_args.args[0] - self.assertIn("mke2fs", argv) - self.assertIn(str(vol), argv) - - class TestEnsureBuilt(unittest.TestCase): def test_default_pulls_artifact_without_docker(self): # PRD 0069 Stage 2: the launch host pulls the prebuilt rootfs; no Docker. @@ -204,30 +132,10 @@ class TestEnsureBuilt(unittest.TestCase): self.assertIn(infra_vm._GATEWAY_IMAGE, tags[:-1]) -class TestWaitForHealth(unittest.TestCase): - def _infra(self, alive: bool = True) -> infra_vm.InfraVm: - vm = MagicMock() - vm.is_alive.return_value = alive - return infra_vm.InfraVm(vm=vm, guest_ip="10.0.0.1", private_key=Path("/k")) - - def test_returns_on_200(self): - infra = self._infra() - cm = MagicMock() - cm.__enter__.return_value.status = 200 - with patch.object(infra_vm.urllib.request, "urlopen", return_value=cm): - infra_vm.wait_for_health(infra, timeout=5) # must not raise - - def test_dies_when_vm_exits(self): - infra = self._infra(alive=False) - assert infra.vm is not None # narrow for the type checker (it's a mock) - infra.vm.process.returncode = 1 - with patch.object(infra_vm.firecracker_vm, "_console_tail", return_value=""), \ - self.assertRaises(SystemExit): - infra_vm.wait_for_health(infra, timeout=5) - - -# The gateway service is imported lazily inside ensure_running / _adopt (to -# break the infra_vm <-> gateway import cycle), so patch it at its home module. +# The orchestrator + gateway services are imported lazily inside ensure_running +# / _adopt (to break the infra_vm <-> service import cycle), so patch them at +# their home modules. +_ORCH_CLS = "bot_bottle.backend.firecracker.orchestrator.FirecrackerOrchestrator" _GW_CLS = "bot_bottle.backend.firecracker.gateway.FirecrackerGateway" @@ -236,6 +144,7 @@ class TestEnsureRunningSingleton(unittest.TestCase): # Healthy control plane + live gateway + existing key + matching version # marker -> adopt (no boot). from bot_bottle.backend.firecracker.gateway import FirecrackerGateway + from bot_bottle.backend.firecracker.orchestrator import FirecrackerOrchestrator import tempfile with tempfile.TemporaryDirectory() as td: d = Path(td) @@ -244,14 +153,13 @@ class TestEnsureRunningSingleton(unittest.TestCase): with patch.object(infra_vm, "_infra_dir", return_value=d), \ patch.object(infra_vm, "_expected_version", return_value="v-current"), \ patch.object(infra_vm, "_health_ok", return_value=True), \ - patch.object(infra_vm, "_pidfile_alive", return_value=True), \ - patch.object(infra_vm, "boot_orchestrator") as boot_o: + patch.object(infra_vm, "_pidfile_alive", return_value=True): ep = infra_vm.ensure_running() - boot_o.assert_not_called() - self.assertIsNone(ep.orchestrator.vm) - # The orchestrator handle + the adopted gateway service address the two - # distinct infra links. - self.assertEqual(infra_vm.netpool.orch_slot().guest_ip, ep.orchestrator.guest_ip) + # Adopted service handles addressing the two distinct infra links. + self.assertIsInstance(ep.orchestrator, FirecrackerOrchestrator) + self.assertEqual( + infra_vm.netpool.orch_slot().guest_ip, + ep.orchestrator_url.split("://")[1].split(":")[0]) self.assertIsInstance(ep.gateway, FirecrackerGateway) self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, ep.gateway.address()) @@ -269,18 +177,15 @@ class TestEnsureRunningSingleton(unittest.TestCase): patch.object(infra_vm, "_pidfile_alive", return_value=True), \ patch.object(infra_vm, "stop") as stop, \ patch.object(infra_vm, "ensure_built"), \ - patch.object(infra_vm, "wait_for_health"), \ - patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \ - patch.object(infra_vm, "mint", return_value="gw.jwt"), \ - patch.object(infra_vm, "boot_orchestrator") as boot_o, \ + patch(_ORCH_CLS) as orch_cls, \ patch(_GW_CLS) as gw_cls: - boot_o.return_value = infra_vm.InfraVm( - guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock()) + orch_cls.return_value.gateway_url.return_value = "http://10.243.255.1:8099" + orch_cls.return_value.mint_gateway_token.return_value = "gw.jwt" infra_vm.ensure_running() stop.assert_called_once() # dislodge the outdated pair - boot_o.assert_called_once() - # The gateway service is bound to the orchestrator's URL, carrying - # the host-minted `gateway` token. + orch_cls.return_value.ensure_running.assert_called_once() + # The gateway service is bound to the orchestrator's gateway URL, + # carrying the orchestrator-minted `gateway` token. gw_cls.return_value.connect_to_orchestrator.assert_called_once_with( "http://10.243.255.1:8099", "gw.jwt") # The fresh boot records the current version for the next launcher. @@ -300,16 +205,11 @@ class TestEnsureRunningSingleton(unittest.TestCase): patch.object(infra_vm, "_pidfile_alive", return_value=False), \ patch.object(infra_vm, "stop") as stop, \ patch.object(infra_vm, "ensure_built"), \ - patch.object(infra_vm, "wait_for_health"), \ - patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \ - patch.object(infra_vm, "mint", return_value="gw.jwt"), \ - patch.object(infra_vm, "boot_orchestrator") as boot_o, \ + patch(_ORCH_CLS) as orch_cls, \ patch(_GW_CLS) as gw_cls: - boot_o.return_value = infra_vm.InfraVm( - guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock()) infra_vm.ensure_running() stop.assert_called_once() - boot_o.assert_called_once() + orch_cls.return_value.ensure_running.assert_called_once() gw_cls.return_value.connect_to_orchestrator.assert_called_once() def test_boots_both_when_unhealthy(self): @@ -320,20 +220,14 @@ class TestEnsureRunningSingleton(unittest.TestCase): patch.object(infra_vm, "_health_ok", return_value=False), \ patch.object(infra_vm, "stop") as stop, \ patch.object(infra_vm, "ensure_built") as built, \ - patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \ - patch.object(infra_vm, "mint", return_value="gw.jwt"), \ - patch.object(infra_vm, "boot_orchestrator") as boot_o, \ - patch(_GW_CLS) as gw_cls, \ - patch.object(infra_vm, "wait_for_health") as wait: - boot_o.return_value = infra_vm.InfraVm( - guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock()) + patch(_ORCH_CLS) as orch_cls, \ + patch(_GW_CLS) as gw_cls: infra_vm.ensure_running() stop.assert_called_once() # clear stale VMs first built.assert_called_once() - # Orchestrator boots + becomes healthy BEFORE the gateway is connected - # (its daemons reach the control plane at startup). - boot_o.assert_called_once() - wait.assert_called_once() + # Orchestrator is brought up BEFORE the gateway is connected (its daemons + # reach the control plane at startup). + orch_cls.return_value.ensure_running.assert_called_once() gw_cls.return_value.connect_to_orchestrator.assert_called_once() diff --git a/tests/unit/test_firecracker_orchestrator.py b/tests/unit/test_firecracker_orchestrator.py new file mode 100644 index 00000000..28fa2c43 --- /dev/null +++ b/tests/unit/test_firecracker_orchestrator.py @@ -0,0 +1,146 @@ +"""Unit: the Firecracker orchestrator (control plane) microVM lifecycle (PRD 0070). + +The control plane's host-side logic — boot on the orchestrator link with the +registry volume, seed the signing key, wait for /health — lives in +`FirecrackerOrchestrator`, composing the shared `infra_vm` VM substrate. +""" + +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +from bot_bottle.backend.firecracker import infra_vm +from bot_bottle.backend.firecracker.orchestrator import ( + ORCHESTRATOR_NAME, + FirecrackerOrchestrator, + registry_volume_path, +) +from bot_bottle.backend.firecracker.infra_vm import ORCHESTRATOR_PORT + +_ORCH = "bot_bottle.backend.firecracker.orchestrator" + + +class TestUrls(unittest.TestCase): + def test_url_and_gateway_url_are_the_orch_link_guest_ip(self) -> None: + orch = FirecrackerOrchestrator() + ip = infra_vm.netpool.orch_slot().guest_ip + self.assertEqual(f"http://{ip}:{ORCHESTRATOR_PORT}", orch.url()) + self.assertEqual(orch.url(), orch.gateway_url()) + + def test_ssh_target_is_stable_key_and_orch_guest_ip(self) -> None: + orch = FirecrackerOrchestrator() + with patch.object(infra_vm, "_infra_dir", return_value=Path("/infra")): + key, ip = orch.ssh_target() + self.assertEqual(Path("/infra/id_ed25519"), key) + self.assertEqual(infra_vm.netpool.orch_slot().guest_ip, ip) + + +class TestEnsureRunning(unittest.TestCase): + def test_boots_orchestrator_role_with_mem_and_registry_then_pushes_key(self) -> None: + orch = FirecrackerOrchestrator() + vm = infra_vm.InfraVm(guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock()) + with patch.object(infra_vm, "boot_vm", return_value=vm) as boot, \ + patch.object(infra_vm, "push_secret") as push, \ + patch(f"{_ORCH}.host_orchestrator_token", return_value="host-key"), \ + patch.object(orch, "is_running", return_value=False), \ + patch.object(orch, "_ensure_registry_volume", return_value=Path("/reg")), \ + patch.object(orch, "_wait_for_health"): + orch.ensure_running() + kw = boot.call_args.kwargs + self.assertEqual("orchestrator", kw["role"]) + self.assertEqual(4096, kw["mem_mib"]) # orchestrator keeps build headroom + self.assertEqual(Path("/reg"), kw["data_drive"]) # DB volume on the CP + # The host-canonical signing key is pushed to the guest signing-key path. + self.assertEqual("host-key", push.call_args.args[1]) + self.assertEqual(infra_vm._GUEST_SIGNING_KEY_PATH, push.call_args.args[2]) + + def test_noop_when_running_and_healthy(self) -> None: + orch = FirecrackerOrchestrator() + with patch.object(orch, "is_running", return_value=True), \ + patch.object(orch, "is_healthy", return_value=True), \ + patch.object(infra_vm, "boot_vm") as boot: + orch.ensure_running() + boot.assert_not_called() + + +class TestWaitForHealth(unittest.TestCase): + def _vm(self, alive: bool = True) -> infra_vm.InfraVm: + h = MagicMock() + h.is_alive.return_value = alive + return infra_vm.InfraVm(vm=h, guest_ip="10.0.0.1", private_key=Path("/k")) + + def test_returns_when_healthy(self) -> None: + orch = FirecrackerOrchestrator() + with patch.object(orch, "is_healthy", return_value=True): + orch._wait_for_health(self._vm(), timeout=5) # must not raise + + def test_dies_when_vm_exits(self) -> None: + orch = FirecrackerOrchestrator() + vm = self._vm(alive=False) + assert vm.vm is not None + vm.vm.process.returncode = 1 + with patch(f"{_ORCH}.firecracker_vm._console_tail", return_value=""), \ + patch(f"{_ORCH}.die", side_effect=SystemExit) as die: + with self.assertRaises(SystemExit): + orch._wait_for_health(vm, timeout=5) + die.assert_called_once() + + +class TestRegistryVolume(unittest.TestCase): + def test_reuses_existing_volume(self) -> None: + orch = FirecrackerOrchestrator() + with tempfile.TemporaryDirectory() as td: + vol = Path(td) / "registry.ext4" + vol.write_bytes(b"") # already present + with patch(f"{_ORCH}.registry_volume_path", return_value=vol), \ + patch(f"{_ORCH}.subprocess.run") as run: + out = orch._ensure_registry_volume() + run.assert_not_called() # no mke2fs when it exists + self.assertEqual(vol, out) + + def test_creates_volume_when_missing(self) -> None: + from subprocess import CompletedProcess + orch = FirecrackerOrchestrator() + with tempfile.TemporaryDirectory() as td: + vol = Path(td) / "registry.ext4" + with patch(f"{_ORCH}.registry_volume_path", return_value=vol), \ + patch(f"{_ORCH}.subprocess.run", + return_value=CompletedProcess([], 0)) as run: + orch._ensure_registry_volume() + argv = run.call_args.args[0] + self.assertIn("mke2fs", argv) + self.assertIn(str(vol), argv) + + def test_registry_volume_path_under_infra_dir(self) -> None: + with patch.object(infra_vm, "_infra_dir", return_value=Path("/infra")): + self.assertEqual(Path("/infra/registry.ext4"), registry_volume_path()) + + +class TestLifecycle(unittest.TestCase): + def test_is_running_reads_the_orch_pidfile(self) -> None: + orch = FirecrackerOrchestrator() + with patch.object(infra_vm, "_pidfile_alive", return_value=True) as alive, \ + patch.object(infra_vm, "_orch_dir", return_value=Path("/orch")): + self.assertTrue(orch.is_running()) + alive.assert_called_once_with(Path("/orch")) + + def test_stop_kills_only_the_orch_pidfile(self) -> None: + orch = FirecrackerOrchestrator() + with tempfile.TemporaryDirectory() as td: + d = Path(td) + (d / "vm.pid").write_text("123") + with patch.object(infra_vm, "_orch_dir", return_value=d), \ + patch.object(infra_vm, "_kill_pidfile") as kill: + orch.stop() + kill.assert_called_once_with(d) + self.assertFalse((d / "vm.pid").exists()) + + def test_name_is_the_orchestrator_vm(self) -> None: + self.assertEqual(ORCHESTRATOR_NAME, FirecrackerOrchestrator().name) + + +if __name__ == "__main__": + unittest.main()