diff --git a/bot_bottle/backend/firecracker/consolidated_launch.py b/bot_bottle/backend/firecracker/consolidated_launch.py index 24e8232..d8468f3 100644 --- a/bot_bottle/backend/firecracker/consolidated_launch.py +++ b/bot_bottle/backend/firecracker/consolidated_launch.py @@ -1,22 +1,23 @@ """Consolidated bottle launch sequence for the Firecracker backend (PRD 0070, Stage B). -The shared gateway + orchestrator control plane run in a single persistent -per-host **infra VM** (`infra_vm.py`), not Docker containers. Agent VMs reach -the gateway's egress / supervise / git-http ports at the infra VM via a -PREROUTING DNAT on their own host-side TAP IP (see -`scripts/firecracker-netpool.sh`), and the host CLI reaches the control plane -over HTTP at the infra VM's guest IP. +The orchestrator control plane and the shared gateway run in **two** persistent +per-host **infra VMs** (`infra_vm.py`), split per PRD 0070 — not Docker +containers. Agent VMs reach the gateway's egress / supervise / git-http ports +via a PREROUTING DNAT on their own host-side TAP IP that redirects to the +**gateway** VM (see `scripts/firecracker-netpool.sh`) — never the orchestrator, +so a breached agent has no L3 route to the control plane. The host CLI reaches +the control plane over HTTP at the orchestrator VM's guest IP. Attribution is by the agent VM's guest IP, unspoofable by construction: the /31 point-to-point TAP + the `bot_bottle_fc` nft table ensure only the expected VM can source-IP that address. Sequence: - 1. ensure the infra VM (control plane + gateway) is up (a singleton — a - prior launcher may already have booted it); - 2. register the bottle by its guest IP (attribution key) → bottle id + - identity token; + 1. ensure the infra VMs (orchestrator control plane + gateway data plane) + are up (a singleton pair — a prior launcher may already have booted them); + 2. register the bottle on the orchestrator by its guest IP (attribution key) + → bottle id + identity token; 3. provision its git-gate repos/creds into the gateway VM (over SSH); 4. fetch the shared gateway CA for the provisioner to install in the rootfs. @@ -150,7 +151,7 @@ def teardown_consolidated( ) -> None: """Deregister the bottle and remove its git-gate state from the gateway VM. Both steps are idempotent so this is safe from a cleanup trap. Does - NOT stop the infra VM — it's a persistent per-host singleton shared by + NOT stop the infra VMs — they're persistent per-host singletons shared by every bottle.""" _teardown_util(bottle_id, infra_vm.gateway_transport(), orchestrator_url=orchestrator_url, timeout=timeout) diff --git a/bot_bottle/backend/firecracker/firecracker_vm.py b/bot_bottle/backend/firecracker/firecracker_vm.py index 5c372b7..129a96c 100644 --- a/bot_bottle/backend/firecracker/firecracker_vm.py +++ b/bot_bottle/backend/firecracker/firecracker_vm.py @@ -60,12 +60,18 @@ class VmHandle: self.process.wait(timeout=5) -def _boot_args(guest_ip: str, host_ip: str, pubkey: str) -> str: +def _boot_args( + guest_ip: str, host_ip: str, pubkey: str, extra: str = "", +) -> str: # ip=::::::off — /31 point-to-point link, # so netmask is 255.255.255.254 and the gateway is the host TAP IP. ip_arg = f"ip={guest_ip}::{host_ip}:255.255.255.254::eth0:off" pub_b64 = base64.b64encode(pubkey.encode()).decode() - return f"{_BASE_BOOT_ARGS} {ip_arg} bb_pubkey={pub_b64}" + args = f"{_BASE_BOOT_ARGS} {ip_arg} bb_pubkey={pub_b64}" + # `extra` carries caller-supplied cmdline params the guest init reads + # (e.g. `bb_role=orchestrator|gateway` selecting which infra plane a + # shared-rootfs infra VM runs). Agent VMs pass nothing. + return f"{args} {extra}".rstrip() if extra else args def _config( @@ -79,6 +85,7 @@ def _config( mem_mib: int, guest_mac: str, data_drive: Path | None = None, + extra_boot_args: str = "", ) -> dict[str, object]: drives: list[dict[str, object]] = [ { @@ -101,7 +108,7 @@ def _config( return { "boot-source": { "kernel_image_path": str(util.kernel_path()), - "boot_args": _boot_args(guest_ip, host_ip, pubkey), + "boot_args": _boot_args(guest_ip, host_ip, pubkey, extra_boot_args), }, "drives": drives, "network-interfaces": [ @@ -132,6 +139,7 @@ def boot( guest_mac: str = "06:00:AC:10:00:02", detached: bool = False, data_drive: Path | None = None, + extra_boot_args: str = "", ) -> VmHandle: """Write the config and launch the VMM. Returns once the process is spawned; callers wait for SSH readiness separately. @@ -147,7 +155,7 @@ def boot( _config( rootfs=rootfs, tap=tap, guest_ip=guest_ip, host_ip=host_ip, pubkey=pubkey, vcpus=vcpus, mem_mib=mem_mib, guest_mac=guest_mac, - data_drive=data_drive, + data_drive=data_drive, extra_boot_args=extra_boot_args, ), indent=2, )) diff --git a/bot_bottle/backend/firecracker/image_builder.py b/bot_bottle/backend/firecracker/image_builder.py index d54a8f8..55b4e12 100644 --- a/bot_bottle/backend/firecracker/image_builder.py +++ b/bot_bottle/backend/firecracker/image_builder.py @@ -1,17 +1,17 @@ """Docker-free agent-image builds for the Firecracker backend (PRD 0069 Stage 3). -Agent Dockerfiles build **inside the persistent per-host infra VM** +Agent Dockerfiles build **inside the persistent per-host orchestrator VM** (`infra_vm.py`), which carries buildah (rootless, daemonless): no host Docker daemon, no root-equivalent `docker` group. The build runs over SSH against the -infra VM and its rootfs streams back to the host, where the existing +orchestrator VM and its rootfs streams back to the host, where the existing `mke2fs -d` path (`util.build_rootfs_ext4`) turns it into a bootable ext4. -Building in the infra VM — rather than a throwaway builder VM — means there is -one buildah image (`bot-bottle-infra`) and no contention for the orchestrator -TAP. Tradeoff: an untrusted Dockerfile's `RUN` steps share the VM with the -control plane + gateway (buildah `--isolation chroot` isn't a hard boundary) — -the accepted single-VM blast-radius tradeoff, re-splittable into a disposable -builder (booted from this same image on its own TAP) later. +Builds run on the control-plane (orchestrator) side — not the data-plane +gateway — per PRD 0070 v1 ("builds stay in the orchestrator"): it owns +launches and already carries buildah. Tradeoff: an untrusted Dockerfile's +`RUN` steps share the VM with the control plane (buildah `--isolation chroot` +isn't a hard boundary) — the accepted blast-radius tradeoff, re-splittable +into a disposable builder (booted from this same image on its own TAP) later. """ from __future__ import annotations @@ -121,11 +121,12 @@ def _build_lock() -> Generator[None, None, None]: def _build_in_infra( dockerfile: Path, base: Path, smoke_test: tuple[str, ...], digest: str, ) -> None: - """Ensure the infra VM is up, `buildah build` the Dockerfile in it, smoke - test the image, and stream its rootfs into `base`. The infra VM persists; - only the per-build container/image/context are cleaned up.""" - infra = infra_vm.ensure_running() - key, ip = infra.private_key, infra.guest_ip + """Ensure the infra VMs are up, `buildah build` the Dockerfile in the + orchestrator VM (the build-capable control plane), smoke test the image, + and stream its rootfs into `base`. The infra VMs persist; only the + per-build container/image/context are cleaned up.""" + orchestrator = infra_vm.ensure_running().orchestrator + key, ip = orchestrator.private_key, orchestrator.guest_ip tag = f"bot-bottle-agent-build-{digest}" ctx = f"/tmp/agent-build-{digest}" smoke_ctr, export_ctr = f"{tag}-smoke", f"{tag}-export" diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py index b63f9a5..5eeba34 100644 --- a/bot_bottle/backend/firecracker/infra_vm.py +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -1,18 +1,30 @@ -"""The per-host infra VM for the Firecracker backend (PRD 0070 Stage B). +"""The per-host infra VMs for the Firecracker backend (PRD 0070). -A single persistent microVM that runs the orchestrator **control plane** (and, -in a following step, the gateway **data plane**) — the trusted per-host service -the docker backend runs as containers. It boots on the NAT'd orchestrator link -(`netpool.orch_slot()`): the host CLI reaches its control plane over HTTP at the -guest IP, and agent VMs reach its gateway ports over VM-to-VM routing. +Two persistent microVMs, split now that #469 got the DB off the data plane +(PRD 0070 "Separating the planes"): -Build-from-source (the default while the design churns): the rootfs is exported -from the locally built orchestrator image, which bakes the stdlib-only -control-plane source. A pull-from-registry mode (Gitea's OCI registry) becomes -the default later. + * **orchestrator VM** — the control plane. Boots on the NAT'd orchestrator + link (`netpool.orch_slot()`); the host CLI reaches its `/health` + + operator routes over HTTP at the guest IP. Sole opener of `bot-bottle.db` + (on its persistent /dev/vdb registry volume); holds the host-canonical + signing key (pushed post-boot). Also carries buildah, so in-VM agent-image + builds run here (PRD 0070 v1: builds stay with the control plane). + * **gateway VM** — the data plane. Boots on its own NAT'd link + (`netpool.gw_slot()`); runs the egress / git-http / supervise daemons that + agent VMs reach (their gateway-port traffic is DNAT'd here — never to the + orchestrator, so a breached agent has no L3 route to the control plane). + Holds the mitmproxy CA + a pre-minted `gateway` JWT (never the signing + key); reaches the orchestrator's control plane at `orch_guest:8099` over + the one nft forward rule that link allows. -SSH is left enabled for debugging; the control plane is the load-bearing -surface. +Both VMs boot the **same** shared infra rootfs (gateway + orchestrator + +buildah); a `bb_role=` kernel-cmdline arg selects which plane a VM's PID-1 +init starts, so there is still a single published artifact to build/pull. The +gateway VM's slimmer memory ceiling (buildah is present on disk but unused +there) is set at boot. + +SSH is left enabled for debugging + provisioning; the control plane is the +load-bearing surface. """ from __future__ import annotations @@ -33,22 +45,27 @@ from pathlib import Path from typing import Generator from ...log import die, info +from ...orchestrator_auth import ROLE_GATEWAY, mint from ...paths import host_orchestrator_token from .. import util as backend_util from ..docker import util as docker_mod from ..docker.gateway_provision import GatewayProvisionError from . import firecracker_vm, infra_artifact, netpool, util -# The infra VM's control-plane signing-key path on its persistent /dev/vdb -# volume (mounted at BOT_BOTTLE_ROOT=/var/lib/bot-bottle). The launcher seeds -# this file with the host-canonical key BEFORE boot, so the VM verifies tokens -# with the same key the host CLI signs with — the host token file stays the -# single source of truth, never clobbered per-backend (issue #469 review). +# The orchestrator VM's signing-key path on its persistent /dev/vdb volume +# (mounted at BOT_BOTTLE_ROOT=/var/lib/bot-bottle). The launcher seeds this +# file with the host-canonical key AFTER boot (over SSH), so the VM verifies +# tokens with the same key the host CLI signs with — the host token file stays +# the single source of truth, never clobbered per-backend (issue #469 review). _GUEST_SIGNING_KEY_PATH = "/var/lib/bot-bottle/orchestrator-token" +# The gateway VM's pre-minted `gateway` JWT path (rootfs, not /dev/vdb — the +# data plane has no registry volume and never opens the DB). Pushed post-boot; +# the gateway daemons present it to the orchestrator, and never see the key. +_GUEST_GATEWAY_JWT_PATH = "/var/lib/bot-bottle/gateway-jwt" -# The single infra-VM image: gateway data plane + baked control-plane source -# (Dockerfile.infra FROM the gateway image). Built from source by default; -# a pull-from-registry mode lands later. +# The single shared infra image: gateway data plane + baked control-plane +# source + buildah (Dockerfile.infra.fc). Built from source by default; a +# pull-from-registry mode lands later. Both VMs boot from it. _INFRA_IMAGE = "bot-bottle-infra:latest" _GATEWAY_IMAGE = "bot-bottle-gateway:latest" _ORCHESTRATOR_IMAGE = "bot-bottle-orchestrator:latest" @@ -56,7 +73,7 @@ _REPO_ROOT = Path(__file__).resolve().parents[3] ORCHESTRATOR_PORT = 8099 # Gateway data-plane ports (agent-facing): egress proxy, supervise MCP, -# git-http. Reached by agent VMs over VM-to-VM routing (added next). +# git-http. Reached by agent VMs via the PREROUTING DNAT to the gateway VM. EGRESS_PORT = 9099 SUPERVISE_PORT = 9100 GIT_HTTP_PORT = 9420 @@ -64,7 +81,14 @@ GIT_HTTP_PORT = 9420 # the gateway's TLS interception. _GATEWAY_CA_PATH = "/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem" -# The infra VM makes direct upstream connections (gateway egress, and buildah +# Memory ceilings (fixed at boot, demand-paged — no balloon reclaim). The +# orchestrator keeps the build headroom (buildah's 2-4 GB working set during +# in-VM agent builds); the gateway is the slim unit (mitmproxy TLS bump + DLP +# body buffering, ~1 GB) — PRD 0070 "Memory: fixed ceilings". +_ORCH_MEM_MIB = 4096 +_GW_MEM_MIB = 2048 + +# The infra VMs make direct upstream connections (gateway egress, and buildah # during builds), and the kernel `ip=` cmdline sets no resolver. Public for # now; routing DNS through a filtered path is a later refinement. _INFRA_RESOLVER = "1.1.1.1" @@ -72,16 +96,16 @@ _INFRA_RESOLVER = "1.1.1.1" _HEALTH_TIMEOUT_SECONDS = 45.0 _HEALTH_POLL_SECONDS = 0.5 _CA_TIMEOUT_SECONDS = 30.0 -# How long the launcher retries pushing the signing key while the guest's SSH -# comes up. Below the init's own wait window, so a failed push dies here first. -_SIGNING_KEY_PUSH_TIMEOUT_SECONDS = 30.0 -_SIGNING_KEY_PUSH_POLL_SECONDS = 0.5 +# How long the launcher retries pushing a secret while the guest's SSH comes +# up. Below the init's own wait window, so a failed push dies here first. +_SECRET_PUSH_TIMEOUT_SECONDS = 30.0 +_SECRET_PUSH_POLL_SECONDS = 0.5 @dataclass class InfraVm: - """A handle to the per-host infra VM: its guest IP and the stable SSH key - used to fetch the gateway CA / provision git-gate. `vm` is the live VMM + """A handle to one infra VM: its guest IP and the stable SSH key used to + seed secrets / fetch the CA / provision git-gate. `vm` is the live VMM handle when this process booted it, and None when adopting a singleton a prior launcher started (teardown then goes through the PID file).""" @@ -93,19 +117,11 @@ class InfraVm: def orchestrator_url(self) -> str: return f"http://{self.guest_ip}:{ORCHESTRATOR_PORT}" - def terminate(self) -> None: - """Stop the infra VM — via the live handle if we booted it, else the - PID file (adopting-process case).""" - if self.vm is not None: - self.vm.terminate() - else: - _kill_pidfile() - _pid_file().unlink(missing_ok=True) - def gateway_ca_pem(self, *, timeout: float = _CA_TIMEOUT_SECONDS) -> str: """The gateway's mitmproxy CA (PEM) that agents install to trust its TLS interception. Generated a moment after boot, so this polls over - SSH until it appears (mirrors DockerGateway.ca_cert_pem).""" + SSH until it appears (mirrors DockerGateway.ca_cert_pem). Call on the + gateway VM handle.""" def _fetch() -> str | None: proc = subprocess.run( util.ssh_base_argv(self.private_key, self.guest_ip) @@ -120,13 +136,31 @@ class InfraVm: die(str(exc)) +@dataclass +class InfraEndpoint: + """How to reach the running per-host pair. `orchestrator` is the control + plane (host CLI + registration + in-VM builds); `gateway` is the data + plane (agent egress / git-http / supervise, CA fetch, git-gate + provisioning). Mirrors the docker/macOS `InfraEndpoint` shape.""" + + orchestrator: InfraVm + gateway: InfraVm + + @property + def orchestrator_url(self) -> str: + return self.orchestrator.orchestrator_url + + def gateway_ca_pem(self, *, timeout: float = _CA_TIMEOUT_SECONDS) -> str: + return self.gateway.gateway_ca_pem(timeout=timeout) + + def ensure_built() -> None: """Ensure the infra rootfs is available before boot. Default (docker-free, PRD 0069 Stage 2): download + verify the prebuilt rootfs artifact matching this code version (see `infra_artifact`); the launch host needs no Docker. `BOT_BOTTLE_INFRA_BUILD=local` instead builds - the three fixed images from source with host Docker — the infra image + the fixed images from source with host Docker — the infra image `COPY --from`s the orchestrator image and is `FROM` the gateway image, so both must exist first — for iterating on the Dockerfiles.""" if infra_artifact.local_build_requested(): @@ -153,10 +187,10 @@ def build_infra_images_with_docker() -> None: def build_infra_rootfs_dir() -> Path: - """The infra VM's base rootfs: the infra image prepared with the - control-plane + gateway init as PID 1. The init's content is folded into - the cache key so an init change rebuilds the rootfs (the base image digest - alone wouldn't catch it).""" + """The infra VMs' shared base rootfs: the infra image prepared with the + role-branched init as PID 1. The init's content is folded into the cache + key so an init change rebuilds the rootfs (the base image digest alone + wouldn't catch it).""" init = _infra_init() tag = hashlib.sha256(init.encode()).hexdigest()[:8] return util.build_base_rootfs_dir( @@ -164,41 +198,55 @@ def build_infra_rootfs_dir() -> Path: ) -def ensure_running() -> InfraVm: - """Idempotent per-host singleton. Adopt the infra VM if its control plane - is already healthy (a prior launcher booted it — it outlives short-lived - `start` processes); otherwise clear any stale VM and boot a fresh one. - Returns a handle usable for CA fetch / git-gate provisioning. +def ensure_running() -> InfraEndpoint: + """Idempotent per-host singleton pair. Adopt the running VMs if the + orchestrator's control plane is healthy AND the gateway VM is alive AND + both booted the CURRENT code version (a prior launcher booted them — they + outlive short-lived `start` processes); otherwise clear any stale VMs and + boot a fresh pair (orchestrator first, then the gateway that resolves + policy against it). Returns handles usable for builds / CA fetch / git-gate + provisioning. Concurrency-safe: the cold stop/build/boot path is serialized by a host flock, so two simultaneous first launches don't both boot on the same - rootfs/PID. The healthy fast-path takes no lock.""" - slot = netpool.orch_slot() - url = f"http://{slot.guest_ip}:{ORCHESTRATOR_PORT}" + links/PIDs. The healthy fast-path takes no lock.""" key = _infra_dir() / "id_ed25519" + url = f"http://{netpool.orch_slot().guest_ip}:{ORCHESTRATOR_PORT}" want = _expected_version() if _adoptable(key, url, want): - info(f"adopting running infra VM at {url}") - return InfraVm(guest_ip=slot.guest_ip, private_key=key) + info(f"adopting running infra VMs (orchestrator at {url})") + return _adopt(key) with _singleton_lock(): - # Re-check under the lock: another launcher may have booted it while + # Re-check under the lock: another launcher may have booted them while # we waited for the lock (double-checked, so we adopt not re-boot). if _adoptable(key, url, want): - info(f"adopting running infra VM at {url}") - return InfraVm(guest_ip=slot.guest_ip, private_key=key) - # Clear a stale/hung/OUTDATED VM holding the link before booting fresh. + info(f"adopting running infra VMs (orchestrator at {url})") + return _adopt(key) + # Clear stale/hung/OUTDATED VMs holding either link before booting fresh. stop() ensure_built() - infra = boot() - wait_for_health(infra) + orchestrator = boot_orchestrator() + wait_for_health(orchestrator) + gateway = boot_gateway(orchestrator.guest_ip) _record_booted_version(want) - return infra + return InfraEndpoint(orchestrator=orchestrator, gateway=gateway) + + +def _adopt(key: Path) -> InfraEndpoint: + """Build handles to the already-running pair from the stable key + the + fixed links (vm=None — teardown then goes through the PID files).""" + return InfraEndpoint( + orchestrator=InfraVm( + guest_ip=netpool.orch_slot().guest_ip, private_key=key), + gateway=InfraVm( + guest_ip=netpool.gw_slot().guest_ip, private_key=key), + ) @contextmanager def _singleton_lock() -> Generator[None, None, None]: - """Host-level exclusive lock serializing the infra VM's cold create path + """Host-level exclusive lock serializing the infra pair's cold create path (`stop`/`ensure_built`/`boot`). flock auto-releases if the launcher crashes, so the lock is never leaked.""" lock_path = _infra_dir() / "singleton.lock" @@ -211,26 +259,72 @@ def _singleton_lock() -> Generator[None, None, None]: def stop() -> None: - """Stop the infra VM singleton (idempotent — absent is success). Reaps the - recorded VMM AND any orphaned firecracker still bound to the infra config — - the PID file drifts after crashes / out-of-band kills, and a survivor would - hold the orchestrator TAP so the next boot dies with "tap … Resource busy". - Drops the version marker so a stopped VM is never treated as adoptable.""" - _kill_pidfile() + """Stop BOTH infra VMs (idempotent — absent is success). Reaps the + recorded VMMs AND any orphaned firecracker still bound to either infra + config — the PID files drift after crashes / out-of-band kills, and a + survivor would hold a link's TAP so the next boot dies with "tap … + Resource busy". Drops the version marker so a stopped pair is never + treated as adoptable.""" + _kill_pidfile(_orch_dir()) + _kill_pidfile(_gw_dir()) _kill_infra_firecrackers() - _pid_file().unlink(missing_ok=True) + _pid_file(_orch_dir()).unlink(missing_ok=True) + _pid_file(_gw_dir()).unlink(missing_ok=True) _version_file().unlink(missing_ok=True) -def boot() -> InfraVm: - """Boot the infra VM (detached, so it outlives the launcher) on the - orchestrator link, recording its PID. Prefer `ensure_running`.""" +def boot_orchestrator() -> InfraVm: + """Boot the orchestrator (control-plane) VM (detached, so it outlives the + launcher) on the orchestrator link, and seed its signing key. Prefer + `ensure_running`.""" slot = netpool.orch_slot() + orch = _boot_vm( + name="bot-bottle-orchestrator", slot=slot, run_dir=_orch_dir(), + role="orchestrator", mem_mib=_ORCH_MEM_MIB, + data_drive=_ensure_registry_volume(), + ) + # Push the host-canonical signing key over SSH (the init waits for it + # before starting the control plane). The host token file stays the single + # source of truth, so a co-running Docker/macOS control plane keeps working. + _push_signing_key(orch) + return orch + + +def boot_gateway(orchestrator_guest_ip: str) -> InfraVm: + """Boot the gateway (data-plane) VM (detached) on the gateway link, and + seed its pre-minted `gateway` JWT. Its daemons resolve policy against the + orchestrator at `orchestrator_guest_ip:8099` (passed on the cmdline, so the + baked init stays IP-independent). Boot the orchestrator first — the gateway + daemons reach it at startup. Prefer `ensure_running`.""" + slot = netpool.gw_slot() + gateway = _boot_vm( + name="bot-bottle-gateway", slot=slot, run_dir=_gw_dir(), + role="gateway", mem_mib=_GW_MEM_MIB, + extra_boot_args=f"bb_orch={orchestrator_guest_ip}", + ) + # Push the pre-minted `gateway` JWT (never the signing key — issue #469). + # The init waits for it before starting the data plane. + _push_gateway_jwt(gateway) + return gateway + + +def _boot_vm( + *, + name: str, + slot: netpool.Slot, + run_dir: Path, + role: str, + mem_mib: int, + data_drive: Path | None = None, + extra_boot_args: str = "", +) -> InfraVm: + """Boot one infra VM from the shared rootfs on `slot`'s link, tagged with + its `bb_role` so the guest init starts the right plane. Records the PID.""" if not netpool.tap_present(slot.iface): - die(f"orchestrator link {slot.iface} not present.\n" + die(f"infra link {slot.iface} not present.\n" f" ./cli.py backend setup --backend=firecracker") - run_dir = _infra_dir() + run_dir.mkdir(parents=True, exist_ok=True) rootfs = run_dir / "rootfs.ext4" if infra_artifact.local_build_requested(): util.build_rootfs_ext4(build_infra_rootfs_dir(), rootfs, slack_mib=8192) @@ -241,21 +335,18 @@ def boot() -> InfraVm: infra_artifact.infra_artifact_version(_infra_init()), rootfs) private_key, pubkey = _stable_keypair() - info(f"booting infra VM on {slot.iface} (guest {slot.guest_ip})") + info(f"booting {role} VM on {slot.iface} (guest {slot.guest_ip})") + boot_args = f"bb_role={role}" + if extra_boot_args: + boot_args = f"{boot_args} {extra_boot_args}" vm = firecracker_vm.boot( - name="bot-bottle-infra", rootfs=rootfs, tap=slot.iface, + name=name, rootfs=rootfs, tap=slot.iface, guest_ip=slot.guest_ip, host_ip=slot.host_ip, pubkey=pubkey, - run_dir=run_dir, mem_mib=4096, detached=True, - data_drive=_ensure_registry_volume(), + run_dir=run_dir, mem_mib=mem_mib, detached=True, + data_drive=data_drive, extra_boot_args=boot_args, ) - _pid_file().write_text(str(vm.process.pid)) - infra = InfraVm(guest_ip=slot.guest_ip, private_key=private_key, vm=vm) - # Push the host-canonical control-plane signing key into the guest over SSH - # (the init waits for it before starting the control plane). The host token - # file stays the single source of truth, so a co-running Docker/macOS control - # plane keeps working (issue #469 review). - _push_signing_key(infra) - return infra + _pid_file(run_dir).write_text(str(vm.process.pid)) + return InfraVm(guest_ip=slot.guest_ip, private_key=private_key, vm=vm) def _infra_dir() -> Path: @@ -264,16 +355,29 @@ def _infra_dir() -> Path: return d -def _pid_file() -> Path: - return _infra_dir() / "vm.pid" +def _orch_dir() -> Path: + d = _infra_dir() / "orchestrator" + d.mkdir(parents=True, exist_ok=True) + return d + + +def _gw_dir() -> Path: + d = _infra_dir() / "gateway" + d.mkdir(parents=True, exist_ok=True) + return d + + +def _pid_file(run_dir: Path) -> Path: + return run_dir / "vm.pid" def _version_file() -> Path: - """Records the infra-artifact version the *running* VM booted from, so a - later launcher can tell whether the singleton it found is the current code. - Without it, a healthy VM built from an older image gets adopted forever and - the new code never boots — every infra change would need an out-of-band - kill to dislodge the stale VM (and races whatever launched next).""" + """Records the infra-artifact version the *running* pair booted from, so a + later launcher can tell whether the singletons it found are the current + code. Without it, a healthy pair built from an older image gets adopted + forever and the new code never boots — every infra change would need an + out-of-band kill to dislodge the stale VMs (and races whatever launched + next). Both VMs boot the same artifact, so one marker covers the pair.""" return _infra_dir() / "booted-version" @@ -282,28 +386,35 @@ def _expected_version() -> str: def _adoptable(key: Path, url: str, want: str) -> bool: - """Adopt a running infra VM only if it booted from the CURRENT version and - its control plane is healthy. A missing/mismatched marker means a prior - launcher booted an older infra image — reboot rather than reuse stale code.""" + """Adopt the running pair only if it booted from the CURRENT version, the + orchestrator's control plane is healthy, and the gateway VM is still alive. + A missing/mismatched marker means a prior launcher booted an older infra + image; a dead gateway means the pair is half-down — reboot both rather than + reuse stale or partial state.""" if not key.exists(): return False try: booted = _version_file().read_text(encoding="utf-8").strip() except OSError: return False - return booted == want and _health_ok(url) + if booted != want: + return False + if not _health_ok(url): + return False + return _pidfile_alive(_gw_dir()) def _record_booted_version(version: str) -> None: _version_file().write_text(version + "\n", encoding="utf-8") -# The registry "volume": a host-side ext4 file attached to the infra VM as a -# second virtio-block device (guest /dev/vdb), mounted at the control plane's -# DB dir. It outlives the ephemeral rootfs, so the bottle registry survives an -# infra-VM restart — the firecracker analogue of a docker volume. It is a -# plain ext4 file: `sudo mount -o loop ` on the host (with the VM -# stopped) to inspect bot-bottle.db directly. +# The registry "volume": a host-side ext4 file attached to the orchestrator VM +# as a second virtio-block device (guest /dev/vdb), mounted at the control +# plane's DB dir. It outlives the ephemeral rootfs, so the bottle registry +# survives an orchestrator-VM restart — the firecracker analogue of a docker +# volume. It is a plain ext4 file: `sudo mount -o loop ` on the host +# (with the VM stopped) to inspect bot-bottle.db directly. The gateway VM has +# no such volume — the data plane never opens the DB (#469). _REGISTRY_SIZE = "512M" @@ -328,36 +439,55 @@ def _ensure_registry_volume() -> Path: def _push_signing_key(infra: InfraVm) -> None: - """Push the host-canonical control-plane signing key into the freshly booted - infra VM over SSH (atomic write), so its orchestrator verifies tokens with - the same key the host CLI signs from. Mirrors the existing - `persist_env_var_secret` transport. Retries until the guest's SSH is up (it - comes up before the init waits for this file); dies if it never lands, since - the VM then refuses to start its control plane rather than run OPEN.""" - key = host_orchestrator_token() - push = ( - f"umask 077; cat > {_GUEST_SIGNING_KEY_PATH}.tmp " - f"&& mv {_GUEST_SIGNING_KEY_PATH}.tmp {_GUEST_SIGNING_KEY_PATH}" + """Push the host-canonical signing key into the freshly booted orchestrator + VM over SSH (atomic write), so its control plane verifies tokens with the + same key the host CLI signs from. Retries until the guest's SSH is up (it + comes up before the init waits for this file); dies if it never lands, + since the VM then refuses to start its control plane rather than run OPEN.""" + _push_secret( + infra, host_orchestrator_token(), _GUEST_SIGNING_KEY_PATH, + "the control-plane signing key to the orchestrator VM " + "(its control plane will not start)", ) - deadline = time.monotonic() + _SIGNING_KEY_PUSH_TIMEOUT_SECONDS + + +def _push_gateway_jwt(infra: InfraVm) -> None: + """Push the pre-minted `gateway` JWT into the freshly booted gateway VM + over SSH (atomic write). Minted on the HOST from the canonical signing key + so the data plane never holds the key itself (issue #469); the gateway + daemons present it to the orchestrator. Retries until SSH is up; dies if it + never lands, since the VM then refuses to start the data plane.""" + _push_secret( + infra, mint(ROLE_GATEWAY, host_orchestrator_token()), + _GUEST_GATEWAY_JWT_PATH, + "the gateway JWT to the gateway VM (its data plane will not start)", + ) + + +def _push_secret(infra: InfraVm, secret: str, dest: str, what: str) -> None: + """Pipe `secret` to an atomic write of `dest` in the guest over SSH, + retrying while the guest's SSH comes up; die (naming `what`) if it never + succeeds. Bare-pipe input keeps the value off argv.""" + push = f"umask 077; cat > {dest}.tmp && mv {dest}.tmp {dest}" + deadline = time.monotonic() + _SECRET_PUSH_TIMEOUT_SECONDS last = "" while time.monotonic() < deadline: proc = subprocess.run( util.ssh_base_argv(infra.private_key, infra.guest_ip) + [push], - input=key, capture_output=True, text=True, check=False, + input=secret, capture_output=True, text=True, check=False, ) if proc.returncode == 0: return last = proc.stderr.strip() - time.sleep(_SIGNING_KEY_PUSH_POLL_SECONDS) - die("could not push the control-plane signing key to the infra VM " - f"(its control plane will not start): {last or ''}") + time.sleep(_SECRET_PUSH_POLL_SECONDS) + die(f"could not push {what}: {last or ''}") def _stable_keypair() -> tuple[Path, str]: - """The infra VM's SSH keypair — generated once and reused, so any later - launcher can SSH in (fetch CA / provision) even though a different process - booted the VM. The pubkey is re-injected on every boot via the cmdline.""" + """The infra VMs' shared SSH keypair — generated once and reused, so any + later launcher can SSH in (seed secrets / fetch CA / provision) even though + a different process booted the VMs. Both VMs get the same pubkey re-injected + on every boot via the cmdline.""" d = _infra_dir() key, pub = d / "id_ed25519", d / "id_ed25519.pub" if key.exists() and pub.exists(): @@ -372,11 +502,24 @@ def _stable_keypair() -> tuple[Path, str]: return key, pub.read_text().strip() -def _kill_pidfile() -> None: - """SIGTERM (then SIGKILL) the recorded infra VMM, if it's still ours. +def _pidfile_alive(run_dir: Path) -> bool: + """True iff `run_dir`'s recorded VMM is still a live firecracker (guards a + recycled PID by checking `comm`).""" + try: + pid = int(_pid_file(run_dir).read_text().strip()) + except (OSError, ValueError): + return False + try: + return Path(f"/proc/{pid}/comm").read_text().strip() == "firecracker" + except OSError: + return False + + +def _kill_pidfile(run_dir: Path) -> None: + """SIGTERM (then SIGKILL) the VMM recorded in `run_dir`, if it's still ours. Guards against a recycled PID by checking the process is firecracker.""" try: - pid = int(_pid_file().read_text().strip()) + pid = int(_pid_file(run_dir).read_text().strip()) except (OSError, ValueError): return try: @@ -397,11 +540,12 @@ def _kill_pidfile() -> None: def _kill_infra_firecrackers(proc_root: Path = Path("/proc")) -> None: - """SIGKILL any firecracker VMM whose `--config-file` is this host's infra - config, independent of the PID file — reaps orphans it lost track of so the - orchestrator TAP is free to rebind. Scoped to the infra config path, so the - interactive pool's agent/infra VMs (other config paths) are untouched.""" - cfg = str(_infra_dir() / "config.json") + """SIGKILL any firecracker VMM whose `--config-file` is one of this host's + infra configs (orchestrator or gateway), independent of the PID files — + reaps orphans it lost track of so both links' TAPs are free to rebind. + Scoped to the two infra config paths, so the interactive pool's agent VMs + (other config paths) are untouched.""" + cfgs = {str(_orch_dir() / "config.json"), str(_gw_dir() / "config.json")} for entry in proc_root.iterdir(): if not entry.name.isdigit(): continue @@ -411,7 +555,7 @@ def _kill_infra_firecrackers(proc_root: Path = Path("/proc")) -> None: args = (entry / "cmdline").read_bytes().split(b"\0") except OSError: continue # process vanished / not ours - if any(a.decode("utf-8", "replace") == cfg for a in args): + if any(a.decode("utf-8", "replace") in cfgs for a in args): try: os.kill(int(entry.name), signal.SIGKILL) except (OSError, ValueError): @@ -427,8 +571,8 @@ def _health_ok(url: str) -> bool: class SshGatewayTransport: - """`GatewayTransport` for the gateway running in the infra VM — the docker - exec/cp equivalents over SSH (dropbear + the stable infra key).""" + """`GatewayTransport` for the gateway running in the gateway VM — the + docker exec/cp equivalents over SSH (dropbear + the stable infra key).""" def __init__(self, private_key: Path, guest_ip: str) -> None: self._key = private_key @@ -461,42 +605,51 @@ class SshGatewayTransport: def gateway_transport() -> SshGatewayTransport: - """git-gate provisioning transport for the gateway in the infra VM, built - from the stable key + the orchestrator link's guest IP. Needs no live VM - handle, so teardown can use it too.""" + """git-gate provisioning transport for the gateway VM, built from the + stable key + the gateway link's guest IP. Needs no live VM handle, so + teardown can use it too.""" return SshGatewayTransport( - _infra_dir() / "id_ed25519", netpool.orch_slot().guest_ip) + _infra_dir() / "id_ed25519", netpool.gw_slot().guest_ip) def wait_for_health( infra: InfraVm, *, timeout: float = _HEALTH_TIMEOUT_SECONDS, ) -> None: - """Poll the control plane's /health until it answers 200 or the deadline + """Poll the orchestrator's /health until it answers 200 or the deadline passes. Dies (with the console tail) if the VMM exits early.""" url = f"{infra.orchestrator_url}/health" deadline = time.monotonic() + timeout while time.monotonic() < deadline: if infra.vm is not None and not infra.vm.is_alive(): - die(f"infra VM exited during boot (rc={infra.vm.process.returncode}).\n" + die(f"orchestrator VM exited during boot (rc={infra.vm.process.returncode}).\n" f"{firecracker_vm._console_tail(infra.vm.console_log)}") try: with urllib.request.urlopen(url, timeout=1.0) as resp: if resp.status == 200: - info(f"infra control plane healthy at {infra.orchestrator_url}") + info(f"orchestrator control plane healthy at {infra.orchestrator_url}") return except (urllib.error.URLError, TimeoutError, OSError): pass time.sleep(_HEALTH_POLL_SECONDS) tail = (firecracker_vm._console_tail(infra.vm.console_log) if infra.vm is not None else "") - die(f"infra control plane at {url} did not become healthy within " + die(f"orchestrator control plane at {url} did not become healthy within " f"{timeout:.0f}s.\n{tail}") def _infra_init() -> str: - """PID-1 init for the infra VM: mount the pseudo-filesystems, wire a - resolver, start dropbear (debug SSH), then launch the control plane and - the gateway data plane (multi-tenant against the local control plane).""" + """PID-1 init for the infra VMs. Shared setup (pseudo-filesystems, PATH, + resolver, debug SSH), then a `bb_role` cmdline branch selecting the plane: + + * orchestrator — mount the persistent registry volume, wait for the + host-seeded signing key, start ONLY the control plane; + * gateway — wait for the host-seeded `gateway` JWT, start ONLY the + data-plane daemons (multi-tenant against the orchestrator at the + `bb_orch` cmdline address). + + Both VMs boot this same init (one published artifact); the cmdline selects + the role, so no orchestrator IP is baked in (an IP_BASE override doesn't + change the artifact version).""" return f"""#!/bin/sh # bot-bottle Firecracker infra VM init (PID 1). mount -t proc proc /proc 2>/dev/null @@ -524,50 +677,62 @@ fi chown -R 0:0 /root 2>/dev/null || true mkdir -p /etc/dropbear /run /var/lib/bot-bottle -# Persistent registry volume (second virtio-block device, /dev/vdb) mounted -# at the control plane's DB dir, so bot-bottle.db survives infra-VM restarts. -mount -t ext4 /dev/vdb /var/lib/bot-bottle 2>/dev/null || true - /bb-dropbear -R -E -p 22 & -# Control plane. Source is baked at /app; the package is stdlib-only. +ROLE=$(sed -n 's/.*bb_role=\\([^ ]*\\).*/\\1/p' /proc/cmdline) cd /app -# Wait for the launcher to push the host-canonical control-plane signing key -# over SSH, then hand it ONLY to the orchestrator (to verify tokens); the -# data-plane daemons get a pre-minted `gateway` JWT they present, never the key. -# If it never arrives, REFUSE to start the control plane rather than run OPEN — -# open mode would grant every unauthenticated caller the `cli` role, and a -# compromised egress / supervise / git-http daemon in this same VM (reaching the -# orchestrator over 127.0.0.1, past the nft boundary that only fences off the -# separate agent VM) could then drive the operator routes (issue #469). -CP_KEY="" -i=0 -while [ "$i" -lt 600 ]; do - CP_KEY=$(cat /var/lib/bot-bottle/orchestrator-token 2>/dev/null) - [ -n "$CP_KEY" ] && break - i=$((i + 1)) - sleep 0.1 -done -if [ -z "$CP_KEY" ]; then - echo "infra: control-plane signing key never arrived; refusing to start the control plane (would run OPEN)" >&2 -else - chmod 600 /var/lib/bot-bottle/orchestrator-token 2>/dev/null || true - GW_JWT=$(BB_SIGNING_KEY="$CP_KEY" python3 -c 'import os; from bot_bottle.orchestrator_auth import mint, ROLE_GATEWAY; print(mint(ROLE_GATEWAY, os.environ["BB_SIGNING_KEY"]))') - BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_ORCHESTRATOR_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator \\ - --host 0.0.0.0 --port {ORCHESTRATOR_PORT} --broker stub & +if [ "$ROLE" = gateway ]; then # Gateway data plane, multi-tenant: each request resolves source-IP -> - # policy against the local control plane. The VM backend reaches git over - # git-http (9420), so the git:// daemon (git-gate, needs a per-bottle - # entrypoint the consolidated model doesn't use) is left out. No - # SUPERVISE_DB_PATH: the data plane reaches the supervise queue over the - # control-plane RPC and never opens bot-bottle.db (PRD 0070 / #469). It - # presents the pre-minted `gateway` JWT; gateway_init keeps the signing key - # out of the data-plane daemons' env. - BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise \\ - BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{ORCHESTRATOR_PORT} \\ - BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT="$GW_JWT" \\ - python3 -m bot_bottle.gateway.bootstrap & + # policy against the orchestrator VM (its guest IP is on the cmdline as + # bb_orch). The VM backend reaches git over git-http (9420), so the git:// + # daemon (git-gate, needs a per-bottle entrypoint the consolidated model + # doesn't use) is left out. No SUPERVISE_DB_PATH: the data plane reaches the + # supervise queue over the control-plane RPC and never opens bot-bottle.db + # (PRD 0070 / #469). It presents the pre-minted `gateway` JWT the launcher + # pushed; if it never arrives, REFUSE to start rather than run without auth. + ORCH=$(sed -n 's/.*bb_orch=\\([^ ]*\\).*/\\1/p' /proc/cmdline) + GW_JWT="" + i=0 + while [ "$i" -lt 600 ]; do + GW_JWT=$(cat {_GUEST_GATEWAY_JWT_PATH} 2>/dev/null) + [ -n "$GW_JWT" ] && break + i=$((i + 1)) + sleep 0.1 + done + if [ -z "$GW_JWT" ]; then + echo "infra gateway: gateway JWT never arrived; refusing to start the data plane" >&2 + else + chmod 600 {_GUEST_GATEWAY_JWT_PATH} 2>/dev/null || true + BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise \\ + BOT_BOTTLE_ORCHESTRATOR_URL=http://$ORCH:{ORCHESTRATOR_PORT} \\ + BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT="$GW_JWT" \\ + python3 -m bot_bottle.gateway.bootstrap & + fi +else + # Control plane. Source is baked at /app; the package is stdlib-only. + # Persistent registry volume (second virtio-block device, /dev/vdb) mounted + # at the DB dir, so bot-bottle.db survives orchestrator-VM restarts. + mount -t ext4 /dev/vdb /var/lib/bot-bottle 2>/dev/null || true + # Wait for the launcher to push the host-canonical signing key over SSH, + # then hand it ONLY to the orchestrator (to verify tokens). If it never + # arrives, REFUSE to start rather than run OPEN — open mode would grant + # every unauthenticated caller the `cli` role (issue #469). + CP_KEY="" + i=0 + while [ "$i" -lt 600 ]; do + CP_KEY=$(cat {_GUEST_SIGNING_KEY_PATH} 2>/dev/null) + [ -n "$CP_KEY" ] && break + i=$((i + 1)) + sleep 0.1 + done + if [ -z "$CP_KEY" ]; then + echo "infra: control-plane signing key never arrived; refusing to start the control plane (would run OPEN)" >&2 + else + chmod 600 {_GUEST_SIGNING_KEY_PATH} 2>/dev/null || true + BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_ORCHESTRATOR_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator \\ + --host 0.0.0.0 --port {ORCHESTRATOR_PORT} --broker stub & + fi fi # Reap as PID 1; children are backgrounded, so `wait` blocks. diff --git a/bot_bottle/backend/firecracker/launch.py b/bot_bottle/backend/firecracker/launch.py index 007724c..5c22d89 100644 --- a/bot_bottle/backend/firecracker/launch.py +++ b/bot_bottle/backend/firecracker/launch.py @@ -11,15 +11,16 @@ Per bottle: 6. provision (shared gateway CA, prompt, skills, workspace, git, supervise) over SSH. -The per-bottle Docker sidecar bundle is gone. The shared gateway handles -egress / git-gate / supervise for every VM; Docker's PREROUTING DNAT routes -the VMs' traffic to it, and the nft table's `ct status dnat accept` rule -in the forward chain lets it pass. The VM still sends to `host_tap_ip:PORT` -— the address its world is, by nft design, limited to. +The per-bottle Docker sidecar bundle is gone. The shared gateway (its own +data-plane VM) handles egress / git-gate / supervise for every VM; the nft +netpool's PREROUTING DNAT routes the VMs' gateway-port traffic to that gateway +VM, and the nft table's `ct status dnat accept` rule in the forward chain lets +it pass. The VM still sends to `host_tap_ip:PORT` — the address its world is, +by nft design, limited to. Isolation is enforced by the operator-provisioned nft table (checked -fail-closed in preflight): a VM reaches only the sidecar (DNAT'd from -the host TAP IP) and nothing else. +fail-closed in preflight): a VM reaches only the gateway (DNAT'd from the host +TAP IP) and nothing else — not even the orchestrator control plane. """ from __future__ import annotations @@ -136,7 +137,7 @@ def launch( ) # Point the agent's git-gate insteadOf rewrites and supervise MCP URL # at the shared gateway (reached at the slot's host TAP IP — the VM - # sends there and Docker DNAT routes to the gateway container). + # sends there and the nft netpool DNAT routes to the gateway VM). git_gate_url = ( f"http://{slot.host_ip}:{_GIT_HTTP_PORT}" if git_gate_plan.upstreams else "" ) diff --git a/bot_bottle/backend/firecracker/netpool.defaults.env b/bot_bottle/backend/firecracker/netpool.defaults.env index 2277f1f..536d1b3 100644 --- a/bot_bottle/backend/firecracker/netpool.defaults.env +++ b/bot_bottle/backend/firecracker/netpool.defaults.env @@ -15,11 +15,19 @@ BOT_BOTTLE_FC_POOL_SIZE=8 BOT_BOTTLE_FC_IP_BASE=10.243.0.0 BOT_BOTTLE_FC_IFACE_PREFIX=bbfc BOT_BOTTLE_FC_NFT_TABLE=bot_bottle_fc -# The orchestrator/gateway VM's own TAP — a dedicated link OUTSIDE the -# bbfc* agent pool. Unlike agent VMs (which reach only their gateway), -# the orchestrator is trusted infra that needs real NAT'd internet -# egress: to FROM-pull + apt/npm during in-VM agent-image builds -# (buildah) and to forward agent egress upstream (Stage B gateway). Its +# The orchestrator VM's own TAP — a dedicated link OUTSIDE the bbfc* +# agent pool. Unlike agent VMs (which reach only their gateway), the +# orchestrator is trusted infra that needs real NAT'd internet egress: +# to FROM-pull + apt/npm during in-VM agent-image builds (buildah). Its # /31 is the top of the IP_BASE /16 (host x.y.255.0, guest x.y.255.1), # clear of the pool near the bottom of the block. BOT_BOTTLE_FC_ORCH_IFACE=bborch0 +# The gateway (data-plane) VM's own TAP — the second infra link, split +# from the orchestrator now that #469 got the DB off the data plane (PRD +# 0070 "Separating the planes"). It mirrors the orchestrator link: NAT'd +# internet egress (the gateway forwards agent egress upstream) plus a +# forward path to reach the orchestrator's control plane. Its /31 is the +# next one above the orchestrator link (host x.y.255.2, guest x.y.255.3). +# Agents DNAT their gateway-port traffic here (not to the orchestrator), +# so a breached agent has no L3 route to the control plane. +BOT_BOTTLE_FC_GW_IFACE=bbgw0 diff --git a/bot_bottle/backend/firecracker/netpool.py b/bot_bottle/backend/firecracker/netpool.py index 3ee6653..e3c9508 100644 --- a/bot_bottle/backend/firecracker/netpool.py +++ b/bot_bottle/backend/firecracker/netpool.py @@ -79,12 +79,19 @@ def _cfg(key: str) -> str: IFACE_PREFIX = _cfg("BOT_BOTTLE_FC_IFACE_PREFIX") NFT_TABLE = _cfg("BOT_BOTTLE_FC_NFT_TABLE") -# The orchestrator/gateway VM's dedicated TAP — outside the bbfc* agent -# pool and, unlike it, NAT'd to the internet (see `orch_slot`). The +# The orchestrator VM's dedicated TAP — outside the bbfc* agent pool +# and, unlike it, NAT'd to the internet (see `orch_slot`). The # orchestrator is trusted infra: it builds agent images in-VM (buildah -# needs to FROM-pull + apt/npm) and forwards agent egress upstream. +# needs to FROM-pull + apt/npm). ORCH_IFACE = _cfg("BOT_BOTTLE_FC_ORCH_IFACE") +# The gateway (data-plane) VM's dedicated TAP — the second infra link +# (see `gw_slot`), split from the orchestrator per PRD 0070. Like the +# orchestrator link it is NAT'd to the internet (the gateway forwards +# agent egress upstream); unlike it, agents DNAT here, never to the +# orchestrator, so a breached agent has no L3 route to the control plane. +GW_IFACE = _cfg("BOT_BOTTLE_FC_GW_IFACE") + def pool_size() -> int: return int(_cfg("BOT_BOTTLE_FC_POOL_SIZE")) @@ -130,12 +137,12 @@ def all_slots() -> list[Slot]: def orch_slot() -> Slot: - """The orchestrator/gateway VM's dedicated link — its own TAP - (`ORCH_IFACE`) on a /31 at the TOP of the IP_BASE /16 (host - x.y.255.0, guest x.y.255.1), well clear of the agent pool near the - bottom of the block. Unlike a pool `Slot`, this link is NAT'd out to - the internet by the setup (the orchestrator is trusted infra), so it - is deliberately *not* one of the isolated `bbfc*` slots. + """The orchestrator VM's dedicated link — its own TAP (`ORCH_IFACE`) + on a /31 at the TOP of the IP_BASE /16 (host x.y.255.0, guest + x.y.255.1), well clear of the agent pool near the bottom of the + block. Unlike a pool `Slot`, this link is NAT'd out to the internet + by the setup (the orchestrator is trusted infra), so it is + deliberately *not* one of the isolated `bbfc*` slots. `index` is -1 (sentinel: not a pool index).""" base16 = int(ipaddress.IPv4Address(ip_base())) & 0xFFFF0000 @@ -148,6 +155,25 @@ def orch_slot() -> Slot: ) +def gw_slot() -> Slot: + """The gateway (data-plane) VM's dedicated link — its own TAP + (`GW_IFACE`) on the /31 immediately above the orchestrator link (host + x.y.255.2, guest x.y.255.3), still clear of the agent pool at the + bottom of the block. Mirrors `orch_slot`: NAT'd to the internet (the + gateway forwards agent egress upstream), not one of the isolated + `bbfc*` slots. Agents DNAT their gateway-port traffic to this guest. + + `index` is -2 (sentinel: not a pool index).""" + base16 = int(ipaddress.IPv4Address(ip_base())) & 0xFFFF0000 + host = base16 + 0xFF02 + return Slot( + index=-2, + iface=GW_IFACE, + host_ip=str(ipaddress.IPv4Address(host)), + guest_ip=str(ipaddress.IPv4Address(host + 1)), + ) + + # --- fail-closed verification --------------------------------------- def _run_ok(argv: list[str]) -> bool: diff --git a/nix/firecracker-netpool.nix b/nix/firecracker-netpool.nix index 8e63cc1..f2118d5 100644 --- a/nix/firecracker-netpool.nix +++ b/nix/firecracker-netpool.nix @@ -67,6 +67,7 @@ let BOT_BOTTLE_FC_IFACE_PREFIX = cfg.ifacePrefix; BOT_BOTTLE_FC_NFT_TABLE = cfg.tableName; BOT_BOTTLE_FC_ORCH_IFACE = cfg.orchIface; + BOT_BOTTLE_FC_GW_IFACE = cfg.gwIface; } // ownEnv; in { @@ -135,11 +136,24 @@ in default = defaults.BOT_BOTTLE_FC_ORCH_IFACE; defaultText = lib.literalMD "the shared `netpool.defaults.env` value"; description = '' - TAP name for the orchestrator/gateway VM's dedicated link. Unlike - the isolated bbfc* agent pool, this link is NAT'd to the internet - (the orchestrator is trusted infra that builds agent images in-VM - and forwards agent egress upstream). Must match - BOT_BOTTLE_FC_ORCH_IFACE / netpool.ORCH_IFACE. + TAP name for the orchestrator VM's dedicated link. Unlike the + isolated bbfc* agent pool, this link is NAT'd to the internet + (the orchestrator is trusted infra that builds agent images + in-VM). Must match BOT_BOTTLE_FC_ORCH_IFACE / netpool.ORCH_IFACE. + ''; + }; + + gwIface = lib.mkOption { + type = lib.types.str; + default = defaults.BOT_BOTTLE_FC_GW_IFACE; + defaultText = lib.literalMD "the shared `netpool.defaults.env` value"; + description = '' + TAP name for the gateway (data-plane) VM's dedicated link — the + second infra link, split from the orchestrator per PRD 0070. Like + the orchestrator link it is NAT'd to the internet (the gateway + forwards agent egress upstream); agents DNAT here, never to the + orchestrator, so a breached agent has no L3 route to the control + plane. Must match BOT_BOTTLE_FC_GW_IFACE / netpool.GW_IFACE. ''; }; @@ -193,6 +207,7 @@ in BOT_BOTTLE_FC_IFACE_PREFIX=${cfg.ifacePrefix} BOT_BOTTLE_FC_NFT_TABLE=${cfg.tableName} BOT_BOTTLE_FC_ORCH_IFACE=${cfg.orchIface} + BOT_BOTTLE_FC_GW_IFACE=${cfg.gwIface} ''; }; }; diff --git a/scripts/firecracker-netpool.sh b/scripts/firecracker-netpool.sh index 356947d..5178ea9 100755 --- a/scripts/firecracker-netpool.sh +++ b/scripts/firecracker-netpool.sh @@ -64,12 +64,13 @@ IP_BASE="${BOT_BOTTLE_FC_IP_BASE:-$(_default BOT_BOTTLE_FC_IP_BASE)}" PREFIX="${BOT_BOTTLE_FC_IFACE_PREFIX:-$(_default BOT_BOTTLE_FC_IFACE_PREFIX)}" TABLE="${BOT_BOTTLE_FC_NFT_TABLE:-$(_default BOT_BOTTLE_FC_NFT_TABLE)}" ORCH_IFACE="${BOT_BOTTLE_FC_ORCH_IFACE:-$(_default BOT_BOTTLE_FC_ORCH_IFACE)}" +GW_IFACE="${BOT_BOTTLE_FC_GW_IFACE:-$(_default BOT_BOTTLE_FC_GW_IFACE)}" OWNER="${BOT_BOTTLE_FC_OWNER:-${SUDO_USER:-$USER}}" GROUP="${BOT_BOTTLE_FC_GROUP:-}" # Fail loudly rather than provisioning a half/empty range if a value # resolved to nothing (env unset AND the shared file unreadable). -for _v in POOL_SIZE IP_BASE PREFIX TABLE ORCH_IFACE; do +for _v in POOL_SIZE IP_BASE PREFIX TABLE ORCH_IFACE GW_IFACE; do [ -n "${!_v}" ] || { echo "error: $_v unresolved (set BOT_BOTTLE_FC_* or fix $_DEFAULTS)" >&2; exit 1; } done @@ -90,13 +91,20 @@ host_ip() { _int_to_ip $(( $(_ip_to_int "$IP_BASE") + 2*$1 )); } guest_ip() { _int_to_ip $(( $(_ip_to_int "$IP_BASE") + 2*$1 + 1 )); } iface() { echo "${PREFIX}$1"; } -# Orchestrator/gateway VM link: a /31 at the TOP of the IP_BASE /16 -# (host x.y.255.0, guest x.y.255.1), well clear of the agent pool near -# the bottom of the block. Must match netpool.py:orch_slot(). +# Orchestrator VM link: a /31 at the TOP of the IP_BASE /16 (host +# x.y.255.0, guest x.y.255.1), well clear of the agent pool near the +# bottom of the block. Must match netpool.py:orch_slot(). _orch_base() { echo $(( ($(_ip_to_int "$IP_BASE") & 0xFFFF0000) + 0xFF00 )); } orch_host() { _int_to_ip "$(_orch_base)"; } orch_guest() { _int_to_ip $(( $(_orch_base) + 1 )); } +# Gateway (data-plane) VM link: the /31 immediately above the +# orchestrator link (host x.y.255.2, guest x.y.255.3). Must match +# netpool.py:gw_slot(). +_gw_base() { echo $(( ($(_ip_to_int "$IP_BASE") & 0xFFFF0000) + 0xFF02 )); } +gw_host() { _int_to_ip "$(_gw_base)"; } +gw_guest() { _int_to_ip $(( $(_gw_base) + 1 )); } + require_root() { if [ "$(id -u)" -ne 0 ]; then echo "error: '$1' needs root (run under sudo)" >&2 @@ -133,21 +141,26 @@ cmd_up() { echo " $dev host=$host guest=$(guest_ip "$i") $own_desc" done - # The orchestrator/gateway VM's dedicated link. Same rootless-open - # ownership as the pool, but NAT'd to the internet (below) — it is - # trusted infra, not an isolated agent slot. - ip link show "$ORCH_IFACE" >/dev/null 2>&1 \ - || ip tuntap add dev "$ORCH_IFACE" mode tap "${own_args[@]}" - ip addr replace "$(orch_host)/31" dev "$ORCH_IFACE" - ip link set "$ORCH_IFACE" up + # The two infra VM links (orchestrator control plane + gateway data + # plane). Same rootless-open ownership as the pool, but NAT'd to the + # internet (below) — trusted infra, not isolated agent slots. + local pair ifc iaddr + for pair in "$ORCH_IFACE:$(orch_host)" "$GW_IFACE:$(gw_host)"; do + ifc="${pair%%:*}" ; iaddr="${pair##*:}" + ip link show "$ifc" >/dev/null 2>&1 \ + || ip tuntap add dev "$ifc" mode tap "${own_args[@]}" + ip addr replace "$iaddr/31" dev "$ifc" + ip link set "$ifc" up + done echo " $ORCH_IFACE host=$(orch_host) guest=$(orch_guest) (NAT'd egress) $own_desc" + echo " $GW_IFACE host=$(gw_host) guest=$(gw_guest) (NAT'd egress) $own_desc" _install_nft echo "nftables table inet $TABLE installed (fail-closed boundary)" - _install_orch_egress - echo "orchestrator egress installed ($ORCH_IFACE -> NAT out)" + _install_infra_egress + echo "infra egress installed ($ORCH_IFACE, $GW_IFACE -> NAT out; $GW_IFACE -> $ORCH_IFACE)" _install_gateway_route - echo "agent->gateway route installed (${PREFIX}* :$GATEWAY_PORTS -> $(orch_guest))" + echo "agent->gateway route installed (${PREFIX}* :$GATEWAY_PORTS -> $(gw_guest))" echo "done." } @@ -200,22 +213,27 @@ ${antispoof} ct state established,related accept EOF } -# Give the orchestrator/gateway VM real internet egress (agent VMs get -# none — that's the isolation table above). Three parts, because the -# path must work both during bootstrap (Docker still present) and after -# Docker is removed: -# * masquerade — SNAT the orch guest /31 out the host uplink so its -# RFC-1918 address can reach the internet. -# * nft forward — accept the orch link's forward path (load-bearing +# Give the two infra VMs (orchestrator control plane + gateway data +# plane) real internet egress, and let the gateway reach the +# orchestrator (agent VMs get neither — that's the isolation table +# above). Three parts, because the path must work both during bootstrap +# (Docker still present) and after Docker is removed: +# * masquerade — SNAT each infra guest /31 out the host uplink so its +# RFC-1918 address can reach the internet. Excludes +# BOTH infra links, so gateway<->orchestrator traffic +# keeps its real source IP (never SNAT'd internally). +# * nft forward — accept each infra link's forward path (load-bearing # on a pure-nft host whose FORWARD policy drops; a -# harmless no-op where forwarding is already open). -# It never drops, so it can't weaken the isolation -# table's agent drops. +# harmless no-op where forwarding is already open). The +# gateway link's `iifname "$GW_IFACE" accept` is what +# lets the gateway VM reach the orchestrator's control +# plane at orch_guest:8099. It never drops, so it can't +# weaken the isolation table's agent drops. # * DOCKER-USER — during bootstrap Docker's FORWARD chain policy is # DROP; its sanctioned DOCKER-USER hook is the only # place a user ACCEPT survives. Best-effort + guarded # (skipped once Docker is gone). -_install_orch_egress() { +_install_infra_egress() { nft -f - </dev/null 2>&1 || return 0 iptables -t filter -L DOCKER-USER >/dev/null 2>&1 || return 0 - for flag in "-i" "-o"; do - if [ "$op" = add ]; then - iptables -C DOCKER-USER "$flag" "$ORCH_IFACE" -j ACCEPT 2>/dev/null \ - || iptables -I DOCKER-USER "$flag" "$ORCH_IFACE" -j ACCEPT - else - iptables -D DOCKER-USER "$flag" "$ORCH_IFACE" -j ACCEPT 2>/dev/null || true - fi + for ifc in "$ORCH_IFACE" "$GW_IFACE"; do + for flag in "-i" "-o"; do + if [ "$op" = add ]; then + iptables -C DOCKER-USER "$flag" "$ifc" -j ACCEPT 2>/dev/null \ + || iptables -I DOCKER-USER "$flag" "$ifc" -j ACCEPT + else + iptables -D DOCKER-USER "$flag" "$ifc" -j ACCEPT 2>/dev/null || true + fi + done done } # Agent -> gateway VM routing. The shared gateway (egress / supervise / -# git-http) runs in the orchestrator/infra VM at $(orch_guest). Agents keep -# addressing their own host-side TAP IP on the gateway ports; a PREROUTING -# DNAT redirects that to the infra VM, and the isolation table's -# `ct status dnat accept` forward rule lets it through — every other agent -# egress stays dropped. Source IP is deliberately NOT masqueraded: the -# gateway attributes each request to the originating bottle by its (nft + -# /31 unspoofable) guest IP. +# git-http) runs in its own data-plane VM at $(gw_guest), split from the +# orchestrator per PRD 0070 so a breached agent has no L3 route to the +# control plane. Agents keep addressing their own host-side TAP IP on the +# gateway ports; a PREROUTING DNAT redirects that to the gateway VM, and +# the isolation table's `ct status dnat accept` forward rule lets it +# through — every other agent egress stays dropped, including any packet +# aimed at the orchestrator link. Source IP is deliberately NOT +# masqueraded: the gateway attributes each request to the originating +# bottle by its (nft + /31 unspoofable) guest IP. _install_gateway_route() { nft -f - </dev/null || true nft delete table inet "${TABLE}_nat" 2>/dev/null || true - if ip link show "$ORCH_IFACE" >/dev/null 2>&1; then - ip link set "$ORCH_IFACE" down 2>/dev/null || true - ip tuntap del dev "$ORCH_IFACE" mode tap 2>/dev/null || true - echo " removed $ORCH_IFACE" - fi + local ifc + for ifc in "$ORCH_IFACE" "$GW_IFACE"; do + if ip link show "$ifc" >/dev/null 2>&1; then + ip link set "$ifc" down 2>/dev/null || true + ip tuntap del dev "$ifc" mode tap 2>/dev/null || true + echo " removed $ifc" + fi + done nft delete table inet "$TABLE" 2>/dev/null || true for i in $(seq 0 $((POOL_SIZE-1))); do local dev ; dev="$(iface "$i")" @@ -296,7 +324,7 @@ cmd_down() { cmd_status() { echo "table inet $TABLE:" nft list table inet "$TABLE" 2>/dev/null || echo " (absent)" - echo "table inet ${TABLE}_nat (orchestrator egress):" + echo "table inet ${TABLE}_nat (infra egress: orchestrator + gateway):" nft list table inet "${TABLE}_nat" 2>/dev/null || echo " (absent)" echo "table ip ${TABLE}_gw (agent->gateway route):" nft list table ip "${TABLE}_gw" 2>/dev/null || echo " (absent)" @@ -307,9 +335,12 @@ cmd_status() { ip -brief addr show "$dev" | sed 's/^/ /' fi done - if ip -brief addr show "$ORCH_IFACE" >/dev/null 2>&1; then - ip -brief addr show "$ORCH_IFACE" | sed 's/^/ /' - fi + local ifc + for ifc in "$ORCH_IFACE" "$GW_IFACE"; do + if ip -brief addr show "$ifc" >/dev/null 2>&1; then + ip -brief addr show "$ifc" | sed 's/^/ /' + fi + done } case "${1:-}" in diff --git a/tests/unit/test_firecracker_helpers.py b/tests/unit/test_firecracker_helpers.py index ef63863..5ae6062 100644 --- a/tests/unit/test_firecracker_helpers.py +++ b/tests/unit/test_firecracker_helpers.py @@ -85,6 +85,22 @@ class TestNetpoolProbes(unittest.TestCase): pool_guests = {sl.guest_ip for sl in netpool.all_slots()} self.assertNotIn(s.guest_ip, pool_guests) + def test_gw_slot_is_second_link_above_orch(self): + # Dedicated gateway (data-plane) link: the /31 immediately above the + # orchestrator link. Must match the shell script's gw_host()/gw_guest() + # and be a distinct non-pool index that never collides with the + # orchestrator link or the agent pool. + with patch.dict("os.environ", {"BOT_BOTTLE_FC_IP_BASE": "10.243.0.0"}): + s = netpool.gw_slot() + orch = netpool.orch_slot() + pool_guests = {sl.guest_ip for sl in netpool.all_slots()} + self.assertEqual(netpool.GW_IFACE, s.iface) + self.assertEqual("10.243.255.2", s.host_ip) + self.assertEqual("10.243.255.3", s.guest_ip) + self.assertEqual(-2, s.index) + self.assertNotEqual(orch.guest_ip, s.guest_ip) # distinct from the orch link + self.assertNotIn(s.guest_ip, pool_guests) # distinct from the pool + class TestConsoleTail(unittest.TestCase): def test_reads_tail(self): diff --git a/tests/unit/test_firecracker_infra_vm.py b/tests/unit/test_firecracker_infra_vm.py index 44d51a1..e9c1058 100644 --- a/tests/unit/test_firecracker_infra_vm.py +++ b/tests/unit/test_firecracker_infra_vm.py @@ -1,8 +1,8 @@ -"""Unit tests for the Firecracker infra VM (control-plane VM boot). +"""Unit tests for the Firecracker infra VMs (orchestrator + gateway boot). -The KVM boot / HTTP reachability is integration-tested on a KVM host; here -we cover the URL shape, the rootfs-variant wiring, and the health-poll -decisions that must hold without a VM. +The KVM boot / HTTP reachability is integration-tested on a KVM host; here we +cover the URL shape, the shared-rootfs role wiring, the secret-push decisions, +and the two-VM singleton lifecycle that must hold without a VM. """ from __future__ import annotations @@ -15,15 +15,16 @@ from unittest.mock import MagicMock, patch from bot_bottle.backend.firecracker import infra_vm -class TestPushSigningKey(unittest.TestCase): - """`_push_signing_key` pushes the host-canonical key into the guest over SSH - (the init waits for it). The host token file stays the single source of - truth, never clobbered per-backend (issue #469).""" +class TestPushSecrets(unittest.TestCase): + """`_push_signing_key` (orchestrator) and `_push_gateway_jwt` (gateway) + seed their guest's secret over SSH via `_push_secret`. The host token file + stays the single source of truth, never clobbered per-backend (#469); the + gateway gets a pre-minted `gateway` JWT, never the signing key.""" def _infra(self) -> infra_vm.InfraVm: return infra_vm.InfraVm(guest_ip="10.0.0.1", private_key=Path("/k")) - def test_writes_host_key_over_ssh_atomically(self): + def test_signing_key_written_atomically(self): proc = MagicMock(returncode=0, stderr="") with patch.object(infra_vm, "host_orchestrator_token", return_value="host-key"), \ patch.object(infra_vm.subprocess, "run", return_value=proc) as run: @@ -35,11 +36,24 @@ class TestPushSigningKey(unittest.TestCase): self.assertIn(f"mv {infra_vm._GUEST_SIGNING_KEY_PATH}.tmp " f"{infra_vm._GUEST_SIGNING_KEY_PATH}", remote_cmd) + def test_gateway_jwt_is_minted_gateway_role_not_the_key(self): + proc = MagicMock(returncode=0, stderr="") + with patch.object(infra_vm, "host_orchestrator_token", return_value="host-key"), \ + patch.object(infra_vm, "mint", return_value="gw.jwt") as mint, \ + patch.object(infra_vm.subprocess, "run", return_value=proc) as run: + infra_vm._push_gateway_jwt(self._infra()) + # Minted on the HOST from the signing key with the gateway role, and it + # is the JWT (never the key itself) that lands in the gateway VM. + mint.assert_called_once_with(infra_vm.ROLE_GATEWAY, "host-key") + self.assertEqual("gw.jwt", run.call_args.kwargs["input"]) + remote_cmd = run.call_args.args[0][-1] + self.assertIn(f"cat > {infra_vm._GUEST_GATEWAY_JWT_PATH}.tmp", remote_cmd) + def test_dies_when_push_never_succeeds(self): proc = MagicMock(returncode=255, stderr="ssh: connect refused") with patch.object(infra_vm, "host_orchestrator_token", return_value="host-key"), \ - patch.object(infra_vm, "_SIGNING_KEY_PUSH_TIMEOUT_SECONDS", 0.05), \ - patch.object(infra_vm, "_SIGNING_KEY_PUSH_POLL_SECONDS", 0.0), \ + patch.object(infra_vm, "_SECRET_PUSH_TIMEOUT_SECONDS", 0.05), \ + patch.object(infra_vm, "_SECRET_PUSH_POLL_SECONDS", 0.0), \ patch.object(infra_vm.subprocess, "run", return_value=proc), \ patch.object(infra_vm, "die", side_effect=SystemExit) as die: with self.assertRaises(SystemExit): @@ -57,8 +71,30 @@ class TestOrchestratorUrl(unittest.TestCase): ) +class TestInfraEndpoint(unittest.TestCase): + """The endpoint mirrors docker/macOS: it exposes the orchestrator's URL and + delegates the CA fetch to the gateway VM handle.""" + + def _endpoint(self) -> infra_vm.InfraEndpoint: + return infra_vm.InfraEndpoint( + orchestrator=infra_vm.InfraVm(guest_ip="10.243.255.1", private_key=Path("/k")), + gateway=infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k")), + ) + + def test_orchestrator_url_is_the_orchestrator_vm(self): + ep = self._endpoint() + self.assertEqual( + f"http://10.243.255.1:{infra_vm.ORCHESTRATOR_PORT}", ep.orchestrator_url) + + def test_gateway_ca_pem_delegates_to_the_gateway_vm(self): + ep = self._endpoint() + with patch.object(ep.gateway, "gateway_ca_pem", return_value="PEM") as ca: + self.assertEqual("PEM", ep.gateway_ca_pem(timeout=1)) + ca.assert_called_once_with(timeout=1) + + class TestBuildInfraRootfs(unittest.TestCase): - def test_uses_infra_variant_and_init(self): + def test_uses_infra_variant_and_role_branched_init(self): with patch.object(infra_vm.util, "build_base_rootfs_dir") as build: build.return_value = Path("/cache/rootfs/x-infra") infra_vm.build_infra_rootfs_dir() @@ -66,32 +102,90 @@ class TestBuildInfraRootfs(unittest.TestCase): self.assertEqual(infra_vm._INFRA_IMAGE, build.call_args.args[0]) # variant is "-infra-" so an init change rebuilds the rootfs. self.assertTrue(build.call_args.kwargs["variant"].startswith("-infra-")) - # The init runs BOTH the control plane and the gateway data plane, - # and exports PATH so gateway_init's subprocess daemons find python3. init = build.call_args.kwargs["init_script"] + # One shared init, role-branched off the kernel cmdline: it starts the + # control plane OR the gateway data plane, and exports PATH so the + # gateway daemons' subprocesses find python3. + self.assertIn("bb_role=", init) + self.assertIn('if [ "$ROLE" = gateway ]; then', init) self.assertIn("bot_bottle.orchestrator", init) - # Gateway launches via the installed package (there is no - # /app/gateway_init.py file since the daemons moved into bot_bottle). self.assertIn("bot_bottle.gateway.bootstrap", init) self.assertIn("export PATH=", init) - # Persistent registry volume mounted at the DB dir before the CP starts. + # Persistent registry volume mounted at the DB dir on the orchestrator. self.assertIn("/dev/vdb", init) # VM backend uses git-http (9420); the git:// daemon is left out. self.assertIn("BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise", init) - # Role-scoped control-plane auth (issue #469 review): the orchestrator - # gets the host-seeded signing key, the gateway daemons get a pre-minted - # `gateway` JWT, and the VM refuses to run OPEN if the key is missing. - self.assertIn("cat /var/lib/bot-bottle/orchestrator-token", init) # host-seeded key - self.assertIn("refusing to start the control plane", init) # no open mode - self.assertIn("mint, ROLE_GATEWAY", init) # gateway JWT + # Role-scoped auth (#469): the orchestrator gets the host-seeded signing + # key; the gateway gets the host-minted `gateway` JWT (NOT the key — the + # JWT is minted on the host now, so the init never touches the key to + # mint it); each plane refuses to start without its secret. + self.assertIn(f"cat {infra_vm._GUEST_SIGNING_KEY_PATH}", init) # host-seeded key + self.assertIn(f"cat {infra_vm._GUEST_GATEWAY_JWT_PATH}", init) # host-minted JWT + self.assertNotIn("ROLE_GATEWAY", init) # minting moved to the host + self.assertIn("refusing to start the control plane", init) # no open mode + self.assertIn("refusing to start the data plane", init) # gateway fail-closed self.assertIn('BOT_BOTTLE_ORCHESTRATOR_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator', init) # key -> orchestrator only self.assertIn('BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT="$GW_JWT"', init) # JWT -> gateway daemons + # The gateway resolves the orchestrator's address off the cmdline + # (bb_orch), so no IP is baked into the artifact. + self.assertIn("bb_orch=", init) + self.assertIn("BOT_BOTTLE_ORCHESTRATOR_URL=http://$ORCH:", init) + + +class TestBootRole(unittest.TestCase): + """`boot_orchestrator` / `boot_gateway` boot the shared rootfs with the + right `bb_role` cmdline, memory ceiling, and (orchestrator-only) registry + volume; each seeds its own secret.""" + + def _boot_env(self): + vm = MagicMock() + vm.process.pid = 4242 + return vm + + def test_orchestrator_role_mem_and_registry(self): + import tempfile + vm = self._boot_env() + with tempfile.TemporaryDirectory() as td, \ + patch.object(infra_vm.netpool, "tap_present", return_value=True), \ + patch.object(infra_vm.infra_artifact, "local_build_requested", return_value=False), \ + patch.object(infra_vm.infra_artifact, "materialize_ext4"), \ + patch.object(infra_vm.infra_artifact, "infra_artifact_version", return_value="v"), \ + patch.object(infra_vm, "_stable_keypair", return_value=(Path("/k"), "pub")), \ + patch.object(infra_vm, "_ensure_registry_volume", return_value=Path("/reg")), \ + patch.object(infra_vm, "_push_signing_key") as push, \ + patch.object(infra_vm.firecracker_vm, "boot", return_value=vm) as boot, \ + patch.object(infra_vm, "_orch_dir", return_value=Path(td)): + infra_vm.boot_orchestrator() + kw = boot.call_args.kwargs + self.assertIn("bb_role=orchestrator", kw["extra_boot_args"]) + self.assertEqual(infra_vm._ORCH_MEM_MIB, kw["mem_mib"]) + self.assertEqual(Path("/reg"), kw["data_drive"]) # DB volume on the CP + push.assert_called_once() # signing key seeded + + def test_gateway_role_carries_orch_addr_and_no_registry(self): + import tempfile + vm = self._boot_env() + with tempfile.TemporaryDirectory() as td, \ + patch.object(infra_vm.netpool, "tap_present", return_value=True), \ + patch.object(infra_vm.infra_artifact, "local_build_requested", return_value=False), \ + patch.object(infra_vm.infra_artifact, "materialize_ext4"), \ + patch.object(infra_vm.infra_artifact, "infra_artifact_version", return_value="v"), \ + patch.object(infra_vm, "_stable_keypair", return_value=(Path("/k"), "pub")), \ + patch.object(infra_vm, "_push_gateway_jwt") as push, \ + patch.object(infra_vm.firecracker_vm, "boot", return_value=vm) as boot, \ + patch.object(infra_vm, "_gw_dir", return_value=Path(td)): + infra_vm.boot_gateway("10.243.255.1") + kw = boot.call_args.kwargs + self.assertIn("bb_role=gateway", kw["extra_boot_args"]) + self.assertIn("bb_orch=10.243.255.1", kw["extra_boot_args"]) # reaches the CP + self.assertEqual(infra_vm._GW_MEM_MIB, kw["mem_mib"]) + self.assertIsNone(kw["data_drive"]) # data plane never opens the DB + push.assert_called_once() # gateway JWT seeded class TestSshGatewayTransport(unittest.TestCase): def test_cp_into_preserves_source_mode(self): - import os import tempfile from subprocess import CompletedProcess with tempfile.NamedTemporaryFile() as f: @@ -112,14 +206,22 @@ class TestSshGatewayTransport(unittest.TestCase): t.exec(["mkdir", "-p", "/git-gate"]) +class TestGatewayTransportTargetsGatewayVm(unittest.TestCase): + def test_transport_uses_the_gateway_link_guest_ip(self): + # git-gate provisioning goes to the gateway VM (gw_slot), not the + # orchestrator, now that the planes are split. + t = infra_vm.gateway_transport() + self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, t._ip) + + class TestGatewayCaPem(unittest.TestCase): def test_dies_when_cert_never_appears(self) -> None: from subprocess import CompletedProcess - infra = infra_vm.InfraVm(vm=None, guest_ip="10.0.0.1", private_key=Path("/k")) + gw = infra_vm.InfraVm(vm=None, guest_ip="10.0.0.1", private_key=Path("/k")) with patch.object(infra_vm.subprocess, "run", return_value=CompletedProcess([], 1, stdout="", stderr="")), \ self.assertRaises(SystemExit): - infra.gateway_ca_pem(timeout=0) + gw.gateway_ca_pem(timeout=0) class TestRegistryVolume(unittest.TestCase): @@ -195,9 +297,9 @@ class TestWaitForHealth(unittest.TestCase): class TestEnsureRunningSingleton(unittest.TestCase): - def test_adopts_when_healthy_and_version_matches(self): - # Healthy control plane + existing key + matching version marker - # -> adopt (no boot), vm=None. + def test_adopts_when_healthy_alive_and_version_matches(self): + # Healthy control plane + live gateway + existing key + matching version + # marker -> adopt (no boot); both VM handles have vm=None. import tempfile with tempfile.TemporaryDirectory() as td: d = Path(td) @@ -206,13 +308,20 @@ class TestEnsureRunningSingleton(unittest.TestCase): with patch.object(infra_vm, "_infra_dir", return_value=d), \ patch.object(infra_vm, "_expected_version", return_value="v-current"), \ patch.object(infra_vm, "_health_ok", return_value=True), \ - patch.object(infra_vm, "boot") as boot: - infra = infra_vm.ensure_running() - boot.assert_not_called() - self.assertIsNone(infra.vm) + patch.object(infra_vm, "_pidfile_alive", return_value=True), \ + patch.object(infra_vm, "boot_orchestrator") as boot_o, \ + patch.object(infra_vm, "boot_gateway") as boot_g: + ep = infra_vm.ensure_running() + boot_o.assert_not_called() + boot_g.assert_not_called() + self.assertIsNone(ep.orchestrator.vm) + self.assertIsNone(ep.gateway.vm) + # The two handles address the two distinct infra links. + self.assertEqual(infra_vm.netpool.orch_slot().guest_ip, ep.orchestrator.guest_ip) + self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, ep.gateway.guest_ip) - def test_reboots_when_version_stale(self): - # Healthy control plane but the running VM booted an OLDER image + def test_reboots_both_when_version_stale(self): + # Healthy control plane but the running pair booted an OLDER image # (marker mismatch) -> reboot rather than adopt stale code. import tempfile with tempfile.TemporaryDirectory() as td: @@ -222,19 +331,52 @@ class TestEnsureRunningSingleton(unittest.TestCase): with patch.object(infra_vm, "_infra_dir", return_value=d), \ patch.object(infra_vm, "_expected_version", return_value="v-current"), \ patch.object(infra_vm, "_health_ok", return_value=True), \ + patch.object(infra_vm, "_pidfile_alive", return_value=True), \ patch.object(infra_vm, "stop") as stop, \ patch.object(infra_vm, "ensure_built"), \ patch.object(infra_vm, "wait_for_health"), \ - patch.object(infra_vm, "boot") as boot: - boot.return_value = infra_vm.InfraVm( + patch.object(infra_vm, "boot_orchestrator") as boot_o, \ + patch.object(infra_vm, "boot_gateway") as boot_g: + boot_o.return_value = infra_vm.InfraVm( guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock()) + boot_g.return_value = infra_vm.InfraVm( + guest_ip="10.243.255.3", private_key=Path("/k"), vm=MagicMock()) infra_vm.ensure_running() - stop.assert_called_once() # dislodge the outdated VM - boot.assert_called_once() + stop.assert_called_once() # dislodge the outdated pair + boot_o.assert_called_once() + boot_g.assert_called_once() + # The gateway is booted pointing at the orchestrator's guest IP. + self.assertEqual("10.243.255.1", boot_g.call_args.args[0]) # The fresh boot records the current version for the next launcher. self.assertEqual("v-current\n", (d / "booted-version").read_text()) - def test_boots_when_unhealthy(self): + def test_reboots_when_gateway_dead(self): + # Orchestrator healthy + marker current, but the gateway VM is gone -> + # the pair is half-down, so reboot both rather than adopt partial state. + import tempfile + with tempfile.TemporaryDirectory() as td: + d = Path(td) + (d / "id_ed25519").write_text("k") + (d / "booted-version").write_text("v-current\n") + with patch.object(infra_vm, "_infra_dir", return_value=d), \ + patch.object(infra_vm, "_expected_version", return_value="v-current"), \ + patch.object(infra_vm, "_health_ok", return_value=True), \ + patch.object(infra_vm, "_pidfile_alive", return_value=False), \ + patch.object(infra_vm, "stop") as stop, \ + patch.object(infra_vm, "ensure_built"), \ + patch.object(infra_vm, "wait_for_health"), \ + patch.object(infra_vm, "boot_orchestrator") as boot_o, \ + patch.object(infra_vm, "boot_gateway") as boot_g: + boot_o.return_value = infra_vm.InfraVm( + guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock()) + boot_g.return_value = infra_vm.InfraVm( + guest_ip="10.243.255.3", private_key=Path("/k"), vm=MagicMock()) + infra_vm.ensure_running() + stop.assert_called_once() + boot_o.assert_called_once() + boot_g.assert_called_once() + + def test_boots_both_when_unhealthy(self): import tempfile with tempfile.TemporaryDirectory() as td: with patch.object(infra_vm, "_infra_dir", return_value=Path(td)), \ @@ -242,37 +384,71 @@ class TestEnsureRunningSingleton(unittest.TestCase): patch.object(infra_vm, "_health_ok", return_value=False), \ patch.object(infra_vm, "stop") as stop, \ patch.object(infra_vm, "ensure_built") as built, \ - patch.object(infra_vm, "boot") as boot, \ + patch.object(infra_vm, "boot_orchestrator") as boot_o, \ + patch.object(infra_vm, "boot_gateway") as boot_g, \ patch.object(infra_vm, "wait_for_health") as wait: - boot.return_value = infra_vm.InfraVm( + boot_o.return_value = infra_vm.InfraVm( guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock()) + boot_g.return_value = infra_vm.InfraVm( + guest_ip="10.243.255.3", private_key=Path("/k"), vm=MagicMock()) infra_vm.ensure_running() - stop.assert_called_once() # clear a stale VM first + stop.assert_called_once() # clear stale VMs first built.assert_called_once() - boot.assert_called_once() + # Orchestrator boots + becomes healthy BEFORE the gateway (its daemons + # reach the control plane at startup). + boot_o.assert_called_once() wait.assert_called_once() + boot_g.assert_called_once() + + +class TestStop(unittest.TestCase): + def test_stops_both_vms_and_drops_marker(self): + import tempfile + with tempfile.TemporaryDirectory() as td: + d = Path(td) + (d / "booted-version").write_text("v\n") + with patch.object(infra_vm, "_infra_dir", return_value=d), \ + patch.object(infra_vm, "_orch_dir", return_value=d / "orchestrator"), \ + patch.object(infra_vm, "_gw_dir", return_value=d / "gateway"), \ + patch.object(infra_vm, "_kill_pidfile") as kill, \ + patch.object(infra_vm, "_kill_infra_firecrackers"): + infra_vm.stop() + # Both VMs' pidfiles are reaped, and the version marker is gone so a + # stopped pair is never adopted. + self.assertEqual(2, kill.call_count) + self.assertFalse((d / "booted-version").exists()) class TestKillPidfile(unittest.TestCase): def test_noop_when_no_pidfile(self): import tempfile with tempfile.TemporaryDirectory() as td: - with patch.object(infra_vm, "_pid_file", return_value=Path(td) / "vm.pid"), \ - patch.object(infra_vm.os, "kill") as kill: - infra_vm._kill_pidfile() # must not raise + with patch.object(infra_vm.os, "kill") as kill: + infra_vm._kill_pidfile(Path(td)) # no vm.pid -> must not raise kill.assert_not_called() def test_skips_dead_or_recycled_pid(self): import tempfile with tempfile.TemporaryDirectory() as td: - pidf = Path(td) / "vm.pid" - pidf.write_text("999999") # a PID that isn't a live firecracker - with patch.object(infra_vm, "_pid_file", return_value=pidf), \ - patch.object(infra_vm.os, "kill") as kill: - infra_vm._kill_pidfile() + (Path(td) / "vm.pid").write_text("999999") # not a live firecracker + with patch.object(infra_vm.os, "kill") as kill: + infra_vm._kill_pidfile(Path(td)) kill.assert_not_called() +class TestPidfileAlive(unittest.TestCase): + def test_false_when_no_pidfile(self): + import tempfile + with tempfile.TemporaryDirectory() as td: + self.assertFalse(infra_vm._pidfile_alive(Path(td))) + + def test_false_when_pid_not_firecracker(self): + import tempfile + with tempfile.TemporaryDirectory() as td: + (Path(td) / "vm.pid").write_text("999999") + self.assertFalse(infra_vm._pidfile_alive(Path(td))) + + class TestAdoptable(unittest.TestCase): def _dir(self, td: str, *, key: bool = True, version: str | None = None) -> Path: d = Path(td) @@ -282,12 +458,13 @@ class TestAdoptable(unittest.TestCase): (d / "booted-version").write_text(version + "\n") return d - def test_true_when_key_version_and_health(self): + def test_true_when_key_version_health_and_gateway_alive(self): import tempfile with tempfile.TemporaryDirectory() as td: d = self._dir(td, version="v1") with patch.object(infra_vm, "_infra_dir", return_value=d), \ - patch.object(infra_vm, "_health_ok", return_value=True): + patch.object(infra_vm, "_health_ok", return_value=True), \ + patch.object(infra_vm, "_pidfile_alive", return_value=True): self.assertTrue(infra_vm._adoptable(d / "id_ed25519", "u", "v1")) def test_false_when_key_missing(self): @@ -309,7 +486,17 @@ class TestAdoptable(unittest.TestCase): with tempfile.TemporaryDirectory() as td: d = self._dir(td, version="v-old") with patch.object(infra_vm, "_infra_dir", return_value=d), \ - patch.object(infra_vm, "_health_ok", return_value=True): + patch.object(infra_vm, "_health_ok", return_value=True), \ + patch.object(infra_vm, "_pidfile_alive", return_value=True): + self.assertFalse(infra_vm._adoptable(d / "id_ed25519", "u", "v1")) + + def test_false_when_gateway_dead(self): + import tempfile + with tempfile.TemporaryDirectory() as td: + d = self._dir(td, version="v1") + with patch.object(infra_vm, "_infra_dir", return_value=d), \ + patch.object(infra_vm, "_health_ok", return_value=True), \ + patch.object(infra_vm, "_pidfile_alive", return_value=False): self.assertFalse(infra_vm._adoptable(d / "id_ed25519", "u", "v1")) @@ -320,26 +507,31 @@ class TestKillInfraFirecrackers(unittest.TestCase): (p / "comm").write_text(comm + "\n") (p / "cmdline").write_bytes(b"\0".join(a.encode() for a in cmdline) + b"\0") - def test_kills_only_matching_infra_firecracker(self): + def test_kills_both_infra_firecrackers_only(self): import tempfile with tempfile.TemporaryDirectory() as td, \ tempfile.TemporaryDirectory() as proc: infra_dir = Path(td) - cfg = str(infra_dir / "config.json") + orch_cfg = str(infra_dir / "orchestrator" / "config.json") + gw_cfg = str(infra_dir / "gateway" / "config.json") root = Path(proc) - # target: firecracker bound to the infra config -> killed + # both infra VMs -> killed self._fake_proc(root, 111, "firecracker", - ["firecracker", "--no-api", "--config-file", cfg]) + ["firecracker", "--no-api", "--config-file", orch_cfg]) + self._fake_proc(root, 112, "firecracker", + ["firecracker", "--no-api", "--config-file", gw_cfg]) # a firecracker for a different (interactive) VM -> spared self._fake_proc(root, 222, "firecracker", ["firecracker", "--config-file", "/home/u/other.json"]) - # a non-firecracker process on the same config path -> spared - self._fake_proc(root, 333, "python3", ["python3", cfg]) + # a non-firecracker process on an infra config path -> spared + self._fake_proc(root, 333, "python3", ["python3", orch_cfg]) (root / "not-a-pid").mkdir() - with patch.object(infra_vm, "_infra_dir", return_value=infra_dir), \ + with patch.object(infra_vm, "_orch_dir", return_value=infra_dir / "orchestrator"), \ + patch.object(infra_vm, "_gw_dir", return_value=infra_dir / "gateway"), \ patch.object(infra_vm.os, "kill") as kill: infra_vm._kill_infra_firecrackers(proc_root=root) - kill.assert_called_once_with(111, infra_vm.signal.SIGKILL) + killed = {c.args[0] for c in kill.call_args_list} + self.assertEqual({111, 112}, killed) if __name__ == "__main__":