"""The Firecracker gateway data plane as a microVM (PRD 0070). `FirecrackerGateway` is the Firecracker implementation of the backend-neutral `Gateway` service, and owns the gateway's host-side logic directly: booting the gateway microVM on its NAT'd link, seeding the pre-minted `gateway` token, fetching the mitmproxy CA, and the SSH exec/cp provisioning transport. The gateway VM never opens `bot-bottle.db` (#469), so it holds no signing key — only the token the host hands it via `connect_to_orchestrator`. The plane-agnostic VM substrate the orchestrator VM also uses stays in `infra_vm` — booting a VM from a per-plane rootfs (`boot_vm`), the stable SSH keypair, the secret-push retry loop, the PID lifecycle, and the adoption/version helpers. The guest-side gateway daemon startup lives in the gateway rootfs's guest init (`_gateway_init` in `infra_vm`), so it can't live in a host-side method either. The pair coordinator (orchestrator-first, under a singleton flock) is `FirecrackerInfraService` (`infra.py`). """ from __future__ import annotations import subprocess from pathlib import Path from urllib.parse import urlparse from ...gateway import ( DEFAULT_CA_TIMEOUT_SECONDS, Gateway, GatewayError, GatewayTransport, ) from ...log import die, info from .. import util as backend_util from . import infra_vm, netpool, util from .gateway_transport import FirecrackerGatewayTransport # The gateway microVM's name (its run dir). Fixed per host — one gateway VM, # shared by every agent VM. GATEWAY_NAME = "bot-bottle-gateway" # The gateway VM's slim memory ceiling — the data plane carries no build # tooling (buildah lives only on the orchestrator rootfs) — PRD 0070 # "Memory: fixed ceilings". _GW_MEM_MIB = 2048 # The pre-minted `gateway` JWT path in the guest. The gateway init (in # `infra_vm`) waits for it before starting the data plane, so its canonical # definition lives with that init; imported here for the push so the # load-bearing path isn't duplicated. _GUEST_GATEWAY_JWT_PATH = infra_vm._GUEST_GATEWAY_JWT_PATH # mitmproxy writes its CA here a beat after start; agents install it to trust # the gateway's TLS interception. Host-side only (SSH cat), so it lives here. _GATEWAY_CA_PATH = "/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem" # The gateway VM's persistent CA volume — a small ext4 file the gateway init # mounts at mitmproxy's confdir (see `infra_vm._GATEWAY_CA_MOUNT`) so the # self-generated CA SURVIVES a gateway-VM rebuild/restart. Without it the CA # lives only in the ephemeral per-boot rootfs, so every rebuild mints a fresh CA # that every already-running bottle distrusts, failing the TLS handshake (the # firecracker analogue of the docker fix's persistent CA bind-mount — issue # #450). Co-located with the orchestrator's registry volume under the infra dir, # which outlives the ephemeral rootfs. mitmproxy is tiny; 16M is ample. _CA_VOLUME_SIZE = "16M" # The gateway VM's persistent git-gate volume — a (sparse) ext4 file the gateway # init mounts, then bind-mounts onto /git + /git-gate/creds (see # `infra_vm._GATEWAY_GIT_MOUNT`), so per-bottle bare repos and deploy creds # SURVIVE a gateway-VM rebuild. Without it a restart drops every already-running # bottle's git-gate state and its agent 404s on fetch/push (same class as the CA # — issue #512). Bare repos hold upstream history, so this is sized far larger # than the CA volume; mke2fs leaves the file sparse, so the host only stores # blocks actually used. _GIT_VOLUME_SIZE = "8G" _CA_FETCH_TIMEOUT_SECONDS = 15.0 class FirecrackerGateway(Gateway): """The consolidated gateway as a Firecracker microVM on the gateway link. The gateway rootfs is built/downloaded by `infra_vm.ensure_built` (the ABC's `ensure_built` no-op here); `connect_to_orchestrator` boots the VM resolving policy against the orchestrator and seeds the pre-minted `gateway` token.""" name = GATEWAY_NAME def __init__(self, vm: infra_vm.InfraVm | None = None) -> None: # The live VM handle when this process booted it; None when adapting an # already-running gateway (CA fetch / provisioning go through the stable # key + fixed link, so no live handle is needed). self._vm = vm self._orchestrator_url = "" self._gateway_token = "" def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None: """Boot the gateway VM on its link resolving policy against `orchestrator_url`, then seed `gateway_token` over SSH. The orchestrator's guest IP is parsed from the URL and passed on the cmdline (`bb_orch=`), so the baked init stays IP-independent. Boot the orchestrator first — the gateway daemons reach it at startup. The gateway never mints, so it holds no signing key, only this token (#469).""" self._orchestrator_url = orchestrator_url self._gateway_token = gateway_token if not self._orchestrator_url: raise GatewayError( "gateway requires an orchestrator URL to run " "(resolver-only data plane; no single-tenant fallback)" ) if not self._gateway_token: raise GatewayError( "gateway requires a pre-minted `gateway` token to run " "(the orchestrator mints it; the gateway never holds the key)" ) orchestrator_guest_ip = urlparse(self._orchestrator_url).hostname or "" if not orchestrator_guest_ip: raise GatewayError( f"cannot resolve orchestrator guest IP from {self._orchestrator_url!r}" ) # Boot on the gateway link from the gateway rootfs, then push the token # the init waits for before starting the data plane. Two persistent # volumes ride along, attached in a FIXED order the gateway init depends # on: the CA volume as /dev/vdb (mounted at mitmproxy's confdir) and the # git-gate volume as /dev/vdc (bind-mounted onto /git + /git-gate/creds). # Both keep gateway-side state STABLE across rebuilds so already-running # bottles keep working — TLS interception (issue #450) and git-gate fetch # /push (issue #512) respectively. vm = infra_vm.boot_vm( name=GATEWAY_NAME, slot=netpool.gw_slot(), run_dir=infra_vm._gw_dir(), role="gateway", mem_mib=_GW_MEM_MIB, extra_boot_args=f"bb_orch={orchestrator_guest_ip}", data_drives=(self._ensure_ca_volume(), self._ensure_git_volume()), ) infra_vm.push_secret( vm, self._gateway_token, _GUEST_GATEWAY_JWT_PATH, "the gateway JWT to the gateway VM (its data plane will not start)", ) self._vm = vm def is_running(self) -> bool: return infra_vm._pidfile_alive(infra_vm._gw_dir()) def stop(self) -> None: """Stop only the gateway VM (idempotent — absent is success). A dead gateway already fails `infra_vm.adoptable`, so no version marker to clear here.""" infra_vm._kill_pidfile(infra_vm._gw_dir()) infra_vm._pid_file(infra_vm._gw_dir()).unlink(missing_ok=True) def _ensure_ca_volume(self) -> Path: """Create the empty ext4 CA volume on first use; reuse it after. A fresh (empty) volume makes mitmproxy generate a CA into it on first boot; every later boot reuses the CA already on the volume — which is what keeps the CA stable across gateway-VM rebuilds. The mirror of `FirecrackerOrchestrator._ensure_registry_volume`.""" vol = infra_vm._gw_dir() / "gateway-ca.ext4" if vol.exists(): return vol info(f"creating gateway CA volume {vol} ({_CA_VOLUME_SIZE})") proc = subprocess.run( ["mke2fs", "-q", "-t", "ext4", "-F", str(vol), _CA_VOLUME_SIZE], capture_output=True, text=True, check=False, ) if proc.returncode != 0: vol.unlink(missing_ok=True) die(f"creating gateway CA volume failed: {proc.stderr.strip()}") return vol def _ensure_git_volume(self) -> Path: """Create the empty ext4 git-gate volume on first use; reuse it after. Empty on first boot (the gateway init lays out `git/` + `creds/` subdirs and bind-mounts them); every later boot reuses whatever repos + creds the volume already holds — which is what keeps git-gate state stable across a gateway-VM rebuild (issue #512). Sibling of `_ensure_ca_volume`.""" vol = infra_vm._gw_dir() / "gateway-git.ext4" if vol.exists(): return vol info(f"creating gateway git-gate volume {vol} ({_GIT_VOLUME_SIZE})") proc = subprocess.run( ["mke2fs", "-q", "-t", "ext4", "-F", str(vol), _GIT_VOLUME_SIZE], capture_output=True, text=True, check=False, ) if proc.returncode != 0: vol.unlink(missing_ok=True) die(f"creating gateway git-gate volume failed: {proc.stderr.strip()}") return vol def address(self) -> str: """The gateway VM's guest IP — the agent-facing target agent VMs' gateway-port traffic is DNAT'd to.""" return netpool.gw_slot().guest_ip def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str: """The gateway's mitmproxy CA (PEM) agents install to trust its TLS interception. Fetched from the gateway VM over SSH; polls because mitmproxy writes it a beat after boot. Works from any later launcher — falls back to the stable key + fixed link when this process didn't boot the VM.""" key = self._vm.private_key if self._vm else infra_vm._infra_dir() / "id_ed25519" ip = self._vm.guest_ip if self._vm else netpool.gw_slot().guest_ip def _fetch() -> str | None: proc = subprocess.run( util.ssh_base_argv(key, ip) + [f"cat {_GATEWAY_CA_PATH}"], capture_output=True, text=True, timeout=_CA_FETCH_TIMEOUT_SECONDS, check=False, ) ok = proc.returncode == 0 and "BEGIN CERTIFICATE" in proc.stdout return proc.stdout if ok else None try: return backend_util.poll_ca_cert(_fetch, timeout=timeout) except TimeoutError as exc: raise GatewayError(str(exc)) from exc def provisioning_transport(self) -> GatewayTransport: """The exec/cp transport git-gate provisioning stages per-bottle repos + deploy keys through (over SSH to the gateway VM). Needs no live handle — built from the stable key + the gateway link's guest IP, so teardown can use it too.""" return FirecrackerGatewayTransport( infra_vm._infra_dir() / "id_ed25519", netpool.gw_slot().guest_ip) __all__ = ["FirecrackerGateway", "FirecrackerGatewayTransport", "GATEWAY_NAME"]