diff --git a/bot_bottle/backend/firecracker/consolidated_launch.py b/bot_bottle/backend/firecracker/consolidated_launch.py index 4868bc7..e2b46d5 100644 --- a/bot_bottle/backend/firecracker/consolidated_launch.py +++ b/bot_bottle/backend/firecracker/consolidated_launch.py @@ -130,9 +130,8 @@ def launch_consolidated( client = OrchestratorClient(url) _reprovision_running_bottles(client) - # Read the gateway's provisioning transport + CA off the Gateway service - # (adapting the running gateway VM). - gateway = FirecrackerGateway(infra.gateway) + # Read the gateway's provisioning transport + CA off the Gateway service. + gateway = infra.gateway transport = gateway.provisioning_transport() reg = provision_bottle( client, guest_ip, egress_plan, git_gate_plan, transport, diff --git a/bot_bottle/backend/firecracker/gateway.py b/bot_bottle/backend/firecracker/gateway.py index 5fa977c..23a0778 100644 --- a/bot_bottle/backend/firecracker/gateway.py +++ b/bot_bottle/backend/firecracker/gateway.py @@ -1,38 +1,100 @@ """The Firecracker gateway data plane as a microVM (PRD 0070). `FirecrackerGateway` is the Firecracker implementation of the backend-neutral -`Gateway` service. The gateway here is a whole microVM — the slim data-plane -unit (mitmproxy TLS bump + DLP + git-http) booted on its own NAT'd link; agent -VMs' gateway-port traffic is DNAT'd to it and it never opens `bot-bottle.db` -(#469), so it holds no signing key, only the pre-minted `gateway` token the -host hands it via `connect_to_orchestrator`. +`Gateway` service, and owns the gateway's host-side logic directly: booting the +gateway microVM on its NAT'd link, seeding the pre-minted `gateway` token, +fetching the mitmproxy CA, and the SSH exec/cp provisioning transport. The +gateway VM never opens `bot-bottle.db` (#469), so it holds no signing key — +only the token the host hands it via `connect_to_orchestrator`. -The VM lifecycle primitives (boot, SSH secret push, netpool links, nft, PID -files) stay in `infra_vm`, which drives the orchestrator+gateway pair as a -singleton; this adapter composes them into the uniform `Gateway` surface. +What deliberately stays in `infra_vm` (not moved here): the plane-agnostic VM +substrate the orchestrator VM shares — booting a VM from the shared rootfs +(`_boot_vm`), the stable SSH keypair, the secret-push retry loop, and the PID +lifecycle — plus the pair coordinator (`ensure_running`: orchestrator-first +health gate, singleton lock, adoption/version marker). The guest-side gateway +daemon startup lives in the shared `bb_role=gateway` init branch baked into the +one published rootfs both VMs boot, so it can't live in a host-side method +either. These are the shared seam that moves to a neutral module when the +Orchestrator service lands and `infra_vm` is dissolved. """ from __future__ import annotations +import os +import shlex +import stat +import subprocess +from pathlib import Path from urllib.parse import urlparse from ...gateway import ( DEFAULT_CA_TIMEOUT_SECONDS, Gateway, GatewayError, + GatewayProvisionError, GatewayTransport, ) -from . import infra_vm +from .. import util as backend_util +from . import infra_vm, netpool, util # The gateway microVM's name (the `bb_role=gateway` boot tag / run dir). Fixed # per host — one gateway VM, shared by every agent VM. GATEWAY_NAME = "bot-bottle-gateway" +# The gateway VM's slim memory ceiling (buildah is present on the shared rootfs +# but unused on the data plane) — PRD 0070 "Memory: fixed ceilings". +_GW_MEM_MIB = 2048 + +# The pre-minted `gateway` JWT path in the guest. The *shared* init (both planes +# boot one rootfs) waits for it before starting the data plane, so its canonical +# definition lives with that init in `infra_vm`; imported here for the push so +# the load-bearing path isn't duplicated. +_GUEST_GATEWAY_JWT_PATH = infra_vm._GUEST_GATEWAY_JWT_PATH +# mitmproxy writes its CA here a beat after start; agents install it to trust +# the gateway's TLS interception. Host-side only (SSH cat), so it lives here. +_GATEWAY_CA_PATH = "/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem" + +_CA_FETCH_TIMEOUT_SECONDS = 15.0 + + +class SshGatewayTransport: + """`GatewayTransport` for the gateway running in the gateway VM — the docker + exec/cp equivalents over SSH (dropbear + the stable infra key).""" + + def __init__(self, private_key: Path, guest_ip: str) -> None: + self._key = private_key + self._ip = guest_ip + + def exec(self, argv: list[str]) -> None: + proc = subprocess.run( + util.ssh_base_argv(self._key, self._ip) + [shlex.join(argv)], + capture_output=True, text=True, timeout=60, check=False, + ) + if proc.returncode != 0: + raise GatewayProvisionError( + f"infra gateway exec {argv!r} failed: {proc.stderr.strip()}") + + def cp_into(self, src: str, dest: str) -> None: + # Preserve the source mode (docker cp does): the access-hook is staged + # 0700 and git-http execs it directly — a plain `cat >` would land it + # 0644 and the exec fails with EACCES; keys stay 0600. + mode = stat.S_IMODE(os.stat(src).st_mode) + q = shlex.quote(dest) + proc = subprocess.run( + util.ssh_base_argv(self._key, self._ip) + + [f"cat > {q} && chmod {mode:o} {q}"], + input=Path(src).read_bytes(), capture_output=True, timeout=30, check=False, + ) + if proc.returncode != 0: + raise GatewayProvisionError( + f"infra gateway cp {src} -> {dest} failed: " + f"{proc.stderr.decode(errors='replace').strip()}") + class FirecrackerGateway(Gateway): """The consolidated gateway as a Firecracker microVM on the gateway link. - The rootfs is built/downloaded by `infra_vm.ensure_built` (the ABC's + The shared rootfs is built/downloaded by `infra_vm.ensure_built` (the ABC's `ensure_built` no-op here); `connect_to_orchestrator` boots the VM resolving policy against the orchestrator and seeds the pre-minted `gateway` token.""" @@ -47,10 +109,12 @@ class FirecrackerGateway(Gateway): self._gateway_token = "" def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None: - """Boot the gateway VM resolving policy against `orchestrator_url` and - seed `gateway_token`. The orchestrator's guest IP is parsed from the URL - and passed on the cmdline, so the baked init stays IP-independent. Boot - the orchestrator first — the gateway daemons reach it at startup.""" + """Boot the gateway VM on its link resolving policy against + `orchestrator_url`, then seed `gateway_token` over SSH. The orchestrator's + guest IP is parsed from the URL and passed on the cmdline (`bb_orch=`), so + the baked init stays IP-independent. Boot the orchestrator first — the + gateway daemons reach it at startup. The gateway never mints, so it holds + no signing key, only this token (#469).""" self._orchestrator_url = orchestrator_url self._gateway_token = gateway_token if not self._orchestrator_url: @@ -68,31 +132,64 @@ class FirecrackerGateway(Gateway): raise GatewayError( f"cannot resolve orchestrator guest IP from {self._orchestrator_url!r}" ) - self._vm = infra_vm.boot_gateway(orchestrator_guest_ip, self._gateway_token) + # Boot on the gateway link from the shared rootfs (bb_role=gateway), then + # push the token the init waits for before starting the data plane. + vm = infra_vm._boot_vm( + name=GATEWAY_NAME, slot=netpool.gw_slot(), run_dir=infra_vm._gw_dir(), + role="gateway", mem_mib=_GW_MEM_MIB, + extra_boot_args=f"bb_orch={orchestrator_guest_ip}", + ) + infra_vm._push_secret( + vm, self._gateway_token, _GUEST_GATEWAY_JWT_PATH, + "the gateway JWT to the gateway VM (its data plane will not start)", + ) + self._vm = vm def is_running(self) -> bool: - return infra_vm.gateway_running() + return infra_vm._pidfile_alive(infra_vm._gw_dir()) def stop(self) -> None: - """Stop only the gateway VM (idempotent).""" - infra_vm.stop_gateway() + """Stop only the gateway VM (idempotent — absent is success). A dead + gateway already fails `infra_vm._adoptable`, so no version marker to + clear here.""" + infra_vm._kill_pidfile(infra_vm._gw_dir()) + infra_vm._pid_file(infra_vm._gw_dir()).unlink(missing_ok=True) def address(self) -> str: """The gateway VM's guest IP — the agent-facing target agent VMs' gateway-port traffic is DNAT'd to.""" - return infra_vm.gateway_guest_ip() + return netpool.gw_slot().guest_ip def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str: """The gateway's mitmproxy CA (PEM) agents install to trust its TLS interception. Fetched from the gateway VM over SSH; polls because - mitmproxy writes it a beat after boot.""" - vm = self._vm or infra_vm.adopted_gateway_vm() - return vm.gateway_ca_pem(timeout=timeout) + mitmproxy writes it a beat after boot. Works from any later launcher — + falls back to the stable key + fixed link when this process didn't boot + the VM.""" + key = self._vm.private_key if self._vm else infra_vm._infra_dir() / "id_ed25519" + ip = self._vm.guest_ip if self._vm else netpool.gw_slot().guest_ip + + def _fetch() -> str | None: + proc = subprocess.run( + util.ssh_base_argv(key, ip) + [f"cat {_GATEWAY_CA_PATH}"], + capture_output=True, text=True, + timeout=_CA_FETCH_TIMEOUT_SECONDS, check=False, + ) + ok = proc.returncode == 0 and "BEGIN CERTIFICATE" in proc.stdout + return proc.stdout if ok else None + + try: + return backend_util.poll_ca_cert(_fetch, timeout=timeout) + except TimeoutError as exc: + raise GatewayError(str(exc)) from exc def provisioning_transport(self) -> GatewayTransport: """The exec/cp transport git-gate provisioning stages per-bottle repos + - deploy keys through (over SSH to the gateway VM).""" - return infra_vm.gateway_transport() + deploy keys through (over SSH to the gateway VM). Needs no live handle — + built from the stable key + the gateway link's guest IP, so teardown can + use it too.""" + return SshGatewayTransport( + infra_vm._infra_dir() / "id_ed25519", netpool.gw_slot().guest_ip) -__all__ = ["FirecrackerGateway", "GATEWAY_NAME"] +__all__ = ["FirecrackerGateway", "SshGatewayTransport", "GATEWAY_NAME"] diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py index 1c0bf66..420f1ca 100644 --- a/bot_bottle/backend/firecracker/infra_vm.py +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -32,9 +32,7 @@ from __future__ import annotations import fcntl import hashlib import os -import shlex import signal -import stat import subprocess import time import urllib.error @@ -42,16 +40,17 @@ import urllib.request from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path -from typing import Generator +from typing import TYPE_CHECKING, Generator from ...log import die, info from ...orchestrator_auth import ROLE_GATEWAY, mint from ...paths import host_orchestrator_token -from .. import util as backend_util from ..docker import util as docker_mod -from ...gateway import GatewayProvisionError from . import firecracker_vm, infra_artifact, netpool, util +if TYPE_CHECKING: + from .gateway import FirecrackerGateway + # The orchestrator VM's signing-key path on its persistent /dev/vdb volume # (mounted at BOT_BOTTLE_ROOT=/var/lib/bot-bottle). The launcher seeds this # file with the host-canonical key AFTER boot (over SSH), so the VM verifies @@ -77,16 +76,11 @@ ORCHESTRATOR_PORT = 8099 EGRESS_PORT = 9099 SUPERVISE_PORT = 9100 GIT_HTTP_PORT = 9420 -# mitmproxy writes its CA here a beat after start; agents install it to trust -# the gateway's TLS interception. -_GATEWAY_CA_PATH = "/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem" - -# Memory ceilings (fixed at boot, demand-paged — no balloon reclaim). The +# Memory ceiling (fixed at boot, demand-paged — no balloon reclaim). The # orchestrator keeps the build headroom (buildah's 2-4 GB working set during -# in-VM agent builds); the gateway is the slim unit (mitmproxy TLS bump + DLP -# body buffering, ~1 GB) — PRD 0070 "Memory: fixed ceilings". +# in-VM agent builds); the gateway's slimmer ceiling is set by the gateway +# service — PRD 0070 "Memory: fixed ceilings". _ORCH_MEM_MIB = 4096 -_GW_MEM_MIB = 2048 # The infra VMs make direct upstream connections (gateway egress, and buildah # during builds), and the kernel `ip=` cmdline sets no resolver. Public for @@ -117,41 +111,23 @@ class InfraVm: def orchestrator_url(self) -> str: return f"http://{self.guest_ip}:{ORCHESTRATOR_PORT}" - def gateway_ca_pem(self, *, timeout: float = _CA_TIMEOUT_SECONDS) -> str: - """The gateway's mitmproxy CA (PEM) that agents install to trust its - TLS interception. Generated a moment after boot, so this polls over - SSH until it appears (mirrors DockerGateway.ca_cert_pem). Call on the - gateway VM handle.""" - def _fetch() -> str | None: - proc = subprocess.run( - util.ssh_base_argv(self.private_key, self.guest_ip) - + [f"cat {_GATEWAY_CA_PATH}"], - capture_output=True, text=True, timeout=15, check=False, - ) - ok = proc.returncode == 0 and "BEGIN CERTIFICATE" in proc.stdout - return proc.stdout if ok else None - try: - return backend_util.poll_ca_cert(_fetch, timeout=timeout) - except TimeoutError as exc: - die(str(exc)) - @dataclass class InfraEndpoint: """How to reach the running per-host pair. `orchestrator` is the control - plane (host CLI + registration + in-VM builds); `gateway` is the data - plane (agent egress / git-http / supervise, CA fetch, git-gate + plane (host CLI + registration + in-VM builds); `gateway` is the data-plane + `Gateway` service (agent egress / git-http / supervise, CA fetch, git-gate provisioning). Mirrors the docker/macOS `InfraEndpoint` shape.""" orchestrator: InfraVm - gateway: InfraVm + gateway: "FirecrackerGateway" @property def orchestrator_url(self) -> str: return self.orchestrator.orchestrator_url def gateway_ca_pem(self, *, timeout: float = _CA_TIMEOUT_SECONDS) -> str: - return self.gateway.gateway_ca_pem(timeout=timeout) + return self.gateway.ca_cert_pem(timeout=timeout) def ensure_built() -> None: @@ -229,22 +205,28 @@ def ensure_running() -> InfraEndpoint: orchestrator = boot_orchestrator() wait_for_health(orchestrator) # The orchestrator holds the signing key; mint the role-scoped `gateway` - # JWT on the host and hand it to the gateway VM, which never sees the key - # (#469). + # JWT on the host and hand it to the gateway service, which never sees + # the key (#469). The gateway owns its own boot — see FirecrackerGateway. + # Local import: the gateway service imports this module for the shared VM + # substrate, so importing it at module scope would cycle. + from .gateway import FirecrackerGateway gateway_token = mint(ROLE_GATEWAY, host_orchestrator_token()) - gateway = boot_gateway(orchestrator.guest_ip, gateway_token) + gateway = FirecrackerGateway() + gateway.connect_to_orchestrator(orchestrator.orchestrator_url, gateway_token) _record_booted_version(want) return InfraEndpoint(orchestrator=orchestrator, gateway=gateway) def _adopt(key: Path) -> InfraEndpoint: """Build handles to the already-running pair from the stable key + the - fixed links (vm=None — teardown then goes through the PID files).""" + fixed links (no live VM handle — teardown / CA fetch go through the PID + files + stable key).""" + from .gateway import FirecrackerGateway return InfraEndpoint( orchestrator=InfraVm( guest_ip=netpool.orch_slot().guest_ip, private_key=key), - gateway=InfraVm( - guest_ip=netpool.gw_slot().guest_ip, private_key=key), + gateway=FirecrackerGateway( + InfraVm(guest_ip=netpool.gw_slot().guest_ip, private_key=key)), ) @@ -294,25 +276,6 @@ def boot_orchestrator() -> InfraVm: return orch -def boot_gateway(orchestrator_guest_ip: str, gateway_token: str) -> InfraVm: - """Boot the gateway (data-plane) VM (detached) on the gateway link, and - seed the pre-minted `gateway` JWT `gateway_token`. Its daemons resolve - policy against the orchestrator at `orchestrator_guest_ip:8099` (passed on - the cmdline, so the baked init stays IP-independent). Boot the orchestrator - first — the gateway daemons reach it at startup. The gateway never holds the - signing key, only this token (#469). Prefer `ensure_running`.""" - slot = netpool.gw_slot() - gateway = _boot_vm( - name="bot-bottle-gateway", slot=slot, run_dir=_gw_dir(), - role="gateway", mem_mib=_GW_MEM_MIB, - extra_boot_args=f"bb_orch={orchestrator_guest_ip}", - ) - # Push the pre-minted `gateway` JWT (never the signing key — issue #469). - # The init waits for it before starting the data plane. - _push_gateway_jwt(gateway, gateway_token) - return gateway - - def _boot_vm( *, name: str, @@ -456,18 +419,6 @@ def _push_signing_key(infra: InfraVm) -> None: ) -def _push_gateway_jwt(infra: InfraVm, gateway_token: str) -> None: - """Push the pre-minted `gateway` JWT into the freshly booted gateway VM - over SSH (atomic write). Minted on the HOST from the canonical signing key - so the data plane never holds the key itself (issue #469); the gateway - daemons present it to the orchestrator. Retries until SSH is up; dies if it - never lands, since the VM then refuses to start the data plane.""" - _push_secret( - infra, gateway_token, _GUEST_GATEWAY_JWT_PATH, - "the gateway JWT to the gateway VM (its data plane will not start)", - ) - - def _push_secret(infra: InfraVm, secret: str, dest: str, what: str) -> None: """Pipe `secret` to an atomic write of `dest` in the guest over SSH, retrying while the guest's SSH comes up; die (naming `what`) if it never @@ -574,77 +525,6 @@ def _health_ok(url: str) -> bool: return False -class SshGatewayTransport: - """`GatewayTransport` for the gateway running in the gateway VM — the - docker exec/cp equivalents over SSH (dropbear + the stable infra key).""" - - def __init__(self, private_key: Path, guest_ip: str) -> None: - self._key = private_key - self._ip = guest_ip - - def exec(self, argv: list[str]) -> None: - proc = subprocess.run( - util.ssh_base_argv(self._key, self._ip) + [shlex.join(argv)], - capture_output=True, text=True, timeout=60, check=False, - ) - if proc.returncode != 0: - raise GatewayProvisionError( - f"infra gateway exec {argv!r} failed: {proc.stderr.strip()}") - - def cp_into(self, src: str, dest: str) -> None: - # Preserve the source mode (docker cp does): the access-hook is staged - # 0700 and git-http execs it directly — a plain `cat >` would land it - # 0644 and the exec fails with EACCES; keys stay 0600. - mode = stat.S_IMODE(os.stat(src).st_mode) - q = shlex.quote(dest) - proc = subprocess.run( - util.ssh_base_argv(self._key, self._ip) - + [f"cat > {q} && chmod {mode:o} {q}"], - input=Path(src).read_bytes(), capture_output=True, timeout=30, check=False, - ) - if proc.returncode != 0: - raise GatewayProvisionError( - f"infra gateway cp {src} -> {dest} failed: " - f"{proc.stderr.decode(errors='replace').strip()}") - - -def gateway_transport() -> SshGatewayTransport: - """git-gate provisioning transport for the gateway VM, built from the - stable key + the gateway link's guest IP. Needs no live VM handle, so - teardown can use it too.""" - return SshGatewayTransport( - _infra_dir() / "id_ed25519", netpool.gw_slot().guest_ip) - - -def gateway_guest_ip() -> str: - """The gateway VM's fixed guest IP on its link — the agent-facing address - agent VMs' gateway-port traffic is DNAT'd to (PRD 0070).""" - return netpool.gw_slot().guest_ip - - -def gateway_running() -> bool: - """True iff the gateway VM is still a live VMM.""" - return _pidfile_alive(_gw_dir()) - - -def stop_gateway() -> None: - """Stop only the gateway VM (idempotent — absent is success). Drops the pair - version marker so a half-stopped pair is never adopted (`_adoptable`).""" - _kill_pidfile(_gw_dir()) - _pid_file(_gw_dir()).unlink(missing_ok=True) - _version_file().unlink(missing_ok=True) - - -def adopted_gateway_vm() -> InfraVm: - """A handle to the already-running gateway VM from the stable key + the - fixed gateway link (vm=None — CA fetch / provisioning work from any later - launcher, not only the process that booted it).""" - return InfraVm( - guest_ip=netpool.gw_slot().guest_ip, - private_key=_infra_dir() / "id_ed25519", - ) - - def wait_for_health( infra: InfraVm, *, timeout: float = _HEALTH_TIMEOUT_SECONDS, ) -> None: diff --git a/tests/unit/test_firecracker_gateway.py b/tests/unit/test_firecracker_gateway.py index 4ce14db..ae1aeec 100644 --- a/tests/unit/test_firecracker_gateway.py +++ b/tests/unit/test_firecracker_gateway.py @@ -1,17 +1,26 @@ -"""Unit: the Firecracker gateway data plane as a microVM (PRD 0070).""" +"""Unit: the Firecracker gateway data plane as a microVM (PRD 0070). + +The gateway service owns its host-side logic directly: booting the gateway VM +(via the shared `infra_vm._boot_vm` substrate), seeding the pre-minted token, +fetching the CA over SSH, and the SSH exec/cp provisioning transport. +""" from __future__ import annotations +import os +import tempfile import unittest from pathlib import Path -from unittest.mock import MagicMock, patch +from subprocess import CompletedProcess +from unittest.mock import patch from bot_bottle.backend.firecracker import infra_vm from bot_bottle.backend.firecracker.gateway import ( GATEWAY_NAME, FirecrackerGateway, + SshGatewayTransport, ) -from bot_bottle.gateway import GatewayError +from bot_bottle.gateway import GatewayError, GatewayProvisionError _GW = "bot_bottle.backend.firecracker.gateway" @@ -25,76 +34,108 @@ class TestFirecrackerGatewayConnect(unittest.TestCase): def test_refuses_without_orchestrator_url(self) -> None: gw = FirecrackerGateway() - with patch.object(infra_vm, "boot_gateway") as boot: + with patch.object(infra_vm, "_boot_vm") as boot: with self.assertRaises(GatewayError): gw.connect_to_orchestrator("", _TOKEN) boot.assert_not_called() def test_refuses_without_gateway_token(self) -> None: gw = FirecrackerGateway() - with patch.object(infra_vm, "boot_gateway") as boot: + with patch.object(infra_vm, "_boot_vm") as boot: with self.assertRaises(GatewayError): gw.connect_to_orchestrator(_ORCH_URL, "") boot.assert_not_called() def test_refuses_a_url_without_a_host(self) -> None: gw = FirecrackerGateway() - with patch.object(infra_vm, "boot_gateway") as boot: + with patch.object(infra_vm, "_boot_vm") as boot: with self.assertRaises(GatewayError): gw.connect_to_orchestrator("http://:8099", _TOKEN) boot.assert_not_called() - def test_boots_the_gateway_vm_against_the_orchestrator_guest_ip(self) -> None: + def test_boots_the_gateway_vm_and_seeds_the_token(self) -> None: gw = FirecrackerGateway() booted = infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k")) - with patch.object(infra_vm, "boot_gateway", return_value=booted) as boot: + with patch.object(infra_vm, "_boot_vm", return_value=booted) as boot, \ + patch.object(infra_vm, "_push_secret") as push: gw.connect_to_orchestrator(_ORCH_URL, _TOKEN) - # The orchestrator guest IP is parsed off the URL; the token is threaded - # through unchanged (the gateway never mints — #469). - boot.assert_called_once_with("10.243.255.1", _TOKEN) + # Booted on the gateway link with the gateway role; the orchestrator's + # guest IP (parsed off the URL) rides the cmdline as bb_orch. + kw = boot.call_args.kwargs + self.assertEqual("gateway", kw["role"]) + self.assertIn("bb_orch=10.243.255.1", kw["extra_boot_args"]) + self.assertIsNone(kw.get("data_drive")) # data plane never opens the DB + # The host-minted token (never the key — #469) is pushed to the guest. + push.assert_called_once() + self.assertEqual(_TOKEN, push.call_args.args[1]) class TestFirecrackerGatewaySurface(unittest.TestCase): def test_is_running_reads_the_gateway_pidfile(self) -> None: gw = FirecrackerGateway() - with patch.object(infra_vm, "gateway_running", return_value=True): + with patch.object(infra_vm, "_pidfile_alive", return_value=True) as alive, \ + patch.object(infra_vm, "_gw_dir", return_value=Path("/gw")): self.assertTrue(gw.is_running()) - with patch.object(infra_vm, "gateway_running", return_value=False): - self.assertFalse(gw.is_running()) + alive.assert_called_once_with(Path("/gw")) - def test_stop_stops_only_the_gateway_vm(self) -> None: + def test_stop_kills_only_the_gateway_pidfile(self) -> None: gw = FirecrackerGateway() - with patch.object(infra_vm, "stop_gateway") as stop: - gw.stop() - stop.assert_called_once_with() + with tempfile.TemporaryDirectory() as td: + d = Path(td) + (d / "vm.pid").write_text("123") + with patch.object(infra_vm, "_gw_dir", return_value=d), \ + patch.object(infra_vm, "_kill_pidfile") as kill: + gw.stop() + kill.assert_called_once_with(d) + self.assertFalse((d / "vm.pid").exists()) # pidfile cleared def test_address_is_the_gateway_link_guest_ip(self) -> None: gw = FirecrackerGateway() - with patch.object(infra_vm, "gateway_guest_ip", return_value="10.243.255.3"): - self.assertEqual("10.243.255.3", gw.address()) + self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, gw.address()) - def test_ca_cert_pem_uses_the_live_handle_when_present(self) -> None: - vm = MagicMock() - vm.gateway_ca_pem.return_value = "PEM" + def test_ca_cert_pem_reads_over_ssh_from_the_handle(self) -> None: + vm = infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k")) gw = FirecrackerGateway(vm) - with patch.object(infra_vm, "adopted_gateway_vm") as adopt: - self.assertEqual("PEM", gw.ca_cert_pem(timeout=1)) - adopt.assert_not_called() - vm.gateway_ca_pem.assert_called_once_with(timeout=1) + pem = "-----BEGIN CERTIFICATE-----\nx\n-----END CERTIFICATE-----\n" + with patch.object(infra_vm.subprocess, "run", + return_value=CompletedProcess([], 0, stdout=pem)) as run: + self.assertEqual(pem, gw.ca_cert_pem(timeout=5)) + # cat the CA path over SSH to the gateway VM's guest IP. + argv = run.call_args.args[0] + self.assertTrue(any("10.243.255.3" in a for a in argv)) + self.assertIn("cat /home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem", argv) - def test_ca_cert_pem_adopts_the_running_gateway_when_no_handle(self) -> None: - vm = MagicMock() - vm.gateway_ca_pem.return_value = "PEM" + def test_ca_cert_pem_raises_gateway_error_when_absent(self) -> None: gw = FirecrackerGateway() - with patch.object(infra_vm, "adopted_gateway_vm", return_value=vm) as adopt: - self.assertEqual("PEM", gw.ca_cert_pem()) - adopt.assert_called_once_with() + with patch.object(infra_vm.subprocess, "run", + return_value=CompletedProcess([], 1, stdout="", stderr="")): + with self.assertRaises(GatewayError): + gw.ca_cert_pem(timeout=0) - def test_provisioning_transport_targets_the_gateway_vm(self) -> None: + def test_provisioning_transport_targets_the_gateway_link(self) -> None: gw = FirecrackerGateway() - sentinel = object() - with patch.object(infra_vm, "gateway_transport", return_value=sentinel): - self.assertIs(sentinel, gw.provisioning_transport()) + transport = gw.provisioning_transport() + assert isinstance(transport, SshGatewayTransport) + self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, transport._ip) + + +class TestSshGatewayTransport(unittest.TestCase): + def test_cp_into_preserves_source_mode(self) -> None: + with tempfile.NamedTemporaryFile() as f: + os.chmod(f.name, 0o700) # like the staged access-hook + t = SshGatewayTransport(Path("/k"), "10.0.0.1") + with patch(f"{_GW}.subprocess.run", + return_value=CompletedProcess([], 0)) as run: + t.cp_into(f.name, "/etc/git-gate/access-hook") + remote_cmd = run.call_args.args[0][-1] + self.assertIn("chmod 700", remote_cmd) # exec bit preserved over SSH + + def test_exec_raises_on_failure(self) -> None: + t = SshGatewayTransport(Path("/k"), "10.0.0.1") + with patch(f"{_GW}.subprocess.run", + return_value=CompletedProcess([], 1, stderr="nope")), \ + self.assertRaises(GatewayProvisionError): + t.exec(["mkdir", "-p", "/git-gate"]) if __name__ == "__main__": diff --git a/tests/unit/test_firecracker_infra_vm.py b/tests/unit/test_firecracker_infra_vm.py index 366a78a..b929626 100644 --- a/tests/unit/test_firecracker_infra_vm.py +++ b/tests/unit/test_firecracker_infra_vm.py @@ -16,10 +16,10 @@ from bot_bottle.backend.firecracker import infra_vm class TestPushSecrets(unittest.TestCase): - """`_push_signing_key` (orchestrator) and `_push_gateway_jwt` (gateway) - seed their guest's secret over SSH via `_push_secret`. The host token file - stays the single source of truth, never clobbered per-backend (#469); the - gateway gets a pre-minted `gateway` JWT, never the signing key.""" + """`_push_signing_key` (orchestrator) seeds the guest's secret over SSH via + `_push_secret`. The host token file stays the single source of truth, never + clobbered per-backend (#469). The gateway's own JWT push lives with the + gateway service (test_firecracker_gateway).""" def _infra(self) -> infra_vm.InfraVm: return infra_vm.InfraVm(guest_ip="10.0.0.1", private_key=Path("/k")) @@ -36,17 +36,6 @@ class TestPushSecrets(unittest.TestCase): self.assertIn(f"mv {infra_vm._GUEST_SIGNING_KEY_PATH}.tmp " f"{infra_vm._GUEST_SIGNING_KEY_PATH}", remote_cmd) - def test_gateway_jwt_is_the_pre_minted_token_not_the_key(self): - # The host mints the `gateway` JWT and hands it in; `_push_gateway_jwt` - # only writes it. It is the JWT (never the key itself) that lands in the - # gateway VM. - proc = MagicMock(returncode=0, stderr="") - with patch.object(infra_vm.subprocess, "run", return_value=proc) as run: - infra_vm._push_gateway_jwt(self._infra(), "gw.jwt") - self.assertEqual("gw.jwt", run.call_args.kwargs["input"]) - remote_cmd = run.call_args.args[0][-1] - self.assertIn(f"cat > {infra_vm._GUEST_GATEWAY_JWT_PATH}.tmp", remote_cmd) - def test_dies_when_push_never_succeeds(self): proc = MagicMock(returncode=255, stderr="ssh: connect refused") with patch.object(infra_vm, "host_orchestrator_token", return_value="host-key"), \ @@ -71,12 +60,14 @@ class TestOrchestratorUrl(unittest.TestCase): class TestInfraEndpoint(unittest.TestCase): """The endpoint mirrors docker/macOS: it exposes the orchestrator's URL and - delegates the CA fetch to the gateway VM handle.""" + delegates the CA fetch to the gateway service.""" def _endpoint(self) -> infra_vm.InfraEndpoint: + from bot_bottle.backend.firecracker.gateway import FirecrackerGateway return infra_vm.InfraEndpoint( orchestrator=infra_vm.InfraVm(guest_ip="10.243.255.1", private_key=Path("/k")), - gateway=infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k")), + gateway=FirecrackerGateway( + infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k"))), ) def test_orchestrator_url_is_the_orchestrator_vm(self): @@ -84,9 +75,9 @@ class TestInfraEndpoint(unittest.TestCase): self.assertEqual( f"http://10.243.255.1:{infra_vm.ORCHESTRATOR_PORT}", ep.orchestrator_url) - def test_gateway_ca_pem_delegates_to_the_gateway_vm(self): + def test_gateway_ca_pem_delegates_to_the_gateway_service(self): ep = self._endpoint() - with patch.object(ep.gateway, "gateway_ca_pem", return_value="PEM") as ca: + with patch.object(ep.gateway, "ca_cert_pem", return_value="PEM") as ca: self.assertEqual("PEM", ep.gateway_ca_pem(timeout=1)) ca.assert_called_once_with(timeout=1) @@ -132,9 +123,10 @@ class TestBuildInfraRootfs(unittest.TestCase): class TestBootRole(unittest.TestCase): - """`boot_orchestrator` / `boot_gateway` boot the shared rootfs with the - right `bb_role` cmdline, memory ceiling, and (orchestrator-only) registry - volume; each seeds its own secret.""" + """`boot_orchestrator` boots the shared rootfs with the `bb_role=orchestrator` + cmdline, the orchestrator memory ceiling + registry volume, and seeds the + signing key. (The gateway's boot lives in the gateway service — + test_firecracker_gateway.)""" def _boot_env(self): vm = MagicMock() @@ -161,67 +153,6 @@ class TestBootRole(unittest.TestCase): self.assertEqual(Path("/reg"), kw["data_drive"]) # DB volume on the CP push.assert_called_once() # signing key seeded - def test_gateway_role_carries_orch_addr_and_no_registry(self): - import tempfile - vm = self._boot_env() - with tempfile.TemporaryDirectory() as td, \ - patch.object(infra_vm.netpool, "tap_present", return_value=True), \ - patch.object(infra_vm.infra_artifact, "local_build_requested", return_value=False), \ - patch.object(infra_vm.infra_artifact, "materialize_ext4"), \ - patch.object(infra_vm.infra_artifact, "infra_artifact_version", return_value="v"), \ - patch.object(infra_vm, "_stable_keypair", return_value=(Path("/k"), "pub")), \ - patch.object(infra_vm, "_push_gateway_jwt") as push, \ - patch.object(infra_vm.firecracker_vm, "boot", return_value=vm) as boot, \ - patch.object(infra_vm, "_gw_dir", return_value=Path(td)): - infra_vm.boot_gateway("10.243.255.1", "gw.jwt") - kw = boot.call_args.kwargs - self.assertIn("bb_role=gateway", kw["extra_boot_args"]) - self.assertIn("bb_orch=10.243.255.1", kw["extra_boot_args"]) # reaches the CP - self.assertEqual(infra_vm._GW_MEM_MIB, kw["mem_mib"]) - self.assertIsNone(kw["data_drive"]) # data plane never opens the DB - # The host-minted token is threaded through to the JWT push. - self.assertEqual("gw.jwt", push.call_args.args[1]) - - -class TestSshGatewayTransport(unittest.TestCase): - def test_cp_into_preserves_source_mode(self): - import tempfile - from subprocess import CompletedProcess - with tempfile.NamedTemporaryFile() as f: - os.chmod(f.name, 0o700) # like the staged access-hook - t = infra_vm.SshGatewayTransport(Path("/k"), "10.0.0.1") - with patch.object(infra_vm.subprocess, "run", - return_value=CompletedProcess([], 0)) as run: - t.cp_into(f.name, "/etc/git-gate/access-hook") - remote_cmd = run.call_args.args[0][-1] - self.assertIn("chmod 700", remote_cmd) # exec bit preserved over SSH - - def test_exec_raises_on_failure(self): - from subprocess import CompletedProcess - t = infra_vm.SshGatewayTransport(Path("/k"), "10.0.0.1") - with patch.object(infra_vm.subprocess, "run", - return_value=CompletedProcess([], 1, stderr="nope")), \ - self.assertRaises(infra_vm.GatewayProvisionError): - t.exec(["mkdir", "-p", "/git-gate"]) - - -class TestGatewayTransportTargetsGatewayVm(unittest.TestCase): - def test_transport_uses_the_gateway_link_guest_ip(self): - # git-gate provisioning goes to the gateway VM (gw_slot), not the - # orchestrator, now that the planes are split. - t = infra_vm.gateway_transport() - self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, t._ip) - - -class TestGatewayCaPem(unittest.TestCase): - def test_dies_when_cert_never_appears(self) -> None: - from subprocess import CompletedProcess - gw = infra_vm.InfraVm(vm=None, guest_ip="10.0.0.1", private_key=Path("/k")) - with patch.object(infra_vm.subprocess, "run", - return_value=CompletedProcess([], 1, stdout="", stderr="")), \ - self.assertRaises(SystemExit): - gw.gateway_ca_pem(timeout=0) - class TestRegistryVolume(unittest.TestCase): def test_reuses_existing_volume(self): @@ -295,10 +226,16 @@ class TestWaitForHealth(unittest.TestCase): infra_vm.wait_for_health(infra, timeout=5) +# The gateway service is imported lazily inside ensure_running / _adopt (to +# break the infra_vm <-> gateway import cycle), so patch it at its home module. +_GW_CLS = "bot_bottle.backend.firecracker.gateway.FirecrackerGateway" + + class TestEnsureRunningSingleton(unittest.TestCase): def test_adopts_when_healthy_alive_and_version_matches(self): # Healthy control plane + live gateway + existing key + matching version - # marker -> adopt (no boot); both VM handles have vm=None. + # marker -> adopt (no boot). + from bot_bottle.backend.firecracker.gateway import FirecrackerGateway import tempfile with tempfile.TemporaryDirectory() as td: d = Path(td) @@ -308,16 +245,15 @@ class TestEnsureRunningSingleton(unittest.TestCase): patch.object(infra_vm, "_expected_version", return_value="v-current"), \ patch.object(infra_vm, "_health_ok", return_value=True), \ patch.object(infra_vm, "_pidfile_alive", return_value=True), \ - patch.object(infra_vm, "boot_orchestrator") as boot_o, \ - patch.object(infra_vm, "boot_gateway") as boot_g: + patch.object(infra_vm, "boot_orchestrator") as boot_o: ep = infra_vm.ensure_running() boot_o.assert_not_called() - boot_g.assert_not_called() self.assertIsNone(ep.orchestrator.vm) - self.assertIsNone(ep.gateway.vm) - # The two handles address the two distinct infra links. + # The orchestrator handle + the adopted gateway service address the two + # distinct infra links. self.assertEqual(infra_vm.netpool.orch_slot().guest_ip, ep.orchestrator.guest_ip) - self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, ep.gateway.guest_ip) + self.assertIsInstance(ep.gateway, FirecrackerGateway) + self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, ep.gateway.address()) def test_reboots_both_when_version_stale(self): # Healthy control plane but the running pair booted an OLDER image @@ -337,19 +273,16 @@ class TestEnsureRunningSingleton(unittest.TestCase): patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \ patch.object(infra_vm, "mint", return_value="gw.jwt"), \ patch.object(infra_vm, "boot_orchestrator") as boot_o, \ - patch.object(infra_vm, "boot_gateway") as boot_g: + patch(_GW_CLS) as gw_cls: boot_o.return_value = infra_vm.InfraVm( guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock()) - boot_g.return_value = infra_vm.InfraVm( - guest_ip="10.243.255.3", private_key=Path("/k"), vm=MagicMock()) infra_vm.ensure_running() stop.assert_called_once() # dislodge the outdated pair boot_o.assert_called_once() - boot_g.assert_called_once() - # The gateway is booted pointing at the orchestrator's guest IP, - # carrying the host-minted `gateway` token. - self.assertEqual("10.243.255.1", boot_g.call_args.args[0]) - self.assertEqual("gw.jwt", boot_g.call_args.args[1]) + # The gateway service is bound to the orchestrator's URL, carrying + # the host-minted `gateway` token. + gw_cls.return_value.connect_to_orchestrator.assert_called_once_with( + "http://10.243.255.1:8099", "gw.jwt") # The fresh boot records the current version for the next launcher. self.assertEqual("v-current\n", (d / "booted-version").read_text()) @@ -371,15 +304,13 @@ class TestEnsureRunningSingleton(unittest.TestCase): patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \ patch.object(infra_vm, "mint", return_value="gw.jwt"), \ patch.object(infra_vm, "boot_orchestrator") as boot_o, \ - patch.object(infra_vm, "boot_gateway") as boot_g: + patch(_GW_CLS) as gw_cls: boot_o.return_value = infra_vm.InfraVm( guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock()) - boot_g.return_value = infra_vm.InfraVm( - guest_ip="10.243.255.3", private_key=Path("/k"), vm=MagicMock()) infra_vm.ensure_running() stop.assert_called_once() boot_o.assert_called_once() - boot_g.assert_called_once() + gw_cls.return_value.connect_to_orchestrator.assert_called_once() def test_boots_both_when_unhealthy(self): import tempfile @@ -392,20 +323,18 @@ class TestEnsureRunningSingleton(unittest.TestCase): patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \ patch.object(infra_vm, "mint", return_value="gw.jwt"), \ patch.object(infra_vm, "boot_orchestrator") as boot_o, \ - patch.object(infra_vm, "boot_gateway") as boot_g, \ + patch(_GW_CLS) as gw_cls, \ patch.object(infra_vm, "wait_for_health") as wait: boot_o.return_value = infra_vm.InfraVm( guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock()) - boot_g.return_value = infra_vm.InfraVm( - guest_ip="10.243.255.3", private_key=Path("/k"), vm=MagicMock()) infra_vm.ensure_running() stop.assert_called_once() # clear stale VMs first built.assert_called_once() - # Orchestrator boots + becomes healthy BEFORE the gateway (its daemons - # reach the control plane at startup). + # Orchestrator boots + becomes healthy BEFORE the gateway is connected + # (its daemons reach the control plane at startup). boot_o.assert_called_once() wait.assert_called_once() - boot_g.assert_called_once() + gw_cls.return_value.connect_to_orchestrator.assert_called_once() class TestStop(unittest.TestCase):