From d54c559a3a4005072420f3d6b732470b2cab6394 Mon Sep 17 00:00:00 2001 From: didericis Date: Sat, 25 Jul 2026 13:12:39 -0400 Subject: [PATCH] refactor(gateway): Firecracker gateway under the Gateway service ABC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `FirecrackerGateway`, the Firecracker implementation of the shared `Gateway` service — completing the trio (docker, macOS, firecracker) behind one uniform contract. The gateway here is a whole microVM, so the adapter composes `infra_vm`'s VM primitives (boot, SSH secret push, netpool links) into the ABC surface: `connect_to_orchestrator(url, token)` parses the orchestrator guest IP off the URL and boots the gateway VM seeding the pre-minted token; `address()` is the gateway link's guest IP; `ca_cert_pem()` fetches over SSH (adapting the running VM when this process didn't boot it); `provisioning_transport()` is the SSH exec/cp transport. Trust-model alignment with the other backends: minting moves out of `_push_gateway_jwt` to the host composition point — `ensure_running` mints the role-scoped `gateway` JWT and threads it through `boot_gateway(orch_ip, token)`, so the push step only writes the token the gateway presents (#469). `infra_vm` keeps driving the orchestrator+gateway pair as a singleton and gains gateway-only helpers (`gateway_running`/`stop_gateway`/`gateway_guest_ip`/ `adopted_gateway_vm`); the consolidated launch reads the gateway's transport + CA off the service. Co-Authored-By: Claude Opus 4.8 --- .../firecracker/consolidated_launch.py | 12 ++- bot_bottle/backend/firecracker/gateway.py | 98 +++++++++++++++++ bot_bottle/backend/firecracker/infra_vm.py | 55 ++++++++-- tests/unit/test_firecracker_gateway.py | 101 ++++++++++++++++++ tests/unit/test_firecracker_infra_vm.py | 29 +++-- 5 files changed, 269 insertions(+), 26 deletions(-) create mode 100644 bot_bottle/backend/firecracker/gateway.py create mode 100644 tests/unit/test_firecracker_gateway.py diff --git a/bot_bottle/backend/firecracker/consolidated_launch.py b/bot_bottle/backend/firecracker/consolidated_launch.py index d8468f3d..4868bc7d 100644 --- a/bot_bottle/backend/firecracker/consolidated_launch.py +++ b/bot_bottle/backend/firecracker/consolidated_launch.py @@ -45,6 +45,7 @@ from ..consolidated_util import ( teardown_consolidated as _teardown_util, ) from . import cleanup, infra_vm, util +from .gateway import FirecrackerGateway _ENV_VAR_SECRET_PATH = "/run/bot-bottle/env-var-secret" @@ -129,18 +130,21 @@ def launch_consolidated( client = OrchestratorClient(url) _reprovision_running_bottles(client) - transport = infra_vm.gateway_transport() + # Read the gateway's provisioning transport + CA off the Gateway service + # (adapting the running gateway VM). + gateway = FirecrackerGateway(infra.gateway) + transport = gateway.provisioning_transport() reg = provision_bottle( client, guest_ip, egress_plan, git_gate_plan, transport, image_ref=image_ref, tokens=tokens, ) # The shared gateway CA every agent on this host trusts for TLS - # interception — fetched from the infra VM over SSH. + # interception — fetched from the gateway VM over SSH. return LaunchContext( bottle_id=reg.bottle_id, identity_token=reg.identity_token, source_ip=guest_ip, - gateway_ca_pem=infra.gateway_ca_pem(), + gateway_ca_pem=gateway.ca_cert_pem(), orchestrator_url=url, env_var_secret=reg.env_var_secret, ) @@ -153,7 +157,7 @@ def teardown_consolidated( VM. Both steps are idempotent so this is safe from a cleanup trap. Does NOT stop the infra VMs — they're persistent per-host singletons shared by every bottle.""" - _teardown_util(bottle_id, infra_vm.gateway_transport(), + _teardown_util(bottle_id, FirecrackerGateway().provisioning_transport(), orchestrator_url=orchestrator_url, timeout=timeout) diff --git a/bot_bottle/backend/firecracker/gateway.py b/bot_bottle/backend/firecracker/gateway.py new file mode 100644 index 00000000..5fa977c6 --- /dev/null +++ b/bot_bottle/backend/firecracker/gateway.py @@ -0,0 +1,98 @@ +"""The Firecracker gateway data plane as a microVM (PRD 0070). + +`FirecrackerGateway` is the Firecracker implementation of the backend-neutral +`Gateway` service. The gateway here is a whole microVM — the slim data-plane +unit (mitmproxy TLS bump + DLP + git-http) booted on its own NAT'd link; agent +VMs' gateway-port traffic is DNAT'd to it and it never opens `bot-bottle.db` +(#469), so it holds no signing key, only the pre-minted `gateway` token the +host hands it via `connect_to_orchestrator`. + +The VM lifecycle primitives (boot, SSH secret push, netpool links, nft, PID +files) stay in `infra_vm`, which drives the orchestrator+gateway pair as a +singleton; this adapter composes them into the uniform `Gateway` surface. +""" + +from __future__ import annotations + +from urllib.parse import urlparse + +from ...gateway import ( + DEFAULT_CA_TIMEOUT_SECONDS, + Gateway, + GatewayError, + GatewayTransport, +) +from . import infra_vm + +# The gateway microVM's name (the `bb_role=gateway` boot tag / run dir). Fixed +# per host — one gateway VM, shared by every agent VM. +GATEWAY_NAME = "bot-bottle-gateway" + + +class FirecrackerGateway(Gateway): + """The consolidated gateway as a Firecracker microVM on the gateway link. + + The rootfs is built/downloaded by `infra_vm.ensure_built` (the ABC's + `ensure_built` no-op here); `connect_to_orchestrator` boots the VM resolving + policy against the orchestrator and seeds the pre-minted `gateway` token.""" + + name = GATEWAY_NAME + + def __init__(self, vm: infra_vm.InfraVm | None = None) -> None: + # The live VM handle when this process booted it; None when adapting an + # already-running gateway (CA fetch / provisioning go through the stable + # key + fixed link, so no live handle is needed). + self._vm = vm + self._orchestrator_url = "" + self._gateway_token = "" + + def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None: + """Boot the gateway VM resolving policy against `orchestrator_url` and + seed `gateway_token`. The orchestrator's guest IP is parsed from the URL + and passed on the cmdline, so the baked init stays IP-independent. Boot + the orchestrator first — the gateway daemons reach it at startup.""" + self._orchestrator_url = orchestrator_url + self._gateway_token = gateway_token + if not self._orchestrator_url: + raise GatewayError( + "gateway requires an orchestrator URL to run " + "(resolver-only data plane; no single-tenant fallback)" + ) + if not self._gateway_token: + raise GatewayError( + "gateway requires a pre-minted `gateway` token to run " + "(the orchestrator mints it; the gateway never holds the key)" + ) + orchestrator_guest_ip = urlparse(self._orchestrator_url).hostname or "" + if not orchestrator_guest_ip: + raise GatewayError( + f"cannot resolve orchestrator guest IP from {self._orchestrator_url!r}" + ) + self._vm = infra_vm.boot_gateway(orchestrator_guest_ip, self._gateway_token) + + def is_running(self) -> bool: + return infra_vm.gateway_running() + + def stop(self) -> None: + """Stop only the gateway VM (idempotent).""" + infra_vm.stop_gateway() + + def address(self) -> str: + """The gateway VM's guest IP — the agent-facing target agent VMs' + gateway-port traffic is DNAT'd to.""" + return infra_vm.gateway_guest_ip() + + def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str: + """The gateway's mitmproxy CA (PEM) agents install to trust its TLS + interception. Fetched from the gateway VM over SSH; polls because + mitmproxy writes it a beat after boot.""" + vm = self._vm or infra_vm.adopted_gateway_vm() + return vm.gateway_ca_pem(timeout=timeout) + + def provisioning_transport(self) -> GatewayTransport: + """The exec/cp transport git-gate provisioning stages per-bottle repos + + deploy keys through (over SSH to the gateway VM).""" + return infra_vm.gateway_transport() + + +__all__ = ["FirecrackerGateway", "GATEWAY_NAME"] diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py index 5eeba346..1c0bf667 100644 --- a/bot_bottle/backend/firecracker/infra_vm.py +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -49,7 +49,7 @@ from ...orchestrator_auth import ROLE_GATEWAY, mint from ...paths import host_orchestrator_token from .. import util as backend_util from ..docker import util as docker_mod -from ..docker.gateway_provision import GatewayProvisionError +from ...gateway import GatewayProvisionError from . import firecracker_vm, infra_artifact, netpool, util # The orchestrator VM's signing-key path on its persistent /dev/vdb volume @@ -228,7 +228,11 @@ def ensure_running() -> InfraEndpoint: ensure_built() orchestrator = boot_orchestrator() wait_for_health(orchestrator) - gateway = boot_gateway(orchestrator.guest_ip) + # The orchestrator holds the signing key; mint the role-scoped `gateway` + # JWT on the host and hand it to the gateway VM, which never sees the key + # (#469). + gateway_token = mint(ROLE_GATEWAY, host_orchestrator_token()) + gateway = boot_gateway(orchestrator.guest_ip, gateway_token) _record_booted_version(want) return InfraEndpoint(orchestrator=orchestrator, gateway=gateway) @@ -290,12 +294,13 @@ def boot_orchestrator() -> InfraVm: return orch -def boot_gateway(orchestrator_guest_ip: str) -> InfraVm: +def boot_gateway(orchestrator_guest_ip: str, gateway_token: str) -> InfraVm: """Boot the gateway (data-plane) VM (detached) on the gateway link, and - seed its pre-minted `gateway` JWT. Its daemons resolve policy against the - orchestrator at `orchestrator_guest_ip:8099` (passed on the cmdline, so the - baked init stays IP-independent). Boot the orchestrator first — the gateway - daemons reach it at startup. Prefer `ensure_running`.""" + seed the pre-minted `gateway` JWT `gateway_token`. Its daemons resolve + policy against the orchestrator at `orchestrator_guest_ip:8099` (passed on + the cmdline, so the baked init stays IP-independent). Boot the orchestrator + first — the gateway daemons reach it at startup. The gateway never holds the + signing key, only this token (#469). Prefer `ensure_running`.""" slot = netpool.gw_slot() gateway = _boot_vm( name="bot-bottle-gateway", slot=slot, run_dir=_gw_dir(), @@ -304,7 +309,7 @@ def boot_gateway(orchestrator_guest_ip: str) -> InfraVm: ) # Push the pre-minted `gateway` JWT (never the signing key — issue #469). # The init waits for it before starting the data plane. - _push_gateway_jwt(gateway) + _push_gateway_jwt(gateway, gateway_token) return gateway @@ -451,15 +456,14 @@ def _push_signing_key(infra: InfraVm) -> None: ) -def _push_gateway_jwt(infra: InfraVm) -> None: +def _push_gateway_jwt(infra: InfraVm, gateway_token: str) -> None: """Push the pre-minted `gateway` JWT into the freshly booted gateway VM over SSH (atomic write). Minted on the HOST from the canonical signing key so the data plane never holds the key itself (issue #469); the gateway daemons present it to the orchestrator. Retries until SSH is up; dies if it never lands, since the VM then refuses to start the data plane.""" _push_secret( - infra, mint(ROLE_GATEWAY, host_orchestrator_token()), - _GUEST_GATEWAY_JWT_PATH, + infra, gateway_token, _GUEST_GATEWAY_JWT_PATH, "the gateway JWT to the gateway VM (its data plane will not start)", ) @@ -612,6 +616,35 @@ def gateway_transport() -> SshGatewayTransport: _infra_dir() / "id_ed25519", netpool.gw_slot().guest_ip) +def gateway_guest_ip() -> str: + """The gateway VM's fixed guest IP on its link — the agent-facing address + agent VMs' gateway-port traffic is DNAT'd to (PRD 0070).""" + return netpool.gw_slot().guest_ip + + +def gateway_running() -> bool: + """True iff the gateway VM is still a live VMM.""" + return _pidfile_alive(_gw_dir()) + + +def stop_gateway() -> None: + """Stop only the gateway VM (idempotent — absent is success). Drops the pair + version marker so a half-stopped pair is never adopted (`_adoptable`).""" + _kill_pidfile(_gw_dir()) + _pid_file(_gw_dir()).unlink(missing_ok=True) + _version_file().unlink(missing_ok=True) + + +def adopted_gateway_vm() -> InfraVm: + """A handle to the already-running gateway VM from the stable key + the + fixed gateway link (vm=None — CA fetch / provisioning work from any later + launcher, not only the process that booted it).""" + return InfraVm( + guest_ip=netpool.gw_slot().guest_ip, + private_key=_infra_dir() / "id_ed25519", + ) + + def wait_for_health( infra: InfraVm, *, timeout: float = _HEALTH_TIMEOUT_SECONDS, ) -> None: diff --git a/tests/unit/test_firecracker_gateway.py b/tests/unit/test_firecracker_gateway.py new file mode 100644 index 00000000..4ce14db5 --- /dev/null +++ b/tests/unit/test_firecracker_gateway.py @@ -0,0 +1,101 @@ +"""Unit: the Firecracker gateway data plane as a microVM (PRD 0070).""" + +from __future__ import annotations + +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +from bot_bottle.backend.firecracker import infra_vm +from bot_bottle.backend.firecracker.gateway import ( + GATEWAY_NAME, + FirecrackerGateway, +) +from bot_bottle.gateway import GatewayError + +_GW = "bot_bottle.backend.firecracker.gateway" + +_ORCH_URL = "http://10.243.255.1:8099" +_TOKEN = "gw-jwt-token" + + +class TestFirecrackerGatewayConnect(unittest.TestCase): + def test_default_name(self) -> None: + self.assertEqual(GATEWAY_NAME, FirecrackerGateway().name) + + def test_refuses_without_orchestrator_url(self) -> None: + gw = FirecrackerGateway() + with patch.object(infra_vm, "boot_gateway") as boot: + with self.assertRaises(GatewayError): + gw.connect_to_orchestrator("", _TOKEN) + boot.assert_not_called() + + def test_refuses_without_gateway_token(self) -> None: + gw = FirecrackerGateway() + with patch.object(infra_vm, "boot_gateway") as boot: + with self.assertRaises(GatewayError): + gw.connect_to_orchestrator(_ORCH_URL, "") + boot.assert_not_called() + + def test_refuses_a_url_without_a_host(self) -> None: + gw = FirecrackerGateway() + with patch.object(infra_vm, "boot_gateway") as boot: + with self.assertRaises(GatewayError): + gw.connect_to_orchestrator("http://:8099", _TOKEN) + boot.assert_not_called() + + def test_boots_the_gateway_vm_against_the_orchestrator_guest_ip(self) -> None: + gw = FirecrackerGateway() + booted = infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k")) + with patch.object(infra_vm, "boot_gateway", return_value=booted) as boot: + gw.connect_to_orchestrator(_ORCH_URL, _TOKEN) + # The orchestrator guest IP is parsed off the URL; the token is threaded + # through unchanged (the gateway never mints — #469). + boot.assert_called_once_with("10.243.255.1", _TOKEN) + + +class TestFirecrackerGatewaySurface(unittest.TestCase): + def test_is_running_reads_the_gateway_pidfile(self) -> None: + gw = FirecrackerGateway() + with patch.object(infra_vm, "gateway_running", return_value=True): + self.assertTrue(gw.is_running()) + with patch.object(infra_vm, "gateway_running", return_value=False): + self.assertFalse(gw.is_running()) + + def test_stop_stops_only_the_gateway_vm(self) -> None: + gw = FirecrackerGateway() + with patch.object(infra_vm, "stop_gateway") as stop: + gw.stop() + stop.assert_called_once_with() + + def test_address_is_the_gateway_link_guest_ip(self) -> None: + gw = FirecrackerGateway() + with patch.object(infra_vm, "gateway_guest_ip", return_value="10.243.255.3"): + self.assertEqual("10.243.255.3", gw.address()) + + def test_ca_cert_pem_uses_the_live_handle_when_present(self) -> None: + vm = MagicMock() + vm.gateway_ca_pem.return_value = "PEM" + gw = FirecrackerGateway(vm) + with patch.object(infra_vm, "adopted_gateway_vm") as adopt: + self.assertEqual("PEM", gw.ca_cert_pem(timeout=1)) + adopt.assert_not_called() + vm.gateway_ca_pem.assert_called_once_with(timeout=1) + + def test_ca_cert_pem_adopts_the_running_gateway_when_no_handle(self) -> None: + vm = MagicMock() + vm.gateway_ca_pem.return_value = "PEM" + gw = FirecrackerGateway() + with patch.object(infra_vm, "adopted_gateway_vm", return_value=vm) as adopt: + self.assertEqual("PEM", gw.ca_cert_pem()) + adopt.assert_called_once_with() + + def test_provisioning_transport_targets_the_gateway_vm(self) -> None: + gw = FirecrackerGateway() + sentinel = object() + with patch.object(infra_vm, "gateway_transport", return_value=sentinel): + self.assertIs(sentinel, gw.provisioning_transport()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_firecracker_infra_vm.py b/tests/unit/test_firecracker_infra_vm.py index e9c10587..366a78a5 100644 --- a/tests/unit/test_firecracker_infra_vm.py +++ b/tests/unit/test_firecracker_infra_vm.py @@ -36,15 +36,13 @@ class TestPushSecrets(unittest.TestCase): self.assertIn(f"mv {infra_vm._GUEST_SIGNING_KEY_PATH}.tmp " f"{infra_vm._GUEST_SIGNING_KEY_PATH}", remote_cmd) - def test_gateway_jwt_is_minted_gateway_role_not_the_key(self): + def test_gateway_jwt_is_the_pre_minted_token_not_the_key(self): + # The host mints the `gateway` JWT and hands it in; `_push_gateway_jwt` + # only writes it. It is the JWT (never the key itself) that lands in the + # gateway VM. proc = MagicMock(returncode=0, stderr="") - with patch.object(infra_vm, "host_orchestrator_token", return_value="host-key"), \ - patch.object(infra_vm, "mint", return_value="gw.jwt") as mint, \ - patch.object(infra_vm.subprocess, "run", return_value=proc) as run: - infra_vm._push_gateway_jwt(self._infra()) - # Minted on the HOST from the signing key with the gateway role, and it - # is the JWT (never the key itself) that lands in the gateway VM. - mint.assert_called_once_with(infra_vm.ROLE_GATEWAY, "host-key") + with patch.object(infra_vm.subprocess, "run", return_value=proc) as run: + infra_vm._push_gateway_jwt(self._infra(), "gw.jwt") self.assertEqual("gw.jwt", run.call_args.kwargs["input"]) remote_cmd = run.call_args.args[0][-1] self.assertIn(f"cat > {infra_vm._GUEST_GATEWAY_JWT_PATH}.tmp", remote_cmd) @@ -175,13 +173,14 @@ class TestBootRole(unittest.TestCase): patch.object(infra_vm, "_push_gateway_jwt") as push, \ patch.object(infra_vm.firecracker_vm, "boot", return_value=vm) as boot, \ patch.object(infra_vm, "_gw_dir", return_value=Path(td)): - infra_vm.boot_gateway("10.243.255.1") + infra_vm.boot_gateway("10.243.255.1", "gw.jwt") kw = boot.call_args.kwargs self.assertIn("bb_role=gateway", kw["extra_boot_args"]) self.assertIn("bb_orch=10.243.255.1", kw["extra_boot_args"]) # reaches the CP self.assertEqual(infra_vm._GW_MEM_MIB, kw["mem_mib"]) self.assertIsNone(kw["data_drive"]) # data plane never opens the DB - push.assert_called_once() # gateway JWT seeded + # The host-minted token is threaded through to the JWT push. + self.assertEqual("gw.jwt", push.call_args.args[1]) class TestSshGatewayTransport(unittest.TestCase): @@ -335,6 +334,8 @@ class TestEnsureRunningSingleton(unittest.TestCase): patch.object(infra_vm, "stop") as stop, \ patch.object(infra_vm, "ensure_built"), \ patch.object(infra_vm, "wait_for_health"), \ + patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \ + patch.object(infra_vm, "mint", return_value="gw.jwt"), \ patch.object(infra_vm, "boot_orchestrator") as boot_o, \ patch.object(infra_vm, "boot_gateway") as boot_g: boot_o.return_value = infra_vm.InfraVm( @@ -345,8 +346,10 @@ class TestEnsureRunningSingleton(unittest.TestCase): stop.assert_called_once() # dislodge the outdated pair boot_o.assert_called_once() boot_g.assert_called_once() - # The gateway is booted pointing at the orchestrator's guest IP. + # The gateway is booted pointing at the orchestrator's guest IP, + # carrying the host-minted `gateway` token. self.assertEqual("10.243.255.1", boot_g.call_args.args[0]) + self.assertEqual("gw.jwt", boot_g.call_args.args[1]) # The fresh boot records the current version for the next launcher. self.assertEqual("v-current\n", (d / "booted-version").read_text()) @@ -365,6 +368,8 @@ class TestEnsureRunningSingleton(unittest.TestCase): patch.object(infra_vm, "stop") as stop, \ patch.object(infra_vm, "ensure_built"), \ patch.object(infra_vm, "wait_for_health"), \ + patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \ + patch.object(infra_vm, "mint", return_value="gw.jwt"), \ patch.object(infra_vm, "boot_orchestrator") as boot_o, \ patch.object(infra_vm, "boot_gateway") as boot_g: boot_o.return_value = infra_vm.InfraVm( @@ -384,6 +389,8 @@ class TestEnsureRunningSingleton(unittest.TestCase): patch.object(infra_vm, "_health_ok", return_value=False), \ patch.object(infra_vm, "stop") as stop, \ patch.object(infra_vm, "ensure_built") as built, \ + patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \ + patch.object(infra_vm, "mint", return_value="gw.jwt"), \ patch.object(infra_vm, "boot_orchestrator") as boot_o, \ patch.object(infra_vm, "boot_gateway") as boot_g, \ patch.object(infra_vm, "wait_for_health") as wait: