diff --git a/bot_bottle/backend/firecracker/gateway.py b/bot_bottle/backend/firecracker/gateway.py index 2c727fa1..b1ea6440 100644 --- a/bot_bottle/backend/firecracker/gateway.py +++ b/bot_bottle/backend/firecracker/gateway.py @@ -19,6 +19,7 @@ singleton flock) is `FirecrackerInfraService` (`infra.py`). from __future__ import annotations import subprocess +from pathlib import Path from urllib.parse import urlparse from ...gateway import ( @@ -27,6 +28,7 @@ from ...gateway import ( GatewayError, GatewayTransport, ) +from ...log import die, info from .. import util as backend_util from . import infra_vm, netpool, util from .gateway_transport import FirecrackerGatewayTransport @@ -49,6 +51,16 @@ _GUEST_GATEWAY_JWT_PATH = infra_vm._GUEST_GATEWAY_JWT_PATH # the gateway's TLS interception. Host-side only (SSH cat), so it lives here. _GATEWAY_CA_PATH = "/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem" +# The gateway VM's persistent CA volume — a small ext4 file the gateway init +# mounts at mitmproxy's confdir (see `infra_vm._GATEWAY_CA_MOUNT`) so the +# self-generated CA SURVIVES a gateway-VM rebuild/restart. Without it the CA +# lives only in the ephemeral per-boot rootfs, so every rebuild mints a fresh CA +# that every already-running bottle distrusts, failing the TLS handshake (the +# firecracker analogue of the docker fix's persistent CA bind-mount — issue +# #450). Co-located with the orchestrator's registry volume under the infra dir, +# which outlives the ephemeral rootfs. mitmproxy is tiny; 16M is ample. +_CA_VOLUME_SIZE = "16M" + _CA_FETCH_TIMEOUT_SECONDS = 15.0 @@ -94,11 +106,15 @@ class FirecrackerGateway(Gateway): f"cannot resolve orchestrator guest IP from {self._orchestrator_url!r}" ) # Boot on the gateway link from the gateway rootfs, then push the token - # the init waits for before starting the data plane. + # the init waits for before starting the data plane. The persistent CA + # volume (/dev/vdb, mounted at mitmproxy's confdir by the gateway init) + # keeps the CA STABLE across rebuilds so already-running bottles keep + # trusting the gateway's TLS interception (issue #450). vm = infra_vm.boot_vm( name=GATEWAY_NAME, slot=netpool.gw_slot(), run_dir=infra_vm._gw_dir(), role="gateway", mem_mib=_GW_MEM_MIB, extra_boot_args=f"bb_orch={orchestrator_guest_ip}", + data_drive=self._ensure_ca_volume(), ) infra_vm.push_secret( vm, self._gateway_token, _GUEST_GATEWAY_JWT_PATH, @@ -116,6 +132,26 @@ class FirecrackerGateway(Gateway): infra_vm._kill_pidfile(infra_vm._gw_dir()) infra_vm._pid_file(infra_vm._gw_dir()).unlink(missing_ok=True) + def _ensure_ca_volume(self) -> Path: + """Create the empty ext4 CA volume on first use; reuse it after. + + A fresh (empty) volume makes mitmproxy generate a CA into it on first + boot; every later boot reuses the CA already on the volume — which is + what keeps the CA stable across gateway-VM rebuilds. The mirror of + `FirecrackerOrchestrator._ensure_registry_volume`.""" + vol = infra_vm._gw_dir() / "gateway-ca.ext4" + if vol.exists(): + return vol + info(f"creating gateway CA volume {vol} ({_CA_VOLUME_SIZE})") + proc = subprocess.run( + ["mke2fs", "-q", "-t", "ext4", "-F", str(vol), _CA_VOLUME_SIZE], + capture_output=True, text=True, check=False, + ) + if proc.returncode != 0: + vol.unlink(missing_ok=True) + die(f"creating gateway CA volume failed: {proc.stderr.strip()}") + return vol + def address(self) -> str: """The gateway VM's guest IP — the agent-facing target agent VMs' gateway-port traffic is DNAT'd to.""" diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py index 0fa03f0f..9fc978a9 100644 --- a/bot_bottle/backend/firecracker/infra_vm.py +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -53,10 +53,16 @@ from . import firecracker_vm, infra_artifact, netpool, util # tokens with the same key the host CLI signs with — the host token file stays # the single source of truth, never clobbered per-backend (issue #469 review). _GUEST_SIGNING_KEY_PATH = "/var/lib/bot-bottle/orchestrator-token" -# The gateway VM's pre-minted `gateway` JWT path (rootfs, not /dev/vdb — the -# data plane has no registry volume and never opens the DB). Pushed post-boot; -# the gateway daemons present it to the orchestrator, and never see the key. +# The gateway VM's pre-minted `gateway` JWT path (rootfs, not /dev/vdb — the JWT +# is re-pushed every boot, so it needn't persist). Pushed post-boot; the gateway +# daemons present it to the orchestrator, and never see the key. _GUEST_GATEWAY_JWT_PATH = "/var/lib/bot-bottle/gateway-jwt" +# The gateway VM's persistent CA volume mount point — mitmproxy's confdir. The +# gateway boots with a persistent /dev/vdb CA volume (see +# `FirecrackerGateway._ensure_ca_volume`) mounted here so the self-generated CA +# survives a gateway-VM rebuild; without it every rebuild mints a fresh CA that +# already-running bottles distrust, breaking the TLS handshake (issue #450). +_GATEWAY_CA_MOUNT = "/home/mitmproxy/.mitmproxy" # The two per-plane rootfs source images. The orchestrator VM boots a control # plane + buildah rootfs (Dockerfile.orchestrator.fc, FROM orchestrator); the @@ -448,6 +454,13 @@ def _gateway_init() -> str: bot-bottle.db (PRD 0070 / #469). If the JWT never arrives, REFUSE to start rather than run without auth.""" return _init_head() + f""" +# Persistent CA volume (second virtio-block device, /dev/vdb) mounted at +# mitmproxy's confdir, so the self-generated mitmproxy CA survives gateway-VM +# rebuilds (issue #450). On first boot the volume is empty and mitmproxy mints a +# CA into it; every later boot reuses it. Must mount BEFORE the data plane (hence +# mitmproxy) starts. +mkdir -p {_GATEWAY_CA_MOUNT} +mount -t ext4 /dev/vdb {_GATEWAY_CA_MOUNT} 2>/dev/null || true ORCH=$(sed -n 's/.*bb_orch=\\([^ ]*\\).*/\\1/p' /proc/cmdline) GW_JWT="" i=0 diff --git a/tests/unit/test_firecracker_gateway.py b/tests/unit/test_firecracker_gateway.py index 2c1b1bfb..c593af97 100644 --- a/tests/unit/test_firecracker_gateway.py +++ b/tests/unit/test_firecracker_gateway.py @@ -59,7 +59,10 @@ class TestFirecrackerGatewayConnect(unittest.TestCase): def test_boots_the_gateway_vm_and_seeds_the_token(self) -> None: gw = FirecrackerGateway() booted = infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k")) + ca_vol = Path("/gw/gateway-ca.ext4") with patch.object(infra_vm, "boot_vm", return_value=booted) as boot, \ + patch.object(FirecrackerGateway, "_ensure_ca_volume", + return_value=ca_vol), \ patch.object(infra_vm, "push_secret") as push: gw.connect_to_orchestrator(_ORCH_URL, _TOKEN) # Booted on the gateway link with the gateway role; the orchestrator's @@ -67,12 +70,43 @@ class TestFirecrackerGatewayConnect(unittest.TestCase): kw = boot.call_args.kwargs self.assertEqual("gateway", kw["role"]) self.assertIn("bb_orch=10.243.255.1", kw["extra_boot_args"]) - self.assertIsNone(kw.get("data_drive")) # data plane never opens the DB + # The persistent CA volume rides as /dev/vdb so the mitmproxy CA + # survives a gateway-VM rebuild (issue #450). + self.assertEqual(ca_vol, kw.get("data_drive")) # The host-minted token (never the key — #469) is pushed to the guest. push.assert_called_once() self.assertEqual(_TOKEN, push.call_args.args[1]) +class TestGatewayCaVolume(unittest.TestCase): + """The persistent CA volume that keeps the mitmproxy CA stable across a + gateway-VM rebuild (issue #450) — the mirror of the orchestrator's registry + volume.""" + + def test_reuses_existing_volume(self) -> None: + gw = FirecrackerGateway() + with tempfile.TemporaryDirectory() as td: + vol = Path(td) / "gateway-ca.ext4" + vol.write_bytes(b"") # already present + with patch.object(infra_vm, "_gw_dir", return_value=Path(td)), \ + patch(f"{_GW}.subprocess.run") as run: + out = gw._ensure_ca_volume() + run.assert_not_called() # no mke2fs when it exists — the CA survives + self.assertEqual(vol, out) + + def test_creates_volume_when_missing(self) -> None: + gw = FirecrackerGateway() + with tempfile.TemporaryDirectory() as td: + with patch.object(infra_vm, "_gw_dir", return_value=Path(td)), \ + patch(f"{_GW}.subprocess.run", + return_value=CompletedProcess([], 0)) as run: + out = gw._ensure_ca_volume() + argv = run.call_args.args[0] + self.assertIn("mke2fs", argv) + self.assertEqual(str(Path(td) / "gateway-ca.ext4"), out.__fspath__()) + self.assertIn(str(out), argv) + + class TestFirecrackerGatewaySurface(unittest.TestCase): def test_is_running_reads_the_gateway_pidfile(self) -> None: gw = FirecrackerGateway() diff --git a/tests/unit/test_firecracker_infra_vm.py b/tests/unit/test_firecracker_infra_vm.py index 52fe6b97..44cc280d 100644 --- a/tests/unit/test_firecracker_infra_vm.py +++ b/tests/unit/test_firecracker_infra_vm.py @@ -61,7 +61,11 @@ class TestRoleInits(unittest.TestCase): init = infra_vm.role_init("gateway") self.assertNotIn("bb_role=", init) self.assertNotIn("bot_bottle.orchestrator", init) - self.assertNotIn("/dev/vdb", init) # no registry volume on the data plane + # Persistent CA volume mounted at mitmproxy's confdir so the CA survives + # a gateway-VM rebuild (issue #450). Mounted BEFORE the data plane starts. + self.assertIn(f"mount -t ext4 /dev/vdb {infra_vm._GATEWAY_CA_MOUNT}", init) + self.assertLess(init.index("/dev/vdb"), + init.index("bot_bottle.gateway.bootstrap")) self.assertIn("export PATH=", init) # shared preamble self.assertIn("BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise", init) self.assertIn(f"cat {infra_vm._GUEST_GATEWAY_JWT_PATH}", init) # host-minted JWT