fix(firecracker): persist the gateway mitmproxy CA across rebuilds

The #450 CA-persistence fix was only implemented for the docker backend
(the host_gateway_ca_dir bind-mount). The firecracker gateway booted with
no persistent volume, so its mitmproxy CA lived only in the ephemeral
per-boot rootfs — every rebuild/restart minted a fresh CA that every
already-running bottle distrusted, failing egress TLS with "SSL
certificate verification failed".

Give the gateway VM a persistent CA volume, mirroring the orchestrator's
registry volume: boot with a small ext4 data_drive (_ensure_ca_volume)
and mount it at mitmproxy's confdir in the gateway guest init, before the
data plane starts, so mitmproxy reuses the on-disk CA instead of
regenerating it.

Closes #510

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

bot_bottle/backend/firecracker/gateway.py

@@ -19,6 +19,7 @@ singleton flock) is `FirecrackerInfraService` (`infra.py`).
 from __future__ import annotations
 
 import subprocess
+from pathlib import Path
 from urllib.parse import urlparse
 
 from ...gateway import (
@@ -27,6 +28,7 @@ from ...gateway import (
     GatewayError,
     GatewayTransport,
 )
+from ...log import die, info
 from .. import util as backend_util
 from . import infra_vm, netpool, util
 from .gateway_transport import FirecrackerGatewayTransport
@@ -49,6 +51,16 @@ _GUEST_GATEWAY_JWT_PATH = infra_vm._GUEST_GATEWAY_JWT_PATH
 # the gateway's TLS interception. Host-side only (SSH cat), so it lives here.
 _GATEWAY_CA_PATH = "/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem"
 
+# The gateway VM's persistent CA volume — a small ext4 file the gateway init
+# mounts at mitmproxy's confdir (see `infra_vm._GATEWAY_CA_MOUNT`) so the
+# self-generated CA SURVIVES a gateway-VM rebuild/restart. Without it the CA
+# lives only in the ephemeral per-boot rootfs, so every rebuild mints a fresh CA
+# that every already-running bottle distrusts, failing the TLS handshake (the
+# firecracker analogue of the docker fix's persistent CA bind-mount — issue
+# #450). Co-located with the orchestrator's registry volume under the infra dir,
+# which outlives the ephemeral rootfs. mitmproxy is tiny; 16M is ample.
+_CA_VOLUME_SIZE = "16M"
+
 _CA_FETCH_TIMEOUT_SECONDS = 15.0
 
 
@@ -94,11 +106,15 @@ class FirecrackerGateway(Gateway):
                 f"cannot resolve orchestrator guest IP from {self._orchestrator_url!r}"
             )
         # Boot on the gateway link from the gateway rootfs, then push the token
-        # the init waits for before starting the data plane.
+        # the init waits for before starting the data plane. The persistent CA
+        # volume (/dev/vdb, mounted at mitmproxy's confdir by the gateway init)
+        # keeps the CA STABLE across rebuilds so already-running bottles keep
+        # trusting the gateway's TLS interception (issue #450).
         vm = infra_vm.boot_vm(
             name=GATEWAY_NAME, slot=netpool.gw_slot(), run_dir=infra_vm._gw_dir(),
             role="gateway", mem_mib=_GW_MEM_MIB,
             extra_boot_args=f"bb_orch={orchestrator_guest_ip}",
+            data_drive=self._ensure_ca_volume(),
         )
         infra_vm.push_secret(
             vm, self._gateway_token, _GUEST_GATEWAY_JWT_PATH,
@@ -116,6 +132,26 @@ class FirecrackerGateway(Gateway):
         infra_vm._kill_pidfile(infra_vm._gw_dir())
         infra_vm._pid_file(infra_vm._gw_dir()).unlink(missing_ok=True)
 
+    def _ensure_ca_volume(self) -> Path:
+        """Create the empty ext4 CA volume on first use; reuse it after.
+
+        A fresh (empty) volume makes mitmproxy generate a CA into it on first
+        boot; every later boot reuses the CA already on the volume — which is
+        what keeps the CA stable across gateway-VM rebuilds. The mirror of
+        `FirecrackerOrchestrator._ensure_registry_volume`."""
+        vol = infra_vm._gw_dir() / "gateway-ca.ext4"
+        if vol.exists():
+            return vol
+        info(f"creating gateway CA volume {vol} ({_CA_VOLUME_SIZE})")
+        proc = subprocess.run(
+            ["mke2fs", "-q", "-t", "ext4", "-F", str(vol), _CA_VOLUME_SIZE],
+            capture_output=True, text=True, check=False,
+        )
+        if proc.returncode != 0:
+            vol.unlink(missing_ok=True)
+            die(f"creating gateway CA volume failed: {proc.stderr.strip()}")
+        return vol
+
     def address(self) -> str:
         """The gateway VM's guest IP — the agent-facing target agent VMs'
         gateway-port traffic is DNAT'd to."""

bot_bottle/backend/firecracker/infra_vm.py

@@ -53,10 +53,16 @@ from . import firecracker_vm, infra_artifact, netpool, util
 # tokens with the same key the host CLI signs with — the host token file stays
 # the single source of truth, never clobbered per-backend (issue #469 review).
 _GUEST_SIGNING_KEY_PATH = "/var/lib/bot-bottle/orchestrator-token"
-# The gateway VM's pre-minted `gateway` JWT path (rootfs, not /dev/vdb — the
-# data plane has no registry volume and never opens the DB). Pushed post-boot;
-# the gateway daemons present it to the orchestrator, and never see the key.
+# The gateway VM's pre-minted `gateway` JWT path (rootfs, not /dev/vdb — the JWT
+# is re-pushed every boot, so it needn't persist). Pushed post-boot; the gateway
+# daemons present it to the orchestrator, and never see the key.
 _GUEST_GATEWAY_JWT_PATH = "/var/lib/bot-bottle/gateway-jwt"
+# The gateway VM's persistent CA volume mount point — mitmproxy's confdir. The
+# gateway boots with a persistent /dev/vdb CA volume (see
+# `FirecrackerGateway._ensure_ca_volume`) mounted here so the self-generated CA
+# survives a gateway-VM rebuild; without it every rebuild mints a fresh CA that
+# already-running bottles distrust, breaking the TLS handshake (issue #450).
+_GATEWAY_CA_MOUNT = "/home/mitmproxy/.mitmproxy"
 
 # The two per-plane rootfs source images. The orchestrator VM boots a control
 # plane + buildah rootfs (Dockerfile.orchestrator.fc, FROM orchestrator); the
@@ -448,6 +454,13 @@ def _gateway_init() -> str:
     bot-bottle.db (PRD 0070 / #469). If the JWT never arrives, REFUSE to start
     rather than run without auth."""
     return _init_head() + f"""
+# Persistent CA volume (second virtio-block device, /dev/vdb) mounted at
+# mitmproxy's confdir, so the self-generated mitmproxy CA survives gateway-VM
+# rebuilds (issue #450). On first boot the volume is empty and mitmproxy mints a
+# CA into it; every later boot reuses it. Must mount BEFORE the data plane (hence
+# mitmproxy) starts.
+mkdir -p {_GATEWAY_CA_MOUNT}
+mount -t ext4 /dev/vdb {_GATEWAY_CA_MOUNT} 2>/dev/null || true
 ORCH=$(sed -n 's/.*bb_orch=\\([^ ]*\\).*/\\1/p' /proc/cmdline)
 GW_JWT=""
 i=0

tests/unit/test_firecracker_gateway.py

@@ -59,7 +59,10 @@ class TestFirecrackerGatewayConnect(unittest.TestCase):
     def test_boots_the_gateway_vm_and_seeds_the_token(self) -> None:
         gw = FirecrackerGateway()
         booted = infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k"))
+        ca_vol = Path("/gw/gateway-ca.ext4")
         with patch.object(infra_vm, "boot_vm", return_value=booted) as boot, \
+                patch.object(FirecrackerGateway, "_ensure_ca_volume",
+                             return_value=ca_vol), \
                 patch.object(infra_vm, "push_secret") as push:
             gw.connect_to_orchestrator(_ORCH_URL, _TOKEN)
         # Booted on the gateway link with the gateway role; the orchestrator's
@@ -67,12 +70,43 @@ class TestFirecrackerGatewayConnect(unittest.TestCase):
         kw = boot.call_args.kwargs
         self.assertEqual("gateway", kw["role"])
         self.assertIn("bb_orch=10.243.255.1", kw["extra_boot_args"])
-        self.assertIsNone(kw.get("data_drive"))  # data plane never opens the DB
+        # The persistent CA volume rides as /dev/vdb so the mitmproxy CA
+        # survives a gateway-VM rebuild (issue #450).
+        self.assertEqual(ca_vol, kw.get("data_drive"))
         # The host-minted token (never the key — #469) is pushed to the guest.
         push.assert_called_once()
         self.assertEqual(_TOKEN, push.call_args.args[1])
 
 
+class TestGatewayCaVolume(unittest.TestCase):
+    """The persistent CA volume that keeps the mitmproxy CA stable across a
+    gateway-VM rebuild (issue #450) — the mirror of the orchestrator's registry
+    volume."""
+
+    def test_reuses_existing_volume(self) -> None:
+        gw = FirecrackerGateway()
+        with tempfile.TemporaryDirectory() as td:
+            vol = Path(td) / "gateway-ca.ext4"
+            vol.write_bytes(b"")  # already present
+            with patch.object(infra_vm, "_gw_dir", return_value=Path(td)), \
+                    patch(f"{_GW}.subprocess.run") as run:
+                out = gw._ensure_ca_volume()
+            run.assert_not_called()  # no mke2fs when it exists — the CA survives
+            self.assertEqual(vol, out)
+
+    def test_creates_volume_when_missing(self) -> None:
+        gw = FirecrackerGateway()
+        with tempfile.TemporaryDirectory() as td:
+            with patch.object(infra_vm, "_gw_dir", return_value=Path(td)), \
+                    patch(f"{_GW}.subprocess.run",
+                          return_value=CompletedProcess([], 0)) as run:
+                out = gw._ensure_ca_volume()
+            argv = run.call_args.args[0]
+            self.assertIn("mke2fs", argv)
+            self.assertEqual(str(Path(td) / "gateway-ca.ext4"), out.__fspath__())
+            self.assertIn(str(out), argv)
+
+
 class TestFirecrackerGatewaySurface(unittest.TestCase):
     def test_is_running_reads_the_gateway_pidfile(self) -> None:
         gw = FirecrackerGateway()

tests/unit/test_firecracker_infra_vm.py

@@ -61,7 +61,11 @@ class TestRoleInits(unittest.TestCase):
         init = infra_vm.role_init("gateway")
         self.assertNotIn("bb_role=", init)
         self.assertNotIn("bot_bottle.orchestrator", init)
-        self.assertNotIn("/dev/vdb", init)  # no registry volume on the data plane
+        # Persistent CA volume mounted at mitmproxy's confdir so the CA survives
+        # a gateway-VM rebuild (issue #450). Mounted BEFORE the data plane starts.
+        self.assertIn(f"mount -t ext4 /dev/vdb {infra_vm._GATEWAY_CA_MOUNT}", init)
+        self.assertLess(init.index("/dev/vdb"),
+                        init.index("bot_bottle.gateway.bootstrap"))
         self.assertIn("export PATH=", init)  # shared preamble
         self.assertIn("BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise", init)
         self.assertIn(f"cat {infra_vm._GUEST_GATEWAY_JWT_PATH}", init)  # host-minted JWT