fix(firecracker): persist the gateway mitmproxy CA across rebuilds

The #450 CA-persistence fix was only implemented for the docker backend
(the host_gateway_ca_dir bind-mount). The firecracker gateway booted with
no persistent volume, so its mitmproxy CA lived only in the ephemeral
per-boot rootfs — every rebuild/restart minted a fresh CA that every
already-running bottle distrusted, failing egress TLS with "SSL
certificate verification failed".

Give the gateway VM a persistent CA volume, mirroring the orchestrator's
registry volume: boot with a small ext4 data_drive (_ensure_ca_volume)
and mount it at mitmproxy's confdir in the gateway guest init, before the
data plane starts, so mitmproxy reuses the on-disk CA instead of
regenerating it.

Closes #510

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

tests/unit/test_firecracker_gateway.py

@@ -59,7 +59,10 @@ class TestFirecrackerGatewayConnect(unittest.TestCase):
     def test_boots_the_gateway_vm_and_seeds_the_token(self) -> None:
         gw = FirecrackerGateway()
         booted = infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k"))
+        ca_vol = Path("/gw/gateway-ca.ext4")
         with patch.object(infra_vm, "boot_vm", return_value=booted) as boot, \
+                patch.object(FirecrackerGateway, "_ensure_ca_volume",
+                             return_value=ca_vol), \
                 patch.object(infra_vm, "push_secret") as push:
             gw.connect_to_orchestrator(_ORCH_URL, _TOKEN)
         # Booted on the gateway link with the gateway role; the orchestrator's
@@ -67,12 +70,43 @@ class TestFirecrackerGatewayConnect(unittest.TestCase):
         kw = boot.call_args.kwargs
         self.assertEqual("gateway", kw["role"])
         self.assertIn("bb_orch=10.243.255.1", kw["extra_boot_args"])
-        self.assertIsNone(kw.get("data_drive"))  # data plane never opens the DB
+        # The persistent CA volume rides as /dev/vdb so the mitmproxy CA
+        # survives a gateway-VM rebuild (issue #450).
+        self.assertEqual(ca_vol, kw.get("data_drive"))
         # The host-minted token (never the key — #469) is pushed to the guest.
         push.assert_called_once()
         self.assertEqual(_TOKEN, push.call_args.args[1])
 
 
+class TestGatewayCaVolume(unittest.TestCase):
+    """The persistent CA volume that keeps the mitmproxy CA stable across a
+    gateway-VM rebuild (issue #450) — the mirror of the orchestrator's registry
+    volume."""
+
+    def test_reuses_existing_volume(self) -> None:
+        gw = FirecrackerGateway()
+        with tempfile.TemporaryDirectory() as td:
+            vol = Path(td) / "gateway-ca.ext4"
+            vol.write_bytes(b"")  # already present
+            with patch.object(infra_vm, "_gw_dir", return_value=Path(td)), \
+                    patch(f"{_GW}.subprocess.run") as run:
+                out = gw._ensure_ca_volume()
+            run.assert_not_called()  # no mke2fs when it exists — the CA survives
+            self.assertEqual(vol, out)
+
+    def test_creates_volume_when_missing(self) -> None:
+        gw = FirecrackerGateway()
+        with tempfile.TemporaryDirectory() as td:
+            with patch.object(infra_vm, "_gw_dir", return_value=Path(td)), \
+                    patch(f"{_GW}.subprocess.run",
+                          return_value=CompletedProcess([], 0)) as run:
+                out = gw._ensure_ca_volume()
+            argv = run.call_args.args[0]
+            self.assertIn("mke2fs", argv)
+            self.assertEqual(str(Path(td) / "gateway-ca.ext4"), out.__fspath__())
+            self.assertIn(str(out), argv)
+
+
 class TestFirecrackerGatewaySurface(unittest.TestCase):
     def test_is_running_reads_the_gateway_pidfile(self) -> None:
         gw = FirecrackerGateway()

tests/unit/test_firecracker_infra_vm.py

@@ -61,7 +61,11 @@ class TestRoleInits(unittest.TestCase):
         init = infra_vm.role_init("gateway")
         self.assertNotIn("bb_role=", init)
         self.assertNotIn("bot_bottle.orchestrator", init)
-        self.assertNotIn("/dev/vdb", init)  # no registry volume on the data plane
+        # Persistent CA volume mounted at mitmproxy's confdir so the CA survives
+        # a gateway-VM rebuild (issue #450). Mounted BEFORE the data plane starts.
+        self.assertIn(f"mount -t ext4 /dev/vdb {infra_vm._GATEWAY_CA_MOUNT}", init)
+        self.assertLess(init.index("/dev/vdb"),
+                        init.index("bot_bottle.gateway.bootstrap"))
         self.assertIn("export PATH=", init)  # shared preamble
         self.assertIn("BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise", init)
         self.assertIn(f"cat {infra_vm._GUEST_GATEWAY_JWT_PATH}", init)  # host-minted JWT