feat(backend): reconcile running bottles' CA on gateway bring-up (0081)

Add `attach_bottled_agents_to_gateway()` as an `@abc.abstractmethod` on
`BottleBackend` and implement the first reconcile step across all three
backends: on a gateway cold boot, replace every already-running agent's
trusted CA with the freshly-minted gateway CA. Reconciling running bottles
is a backend responsibility (only the backend can enumerate its agents and
reach them — firecracker over SSH, docker/macOS over exec/cp), so making it
an abstract method keeps the fix cross-backend by construction.

The gateway rootfs is ephemeral, so a rebuild mints a new CA that every
running bottle distrusts (SSL verification fails — #510). This reconcile is
what distributes the fresh CA, so a routine gateway rebuild now doubles as a
free CA rotation. Per-bottle steps are best-effort: one unreachable or
malformed bottle is logged and skipped, never blocking the others.

Trigger — `Gateway.connect_to_orchestrator` now returns a cold-boot bool
(True when it actually (re)brought the gateway up, False when a healthy
current gateway was left untouched); each infra `ensure_running` gates the
reconcile on it, so it fires exactly on cold boot and never on an adopt.

Git-gate re-provision and egress-token restore fold into this same reconcile
on the same trigger in follow-up PRs (#516).

Refs #516. Addresses #510.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

bot_bottle/backend/firecracker/backend.py

@@ -119,6 +119,10 @@ class FirecrackerBottleBackend(
     def enumerate_active(self) -> Sequence[ActiveAgent]:
         return _enumerate.enumerate_active()
 
+    def attach_bottled_agents_to_gateway(self) -> None:
+        from .consolidated_launch import attach_bottled_agents_to_gateway
+        attach_bottled_agents_to_gateway()
+
     def supervise_mcp_url(self, plan: FirecrackerBottlePlan) -> str:
         return plan.agent_supervise_url
 

bot_bottle/backend/firecracker/consolidated_launch.py

@@ -32,6 +32,7 @@ from dataclasses import dataclass
 from pathlib import Path
 
 from ...egress import EgressPlan
+from ...gateway import GatewayError
 from ...git_gate import GitGatePlan
 from ...log import info
 from ...orchestrator.client import OrchestratorClient, OrchestratorClientError
@@ -41,6 +42,7 @@ from ...orchestrator.lifecycle import (
 from ...orchestrator.reprovision import reprovision_bottles
 from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
 from ..provision_bottle import deprovision_bottle, provision_bottle
+from ..util import AGENT_CA_PATH
 from . import cleanup, util
 from .gateway import FirecrackerGateway
 from .infra import FirecrackerInfraService
@@ -89,6 +91,56 @@ def persist_env_var_secret(private_key: Path, guest_ip: str, secret: str) -> Non
         )
 
 
+def _push_gateway_ca_to_agent(private_key: Path, guest_ip: str, ca_pem: str) -> None:
+    """Replace one running agent VM's trusted gateway CA with `ca_pem` and
+    rebuild its trust store over SSH. Unconditional install — there is one
+    gateway, so no fingerprint match is needed. Bare-pipe input keeps the cert
+    off argv. Raises `ConsolidatedLaunchError` on failure."""
+    install = (
+        "umask 022; mkdir -p /usr/local/share/ca-certificates && "
+        f"cat > {AGENT_CA_PATH} && chmod 644 {AGENT_CA_PATH} && "
+        "update-ca-certificates"
+    )
+    proc = subprocess.run(
+        util.ssh_base_argv(private_key, guest_ip) + [install],
+        input=ca_pem, capture_output=True, text=True, check=False,
+    )
+    if proc.returncode != 0:
+        raise ConsolidatedLaunchError(
+            f"CA push to {guest_ip} failed: "
+            f"{proc.stderr.strip() or '<no stderr>'}"
+        )
+
+
+def attach_bottled_agents_to_gateway() -> None:
+    """Reconcile every running agent VM against the freshly-booted gateway
+    (PRD 0081). Replaces each agent's trusted CA with the current gateway CA —
+    the gateway rootfs is ephemeral, so a cold boot mints a fresh CA and this is
+    what distributes it (a gateway rebuild doubles as a free CA rotation, #510).
+
+    Per-bottle steps are best-effort: one unreachable or malformed VM is logged
+    and skipped so it never blocks the others or the gateway coming up."""
+    gateway = FirecrackerGateway()
+    try:
+        ca_pem = gateway.ca_cert_pem()
+    except GatewayError as exc:
+        info(f"bring-up reconcile skipped: gateway CA unavailable: {exc}")
+        return
+    reconciled = 0
+    for run_dir in cleanup.live_run_dirs():
+        guest_ip = _guest_ip_from_config(run_dir / "config.json")
+        private_key = run_dir / "bottle_id_ed25519"
+        if not guest_ip or not private_key.is_file():
+            continue
+        try:
+            _push_gateway_ca_to_agent(private_key, guest_ip, ca_pem)
+            reconciled += 1
+        except (OSError, ConsolidatedLaunchError) as exc:
+            info(f"CA reconcile skipped for {guest_ip}: {exc}")
+    if reconciled:
+        info(f"reconciled gateway CA into {reconciled} running Firecracker bottle(s)")
+
+
 def _reprovision_running_bottles(client: OrchestratorClient) -> None:
     """Read keys from live agent VMs and restore the restarted gateway."""
     try:
@@ -161,6 +213,7 @@ def deprovision_consolidated(
 __all__ = [
     "LaunchContext",
     "launch_consolidated",
+    "attach_bottled_agents_to_gateway",
     "deprovision_consolidated",
     "ConsolidatedLaunchError",
     "OrchestratorStartError",

bot_bottle/backend/firecracker/gateway.py

@@ -69,13 +69,17 @@ class FirecrackerGateway(Gateway):
         self._orchestrator_url = ""
         self._gateway_token = ""
 
-    def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None:
+    def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> bool:
         """Boot the gateway VM on its link resolving policy against
         `orchestrator_url`, then seed `gateway_token` over SSH. The orchestrator's
         guest IP is parsed from the URL and passed on the cmdline (`bb_orch=`), so
         the baked init stays IP-independent. Boot the orchestrator first — the
         gateway daemons reach it at startup. The gateway never mints, so it holds
-        no signing key, only this token (#469)."""
+        no signing key, only this token (#469).
+
+        Always returns True: the firecracker gateway VM is booted fresh here
+        (this runs only on the infra cold-boot path), so it is always a cold
+        boot that minted a new CA — the caller reconciles running bottles."""
         self._orchestrator_url = orchestrator_url
         self._gateway_token = gateway_token
         if not self._orchestrator_url:
@@ -105,6 +109,7 @@ class FirecrackerGateway(Gateway):
             "the gateway JWT to the gateway VM (its data plane will not start)",
         )
         self._vm = vm
+        return True
 
     def is_running(self) -> bool:
         return infra_vm._pidfile_alive(infra_vm._gw_dir())

bot_bottle/backend/firecracker/infra.py

@@ -67,9 +67,17 @@ class FirecrackerInfraService(InfraService):
             # holds the signing key) mints the role-scoped `gateway` JWT for the
             # gateway, which never sees the key (#469).
             orchestrator.ensure_running(startup_timeout=startup_timeout)
-            self.gateway().connect_to_orchestrator(
+            cold_booted = self.gateway().connect_to_orchestrator(
                 orchestrator.gateway_url(), orchestrator.mint_gateway_token())
             infra_vm.record_booted_version(want)
+            # The fresh gateway rootfs minted a new CA and lost every bottle's
+            # git-gate/token state; reconcile the already-running bottles against
+            # it so a gateway rebuild doesn't silently break their egress
+            # (PRD 0081). Cold-boot only — the adopt fast-paths above never reach
+            # here, so a healthy current gateway is left untouched.
+            if cold_booted:
+                from .backend import FirecrackerBottleBackend
+                FirecrackerBottleBackend().attach_bottled_agents_to_gateway()
             return url
 
     def stop(self) -> None: