feat(backend): reconcile running bottles' CA on gateway bring-up (0081)

Add `attach_bottled_agents_to_gateway()` as an `@abc.abstractmethod` on
`BottleBackend` and implement the first reconcile step across all three
backends: on a gateway cold boot, replace every already-running agent's
trusted CA with the freshly-minted gateway CA. Reconciling running bottles
is a backend responsibility (only the backend can enumerate its agents and
reach them — firecracker over SSH, docker/macOS over exec/cp), so making it
an abstract method keeps the fix cross-backend by construction.

The gateway rootfs is ephemeral, so a rebuild mints a new CA that every
running bottle distrusts (SSL verification fails — #510). This reconcile is
what distributes the fresh CA, so a routine gateway rebuild now doubles as a
free CA rotation. Per-bottle steps are best-effort: one unreachable or
malformed bottle is logged and skipped, never blocking the others.

Trigger — `Gateway.connect_to_orchestrator` now returns a cold-boot bool
(True when it actually (re)brought the gateway up, False when a healthy
current gateway was left untouched); each infra `ensure_running` gates the
reconcile on it, so it fires exactly on cold boot and never on an adopt.

Git-gate re-provision and egress-token restore fold into this same reconcile
on the same trigger in follow-up PRs (#516).

Refs #516. Addresses #510.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

bot_bottle/backend/macos_container/backend.py

@@ -117,5 +117,9 @@ class MacosContainerBottleBackend(
     def enumerate_active(self) -> Sequence[ActiveAgent]:
         return _enumerate.enumerate_active()
 
+    def attach_bottled_agents_to_gateway(self) -> None:
+        from .consolidated_launch import attach_bottled_agents_to_gateway
+        attach_bottled_agents_to_gateway()
+
     def supervise_mcp_url(self, plan: MacosContainerBottlePlan) -> str:
         return plan.agent_supervise_url

bot_bottle/backend/macos_container/consolidated_launch.py

@@ -34,6 +34,7 @@ every real command arrives through `container exec`.
 
 from __future__ import annotations
 
+import tempfile
 from dataclasses import dataclass
 
 from ...egress import EgressPlan
@@ -43,6 +44,7 @@ from ...orchestrator.client import OrchestratorClient, OrchestratorClientError
 from ...orchestrator.reprovision import reprovision_bottles
 from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
 from ..provision_bottle import deprovision_bottle, provision_bottle
+from ..util import AGENT_CA_PATH
 from . import util as container_mod
 from .enumerate import CONTAINER_NAME_PREFIX, EnumerationError, enumerate_active
 from .gateway import GATEWAY_NETWORK
@@ -98,6 +100,66 @@ def ensure_gateway(
     return endpoint
 
 
+def _push_ca_to_container(name: str, ca_pem: str) -> None:
+    """Replace one running agent container's trusted gateway CA with `ca_pem`
+    and rebuild its trust store via `container cp` + `container exec`.
+    Unconditional install — there is one gateway, so no fingerprint match is
+    needed. Raises `ConsolidatedLaunchError` on failure."""
+    mkdir = container_mod.run_container_argv(
+        ["container", "exec", name, "mkdir", "-p",
+         "/usr/local/share/ca-certificates"]
+    )
+    if mkdir.returncode != 0:
+        raise ConsolidatedLaunchError(
+            f"CA push to {name} failed (mkdir): {(mkdir.stderr or '').strip()}"
+        )
+    with tempfile.NamedTemporaryFile("w", suffix=".crt") as tmp:
+        tmp.write(ca_pem)
+        tmp.flush()
+        cp = container_mod.run_container_argv(
+            ["container", "cp", tmp.name, f"{name}:{AGENT_CA_PATH}"]
+        )
+    if cp.returncode != 0:
+        raise ConsolidatedLaunchError(
+            f"CA push to {name} failed (cp): {(cp.stderr or '').strip()}"
+        )
+    ex = container_mod.run_container_argv(
+        ["container", "exec", name, "sh", "-c",
+         f"chmod 644 {AGENT_CA_PATH} && update-ca-certificates"]
+    )
+    if ex.returncode != 0:
+        raise ConsolidatedLaunchError(
+            f"CA push to {name} failed (update-ca-certificates): "
+            f"{(ex.stderr or '').strip()}"
+        )
+
+
+def attach_bottled_agents_to_gateway(ca_pem: str | None = None) -> None:
+    """Reconcile every running agent container against the freshly-(re)created
+    gateway (PRD 0081): replace each agent's trusted CA with the current gateway
+    CA, so a gateway rebuild doesn't silently break their TLS egress (#510).
+
+    Per-container steps are best-effort: one failure is logged and skipped so it
+    never blocks the others or the gateway coming up."""
+    if ca_pem is None:
+        ca_pem = MacosInfraService().ca_cert_pem()
+    try:
+        agents = list(enumerate_active())
+    except (EnumerationError, OSError) as exc:
+        info(f"CA reconcile skipped: {exc}")
+        return
+    reconciled = 0
+    for agent in agents:
+        name = f"{CONTAINER_NAME_PREFIX}{agent.slug}"
+        try:
+            _push_ca_to_container(name, ca_pem)
+            reconciled += 1
+        except (OSError, ConsolidatedLaunchError) as exc:
+            info(f"CA reconcile skipped for {name}: {exc}")
+    if reconciled:
+        info(f"reconciled gateway CA into {reconciled} running macOS bottle(s)")
+
+
 def _reprovision_running_bottles(endpoint: GatewayEndpoint) -> None:
     """Recover keys from live Apple containers and restore gateway tokens."""
     try:
@@ -200,6 +262,7 @@ __all__ = [
     "GatewayEndpoint",
     "LaunchContext",
     "ensure_gateway",
+    "attach_bottled_agents_to_gateway",
     "live_source_ips",
     "register_agent",
     "deprovision_consolidated",

bot_bottle/backend/macos_container/gateway.py

@@ -104,11 +104,15 @@ class MacosGateway(Gateway):
         container_mod.build_image(
             self.image_ref, str(self._repo_root), dockerfile="Dockerfile.gateway")
 
-    def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None:
+    def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> bool:
         """Bind the gateway to this orchestrator and (re)start it, dual-homed on
         the agent + control networks, resolving policy against `orchestrator_url`
         (the orchestrator's control-network address — Apple has no container
-        DNS) and presenting `gateway_token`."""
+        DNS) and presenting `gateway_token`.
+
+        Always returns True: this recreates the gateway container unconditionally
+        (a cold boot), so the caller reconciles running bottles against it
+        (PRD 0081)."""
         self._orchestrator_url = orchestrator_url
         self._gateway_token = gateway_token
         # Fail closed on a missing policy source or token: the resolver-only data
@@ -156,6 +160,7 @@ class MacosGateway(Gateway):
                 f"gateway container failed to start: "
                 f"{(result.stderr or '').strip() or '<no stderr>'}"
             )
+        return True
 
     def is_running(self) -> bool:
         return container_mod.container_is_running(self.name)

bot_bottle/backend/macos_container/infra.py

@@ -138,7 +138,15 @@ class MacosInfraService(InfraService):
         # `gateway` JWT and hands it to the gateway, which never sees the key
         # (#469).
         url = orchestrator.url()
-        gateway.connect_to_orchestrator(url, orchestrator.mint_gateway_token())
+        cold_booted = gateway.connect_to_orchestrator(
+            url, orchestrator.mint_gateway_token())
+        # A (re)created gateway container minted/mounted its CA afresh and lost
+        # every bottle's per-bottle state; reconcile the already-running bottles
+        # against it (PRD 0081). Skipped when a healthy current gateway was left
+        # untouched — no cold boot, nothing to reconcile.
+        if cold_booted:
+            from .backend import MacosContainerBottleBackend
+            MacosContainerBottleBackend().attach_bottled_agents_to_gateway()
         return url
 
     def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str: