fix(orchestrator): poll for the gateway CA (fresh-start readiness race)

On a fresh gateway container, mitmproxy writes mitmproxy-ca-cert.pem a beat
after mitmdump boots, so ca_cert_pem() read it too early and launch died with
'gateway CA cert not available'. It now polls (up to 30s) until mitmproxy has
generated it. Surfaced running the docker backend locally on a cold gateway.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck

bot_bottle/orchestrator/gateway.py

@@ -19,10 +19,16 @@ from __future__ import annotations
 
 import abc
 import os
+import time
 from pathlib import Path
 
 from ..docker_cmd import run_docker
 
+# The gateway's mitmproxy writes its CA a beat after the container starts, so
+# reads poll for it rather than assuming it's there on a fresh launch.
+_CA_POLL_SECONDS = 0.5
+DEFAULT_CA_TIMEOUT_SECONDS = 30.0
+
 GATEWAY_NAME = "bot-bottle-orch-gateway"
 GATEWAY_LABEL = "bot-bottle-orch-gateway=1"
 # The single user-defined network the gateway and every agent bottle share.
@@ -175,17 +181,23 @@ class DockerGateway(Gateway):
         if proc.returncode != 0:
             raise GatewayError(f"gateway failed to start: {proc.stderr.strip()}")
 
-    def ca_cert_pem(self) -> str:
+    def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
         """The gateway's CA certificate (PEM) that agents install to trust its
-        TLS interception. Read from the running container; raises if the
-        gateway hasn't generated it yet (it's created on first mitmproxy
-        start)."""
-        proc = run_docker(["docker", "exec", self.name, "cat", GATEWAY_CA_CERT])
-        if proc.returncode != 0 or not proc.stdout.strip():
-            raise GatewayError(
-                f"gateway CA cert not available: {proc.stderr.strip() or 'empty'}"
-            )
-        return proc.stdout
+        TLS interception. mitmproxy generates it a moment after the container
+        starts, so this **polls** for it (up to `timeout`) rather than assuming
+        it's already there on a fresh gateway — raising only if it never
+        appears."""
+        deadline = time.monotonic() + timeout
+        while True:
+            proc = run_docker(["docker", "exec", self.name, "cat", GATEWAY_CA_CERT])
+            if proc.returncode == 0 and proc.stdout.strip():
+                return proc.stdout
+            if time.monotonic() >= deadline:
+                raise GatewayError(
+                    f"gateway CA cert not available after {timeout:g}s: "
+                    f"{proc.stderr.strip() or 'empty'}"
+                )
+            time.sleep(_CA_POLL_SECONDS)
 
     def stop(self) -> None:
         proc = run_docker(["docker", "rm", "--force", self.name])

tests/unit/test_orchestrator_gateway.py

@@ -81,9 +81,17 @@ class TestDockerGateway(unittest.TestCase):
         self.assertEqual(["docker", "exec", self.sc.name, "cat", GATEWAY_CA_CERT], argv)
 
     def test_ca_cert_pem_raises_when_absent(self) -> None:
+        # timeout=0 → one probe then give up (no polling delay in the test).
         with patch(_RUN_DOCKER, return_value=_proc(returncode=1, stderr="No such file")):
             with self.assertRaises(GatewayError):
-                self.sc.ca_cert_pem()
+                self.sc.ca_cert_pem(timeout=0)
+
+    def test_ca_cert_pem_polls_until_mitmproxy_writes_it(self) -> None:
+        # First read: CA not there yet; second read: present.
+        seq = [_proc(returncode=1, stderr="No such file"), _proc(stdout=_CA_PEM)]
+        with patch(_RUN_DOCKER, side_effect=seq), \
+                patch("bot_bottle.orchestrator.gateway.time.sleep"):
+            self.assertEqual(_CA_PEM, self.sc.ca_cert_pem(timeout=5))
 
     def test_ensure_running_reuses_existing_network(self) -> None:
         calls: list[list[str]] = []