From 4132b09ebcb216bece02f38aff4d198026af72d5 Mon Sep 17 00:00:00 2001 From: didericis Date: Tue, 14 Jul 2026 01:38:22 -0400 Subject: [PATCH] fix(orchestrator): poll for the gateway CA (fresh-start readiness race) On a fresh gateway container, mitmproxy writes mitmproxy-ca-cert.pem a beat after mitmdump boots, so ca_cert_pem() read it too early and launch died with 'gateway CA cert not available'. It now polls (up to 30s) until mitmproxy has generated it. Surfaced running the docker backend locally on a cold gateway. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck --- bot_bottle/orchestrator/gateway.py | 32 +++++++++++++++++-------- tests/unit/test_orchestrator_gateway.py | 10 +++++++- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/bot_bottle/orchestrator/gateway.py b/bot_bottle/orchestrator/gateway.py index 4f0806b..8cc0bbf 100644 --- a/bot_bottle/orchestrator/gateway.py +++ b/bot_bottle/orchestrator/gateway.py @@ -19,10 +19,16 @@ from __future__ import annotations import abc import os +import time from pathlib import Path from ..docker_cmd import run_docker +# The gateway's mitmproxy writes its CA a beat after the container starts, so +# reads poll for it rather than assuming it's there on a fresh launch. +_CA_POLL_SECONDS = 0.5 +DEFAULT_CA_TIMEOUT_SECONDS = 30.0 + GATEWAY_NAME = "bot-bottle-orch-gateway" GATEWAY_LABEL = "bot-bottle-orch-gateway=1" # The single user-defined network the gateway and every agent bottle share. @@ -175,17 +181,23 @@ class DockerGateway(Gateway): if proc.returncode != 0: raise GatewayError(f"gateway failed to start: {proc.stderr.strip()}") - def ca_cert_pem(self) -> str: + def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str: """The gateway's CA certificate (PEM) that agents install to trust its - TLS interception. Read from the running container; raises if the - gateway hasn't generated it yet (it's created on first mitmproxy - start).""" - proc = run_docker(["docker", "exec", self.name, "cat", GATEWAY_CA_CERT]) - if proc.returncode != 0 or not proc.stdout.strip(): - raise GatewayError( - f"gateway CA cert not available: {proc.stderr.strip() or 'empty'}" - ) - return proc.stdout + TLS interception. mitmproxy generates it a moment after the container + starts, so this **polls** for it (up to `timeout`) rather than assuming + it's already there on a fresh gateway — raising only if it never + appears.""" + deadline = time.monotonic() + timeout + while True: + proc = run_docker(["docker", "exec", self.name, "cat", GATEWAY_CA_CERT]) + if proc.returncode == 0 and proc.stdout.strip(): + return proc.stdout + if time.monotonic() >= deadline: + raise GatewayError( + f"gateway CA cert not available after {timeout:g}s: " + f"{proc.stderr.strip() or 'empty'}" + ) + time.sleep(_CA_POLL_SECONDS) def stop(self) -> None: proc = run_docker(["docker", "rm", "--force", self.name]) diff --git a/tests/unit/test_orchestrator_gateway.py b/tests/unit/test_orchestrator_gateway.py index 0f84029..484ca61 100644 --- a/tests/unit/test_orchestrator_gateway.py +++ b/tests/unit/test_orchestrator_gateway.py @@ -81,9 +81,17 @@ class TestDockerGateway(unittest.TestCase): self.assertEqual(["docker", "exec", self.sc.name, "cat", GATEWAY_CA_CERT], argv) def test_ca_cert_pem_raises_when_absent(self) -> None: + # timeout=0 → one probe then give up (no polling delay in the test). with patch(_RUN_DOCKER, return_value=_proc(returncode=1, stderr="No such file")): with self.assertRaises(GatewayError): - self.sc.ca_cert_pem() + self.sc.ca_cert_pem(timeout=0) + + def test_ca_cert_pem_polls_until_mitmproxy_writes_it(self) -> None: + # First read: CA not there yet; second read: present. + seq = [_proc(returncode=1, stderr="No such file"), _proc(stdout=_CA_PEM)] + with patch(_RUN_DOCKER, side_effect=seq), \ + patch("bot_bottle.orchestrator.gateway.time.sleep"): + self.assertEqual(_CA_PEM, self.sc.ca_cert_pem(timeout=5)) def test_ensure_running_reuses_existing_network(self) -> None: calls: list[list[str]] = []