feat(docker): split orchestrator and gateway into separate containers (PRD 0070)

Now that #469 got the DB off the data plane, the docker backend runs the
control plane and data plane as two containers instead of the combined
`bot-bottle-infra` container:

  * bot-bottle-orchestrator — the lean control-plane container (image
    Dockerfile.orchestrator, `-m bot_bottle.orchestrator`). Joins the new
    `bot-bottle-orchestrator` control network (`--internal`) only, plus a
    host-loopback publish for the CLI. Sole opener of bot-bottle.db; holds the
    signing key; no CA, no gateway daemons.
  * bot-bottle-orch-gateway — the data-plane container (DockerGateway),
    dual-homed on the agent `bot-bottle-gateway` network AND the control
    network, so it resolves the orchestrator by name
    (http://bot-bottle-orchestrator:8099) while agents — never on the control
    network — have no route to the control plane (the L3 block, not just the
    JWT). Holds the gateway JWT + the mitmproxy CA.

DockerInfraService now orchestrates the pair (builds gateway + orchestrator
images, brings up the orchestrator then the gateway) and exposes gateway_name;
the launch flow attributes agents against the gateway container. DockerGateway
gains a control_network it joins after run. rotate_ca / the CA read target the
gateway container. The combined Dockerfile.infra is no longer built by docker
(firecracker/macOS still use it until their splits).

pyright 0 errors; unit suite green (2273). Integration tests updated for the
two-container shape but need a real-docker run to validate the networking.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

tests/integration/test_orchestrator_docker_auth.py

@@ -35,7 +35,6 @@ from tests._backend import skip_unless_backend
 # image instead of leaking a new dangling tag on every invocation.
 _TEST_ORCHESTRATOR_IMAGE = "bot-bottle-orchestrator:itest"
 _TEST_GATEWAY_IMAGE = "bot-bottle-gateway:itest"
-_TEST_INFRA_IMAGE = "bot-bottle-infra:itest"
 
 
 @skip_unless_backend("docker")
@@ -71,17 +70,23 @@ class TestDockerOrchestratorAuthIntegration(unittest.TestCase):
         os.environ["BOT_BOTTLE_ROOT"] = cls._tmp.name
         cls.addClassCleanup(_restore_root)
 
-        infra_name = f"bot-bottle-infra-itest-{suffix}"
+        orchestrator_name = f"bot-bottle-orchestrator-itest-{suffix}"
+        gateway_name = f"bot-bottle-gateway-itest-{suffix}"
         network = f"bot-bottle-net-itest-{suffix}"
+        control_network = f"bot-bottle-ctrl-itest-{suffix}"
         host_root = Path(cls._tmp.name)
         cls.addClassCleanup(
-            cls._teardown_docker, infra_name, network, host_root
+            cls._teardown_docker,
+            orchestrator_name, gateway_name, network, control_network, host_root,
         )
 
         cls.svc = DockerInfraService(
-            infra_name=infra_name,
+            orchestrator_name=orchestrator_name,
+            gateway_name=gateway_name,
             network=network,
-            image=_TEST_INFRA_IMAGE,
+            control_network=control_network,
+            orchestrator_image=_TEST_ORCHESTRATOR_IMAGE,
+            gateway_image=_TEST_GATEWAY_IMAGE,
             port=20000 + secrets.randbelow(10000),
             host_root=host_root,
         )
@@ -94,23 +99,26 @@ class TestDockerOrchestratorAuthIntegration(unittest.TestCase):
 
     @staticmethod
     def _teardown_docker(
-        infra_name: str, network: str, host_root: Path
+        orchestrator_name: str, gateway_name: str,
+        network: str, control_network: str, host_root: Path,
     ) -> None:
-        subprocess.run(
-            ["docker", "rm", "--force", infra_name],
-            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
-        )
-        subprocess.run(
-            ["docker", "network", "rm", network],
-            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
-        )
-        # The infra container (no USER directive) wrote the registry
+        for name in (gateway_name, orchestrator_name):
+            subprocess.run(
+                ["docker", "rm", "--force", name],
+                stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
+            )
+        for net in (network, control_network):
+            subprocess.run(
+                ["docker", "network", "rm", net],
+                stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
+            )
+        # The orchestrator container (no USER directive) wrote the registry
         # DB as root into the throwaway host_root; chown it back so the
         # (non-root) tempdir cleanup can remove it. Same workaround
         # test_multitenant_isolation.py uses for the identical bind mount.
         subprocess.run(
             ["docker", "run", "--rm", "-v", f"{host_root}:/r",
-             "--entrypoint", "chown", _TEST_INFRA_IMAGE, "-R",
+             "--entrypoint", "chown", _TEST_ORCHESTRATOR_IMAGE, "-R",
              f"{os.getuid()}:{os.getgid()}", "/r"],
             stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
         )

tests/unit/test_backend_secret_reprovision.py

@@ -82,8 +82,10 @@ class TestMacosReprovision(unittest.TestCase):
 
 class TestDockerReprovision(unittest.TestCase):
     def test_maps_network_containers_to_keys(self) -> None:
+        # The gateway container (excluded — it's on the data network but isn't
+        # an agent) + one agent + a malformed line.
         inspect = _proc(stdout=(
-            "bot-bottle-infra 172.18.0.2/16\n"
+            "bot-bottle-orch-gateway 172.18.0.2/16\n"
             "bot-bottle-a 172.18.0.3/16\n"
             "malformed\n"
         ))

tests/unit/test_docker_infra.py

@@ -1,4 +1,9 @@
-"""Unit: infra container lifecycle — idempotent singleton (PRD 0070)."""
+"""Unit: orchestrator + gateway container lifecycle (PRD 0070 plane split).
+
+`DockerInfraService` brings up two containers — the lean orchestrator on the
+`--internal` control network + host loopback, and the dual-homed gateway. These
+tests isolate the orchestrator-container logic by mocking `_gateway` (the
+gateway runs via `DockerGateway`, exercised in its own test)."""
 
 from __future__ import annotations
 
@@ -10,15 +15,16 @@ from unittest.mock import MagicMock, Mock, patch
 
 from bot_bottle.gateway import GatewayError
 from bot_bottle.backend.docker.infra import (
-    INFRA_NAME,
-    INFRA_SOURCE_HASH_LABEL,
+    ORCHESTRATOR_NAME,
+    ORCHESTRATOR_NETWORK,
+    ORCHESTRATOR_SOURCE_HASH_LABEL,
     DockerInfraService,
 )
+from bot_bottle.gateway import GATEWAY_NAME
 from bot_bottle.orchestrator.lifecycle import (
     OrchestratorStartError,
     source_hash,
 )
-from bot_bottle.paths import GATEWAY_CA_DIRNAME
 from tests.unit import use_bottle_root
 
 _URLOPEN = "bot_bottle.backend.docker.infra.urllib.request.urlopen"
@@ -43,168 +49,147 @@ class TestDockerInfraService(unittest.TestCase):
         self.addCleanup(self._tmp.cleanup)
         self.addCleanup(use_bottle_root(Path(self._tmp.name)))
         self.svc = DockerInfraService(port=8099)
+        # Isolate the orchestrator-container logic: the gateway is brought up by
+        # DockerGateway (its own module/run_docker), tested separately.
+        self.gw = MagicMock()
+        patcher = patch.object(self.svc, "_gateway", return_value=self.gw)
+        patcher.start()
+        self.addCleanup(patcher.stop)
 
     def test_url(self) -> None:
         self.assertEqual("http://127.0.0.1:8099", self.svc.url)
 
+    def test_gateway_name(self) -> None:
+        self.assertEqual(GATEWAY_NAME, self.svc.gateway_name)
+
     def test_is_healthy(self) -> None:
         with patch(_URLOPEN, return_value=_health(200)):
             self.assertTrue(self.svc.is_healthy())
         with patch(_URLOPEN, side_effect=urllib.error.URLError("refused")):
             self.assertFalse(self.svc.is_healthy())
 
-    def test_ensure_running_noop_when_healthy_and_source_unchanged(self) -> None:
-        # A healthy container on current source is left alone — recreating it
-        # on every launch drops in-memory egress tokens (#381).
+    def test_noop_when_healthy_and_source_unchanged(self) -> None:
+        # A healthy orchestrator on current source is left alone — recreating it
+        # on every launch drops in-memory egress tokens (#381). The gateway is
+        # still refreshed (idempotent).
         current = source_hash(self.svc._repo_root)
-        calls: list[list[str]] = []
 
         def fake(argv: list[str], **_kw: object) -> Mock:
-            calls.append(argv)
             if argv[:2] == ["docker", "ps"]:
-                return _proc(stdout=INFRA_NAME)
+                return _proc(stdout=ORCHESTRATOR_NAME)
             if argv[:2] == ["docker", "inspect"]:
                 return _proc(stdout=current)
             return _proc()
 
         with patch(_URLOPEN, return_value=_health(200)), \
-                patch(_RUN, side_effect=fake), patch(_SLEEP):
+                patch(_RUN, side_effect=fake) as run, patch(_SLEEP):
             self.assertEqual(self.svc.url, self.svc.ensure_running())
-        runs = [c for c in calls if c[:2] == ["docker", "run"]]
-        rms = [c for c in calls if c[:3] == ["docker", "rm", "--force"] and INFRA_NAME in c]
+        runs = [c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "run"]]
         self.assertEqual([], runs)
-        self.assertEqual([], rms)
-
-    def test_ensure_running_recreates_when_source_changed(self) -> None:
-        calls: list[list[str]] = []
+        self.gw.ensure_running.assert_called_once_with()
 
+    def test_recreates_orchestrator_when_source_changed(self) -> None:
         def fake(argv: list[str], **_kw: object) -> Mock:
-            calls.append(argv)
             if argv[:2] == ["docker", "ps"]:
-                return _proc(stdout=INFRA_NAME)
+                return _proc(stdout=ORCHESTRATOR_NAME)
             if argv[:2] == ["docker", "inspect"]:
                 return _proc(stdout="stale-hash")
             return _proc()
 
         with patch(_URLOPEN, return_value=_health(200)), \
-                patch(_RUN, side_effect=fake), patch(_SLEEP):
+                patch(_RUN, side_effect=fake) as run, patch(_SLEEP):
             self.assertEqual(self.svc.url, self.svc.ensure_running())
-        runs = [c for c in calls if c[:2] == ["docker", "run"]]
+        runs = [c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "run"]]
         self.assertEqual(1, len(runs))
-        self.assertIn(INFRA_NAME, runs[0])
+        self.assertIn(ORCHESTRATOR_NAME, runs[0])
         current = source_hash(self.svc._repo_root)
-        self.assertIn(f"{INFRA_SOURCE_HASH_LABEL}={current}", runs[0])
-
-    def test_ensure_running_starts_infra_container_when_absent(self) -> None:
-        calls: list[list[str]] = []
+        self.assertIn(f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={current}", runs[0])
 
+    def test_starts_orchestrator_when_absent(self) -> None:
         def fake(argv: list[str], **_kw: object) -> Mock:
-            calls.append(argv)
             if argv[:2] == ["docker", "ps"]:
                 return _proc(stdout="")
             return _proc()
 
         with patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \
-                patch(_RUN, side_effect=fake), patch(_SLEEP):
+                patch(_RUN, side_effect=fake) as run, patch(_SLEEP):
             self.assertEqual(self.svc.url, self.svc.ensure_running())
-        runs = [c for c in calls if c[:2] == ["docker", "run"]]
+        runs = [c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "run"]]
         self.assertEqual(1, len(runs))
         argv = runs[0]
-        self.assertIn(INFRA_NAME, argv)
-        # Published on loopback — not exposed on external interfaces.
+        self.assertIn(ORCHESTRATOR_NAME, argv)
+        # Control network only — agents are never on it.
+        self.assertEqual(ORCHESTRATOR_NETWORK, argv[argv.index("--network") + 1])
+        # Published on loopback for the CLI, mapped to the fixed internal 8099.
         self.assertEqual("127.0.0.1:8099:8099", argv[argv.index("--publish") + 1])
-        # Both processes in one container — no separate entrypoint override.
-        self.assertNotIn("--entrypoint", argv)
-        # Gateway daemons + orchestrator explicitly opted in.
-        daemons_flag = "BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise,orchestrator"
-        self.assertIn("orchestrator", argv[argv.index(daemons_flag)])
-        # The mitmproxy CA persists on a HOST bind-mount under the app-data root
-        # (not a docker named volume `docker volume prune` would wipe — #450), so
-        # a restarted infra container keeps the CA every running bottle trusts.
-        ca_mounts = [a for a in argv if a.endswith(":/home/mitmproxy/.mitmproxy")]
-        self.assertEqual(1, len(ca_mounts))
-        src = ca_mounts[0].rsplit(":", 1)[0]
-        self.assertTrue(src.startswith(self._tmp.name), src)
-        self.assertTrue(src.endswith("/" + GATEWAY_CA_DIRNAME), src)
-
-    def test_ensure_running_builds_all_images(self) -> None:
-        calls: list[list[str]] = []
+        # The lean control plane: no mitmproxy CA mount, no gateway daemons.
+        self.assertFalse([a for a in argv if a.endswith(":/home/mitmproxy/.mitmproxy")])
+        self.assertNotIn("BOT_BOTTLE_GATEWAY_DAEMONS", " ".join(argv))
+        # Orchestrator entrypoint args (image ENTRYPOINT is `-m bot_bottle.orchestrator`).
+        self.assertIn("--broker", argv)
+        self.assertIn("stub", argv)
+        # Gateway is brought up after the control plane is healthy.
+        self.gw.ensure_running.assert_called_once_with()
 
+    def test_builds_gateway_and_orchestrator_images(self) -> None:
         def fake(argv: list[str], **_kw: object) -> Mock:
-            calls.append(argv)
             if argv[:2] == ["docker", "ps"]:
                 return _proc(stdout="")
             return _proc()
 
         with patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \
-                patch(_RUN, side_effect=fake), patch(_SLEEP):
+                patch(_RUN, side_effect=fake) as run, patch(_SLEEP):
             self.svc.ensure_running()
-        builds = [c for c in calls if c[:2] == ["docker", "build"]]
-        # Gateway base + orchestrator intermediate + infra image — all three built.
-        self.assertEqual(3, len(builds))
+        builds = [c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "build"]]
+        # Two images now (the combined Dockerfile.infra is retired).
+        self.assertEqual(2, len(builds))
         dockerfiles = [next(a for a in b if "Dockerfile" in a) for b in builds]
         self.assertIn("Dockerfile.gateway", dockerfiles[0])
         self.assertIn("Dockerfile.orchestrator", dockerfiles[1])
-        self.assertIn("Dockerfile.infra", dockerfiles[2])
-        # All three images are distinct.
-        tags = [b[b.index("-t") + 1] for b in builds]
-        self.assertEqual(3, len(set(tags)))
 
     def test_publish_maps_host_port_to_fixed_internal_port(self) -> None:
-        """A non-default self.port is published to the fixed internal port 8099,
-        not to self.port:self.port — the orchestrator always listens on 8099."""
-        calls: list[list[str]] = []
-
         def fake(argv: list[str], **_kw: object) -> Mock:
-            calls.append(argv)
             if argv[:2] == ["docker", "ps"]:
                 return _proc(stdout="")
             return _proc()
 
         svc = DockerInfraService(port=20001)
-        with patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \
-                patch(_RUN, side_effect=fake), patch(_SLEEP):
+        with patch.object(svc, "_gateway", return_value=MagicMock()), \
+                patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \
+                patch(_RUN, side_effect=fake) as run, patch(_SLEEP):
             svc.ensure_running()
-        runs = [c for c in calls if c[:2] == ["docker", "run"]]
-        argv = runs[0]
+        argv = next(c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "run"])
         self.assertEqual("127.0.0.1:20001:8099", argv[argv.index("--publish") + 1])
-        orch_url = next(a for a in argv if "BOT_BOTTLE_ORCHESTRATOR_URL" in a)
-        self.assertIn(":8099", orch_url)
 
-    def test_ensure_running_raises_on_timeout(self) -> None:
+    def test_raises_on_control_plane_timeout(self) -> None:
         with patch(_URLOPEN, side_effect=urllib.error.URLError("down")), \
-                patch(_RUN, return_value=Mock(returncode=0, stdout="", stderr="")), \
+                patch(_RUN, return_value=_proc()), \
                 patch(_SLEEP), patch(_MONOTONIC, side_effect=[0.0, 0.5, 2.0]):
             with self.assertRaises(OrchestratorStartError):
                 self.svc.ensure_running(startup_timeout=1.0)
 
     def test_noop_when_healthy_and_inspect_fails(self) -> None:
-        """If docker inspect fails (e.g. docker daemon hiccup), leave the
-        working container alone rather than churning it."""
         def fake(argv: list[str], **_kw: object) -> Mock:
             if argv[:2] == ["docker", "ps"]:
-                return _proc(stdout=INFRA_NAME)
+                return _proc(stdout=ORCHESTRATOR_NAME)
             if argv[:2] == ["docker", "inspect"]:
                 return _proc(returncode=1, stderr="daemon error")
             return _proc()
 
         with patch(_URLOPEN, return_value=_health(200)), \
-                patch(_RUN, side_effect=fake), patch(_SLEEP):
+                patch(_RUN, side_effect=fake) as run, patch(_SLEEP):
             self.svc.ensure_running()
-        # no docker run — the working container was left alone
+        runs = [c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "run"]]
+        self.assertEqual([], runs)
 
     def test_build_failure_raises(self) -> None:
         with patch(_URLOPEN, side_effect=urllib.error.URLError("down")), \
-                patch(_RUN, return_value=_proc(returncode=1, stderr="no space left on device")):
+                patch(_RUN, return_value=_proc(returncode=1, stderr="no space left")):
             with self.assertRaises(GatewayError):
                 self.svc.ensure_running()
 
-    def test_ensure_network_creates_if_missing(self) -> None:
-        """If the gateway network doesn't exist yet, create it."""
-        calls: list[list[str]] = []
-
+    def test_creates_control_network_internal(self) -> None:
         def fake(argv: list[str], **_kw: object) -> Mock:
-            calls.append(argv)
             if argv[:3] == ["docker", "network", "inspect"]:
                 return _proc(returncode=1, stderr="not found")
             if argv[:2] == ["docker", "ps"]:
@@ -212,19 +197,23 @@ class TestDockerInfraService(unittest.TestCase):
             return _proc()
 
         with patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \
-                patch(_RUN, side_effect=fake), patch(_SLEEP):
+                patch(_RUN, side_effect=fake) as run, patch(_SLEEP):
             self.svc.ensure_running()
-        creates = [c for c in calls if c[:3] == ["docker", "network", "create"]]
-        self.assertEqual(1, len(creates))
+        creates = [c.args[0] for c in run.call_args_list if c.args[0][:3] == ["docker", "network", "create"]]
+        # The control network is created --internal; the data network isn't.
+        control = [c for c in creates if ORCHESTRATOR_NETWORK in c]
+        self.assertEqual(1, len(control))
+        self.assertIn("--internal", control[0])
 
-    def test_stop_removes_infra_container(self) -> None:
+    def test_stop_removes_both_containers(self) -> None:
         with patch(_RUN) as run:
             self.svc.stop()
         rms = [
             c.args[0] for c in run.call_args_list
             if c.args[0][:3] == ["docker", "rm", "--force"]
         ]
-        self.assertTrue(any(INFRA_NAME in a for a in rms))
+        self.assertTrue(any(GATEWAY_NAME in a for a in rms))
+        self.assertTrue(any(ORCHESTRATOR_NAME in a for a in rms))
 
 
 if __name__ == "__main__":