diff --git a/bot_bottle/backend/macos_container/gateway.py b/bot_bottle/backend/macos_container/gateway.py index fcac8558..8bdfcfd7 100644 --- a/bot_bottle/backend/macos_container/gateway.py +++ b/bot_bottle/backend/macos_container/gateway.py @@ -19,27 +19,40 @@ from . import util as container_mod # on. The agent's address here is the attribution key. Distinct from the docker # names so both backends can coexist on one host. GATEWAY_NETWORK = "bot-bottle-mac-gateway" -# The NAT network that gives the infra container (and only it) a route out. +# The NAT network that gives the gateway (and only it) a route out. GATEWAY_EGRESS_NETWORK = "bot-bottle-mac-egress" +# The control network the gateway reaches the orchestrator over (host-only). +# Only the orchestrator + gateway join it; agents never do, so agents have no +# route to the control plane (PRD 0070 "Separating the planes"). +CONTROL_NETWORK = "bot-bottle-mac-control" GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest") +ORCHESTRATOR_IMAGE = os.environ.get( + "BOT_BOTTLE_ORCHESTRATOR_IMAGE", "bot-bottle-orchestrator:latest" +) DEFAULT_CA_TIMEOUT_SECONDS = 30.0 def ensure_networks( - network: str = GATEWAY_NETWORK, egress_network: str = GATEWAY_EGRESS_NETWORK, + network: str = GATEWAY_NETWORK, + egress_network: str = GATEWAY_EGRESS_NETWORK, + control_network: str = CONTROL_NETWORK, ) -> None: - """Create the shared host-only network + the NAT egress network. Idempotent - — `create_network` tolerates 'already exists'.""" + """Create the shared host-only agent network, the NAT egress network, and + the host-only control network. Idempotent — `create_network` tolerates + 'already exists'.""" container_mod.create_network(egress_network) container_mod.create_network(network, internal=True) + container_mod.create_network(control_network, internal=True) __all__ = [ "GATEWAY_NETWORK", "GATEWAY_EGRESS_NETWORK", + "CONTROL_NETWORK", "GATEWAY_IMAGE", + "ORCHESTRATOR_IMAGE", "GatewayError", "DEFAULT_CA_TIMEOUT_SECONDS", "ensure_networks", diff --git a/bot_bottle/backend/macos_container/infra.py b/bot_bottle/backend/macos_container/infra.py index c2f14cc7..5df0ad33 100644 --- a/bot_bottle/backend/macos_container/infra.py +++ b/bot_bottle/backend/macos_container/infra.py @@ -1,34 +1,22 @@ -"""The per-host infra container for the macOS backend (PRD 0070). +"""The per-host control plane + gateway for the macOS backend (PRD 0070). -A single persistent Apple container that runs BOTH the orchestrator control -plane and the gateway data plane — the macOS analogue of the Firecracker infra -VM (`backend/firecracker/infra_vm.py`), not the docker backend's two separate -containers. +Two Apple containers — the orchestrator (control plane) and the gateway (data +plane) — split now that #469 got the DB off the data plane. The single-container +model existed only because two Apple-Container guests writing one `bot-bottle.db` +over virtiofs would race incoherent `fcntl` locks; with the data plane no longer +opening the DB at all, only the orchestrator does, so the split is safe. -Why one container, not two: Apple Containers are lightweight VMs, each with its -own kernel. The docker backend runs the orchestrator and gateway as two -containers safely because they share the host kernel, so their concurrent -writes to the one `bot-bottle.db` (the orchestrator's registry + the gateway -supervise daemon's queue) are serialized by coherent `fcntl` locks. Across two -*guest* kernels sharing a virtiofs-mounted DB those locks are not coherent, and -concurrent writers can corrupt the file. Firecracker solved this by putting -both services in one guest with the DB on a device only that guest mounts; this -does the same with Apple primitives. + * `bot-bottle-mac-orchestrator` — the lean control plane. Joins the host-only + **control network** (`bot-bottle-mac-control`) only. Sole opener of the + container-only DB volume; holds the signing key. The host CLI reaches it at + its control-network address; the gateway reaches it there too. + * `bot-bottle-mac-infra` — the gateway data plane. Triple-homed: the NAT + egress network (route out), the host-only agent network (agents + CLI reach + the gateway), and the control network (reach the orchestrator by IP — Apple + has no container DNS). Holds the mitmproxy CA + the `gateway` JWT. -Two consequences fall out of the single container, both simplifications: - -- **No DNS dance.** The control plane and the gateway daemons reach each other - over `127.0.0.1`, so nothing depends on Apple's (absent) container DNS and - there is no orchestrator-before-gateway ordering to get right. -- **The DB is never host-shared.** It lives on a container-only volume, so no - host process opens the live file. The host CLI reaches registry + supervise - state through the control-plane HTTP surface (`cli/supervise.py` already uses - `OrchestratorClient`), exactly as it does for firecracker. - -The control-plane source is bind-mounted (like the docker orchestrator), so a -code change takes effect on the next launch without an image rebuild; the -gateway daemons are baked in the gateway image and rebuild through its own -digest check. +Agents sit on the agent network only, never the control network, so they have no +route to the control plane (the L3 block, not just the JWT). """ from __future__ import annotations @@ -59,23 +47,29 @@ from ...paths import ( from .. import util as backend_util from . import util as container_mod from .gateway import ( + CONTROL_NETWORK, DEFAULT_CA_TIMEOUT_SECONDS, GATEWAY_EGRESS_NETWORK, GATEWAY_IMAGE, GATEWAY_NETWORK, GatewayError, + ORCHESTRATOR_IMAGE, ensure_networks, ) -# The one per-host infra container: control plane + gateway data plane. -INFRA_NAME = "bot-bottle-mac-infra" +# The orchestrator (control plane) container + the gateway (data plane) +# container. `INFRA_NAME` is kept — now the gateway container — for callers that +# still import it (probe / reprovision attribute against the gateway). +ORCHESTRATOR_NAME = "bot-bottle-mac-orchestrator" +ORCHESTRATOR_LABEL = "bot-bottle-mac-orchestrator=1" +INFRA_NAME = "bot-bottle-mac-infra" # the gateway container INFRA_LABEL = "bot-bottle-mac-infra=1" -# Container-only volume holding bot-bottle.db. No host bind-mount, so the DB is -# written by exactly one kernel (this container's). Survives recreation. +# Container-only volume holding bot-bottle.db, mounted ONLY into the +# orchestrator. One kernel writes it (never host-shared or cross-guest). INFRA_DB_VOLUME = "bot-bottle-mac-db" -# BOT_BOTTLE_ROOT inside the container; host_db_path() resolves the DB to -# /db/ and the supervise daemon writes the same file. +# BOT_BOTTLE_ROOT inside the orchestrator; host_db_path() resolves the DB to +# /db/. _DB_ROOT_IN_CONTAINER = "/var/lib/bot-bottle" _DB_PATH_IN_CONTAINER = f"{_DB_ROOT_IN_CONTAINER}/db/{HOST_DB_FILENAME}" _SRC_IN_CONTAINER = "/bot-bottle-src" @@ -84,45 +78,23 @@ _REPO_ROOT = Path(__file__).resolve().parents[3] _HEALTH_POLL_SECONDS = 0.25 _HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0 -_CA_POLL_SECONDS = 0.5 # The gateway subset the consolidated model runs (no per-bottle git:// daemon). _GATEWAY_DAEMONS = "egress,git-http,supervise" -def _init_script(port: int) -> str: - """PID-1 init: start the control plane and the gateway daemons, both in - this container, reaching each other over loopback. Backgrounded so `wait` - reaps as PID 1. No `set -e` — a transient daemon failure must not kill the - whole container (gateway_init applies the same 'stay up' policy).""" - return ( - "export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\n" - f"mkdir -p $(dirname {_DB_PATH_IN_CONTAINER})\n" - # Control plane, from the bind-mounted source (stdlib-only package). - f"( cd {_SRC_IN_CONTAINER} && BOT_BOTTLE_ROOT={_DB_ROOT_IN_CONTAINER} " - f"python3 -m bot_bottle.orchestrator --host 0.0.0.0 --port {port} " - "--broker stub ) &\n" - # Gateway data plane, multi-tenant against the local control plane. No - # SUPERVISE_DB_PATH: the data plane reaches the supervise queue over the - # control-plane RPC and never opens bot-bottle.db (PRD 0070 / #469). - f"( cd /app && BOT_BOTTLE_GATEWAY_DAEMONS={_GATEWAY_DAEMONS} " - f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{port} " - f"python3 -m bot_bottle.gateway.bootstrap ) &\n" - "while : ; do wait ; done\n" - ) - - @dataclass(frozen=True) class InfraEndpoint: - """How to reach the running infra container. The control plane and the - gateway are the same container, so one address serves both.""" + """How to reach the running pair. `orchestrator_url` is the orchestrator's + control-network address (host CLI + registration); `gateway_ip` is the + gateway's agent-network address (proxy / git-http / MCP target).""" - orchestrator_url: str # http://:8099 — host CLI + registration - gateway_ip: str # same container; agents' proxy / git-http / MCP target + orchestrator_url: str + gateway_ip: str class MacosInfraService: - """Manages the single per-host infra container. Callers use + """Manages the per-host orchestrator + gateway containers. Callers use `ensure_running()` (returns the endpoint) and `ca_cert_pem()`.""" def __init__( @@ -131,24 +103,41 @@ class MacosInfraService: port: int = DEFAULT_PORT, network: str = GATEWAY_NETWORK, egress_network: str = GATEWAY_EGRESS_NETWORK, - image: str = GATEWAY_IMAGE, + control_network: str = CONTROL_NETWORK, + gateway_image: str = GATEWAY_IMAGE, + orchestrator_image: str = ORCHESTRATOR_IMAGE, repo_root: Path = _REPO_ROOT, - name: str = INFRA_NAME, + orchestrator_name: str = ORCHESTRATOR_NAME, + gateway_name: str = INFRA_NAME, db_volume: str = INFRA_DB_VOLUME, ) -> None: self.port = port self.network = network self.egress_network = egress_network - self.image = image + self.control_network = control_network + self.gateway_image = gateway_image + self.orchestrator_image = orchestrator_image self._repo_root = repo_root - self._name = name + self._orchestrator_name = orchestrator_name + self._gateway_name = gateway_name self._db_volume = db_volume - def _resolve_url(self) -> str: - """The control-plane URL, or "" while the container has no address.""" - ip = container_mod.try_container_ipv4_on_network(self._name, self.network) + @property + def gateway_name(self) -> str: + return self._gateway_name + + def _resolve_orchestrator_url(self) -> str: + """The control-plane URL (orchestrator's control-network address), or "" + while it has no address.""" + ip = container_mod.try_container_ipv4_on_network( + self._orchestrator_name, self.control_network) return f"http://{ip}:{self.port}" if ip else "" + def _resolve_gateway_ip(self) -> str: + """The gateway's agent-network address, or "" while it has none.""" + return container_mod.try_container_ipv4_on_network( + self._gateway_name, self.network) + def is_healthy( self, url: str, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS, ) -> bool: @@ -160,158 +149,165 @@ class MacosInfraService: except (urllib.error.URLError, TimeoutError, OSError): return False - def _source_current(self, current_hash: str) -> bool: - """True iff the running infra container was created from the current - bind-mounted control-plane source. The control-plane process loads that - code at startup and won't reload it, so a stale container keeps serving - OLD code.""" - if not container_mod.container_is_running(self._name): + def _orchestrator_source_current(self, current_hash: str) -> bool: + """True iff the running orchestrator was created from the current + bind-mounted control-plane source (it loads that code at startup and + won't reload it).""" + if not container_mod.container_is_running(self._orchestrator_name): return False - env = container_mod.container_env(self._name) + env = container_mod.container_env(self._orchestrator_name) if not env: return True # can't compare → don't churn a working container return env.get("BOT_BOTTLE_SOURCE_HASH") == current_hash - def _running_healthy_endpoint(self, current_hash: str) -> InfraEndpoint | None: - """The endpoint if the running container is BOTH source-current and - answering /health, else None (→ recreate). Health, not just the source - label, is what lets a wedged-but-current container self-heal instead of - being polled to death forever.""" - if not self._source_current(current_hash): - return None - url = self._resolve_url() - if url and self.is_healthy(url): - return InfraEndpoint(orchestrator_url=url, gateway_ip=_ip_of(url)) - return None - def ensure_built(self) -> None: - """Ensure the gateway data-plane image exists. The control-plane source - is bind-mounted, not baked, so only the gateway image needs building.""" + """Ensure the gateway + orchestrator images exist. The control-plane + source is bind-mounted, so a code change takes effect without a rebuild; + the images still carry the package for their entrypoints.""" container_mod.build_image( - self.image, str(self._repo_root), dockerfile="Dockerfile.gateway", - ) + self.gateway_image, str(self._repo_root), dockerfile="Dockerfile.gateway") + container_mod.build_image( + self.orchestrator_image, str(self._repo_root), + dockerfile="Dockerfile.orchestrator") def ensure_running( self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS, ) -> InfraEndpoint: - """Ensure the single infra container is up; return how to reach it. - Idempotent per-host singleton — a healthy container on current source - is left untouched, so N launches share the one control plane + gateway. - Raises `OrchestratorStartError` on startup timeout.""" - current_hash = source_hash(self._repo_root) - endpoint = self._running_healthy_endpoint(current_hash) - if endpoint is not None: - return endpoint + """Ensure the orchestrator + gateway containers are up; return how to + reach them. Idempotent per-host singleton — a healthy orchestrator on + current source is left untouched. Raises `OrchestratorStartError` on + control-plane startup timeout.""" self.ensure_built() - log.info("starting infra container", context={"name": self._name}) - self._run_container(current_hash) - return self._wait_healthy(startup_timeout) + ensure_networks(self.network, self.egress_network, self.control_network) - def _run_container(self, current_hash: str) -> None: - ensure_networks(self.network, self.egress_network) - container_mod.force_remove_container(self._name) + current_hash = source_hash(self._repo_root) + url = self._resolve_orchestrator_url() + if not (self._orchestrator_source_current(current_hash) + and url and self.is_healthy(url)): + log.info("starting orchestrator container", + context={"name": self._orchestrator_name}) + self._run_orchestrator_container(current_hash) + url = self._wait_healthy(startup_timeout) + + # (Re)ensure the gateway once the control plane it resolves against is + # healthy — it needs the orchestrator's control-network address. + self._ensure_gateway_container(url) + return InfraEndpoint(orchestrator_url=url, gateway_ip=self._resolve_gateway_ip()) + + def _run_orchestrator_container(self, current_hash: str) -> None: + container_mod.force_remove_container(self._orchestrator_name) + _signing_key = host_orchestrator_token() argv = [ "container", "run", "--detach", - "--name", self._name, + "--name", self._orchestrator_name, "--label", "bot-bottle.backend=macos-container", - "--label", INFRA_LABEL, - # NAT network FIRST so the gateway's egress has a default route; - # the host-only network is where agents (and the host CLI) reach it. - "--network", self.egress_network, - "--network", self.network, + "--label", ORCHESTRATOR_LABEL, + # Control network only — agents are never on it (L3-isolated). + "--network", self.control_network, "--dns", container_mod.dns_server(), - # Container-only DB volume: one kernel writes bot-bottle.db, never - # shared with the host or another guest. + # Container-only DB volume: exactly one kernel writes bot-bottle.db. "--volume", f"{self._db_volume}:{_DB_ROOT_IN_CONTAINER}", - # The DB needs a container-only ext4 volume for coherent SQLite - # locking, but the CA has no such constraint. Keep it in the host - # app-data root so infra-container recreation and Apple Container - # volume pruning cannot silently rotate every bottle's trust - # anchor (issue #450). - "--mount", - container_mod.bind_mount_spec( - str(host_gateway_ca_dir()), MITMPROXY_HOME), - # Bind-mount the control-plane source (read-only); a code change - # takes effect on relaunch with no image rebuild. + # Live control-plane source (a code change takes effect on relaunch). "--mount", container_mod.bind_mount_spec( str(self._repo_root), _SRC_IN_CONTAINER, readonly=True), - # Baked onto the container so `_source_current` can detect a real - # control-plane code change and recreate. + "--env", f"PYTHONPATH={_SRC_IN_CONTAINER}", + "--env", f"BOT_BOTTLE_ROOT={_DB_ROOT_IN_CONTAINER}", + # Detect a real control-plane code change and recreate. "--env", f"BOT_BOTTLE_SOURCE_HASH={current_hash}", - # The control-plane signing key (control plane: verifies tokens) and - # the pre-minted `gateway` JWT (the gateway's PolicyResolver: presents - # it) — they share this one container, and gateway_init scopes each to - # its process so a compromised data-plane daemon never sees the key - # (issue #469 review). Bare `--env NAME` inherits the value from the - # run process below, so neither lands on argv or in `container - # inspect`'s command line. The agent runs in a SEPARATE container that - # is never given these vars, which is the whole point. + # The signing key — held ONLY by the orchestrator (issue #469). Bare + # `--env NAME` keeps the value off argv / `container inspect`. "--env", ORCHESTRATOR_TOKEN_ENV, - "--env", ORCHESTRATOR_AUTH_JWT_ENV, - "--entrypoint", "sh", - self.image, - "-c", _init_script(self.port), + self.orchestrator_image, + # Dockerfile.orchestrator ENTRYPOINT is `-m bot_bottle.orchestrator`. + "--host", "0.0.0.0", "--port", str(self.port), "--broker", "stub", ] - _signing_key = host_orchestrator_token() - run_env = { - **os.environ, - ORCHESTRATOR_TOKEN_ENV: _signing_key, - ORCHESTRATOR_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key), - } - result = container_mod.run_container_argv(argv, env=run_env) + result = container_mod.run_container_argv( + argv, env={**os.environ, ORCHESTRATOR_TOKEN_ENV: _signing_key}) if result.returncode != 0: raise OrchestratorStartError( - f"infra container failed to start: " + f"orchestrator container failed to start: " f"{(result.stderr or '').strip() or ''}" ) - def _wait_healthy(self, startup_timeout: float) -> InfraEndpoint: + def _ensure_gateway_container(self, orchestrator_url: str) -> None: + """Start (recreate) the gateway container, dual-homed on the agent + + control networks, resolving policy against `orchestrator_url` (the + orchestrator's control-network address — Apple has no DNS).""" + container_mod.force_remove_container(self._gateway_name) + _signing_key = host_orchestrator_token() + argv = [ + "container", "run", "--detach", + "--name", self._gateway_name, + "--label", "bot-bottle.backend=macos-container", + "--label", INFRA_LABEL, + # NAT egress FIRST (default route out); the host-only agent network + # is where agents reach the gateway; the control network reaches the + # orchestrator. + "--network", self.egress_network, + "--network", self.network, + "--network", self.control_network, + "--dns", container_mod.dns_server(), + # The mitmproxy CA on a host bind-mount (survives recreation + + # volume pruning — issue #450). No DB mount: the data plane never + # opens bot-bottle.db (#469). + "--mount", + container_mod.bind_mount_spec(str(host_gateway_ca_dir()), MITMPROXY_HOME), + "--env", f"BOT_BOTTLE_GATEWAY_DAEMONS={_GATEWAY_DAEMONS}", + "--env", f"BOT_BOTTLE_ORCHESTRATOR_URL={orchestrator_url}", + # The pre-minted `gateway` JWT (never the signing key). Bare + # `--env NAME` inherits the value from run_env below. + "--env", ORCHESTRATOR_AUTH_JWT_ENV, + self.gateway_image, + ] + run_env = {**os.environ, ORCHESTRATOR_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key)} + result = container_mod.run_container_argv(argv, env=run_env) + if result.returncode != 0: + raise GatewayError( + f"gateway container failed to start: " + f"{(result.stderr or '').strip() or ''}" + ) + + def _wait_healthy(self, startup_timeout: float) -> str: deadline = time.monotonic() + startup_timeout while True: - url = self._resolve_url() + url = self._resolve_orchestrator_url() if url and self.is_healthy(url): - log.info("infra container healthy", context={"url": url}) - return InfraEndpoint(orchestrator_url=url, gateway_ip=_ip_of(url)) + log.info("orchestrator healthy", context={"url": url}) + return url if time.monotonic() >= deadline: raise OrchestratorStartError( - f"infra container did not become healthy within " + f"orchestrator did not become healthy within " f"{startup_timeout:g}s" ) time.sleep(_HEALTH_POLL_SECONDS) def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str: """The gateway's mitmproxy CA (PEM) agents install to trust its TLS - interception. Read through the container path backed by the persistent - host CA directory; polls because mitmproxy writes it a beat after - start.""" + interception. Read from the gateway container; polls because mitmproxy + writes it a beat after start.""" def _fetch() -> str | None: result = container_mod.run_container_argv( - ["container", "exec", self._name, "cat", GATEWAY_CA_CERT]) + ["container", "exec", self._gateway_name, "cat", GATEWAY_CA_CERT]) return result.stdout if result.returncode == 0 and result.stdout.strip() else None try: return backend_util.poll_ca_cert(_fetch, timeout=timeout) except TimeoutError as exc: raise GatewayError( - f"gateway CA not available in {self._name} after {timeout:g}s" + f"gateway CA not available in {self._gateway_name} after {timeout:g}s" ) from exc def stop(self) -> None: - """Remove the infra container (idempotent). The DB volume persists.""" - container_mod.force_remove_container(self._name) - - -def _ip_of(url: str) -> str: - """The host from an http://host:port URL.""" - return url.split("://", 1)[-1].rsplit(":", 1)[0] + """Remove both containers (idempotent). The DB volume persists.""" + container_mod.force_remove_container(self._gateway_name) + container_mod.force_remove_container(self._orchestrator_name) def probe_orchestrator_url(port: int = DEFAULT_PORT) -> str: - """The running infra container's control-plane URL, or "" if it isn't up. - Used by host-side control-plane discovery (`discover_orchestrator_url`); - safe to call on any host — returns "" when the container or the `container` - CLI isn't present.""" - ip = container_mod.try_container_ipv4_on_network(INFRA_NAME, GATEWAY_NETWORK) + """The running orchestrator's control-plane URL, or "" if it isn't up. Used + by host-side control-plane discovery; safe on any host (returns "" when the + container or the `container` CLI isn't present).""" + ip = container_mod.try_container_ipv4_on_network(ORCHESTRATOR_NAME, CONTROL_NETWORK) return f"http://{ip}:{port}" if ip else "" @@ -320,6 +316,7 @@ __all__ = [ "InfraEndpoint", "OrchestratorStartError", "GatewayError", + "ORCHESTRATOR_NAME", "INFRA_NAME", "INFRA_DB_VOLUME", ] diff --git a/tests/unit/test_macos_infra.py b/tests/unit/test_macos_infra.py index f1341ecd..a9845730 100644 --- a/tests/unit/test_macos_infra.py +++ b/tests/unit/test_macos_infra.py @@ -1,4 +1,4 @@ -"""Unit: the single macOS infra container (control plane + gateway, PRD 0070).""" +"""Unit: macOS orchestrator + gateway containers (PRD 0070 plane split).""" from __future__ import annotations @@ -24,86 +24,94 @@ def _fail(stderr: str = "boom") -> Mock: return Mock(returncode=1, stdout="", stderr=stderr) -class TestInfraRun(unittest.TestCase): - def _run_container(self, svc: MacosInfraService) -> list[str]: +def _spec(src: str, tgt: str, readonly: bool = False) -> str: + return f"type=bind,source={src},target={tgt}" + (",readonly" if readonly else "") + + +class TestOrchestratorRun(unittest.TestCase): + def _run(self, svc: MacosInfraService) -> list[str]: run = Mock(return_value=_ok()) - - def _spec(src: str, tgt: str, readonly: bool = False) -> str: - return f"type=bind,source={src},target={tgt}" + ( - ",readonly" if readonly else "") - with patch(f"{_INFRA}.container_mod") as mod, \ - patch(f"{_INFRA}.ensure_networks"): + patch(f"{_INFRA}.host_orchestrator_token", return_value="k"): mod.dns_server.return_value = "1.1.1.1" mod.bind_mount_spec.side_effect = _spec mod.run_container_argv = run - svc._run_container("h1") + svc._run_orchestrator_container("h1") return run.call_args.args[0] - def test_single_container_runs_both_processes(self) -> None: - """The whole point: one container starts the control plane AND the - gateway daemons, so one kernel owns the DB.""" - argv = self._run_container(MacosInfraService(repo_root=Path("/r"))) - script = argv[-1] - self.assertIn("bot_bottle.orchestrator", script) - # Gateway launches via the installed package (there is no - # /app/gateway_init.py file since the daemons moved into bot_bottle). - self.assertIn("bot_bottle.gateway.bootstrap", script) - self.assertIn("127.0.0.1", script) # they reach each other on loopback + def test_runs_the_orchestrator_on_the_control_network_only(self) -> None: + argv = self._run(MacosInfraService(repo_root=Path("/r"))) + nets = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"] + self.assertEqual(["bot-bottle-mac-control"], nets) + # Image ENTRYPOINT is `-m bot_bottle.orchestrator`; these are its args. + self.assertIn("--broker", argv) + self.assertIn("stub", argv) + self.assertIn("bot-bottle-orchestrator:latest", argv) def test_db_is_a_container_only_volume(self) -> None: - """No host bind-mount of the DB — a named volume only this container - mounts, so the DB is never written by two kernels.""" - argv = self._run_container(MacosInfraService(repo_root=Path("/r"))) + argv = self._run(MacosInfraService(repo_root=Path("/r"))) vols = [argv[i + 1] for i, a in enumerate(argv) if a == "--volume"] self.assertTrue(any(v.startswith(f"{INFRA_DB_VOLUME}:") for v in vols)) - # The repo source is bind-mounted read-only; the DB is not a bind mount. - mounts = [argv[i + 1] for i, a in enumerate(argv) if a == "--mount"] - self.assertTrue(all("bot-bottle.db" not in m for m in mounts)) - def test_ca_is_persisted_on_the_host_not_the_container_volume(self) -> None: - """The CA survives infra recreation and cannot be removed by Apple - Container's volume-prune command.""" - argv = self._run_container(MacosInfraService(repo_root=Path("/r"))) + def test_orchestrator_has_no_ca_mount(self) -> None: + # The CA lives with the gateway, not the control plane. + argv = self._run(MacosInfraService(repo_root=Path("/r"))) mounts = [argv[i + 1] for i, a in enumerate(argv) if a == "--mount"] - ca_mounts = [ - m for m in mounts - if "target=/home/mitmproxy/.mitmproxy" in m - ] - self.assertEqual(1, len(ca_mounts)) - self.assertIn("source=", ca_mounts[0]) - self.assertIn("/gateway-ca", ca_mounts[0]) - self.assertNotIn(",readonly", ca_mounts[0]) - - def test_nat_network_precedes_the_host_only_network(self) -> None: - argv = self._run_container(MacosInfraService(repo_root=Path("/r"))) - nets = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"] - self.assertEqual(["bot-bottle-mac-egress", "bot-bottle-mac-gateway"], nets) + self.assertFalse([m for m in mounts if "/home/mitmproxy" in m]) def test_source_hash_is_labelled_for_recreate(self) -> None: - argv = self._run_container(MacosInfraService(repo_root=Path("/r"))) + argv = self._run(MacosInfraService(repo_root=Path("/r"))) self.assertIn("BOT_BOTTLE_SOURCE_HASH=h1", argv) def test_start_failure_raises(self) -> None: svc = MacosInfraService(repo_root=Path("/r")) with patch(f"{_INFRA}.container_mod") as mod, \ - patch(f"{_INFRA}.ensure_networks"): + patch(f"{_INFRA}.host_orchestrator_token", return_value="k"): mod.dns_server.return_value = "1.1.1.1" - mod.bind_mount_spec.return_value = "m" mod.run_container_argv = Mock(return_value=_fail()) with self.assertRaises(OrchestratorStartError): - svc._run_container("h1") + svc._run_orchestrator_container("h1") + + +class TestGatewayRun(unittest.TestCase): + def _run(self, svc: MacosInfraService, url: str = "http://10.0.0.5:8099") -> list[str]: + run = Mock(return_value=_ok()) + with patch(f"{_INFRA}.container_mod") as mod, \ + patch(f"{_INFRA}.host_orchestrator_token", return_value="k"), \ + patch(f"{_INFRA}.mint", return_value="jwt"): + mod.dns_server.return_value = "1.1.1.1" + mod.bind_mount_spec.side_effect = _spec + mod.run_container_argv = run + svc._ensure_gateway_container(url) + return run.call_args.args[0] + + def test_gateway_is_triple_homed(self) -> None: + argv = self._run(MacosInfraService(repo_root=Path("/r"))) + nets = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"] + self.assertEqual( + ["bot-bottle-mac-egress", "bot-bottle-mac-gateway", "bot-bottle-mac-control"], + nets, + ) + + def test_gateway_resolves_orchestrator_by_url_and_holds_ca(self) -> None: + argv = self._run(MacosInfraService(repo_root=Path("/r")), "http://10.0.0.5:8099") + self.assertIn("BOT_BOTTLE_ORCHESTRATOR_URL=http://10.0.0.5:8099", argv) + mounts = [argv[i + 1] for i, a in enumerate(argv) if a == "--mount"] + self.assertTrue([m for m in mounts if "target=/home/mitmproxy/.mitmproxy" in m]) + self.assertIn("bot-bottle-gateway:latest", argv) + self.assertIn( + "BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise", argv) class TestInfraEnsureRunning(unittest.TestCase): - def test_current_healthy_container_is_left_alone(self) -> None: - """Idempotent singleton: N launches must not churn the infra container - and drop every live bottle's control plane.""" + def test_current_healthy_orchestrator_left_alone(self) -> None: svc = MacosInfraService(repo_root=Path("/r")) run = Mock() with patch(f"{_INFRA}.container_mod") as mod, \ patch(f"{_INFRA}.source_hash", return_value="h1"), \ - patch.object(svc, "_run_container", run), \ + patch.object(svc, "ensure_built"), \ + patch.object(svc, "_run_orchestrator_container", run), \ + patch.object(svc, "_ensure_gateway_container"), \ patch.object(svc, "is_healthy", return_value=True): mod.container_is_running.return_value = True mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"} @@ -113,13 +121,14 @@ class TestInfraEnsureRunning(unittest.TestCase): self.assertEqual("http://192.168.128.2:8099", endpoint.orchestrator_url) self.assertEqual("192.168.128.2", endpoint.gateway_ip) - def test_changed_source_recreates(self) -> None: + def test_changed_source_recreates_orchestrator(self) -> None: svc = MacosInfraService(repo_root=Path("/r")) run = Mock() with patch(f"{_INFRA}.container_mod") as mod, \ patch(f"{_INFRA}.source_hash", return_value="h2"), \ patch.object(svc, "ensure_built"), \ - patch.object(svc, "_run_container", run), \ + patch.object(svc, "_run_orchestrator_container", run), \ + patch.object(svc, "_ensure_gateway_container"), \ patch.object(svc, "is_healthy", return_value=True): mod.container_is_running.return_value = True mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"} @@ -127,17 +136,15 @@ class TestInfraEnsureRunning(unittest.TestCase): svc.ensure_running() run.assert_called_once() - def test_wedged_but_current_container_is_recreated(self) -> None: - """Current source but a dead HTTP server must be recreated, not polled - to death forever — health, not just the source label, gates reuse.""" + def test_wedged_but_current_orchestrator_recreated(self) -> None: svc = MacosInfraService(repo_root=Path("/r")) run = Mock() - health = Mock(side_effect=[False, True]) with patch(f"{_INFRA}.container_mod") as mod, \ patch(f"{_INFRA}.source_hash", return_value="h1"), \ patch.object(svc, "ensure_built"), \ - patch.object(svc, "_run_container", run), \ - patch.object(svc, "is_healthy", health): + patch.object(svc, "_run_orchestrator_container", run), \ + patch.object(svc, "_ensure_gateway_container"), \ + patch.object(svc, "is_healthy", Mock(side_effect=[False, True])): mod.container_is_running.return_value = True mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"} mod.try_container_ipv4_on_network.return_value = "192.168.128.2" @@ -149,7 +156,8 @@ class TestInfraEnsureRunning(unittest.TestCase): with patch(f"{_INFRA}.container_mod") as mod, \ patch(f"{_INFRA}.source_hash", return_value="h1"), \ patch.object(svc, "ensure_built"), \ - patch.object(svc, "_run_container"), \ + patch.object(svc, "_run_orchestrator_container"), \ + patch.object(svc, "_ensure_gateway_container"), \ patch.object(svc, "is_healthy", return_value=False): mod.container_is_running.return_value = False mod.try_container_ipv4_on_network.return_value = "192.168.128.2" @@ -158,7 +166,7 @@ class TestInfraEnsureRunning(unittest.TestCase): class TestCaCertPem(unittest.TestCase): - def test_reads_ca_out_of_the_container(self) -> None: + def test_reads_ca_out_of_the_gateway_container(self) -> None: svc = MacosInfraService(repo_root=Path("/r")) with patch(f"{_INFRA}.container_mod") as mod: mod.run_container_argv.return_value = _ok("-----BEGIN CERTIFICATE-----\n")