From 546537965457bec08f2e9572c4f47a275489d7ab Mon Sep 17 00:00:00 2001 From: didericis Date: Sat, 25 Jul 2026 14:44:21 -0400 Subject: [PATCH] refactor(orchestrator): macOS orchestrator under the Orchestrator ABC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the Apple-Container control-plane lifecycle out of `MacosInfraService` into `MacosOrchestrator` (backend/macos_container/orchestrator.py): the container run on the host-only control network, the container-only DB volume, the source-hash recreate gate, health polling against the resolved control-network address, and `probe_orchestrator_url`. `url()` == `gateway_url()` here (Apple has no container DNS, so one resolved address serves the CLI and the gateway). `MacosInfraService` now composes `orchestrator()` + `gateway()`: ensure the networks, build both images (each service self-builds via `ensure_built` — added to `MacosGateway` too), bring the orchestrator up first, then connect the gateway with the orchestrator-minted token. `ORCHESTRATOR_IMAGE` + the DB-volume constant move to the orchestrator module (with back-compat aliases where imported). Container-lifecycle tests split into test_macos_orchestrator; test_macos_infra now covers the composition. Co-Authored-By: Claude Opus 4.8 --- bot_bottle/backend/macos_container/gateway.py | 20 +- bot_bottle/backend/macos_container/infra.py | 202 +++++------------ .../backend/macos_container/orchestrator.py | 205 ++++++++++++++++++ tests/unit/test_macos_infra.py | 192 ++++------------ tests/unit/test_macos_orchestrator.py | 168 ++++++++++++++ 5 files changed, 486 insertions(+), 301 deletions(-) create mode 100644 bot_bottle/backend/macos_container/orchestrator.py create mode 100644 tests/unit/test_macos_orchestrator.py diff --git a/bot_bottle/backend/macos_container/gateway.py b/bot_bottle/backend/macos_container/gateway.py index 26c6ec23..e81b8e9c 100644 --- a/bot_bottle/backend/macos_container/gateway.py +++ b/bot_bottle/backend/macos_container/gateway.py @@ -14,6 +14,7 @@ share: the network names, the gateway image, and the network-creation helper. from __future__ import annotations import os +from pathlib import Path from ...gateway import ( DEFAULT_CA_TIMEOUT_SECONDS, @@ -50,9 +51,8 @@ GATEWAY_LABEL = "bot-bottle-mac-infra=1" GATEWAY_DAEMONS = "egress,git-http,supervise" GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest") -ORCHESTRATOR_IMAGE = os.environ.get( - "BOT_BOTTLE_ORCHESTRATOR_IMAGE", "bot-bottle-orchestrator:latest" -) + +_REPO_ROOT = Path(__file__).resolve().parents[3] def ensure_networks( @@ -72,9 +72,9 @@ class MacosGateway(Gateway): """The consolidated gateway as a single Apple container, triple-homed on the NAT egress, host-only agent, and control networks. - Images are built by the infra service (`ensure_built` here is the ABC's - no-op); the networks are ensured there too. `connect_to_orchestrator` runs - the container carrying the mitmproxy CA + the pre-minted `gateway` token.""" + `ensure_built` builds `Dockerfile.gateway`; the networks are ensured by the + composer. `connect_to_orchestrator` runs the container carrying the mitmproxy + CA + the pre-minted `gateway` token.""" def __init__( self, @@ -84,18 +84,25 @@ class MacosGateway(Gateway): network: str = GATEWAY_NETWORK, egress_network: str = GATEWAY_EGRESS_NETWORK, control_network: str = CONTROL_NETWORK, + repo_root: Path = _REPO_ROOT, ) -> None: self.image_ref = image_ref self.name = name self.network = network self.egress_network = egress_network self.control_network = control_network + self._repo_root = repo_root # Set by `connect_to_orchestrator`: the URL the daemons resolve policy # against + the pre-minted `gateway` token they present. The gateway # never mints, so it never holds the signing key (#469). self._orchestrator_url = "" self._gateway_token = "" + def ensure_built(self) -> None: + """Build the data-plane image from `Dockerfile.gateway`.""" + container_mod.build_image( + self.image_ref, str(self._repo_root), dockerfile="Dockerfile.gateway") + def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None: """Bind the gateway to this orchestrator and (re)start it, dual-homed on the agent + control networks, resolving policy against `orchestrator_url` @@ -198,7 +205,6 @@ __all__ = [ "GATEWAY_LABEL", "GATEWAY_DAEMONS", "GATEWAY_IMAGE", - "ORCHESTRATOR_IMAGE", "GatewayError", "DEFAULT_CA_TIMEOUT_SECONDS", "ensure_networks", diff --git a/bot_bottle/backend/macos_container/infra.py b/bot_bottle/backend/macos_container/infra.py index 1eacc793..ea20858f 100644 --- a/bot_bottle/backend/macos_container/infra.py +++ b/bot_bottle/backend/macos_container/infra.py @@ -6,40 +6,29 @@ model existed only because two Apple-Container guests writing one `bot-bottle.db over virtiofs would race incoherent `fcntl` locks; with the data plane no longer opening the DB at all, only the orchestrator does, so the split is safe. - * `bot-bottle-mac-orchestrator` — the lean control plane. Joins the host-only - **control network** (`bot-bottle-mac-control`) only. Sole opener of the - container-only DB volume; holds the signing key. The host CLI reaches it at - its control-network address; the gateway reaches it there too. - * `bot-bottle-mac-infra` — the gateway data plane. Triple-homed: the NAT - egress network (route out), the host-only agent network (agents + CLI reach - the gateway), and the control network (reach the orchestrator by IP — Apple - has no container DNS). Holds the mitmproxy CA + the `gateway` JWT. + * `bot-bottle-mac-orchestrator` — the lean control plane (`MacosOrchestrator`). + Joins the host-only **control network** (`bot-bottle-mac-control`) only. Sole + opener of the container-only DB volume; holds the signing key. The host CLI + reaches it at its control-network address; the gateway reaches it there too. + * `bot-bottle-mac-infra` — the gateway data plane (`MacosGateway`). Triple-homed: + the NAT egress network (route out), the host-only agent network (agents + CLI + reach the gateway), and the control network (reach the orchestrator by IP — + Apple has no container DNS). Holds the mitmproxy CA + the `gateway` JWT. -Agents sit on the agent network only, never the control network, so they have no -route to the control plane (the L3 block, not just the JWT). +`MacosInfraService` composes the two services and brings them up as an idempotent +per-host pair. Agents sit on the agent network only, never the control network, +so they have no route to the control plane (the L3 block, not just the JWT). """ from __future__ import annotations -import os -import time -import urllib.error -import urllib.request from dataclasses import dataclass from pathlib import Path -from ... import log from ...orchestrator.lifecycle import ( DEFAULT_PORT, DEFAULT_STARTUP_TIMEOUT_SECONDS, OrchestratorStartError, - source_hash, -) -from ...orchestrator_auth import ROLE_GATEWAY, mint -from ...paths import ( - ORCHESTRATOR_TOKEN_ENV, - HOST_DB_FILENAME, - host_orchestrator_token, ) from . import util as container_mod from .gateway import ( @@ -51,31 +40,24 @@ from .gateway import ( GATEWAY_NETWORK, GatewayError, MacosGateway, - ORCHESTRATOR_IMAGE, ensure_networks, ) +from .orchestrator import ( + ORCHESTRATOR_DB_VOLUME, + ORCHESTRATOR_IMAGE, + ORCHESTRATOR_NAME, + MacosOrchestrator, + probe_orchestrator_url, +) -# The orchestrator (control plane) container. `INFRA_NAME` is kept — now aliasing -# the gateway container — for callers that still import it (probe / reprovision -# attribute against the gateway). -ORCHESTRATOR_NAME = "bot-bottle-mac-orchestrator" -ORCHESTRATOR_LABEL = "bot-bottle-mac-orchestrator=1" -INFRA_NAME = GATEWAY_NAME # the container agents attribute against is the gateway -# Container-only volume holding bot-bottle.db, mounted ONLY into the -# orchestrator. One kernel writes it (never host-shared or cross-guest). -INFRA_DB_VOLUME = "bot-bottle-mac-db" - -# BOT_BOTTLE_ROOT inside the orchestrator; host_db_path() resolves the DB to -# /db/. -_DB_ROOT_IN_CONTAINER = "/var/lib/bot-bottle" -_DB_PATH_IN_CONTAINER = f"{_DB_ROOT_IN_CONTAINER}/db/{HOST_DB_FILENAME}" -_SRC_IN_CONTAINER = "/bot-bottle-src" +# `INFRA_NAME` is kept — now aliasing the gateway container — for callers that +# still import it (probe / reprovision attribute against the gateway). +INFRA_NAME = GATEWAY_NAME +# Back-compat alias: the DB volume moved to the orchestrator module. +INFRA_DB_VOLUME = ORCHESTRATOR_DB_VOLUME _REPO_ROOT = Path(__file__).resolve().parents[3] -_HEALTH_POLL_SECONDS = 0.25 -_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0 - @dataclass(frozen=True) class InfraEndpoint: @@ -88,7 +70,7 @@ class InfraEndpoint: class MacosInfraService: - """Manages the per-host orchestrator + gateway containers. Callers use + """Composes the per-host orchestrator + gateway containers. Callers use `ensure_running()` (returns the endpoint) and `ca_cert_pem()`.""" def __init__( @@ -103,7 +85,7 @@ class MacosInfraService: repo_root: Path = _REPO_ROOT, orchestrator_name: str = ORCHESTRATOR_NAME, gateway_name: str = INFRA_NAME, - db_volume: str = INFRA_DB_VOLUME, + db_volume: str = ORCHESTRATOR_DB_VOLUME, ) -> None: self.port = port self.network = network @@ -120,6 +102,19 @@ class MacosInfraService: def gateway_name(self) -> str: return self._gateway_name + def orchestrator(self) -> MacosOrchestrator: + """The control-plane service on the host-only control network. Cheap to + reconstruct — the launch flow reads its `url()` / `gateway_url()` / + `mint_gateway_token()` off it.""" + return MacosOrchestrator( + self.orchestrator_image, + name=self._orchestrator_name, + port=self.port, + control_network=self.control_network, + repo_root=self._repo_root, + db_volume=self._db_volume, + ) + def gateway(self) -> MacosGateway: """The data-plane gateway service, triple-homed on the egress + agent + control networks. Cheap to reconstruct — the launch flow reads its @@ -131,51 +126,16 @@ class MacosInfraService: network=self.network, egress_network=self.egress_network, control_network=self.control_network, + repo_root=self._repo_root, ) - def _resolve_orchestrator_url(self) -> str: - """The control-plane URL (orchestrator's control-network address), or "" - while it has no address.""" - ip = container_mod.try_container_ipv4_on_network( - self._orchestrator_name, self.control_network) - return f"http://{ip}:{self.port}" if ip else "" - def _resolve_gateway_ip(self) -> str: """The gateway's agent-network address, or "" while it has none.""" return container_mod.try_container_ipv4_on_network( self._gateway_name, self.network) - def is_healthy( - self, url: str, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS, - ) -> bool: - if not url: - return False - try: - with urllib.request.urlopen(f"{url}/health", timeout=timeout) as resp: - return resp.status == 200 - except (urllib.error.URLError, TimeoutError, OSError): - return False - - def _orchestrator_source_current(self, current_hash: str) -> bool: - """True iff the running orchestrator was created from the current - bind-mounted control-plane source (it loads that code at startup and - won't reload it).""" - if not container_mod.container_is_running(self._orchestrator_name): - return False - env = container_mod.container_env(self._orchestrator_name) - if not env: - return True # can't compare → don't churn a working container - return env.get("BOT_BOTTLE_SOURCE_HASH") == current_hash - - def ensure_built(self) -> None: - """Ensure the gateway + orchestrator images exist. The control-plane - source is bind-mounted, so a code change takes effect without a rebuild; - the images still carry the package for their entrypoints.""" - container_mod.build_image( - self.gateway_image, str(self._repo_root), dockerfile="Dockerfile.gateway") - container_mod.build_image( - self.orchestrator_image, str(self._repo_root), - dockerfile="Dockerfile.orchestrator") + def is_healthy(self) -> bool: + return self.orchestrator().is_healthy() def ensure_running( self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS, @@ -184,77 +144,26 @@ class MacosInfraService: reach them. Idempotent per-host singleton — a healthy orchestrator on current source is left untouched. Raises `OrchestratorStartError` on control-plane startup timeout.""" - self.ensure_built() + # The networks (host-only agent + NAT egress + host-only control) are a + # shared concern — create them before either plane comes up. ensure_networks(self.network, self.egress_network, self.control_network) - current_hash = source_hash(self._repo_root) - url = self._resolve_orchestrator_url() - if not (self._orchestrator_source_current(current_hash) - and url and self.is_healthy(url)): - log.info("starting orchestrator container", - context={"name": self._orchestrator_name}) - self._run_orchestrator_container(current_hash) - url = self._wait_healthy(startup_timeout) + orchestrator = self.orchestrator() + gateway = self.gateway() + orchestrator.ensure_built() + gateway.ensure_built() + + orchestrator.ensure_running(startup_timeout=startup_timeout) # (Re)ensure the gateway once the control plane it resolves against is # healthy — it needs the orchestrator's control-network address. The # orchestrator (which holds the signing key) mints the role-scoped # `gateway` JWT and hands it to the gateway, which never sees the key # (#469). - gateway_token = mint(ROLE_GATEWAY, host_orchestrator_token()) - self.gateway().connect_to_orchestrator(url, gateway_token) + url = orchestrator.url() + gateway.connect_to_orchestrator(url, orchestrator.mint_gateway_token()) return InfraEndpoint(orchestrator_url=url, gateway_ip=self._resolve_gateway_ip()) - def _run_orchestrator_container(self, current_hash: str) -> None: - container_mod.force_remove_container(self._orchestrator_name) - _signing_key = host_orchestrator_token() - argv = [ - "container", "run", "--detach", - "--name", self._orchestrator_name, - "--label", "bot-bottle.backend=macos-container", - "--label", ORCHESTRATOR_LABEL, - # Control network only — agents are never on it (L3-isolated). - "--network", self.control_network, - "--dns", container_mod.dns_server(), - # Container-only DB volume: exactly one kernel writes bot-bottle.db. - "--volume", f"{self._db_volume}:{_DB_ROOT_IN_CONTAINER}", - # Live control-plane source (a code change takes effect on relaunch). - "--mount", - container_mod.bind_mount_spec( - str(self._repo_root), _SRC_IN_CONTAINER, readonly=True), - "--env", f"PYTHONPATH={_SRC_IN_CONTAINER}", - "--env", f"BOT_BOTTLE_ROOT={_DB_ROOT_IN_CONTAINER}", - # Detect a real control-plane code change and recreate. - "--env", f"BOT_BOTTLE_SOURCE_HASH={current_hash}", - # The signing key — held ONLY by the orchestrator (issue #469). Bare - # `--env NAME` keeps the value off argv / `container inspect`. - "--env", ORCHESTRATOR_TOKEN_ENV, - self.orchestrator_image, - # Dockerfile.orchestrator ENTRYPOINT is `-m bot_bottle.orchestrator`. - "--host", "0.0.0.0", "--port", str(self.port), "--broker", "stub", - ] - result = container_mod.run_container_argv( - argv, env={**os.environ, ORCHESTRATOR_TOKEN_ENV: _signing_key}) - if result.returncode != 0: - raise OrchestratorStartError( - f"orchestrator container failed to start: " - f"{(result.stderr or '').strip() or ''}" - ) - - def _wait_healthy(self, startup_timeout: float) -> str: - deadline = time.monotonic() + startup_timeout - while True: - url = self._resolve_orchestrator_url() - if url and self.is_healthy(url): - log.info("orchestrator healthy", context={"url": url}) - return url - if time.monotonic() >= deadline: - raise OrchestratorStartError( - f"orchestrator did not become healthy within " - f"{startup_timeout:g}s" - ) - time.sleep(_HEALTH_POLL_SECONDS) - def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str: """The gateway's mitmproxy CA (PEM) agents install to trust its TLS interception — delegated to the gateway service (reads it out of the @@ -267,14 +176,6 @@ class MacosInfraService: container_mod.force_remove_container(self._orchestrator_name) -def probe_orchestrator_url(port: int = DEFAULT_PORT) -> str: - """The running orchestrator's control-plane URL, or "" if it isn't up. Used - by host-side control-plane discovery; safe on any host (returns "" when the - container or the `container` CLI isn't present).""" - ip = container_mod.try_container_ipv4_on_network(ORCHESTRATOR_NAME, CONTROL_NETWORK) - return f"http://{ip}:{port}" if ip else "" - - __all__ = [ "MacosInfraService", "InfraEndpoint", @@ -283,4 +184,5 @@ __all__ = [ "ORCHESTRATOR_NAME", "INFRA_NAME", "INFRA_DB_VOLUME", + "probe_orchestrator_url", ] diff --git a/bot_bottle/backend/macos_container/orchestrator.py b/bot_bottle/backend/macos_container/orchestrator.py new file mode 100644 index 00000000..32802760 --- /dev/null +++ b/bot_bottle/backend/macos_container/orchestrator.py @@ -0,0 +1,205 @@ +"""The macOS orchestrator (control plane) as an Apple container (PRD 0070). + +`MacosOrchestrator` is the Apple-Container implementation of the backend-neutral +`Orchestrator` service. The lean control-plane container joins the host-only +control network only (agents are never on it), mounts a container-only DB volume +(exactly one kernel writes `bot-bottle.db`), and holds the signing key (#469). +The host CLI and the gateway both reach it at its control-network address — +Apple has no container DNS, so there's a single resolved URL, not docker's +loopback-vs-name split. +""" + +from __future__ import annotations + +import os +import time +import urllib.error +import urllib.request +from pathlib import Path + +from ... import log +from ...paths import ( + ORCHESTRATOR_TOKEN_ENV, + host_orchestrator_token, +) +from ...orchestrator.lifecycle import ( + DEFAULT_HEALTH_TIMEOUT_SECONDS, + DEFAULT_PORT, + DEFAULT_STARTUP_TIMEOUT_SECONDS, + Orchestrator, + OrchestratorStartError, + source_hash, +) +from . import util as container_mod +from .gateway import CONTROL_NETWORK + +ORCHESTRATOR_IMAGE = os.environ.get( + "BOT_BOTTLE_ORCHESTRATOR_IMAGE", "bot-bottle-orchestrator:latest" +) +ORCHESTRATOR_NAME = "bot-bottle-mac-orchestrator" +ORCHESTRATOR_LABEL = "bot-bottle-mac-orchestrator=1" +# Container-only volume holding bot-bottle.db, mounted ONLY into the +# orchestrator. One kernel writes it (never host-shared or cross-guest). +ORCHESTRATOR_DB_VOLUME = "bot-bottle-mac-db" + +# BOT_BOTTLE_ROOT inside the orchestrator; host_db_path() resolves the DB to +# /db/. +_DB_ROOT_IN_CONTAINER = "/var/lib/bot-bottle" +_SRC_IN_CONTAINER = "/bot-bottle-src" + +_HEALTH_POLL_SECONDS = 0.25 +_REPO_ROOT = Path(__file__).resolve().parents[3] + + +class MacosOrchestrator(Orchestrator): + """The control plane as an Apple container on the host-only control network. + `ensure_built` builds `Dockerfile.orchestrator`; `ensure_running` starts it + and blocks until `/health` answers at its control-network address.""" + + def __init__( + self, + image_ref: str = ORCHESTRATOR_IMAGE, + *, + name: str = ORCHESTRATOR_NAME, + label: str = ORCHESTRATOR_LABEL, + port: int = DEFAULT_PORT, + control_network: str = CONTROL_NETWORK, + repo_root: Path = _REPO_ROOT, + db_volume: str = ORCHESTRATOR_DB_VOLUME, + ) -> None: + self.image_ref = image_ref + self.name = name + self.label = label + self.port = port + self.control_network = control_network + self._repo_root = repo_root + self._db_volume = db_volume + + def url(self) -> str: + """The orchestrator's control-network address (host CLI + registration), + or "" while it has no address yet. Apple has no container DNS, so this is + also what the gateway resolves against (`gateway_url`).""" + ip = container_mod.try_container_ipv4_on_network(self.name, self.control_network) + return f"http://{ip}:{self.port}" if ip else "" + + def gateway_url(self) -> str: + """Same address the host uses — the gateway reaches the orchestrator by + control-network IP (Apple has no container DNS).""" + return self.url() + + def is_healthy(self, *, timeout: float = DEFAULT_HEALTH_TIMEOUT_SECONDS) -> bool: + url = self.url() + if not url: + return False + try: + with urllib.request.urlopen(f"{url}/health", timeout=timeout) as resp: + return resp.status == 200 + except (urllib.error.URLError, TimeoutError, OSError): + return False + + def is_running(self) -> bool: + return container_mod.container_is_running(self.name) + + def _source_current(self, current_hash: str) -> bool: + """True iff the running orchestrator was created from the current + bind-mounted control-plane source (it loads that code at startup and + won't reload it).""" + if not self.is_running(): + return False + env = container_mod.container_env(self.name) + if not env: + return True # can't compare → don't churn a working container + return env.get("BOT_BOTTLE_SOURCE_HASH") == current_hash + + def ensure_built(self) -> None: + """Build the control-plane image. The source is bind-mounted so a code + change takes effect without a rebuild; the image still carries the + package for its entrypoint.""" + container_mod.build_image( + self.image_ref, str(self._repo_root), dockerfile="Dockerfile.orchestrator") + + def ensure_running( + self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS, + ) -> None: + """Ensure the control-plane container is up on current source; block + until healthy. Idempotent — a healthy orchestrator on current source is + left untouched. Raises `OrchestratorStartError` on startup timeout. + The control network must already exist (the composer ensures it).""" + current_hash = source_hash(self._repo_root) + if self._source_current(current_hash) and self.is_healthy(): + return + log.info("starting orchestrator container", context={"name": self.name}) + self._run_container(current_hash) + self._wait_healthy(startup_timeout) + + def _run_container(self, current_hash: str) -> None: + container_mod.force_remove_container(self.name) + _signing_key = host_orchestrator_token() + argv = [ + "container", "run", "--detach", + "--name", self.name, + "--label", "bot-bottle.backend=macos-container", + "--label", self.label, + # Control network only — agents are never on it (L3-isolated). + "--network", self.control_network, + "--dns", container_mod.dns_server(), + # Container-only DB volume: exactly one kernel writes bot-bottle.db. + "--volume", f"{self._db_volume}:{_DB_ROOT_IN_CONTAINER}", + # Live control-plane source (a code change takes effect on relaunch). + "--mount", + container_mod.bind_mount_spec( + str(self._repo_root), _SRC_IN_CONTAINER, readonly=True), + "--env", f"PYTHONPATH={_SRC_IN_CONTAINER}", + "--env", f"BOT_BOTTLE_ROOT={_DB_ROOT_IN_CONTAINER}", + # Detect a real control-plane code change and recreate. + "--env", f"BOT_BOTTLE_SOURCE_HASH={current_hash}", + # The signing key — held ONLY by the orchestrator (issue #469). Bare + # `--env NAME` keeps the value off argv / `container inspect`. + "--env", ORCHESTRATOR_TOKEN_ENV, + self.image_ref, + # Dockerfile.orchestrator ENTRYPOINT is `-m bot_bottle.orchestrator`. + "--host", "0.0.0.0", "--port", str(self.port), "--broker", "stub", + ] + result = container_mod.run_container_argv( + argv, env={**os.environ, ORCHESTRATOR_TOKEN_ENV: _signing_key}) + if result.returncode != 0: + raise OrchestratorStartError( + f"orchestrator container failed to start: " + f"{(result.stderr or '').strip() or ''}" + ) + + def _wait_healthy(self, startup_timeout: float) -> None: + deadline = time.monotonic() + startup_timeout + while True: + if self.is_healthy(): + log.info("orchestrator healthy", context={"url": self.url()}) + return + if time.monotonic() >= deadline: + raise OrchestratorStartError( + f"orchestrator did not become healthy within " + f"{startup_timeout:g}s" + ) + time.sleep(_HEALTH_POLL_SECONDS) + + def stop(self) -> None: + """Remove the control-plane container (idempotent). The DB volume + persists.""" + container_mod.force_remove_container(self.name) + + +def probe_orchestrator_url(port: int = DEFAULT_PORT) -> str: + """The running orchestrator's control-plane URL, or "" if it isn't up. Used + by host-side control-plane discovery; safe on any host (returns "" when the + container or the `container` CLI isn't present).""" + ip = container_mod.try_container_ipv4_on_network(ORCHESTRATOR_NAME, CONTROL_NETWORK) + return f"http://{ip}:{port}" if ip else "" + + +__all__ = [ + "MacosOrchestrator", + "ORCHESTRATOR_NAME", + "ORCHESTRATOR_LABEL", + "ORCHESTRATOR_IMAGE", + "ORCHESTRATOR_DB_VOLUME", + "probe_orchestrator_url", +] diff --git a/tests/unit/test_macos_infra.py b/tests/unit/test_macos_infra.py index b319fadf..06ef02cc 100644 --- a/tests/unit/test_macos_infra.py +++ b/tests/unit/test_macos_infra.py @@ -1,153 +1,59 @@ -"""Unit: macOS orchestrator + gateway containers (PRD 0070 plane split).""" +"""Unit: MacosInfraService composes the orchestrator + gateway services. + +`MacosInfraService` no longer owns the container lifecycles — it composes the +`MacosOrchestrator` (control plane) and `MacosGateway` (data plane) services, +each tested in its own module. These tests isolate the *composition*: the +networks are ensured, both images built, the orchestrator comes up first, then +the gateway is connected to it with a freshly minted token.""" from __future__ import annotations import unittest from pathlib import Path -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import MagicMock, patch -from bot_bottle.backend.macos_container.infra import ( - INFRA_DB_VOLUME, - MacosInfraService, - OrchestratorStartError, - probe_orchestrator_url, -) +from bot_bottle.backend.macos_container.infra import MacosInfraService _INFRA = "bot_bottle.backend.macos_container.infra" -def _ok(stdout: str = "") -> Mock: - return Mock(returncode=0, stdout=stdout, stderr="") - - -def _fail(stderr: str = "boom") -> Mock: - return Mock(returncode=1, stdout="", stderr=stderr) - - -def _spec(src: str, tgt: str, readonly: bool = False) -> str: - return f"type=bind,source={src},target={tgt}" + (",readonly" if readonly else "") - - -class TestOrchestratorRun(unittest.TestCase): - def _run(self, svc: MacosInfraService) -> list[str]: - run = Mock(return_value=_ok()) - with patch(f"{_INFRA}.container_mod") as mod, \ - patch(f"{_INFRA}.host_orchestrator_token", return_value="k"): - mod.dns_server.return_value = "1.1.1.1" - mod.bind_mount_spec.side_effect = _spec - mod.run_container_argv = run - svc._run_orchestrator_container("h1") - return run.call_args.args[0] - - def test_runs_the_orchestrator_on_the_control_network_only(self) -> None: - argv = self._run(MacosInfraService(repo_root=Path("/r"))) - nets = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"] - self.assertEqual(["bot-bottle-mac-control"], nets) - # Image ENTRYPOINT is `-m bot_bottle.orchestrator`; these are its args. - self.assertIn("--broker", argv) - self.assertIn("stub", argv) - self.assertIn("bot-bottle-orchestrator:latest", argv) - - def test_db_is_a_container_only_volume(self) -> None: - argv = self._run(MacosInfraService(repo_root=Path("/r"))) - vols = [argv[i + 1] for i, a in enumerate(argv) if a == "--volume"] - self.assertTrue(any(v.startswith(f"{INFRA_DB_VOLUME}:") for v in vols)) - - def test_orchestrator_has_no_ca_mount(self) -> None: - # The CA lives with the gateway, not the control plane. - argv = self._run(MacosInfraService(repo_root=Path("/r"))) - mounts = [argv[i + 1] for i, a in enumerate(argv) if a == "--mount"] - self.assertFalse([m for m in mounts if "/home/mitmproxy" in m]) - - def test_source_hash_is_labelled_for_recreate(self) -> None: - argv = self._run(MacosInfraService(repo_root=Path("/r"))) - self.assertIn("BOT_BOTTLE_SOURCE_HASH=h1", argv) - - def test_start_failure_raises(self) -> None: - svc = MacosInfraService(repo_root=Path("/r")) - with patch(f"{_INFRA}.container_mod") as mod, \ - patch(f"{_INFRA}.host_orchestrator_token", return_value="k"): - mod.dns_server.return_value = "1.1.1.1" - mod.run_container_argv = Mock(return_value=_fail()) - with self.assertRaises(OrchestratorStartError): - svc._run_orchestrator_container("h1") - - class TestInfraEnsureRunning(unittest.TestCase): - """Isolate the orchestrator-container logic; the gateway is brought up by - `MacosGateway` (its own module, tested separately), so mock `svc.gateway`. - `ensure_running` mints the `gateway` token, so `mint` / - `host_orchestrator_token` are stubbed.""" + def setUp(self) -> None: + self.svc = MacosInfraService(repo_root=Path("/r")) + self.orch = MagicMock() + self.orch.url.return_value = "http://192.168.128.2:8099" + self.orch.mint_gateway_token.return_value = "gw.jwt" + self.gw = MagicMock() + for name, mock in (("orchestrator", self.orch), ("gateway", self.gw)): + p = patch.object(self.svc, name, return_value=mock) + p.start() + self.addCleanup(p.stop) - def _patches(self, svc: MacosInfraService): - gw = MagicMock() - return gw, ( - patch.object(svc, "ensure_built"), - patch.object(svc, "gateway", return_value=gw), - patch(f"{_INFRA}.host_orchestrator_token", return_value="k"), - patch(f"{_INFRA}.mint", return_value="jwt"), - ) - - def test_current_healthy_orchestrator_left_alone(self) -> None: - svc = MacosInfraService(repo_root=Path("/r")) - run = Mock() - gw, extra = self._patches(svc) - with patch(f"{_INFRA}.container_mod") as mod, \ - patch(f"{_INFRA}.source_hash", return_value="h1"), \ - patch.object(svc, "_run_orchestrator_container", run), \ - patch.object(svc, "is_healthy", return_value=True), \ - extra[0], extra[1], extra[2], extra[3]: - mod.container_is_running.return_value = True - mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"} - mod.try_container_ipv4_on_network.return_value = "192.168.128.2" - endpoint = svc.ensure_running() - run.assert_not_called() - gw.connect_to_orchestrator.assert_called_once() + def test_composes_networks_builds_and_brings_up_orchestrator_first(self) -> None: + with patch(f"{_INFRA}.ensure_networks") as nets, \ + patch(f"{_INFRA}.container_mod") as mod: + mod.try_container_ipv4_on_network.return_value = "192.168.128.3" + endpoint = self.svc.ensure_running() + nets.assert_called_once() + self.orch.ensure_built.assert_called_once() + self.gw.ensure_built.assert_called_once() + self.orch.ensure_running.assert_called_once() self.assertEqual("http://192.168.128.2:8099", endpoint.orchestrator_url) - self.assertEqual("192.168.128.2", endpoint.gateway_ip) + self.assertEqual("192.168.128.3", endpoint.gateway_ip) - def test_changed_source_recreates_orchestrator(self) -> None: - svc = MacosInfraService(repo_root=Path("/r")) - run = Mock() - _gw, extra = self._patches(svc) - with patch(f"{_INFRA}.container_mod") as mod, \ - patch(f"{_INFRA}.source_hash", return_value="h2"), \ - patch.object(svc, "_run_orchestrator_container", run), \ - patch.object(svc, "is_healthy", return_value=True), \ - extra[0], extra[1], extra[2], extra[3]: - mod.container_is_running.return_value = True - mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"} - mod.try_container_ipv4_on_network.return_value = "192.168.128.2" - svc.ensure_running() - run.assert_called_once() + def test_connects_gateway_with_orchestrator_url_and_minted_token(self) -> None: + with patch(f"{_INFRA}.ensure_networks"), \ + patch(f"{_INFRA}.container_mod") as mod: + mod.try_container_ipv4_on_network.return_value = "192.168.128.3" + self.svc.ensure_running() + self.gw.connect_to_orchestrator.assert_called_once_with( + "http://192.168.128.2:8099", "gw.jwt") + self.orch.mint_gateway_token.assert_called_once() - def test_wedged_but_current_orchestrator_recreated(self) -> None: - svc = MacosInfraService(repo_root=Path("/r")) - run = Mock() - _gw, extra = self._patches(svc) - with patch(f"{_INFRA}.container_mod") as mod, \ - patch(f"{_INFRA}.source_hash", return_value="h1"), \ - patch.object(svc, "_run_orchestrator_container", run), \ - patch.object(svc, "is_healthy", Mock(side_effect=[False, True])), \ - extra[0], extra[1], extra[2], extra[3]: - mod.container_is_running.return_value = True - mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"} - mod.try_container_ipv4_on_network.return_value = "192.168.128.2" - svc.ensure_running() - run.assert_called_once() - - def test_never_healthy_raises(self) -> None: - svc = MacosInfraService(repo_root=Path("/r")) - _gw, extra = self._patches(svc) - with patch(f"{_INFRA}.container_mod") as mod, \ - patch(f"{_INFRA}.source_hash", return_value="h1"), \ - patch.object(svc, "_run_orchestrator_container"), \ - patch.object(svc, "is_healthy", return_value=False), \ - extra[0], extra[1], extra[2], extra[3]: - mod.container_is_running.return_value = False - mod.try_container_ipv4_on_network.return_value = "192.168.128.2" - with self.assertRaises(OrchestratorStartError): - svc.ensure_running(startup_timeout=0.01) + def test_is_healthy_delegates_to_the_orchestrator(self) -> None: + self.orch.is_healthy.return_value = True + self.assertTrue(self.svc.is_healthy()) + self.orch.is_healthy.assert_called_once() class TestCaCertPem(unittest.TestCase): @@ -161,16 +67,14 @@ class TestCaCertPem(unittest.TestCase): gw.ca_cert_pem.assert_called_once_with(timeout=5) -class TestProbeOrchestrator(unittest.TestCase): - def test_returns_url_when_running(self) -> None: +class TestStop(unittest.TestCase): + def test_removes_both_containers(self) -> None: + svc = MacosInfraService(repo_root=Path("/r")) with patch(f"{_INFRA}.container_mod") as mod: - mod.try_container_ipv4_on_network.return_value = "192.168.128.2" - self.assertEqual("http://192.168.128.2:8099", probe_orchestrator_url()) - - def test_empty_when_absent(self) -> None: - with patch(f"{_INFRA}.container_mod") as mod: - mod.try_container_ipv4_on_network.return_value = "" - self.assertEqual("", probe_orchestrator_url()) + svc.stop() + removed = {c.args[0] for c in mod.force_remove_container.call_args_list} + self.assertIn("bot-bottle-mac-infra", removed) # gateway + self.assertIn("bot-bottle-mac-orchestrator", removed) # orchestrator if __name__ == "__main__": diff --git a/tests/unit/test_macos_orchestrator.py b/tests/unit/test_macos_orchestrator.py new file mode 100644 index 00000000..845c099d --- /dev/null +++ b/tests/unit/test_macos_orchestrator.py @@ -0,0 +1,168 @@ +"""Unit: the macOS orchestrator (control plane) container lifecycle (PRD 0070).""" + +from __future__ import annotations + +import unittest +from pathlib import Path +from unittest.mock import Mock, patch + +from bot_bottle.backend.macos_container.orchestrator import ( + ORCHESTRATOR_DB_VOLUME, + ORCHESTRATOR_NAME, + MacosOrchestrator, + probe_orchestrator_url, +) +from bot_bottle.orchestrator.lifecycle import OrchestratorStartError + +_ORCH = "bot_bottle.backend.macos_container.orchestrator" + + +def _ok(stdout: str = "") -> Mock: + return Mock(returncode=0, stdout=stdout, stderr="") + + +def _fail(stderr: str = "boom") -> Mock: + return Mock(returncode=1, stdout="", stderr=stderr) + + +def _spec(src: str, tgt: str, readonly: bool = False) -> str: + return f"type=bind,source={src},target={tgt}" + (",readonly" if readonly else "") + + +class TestMacosOrchestratorRun(unittest.TestCase): + def _run(self) -> list[str]: + run = Mock(return_value=_ok()) + with patch(f"{_ORCH}.container_mod") as mod, \ + patch(f"{_ORCH}.host_orchestrator_token", return_value="k"): + mod.dns_server.return_value = "1.1.1.1" + mod.bind_mount_spec.side_effect = _spec + mod.run_container_argv = run + MacosOrchestrator(repo_root=Path("/r"))._run_container("h1") + return run.call_args.args[0] + + def test_runs_on_the_control_network_only(self) -> None: + argv = self._run() + nets = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"] + self.assertEqual(["bot-bottle-mac-control"], nets) + # Image ENTRYPOINT is `-m bot_bottle.orchestrator`; these are its args. + self.assertIn("--broker", argv) + self.assertIn("stub", argv) + self.assertIn("bot-bottle-orchestrator:latest", argv) + + def test_db_is_a_container_only_volume(self) -> None: + argv = self._run() + vols = [argv[i + 1] for i, a in enumerate(argv) if a == "--volume"] + self.assertTrue(any(v.startswith(f"{ORCHESTRATOR_DB_VOLUME}:") for v in vols)) + + def test_no_ca_mount(self) -> None: + # The CA lives with the gateway, not the control plane. + argv = self._run() + mounts = [argv[i + 1] for i, a in enumerate(argv) if a == "--mount"] + self.assertFalse([m for m in mounts if "/home/mitmproxy" in m]) + + def test_source_hash_is_labelled_for_recreate(self) -> None: + self.assertIn("BOT_BOTTLE_SOURCE_HASH=h1", self._run()) + + def test_start_failure_raises(self) -> None: + with patch(f"{_ORCH}.container_mod") as mod, \ + patch(f"{_ORCH}.host_orchestrator_token", return_value="k"): + mod.dns_server.return_value = "1.1.1.1" + mod.run_container_argv = Mock(return_value=_fail()) + with self.assertRaises(OrchestratorStartError): + MacosOrchestrator(repo_root=Path("/r"))._run_container("h1") + + +class TestMacosOrchestratorUrls(unittest.TestCase): + def test_url_and_gateway_url_are_the_control_net_address(self) -> None: + orch = MacosOrchestrator(port=8099) + with patch(f"{_ORCH}.container_mod") as mod: + mod.try_container_ipv4_on_network.return_value = "192.168.128.2" + self.assertEqual("http://192.168.128.2:8099", orch.url()) + self.assertEqual("http://192.168.128.2:8099", orch.gateway_url()) + + def test_url_empty_when_no_address(self) -> None: + orch = MacosOrchestrator() + with patch(f"{_ORCH}.container_mod") as mod: + mod.try_container_ipv4_on_network.return_value = "" + self.assertEqual("", orch.url()) + + def test_is_healthy_false_when_no_address(self) -> None: + orch = MacosOrchestrator() + with patch(f"{_ORCH}.container_mod") as mod: + mod.try_container_ipv4_on_network.return_value = "" + self.assertFalse(orch.is_healthy()) + + +class TestMacosOrchestratorEnsureRunning(unittest.TestCase): + def _orch(self) -> MacosOrchestrator: + return MacosOrchestrator(repo_root=Path("/r")) + + def test_noop_when_current_and_healthy(self) -> None: + orch = self._orch() + with patch(f"{_ORCH}.source_hash", return_value="h1"), \ + patch.object(orch, "_source_current", return_value=True), \ + patch.object(orch, "is_healthy", return_value=True), \ + patch.object(orch, "_run_container") as run: + orch.ensure_running() + run.assert_not_called() + + def test_recreates_when_source_changed(self) -> None: + orch = self._orch() + with patch(f"{_ORCH}.source_hash", return_value="h2"), \ + patch.object(orch, "_source_current", return_value=False), \ + patch.object(orch, "_run_container") as run, \ + patch.object(orch, "_wait_healthy"): + orch.ensure_running() + run.assert_called_once() + + def test_recreates_when_current_but_wedged(self) -> None: + orch = self._orch() + with patch(f"{_ORCH}.source_hash", return_value="h1"), \ + patch.object(orch, "_source_current", return_value=True), \ + patch.object(orch, "is_healthy", return_value=False), \ + patch.object(orch, "_run_container") as run, \ + patch.object(orch, "_wait_healthy"): + orch.ensure_running() + run.assert_called_once() + + def test_never_healthy_raises(self) -> None: + orch = self._orch() + with patch(f"{_ORCH}.source_hash", return_value="h1"), \ + patch.object(orch, "_source_current", return_value=False), \ + patch.object(orch, "_run_container"), \ + patch.object(orch, "is_healthy", return_value=False), \ + patch(f"{_ORCH}.time.sleep"), \ + patch(f"{_ORCH}.time.monotonic", side_effect=[0.0, 0.5, 2.0]): + with self.assertRaises(OrchestratorStartError): + orch.ensure_running(startup_timeout=1.0) + + +class TestMacosOrchestratorBuildStop(unittest.TestCase): + def test_ensure_built_builds_the_orchestrator_image(self) -> None: + orch = MacosOrchestrator(repo_root=Path("/r")) + with patch(f"{_ORCH}.container_mod") as mod: + orch.ensure_built() + _args, kwargs = mod.build_image.call_args + self.assertEqual("Dockerfile.orchestrator", kwargs["dockerfile"]) + + def test_stop_removes_the_container(self) -> None: + orch = MacosOrchestrator() + with patch(f"{_ORCH}.container_mod") as mod: + orch.stop() + mod.force_remove_container.assert_called_once_with(ORCHESTRATOR_NAME) + + +class TestProbeOrchestrator(unittest.TestCase): + def test_returns_url_when_running(self) -> None: + with patch(f"{_ORCH}.container_mod") as mod: + mod.try_container_ipv4_on_network.return_value = "192.168.128.2" + self.assertEqual("http://192.168.128.2:8099", probe_orchestrator_url()) + + def test_empty_when_absent(self) -> None: + with patch(f"{_ORCH}.container_mod") as mod: + mod.try_container_ipv4_on_network.return_value = "" + self.assertEqual("", probe_orchestrator_url()) + + +if __name__ == "__main__": + unittest.main()