From 4a607ad0988c26797be908bf21cac8aaa7c9e976 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 17 Jul 2026 04:14:14 -0400 Subject: [PATCH] refactor(macos): one infra container (control plane + gateway), fixes shared-DB races MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adopts the firecracker infra-VM pattern for macOS: the orchestrator control plane and the gateway data plane now run in a SINGLE Apple container instead of two. Apple Containers are lightweight VMs with separate kernels, so the prior two-container design had both guests writing one bot-bottle.db over virtiofs, where fcntl locks are not coherent across kernels — concurrent writes (the orchestrator's registry vs the gateway supervise daemon's queue) could corrupt it. One container = one kernel = coherent locking. The DB moves onto a container-only Apple volume (bot-bottle-mac-db), never bind-mounted from the host, so no host process opens the live file either. The host CLI already reaches registry + supervise state over the control-plane HTTP surface (cli/supervise.py uses OrchestratorClient), exactly as firecracker's VM-only DB requires. Two simplifications fall out of the single container: - No DNS dance: the control plane and gateway daemons reach each other over 127.0.0.1, so the orchestrator-before-gateway ordering (a workaround for Apple having no container DNS) is gone, along with the moved-IP recreate logic it needed. - Net -243 lines. Mechanics: the infra container runs from the gateway image with the control-plane source bind-mounted read-only (like the docker orchestrator, so a code change needs no rebuild) and a small sh -c init that starts both processes (mirrors firecracker's _infra_init). Also implements the macOS backend's ensure_orchestrator() and adds it to discover_orchestrator_url, so operator tools (supervise) can bring up / find the control plane on demand — previously the macOS backend died with "no orchestrator control plane". Verified end-to-end on real Apple Container 1.0.0: the single infra container comes up healthy (one address for control plane + gateway), both processes run, the DB is written on the container-only volume, host-side supervise works over HTTP, and a registered agent gets 200 for an allowed host / 403 for a denied one. 1824 unit tests pass with `container` absent (CI parity), pyright clean, pylint 9.89. Co-Authored-By: Claude Fable 5 --- bot_bottle/backend/macos_container/backend.py | 8 + .../macos_container/consolidated_launch.py | 30 +- .../backend/macos_container/enumerate.py | 11 +- bot_bottle/backend/macos_container/gateway.py | 223 +------------ .../macos_container/gateway_provision.py | 10 +- bot_bottle/backend/macos_container/infra.py | 292 +++++++++++++++++ .../macos_container/orchestrator_service.py | 242 -------------- bot_bottle/orchestrator/client.py | 7 + tests/unit/test_backend_selection.py | 15 +- tests/unit/test_macos_consolidated_launch.py | 27 +- tests/unit/test_macos_container_cleanup.py | 11 +- tests/unit/test_macos_gateway.py | 297 ------------------ tests/unit/test_macos_infra.py | 168 ++++++++++ 13 files changed, 549 insertions(+), 792 deletions(-) create mode 100644 bot_bottle/backend/macos_container/infra.py delete mode 100644 bot_bottle/backend/macos_container/orchestrator_service.py delete mode 100644 tests/unit/test_macos_gateway.py create mode 100644 tests/unit/test_macos_infra.py diff --git a/bot_bottle/backend/macos_container/backend.py b/bot_bottle/backend/macos_container/backend.py index c3c38d6..71b4d46 100644 --- a/bot_bottle/backend/macos_container/backend.py +++ b/bot_bottle/backend/macos_container/backend.py @@ -89,6 +89,14 @@ class MacosContainerBottleBackend( with _launch.launch(plan, provision=self.provision) as bottle: yield bottle + def ensure_orchestrator(self) -> str: + """Bring up the per-host infra container (control plane + gateway) and + return its control-plane URL — the on-demand entry point operator tools + (`supervise`) call when no control plane is running yet. Mirrors + firecracker's infra-VM bring-up.""" + from .infra import MacosInfraService + return MacosInfraService().ensure_running().control_plane_url + def prepare_cleanup(self) -> MacosContainerBottleCleanupPlan: return _cleanup.prepare_cleanup() diff --git a/bot_bottle/backend/macos_container/consolidated_launch.py b/bot_bottle/backend/macos_container/consolidated_launch.py index dedf4e8..5b7efc1 100644 --- a/bot_bottle/backend/macos_container/consolidated_launch.py +++ b/bot_bottle/backend/macos_container/consolidated_launch.py @@ -15,6 +15,9 @@ caller has to start the agent in between. `ensure_gateway` runs first because the agent's proxy env needs the gateway's address at `container run` time; the agent's *own* address (the attribution key) only exists afterwards. +The control plane and the gateway are one **infra container** here (see +`infra`), so `gateway_ip` and the control-plane host are the same address. + The consequence for the identity token: it is minted by registration, i.e. *after* the agent container exists, so it cannot be baked into the run-time env the way docker's compose spec does. It is delivered at `container exec` @@ -38,7 +41,7 @@ from ...orchestrator.registration import registration_inputs from ..docker.gateway_provision import deprovision_git_gate, provision_git_gate from .gateway import GATEWAY_NETWORK from .gateway_provision import AppleGatewayTransport -from .orchestrator_service import MacosOrchestratorService, OrchestratorStartError +from .infra import MacosInfraService, OrchestratorStartError class ConsolidatedLaunchError(RuntimeError): @@ -47,7 +50,9 @@ class ConsolidatedLaunchError(RuntimeError): @dataclass(frozen=True) class GatewayEndpoint: - """What the agent `container run` needs to reach the shared gateway.""" + """What the agent `container run` needs to reach the shared gateway (the + infra container). `gateway_ip` is that container's host-only address, the + same host the control-plane URL points at.""" orchestrator_url: str gateway_ip: str # the gateway's address — the agent's proxy target @@ -68,19 +73,18 @@ class LaunchContext: def ensure_gateway( - *, service: MacosOrchestratorService | None = None, + *, service: MacosInfraService | None = None, ) -> GatewayEndpoint: - """Ensure the orchestrator control plane + shared gateway are up, and - report how to reach them. Idempotent — both are per-host singletons, so N - bottle launches share the one pair. Call before starting the agent - container: the agent's proxy env needs `gateway_ip` at run time.""" - service = service or MacosOrchestratorService() - url = service.ensure_running() - gateway = service.gateway(url) + """Ensure the per-host infra container (control plane + gateway) is up and + report how to reach it. Idempotent — one singleton, so N bottle launches + share it. Call before starting the agent container: the agent's proxy env + needs `gateway_ip` at run time.""" + service = service or MacosInfraService() + infra = service.ensure_running() return GatewayEndpoint( - orchestrator_url=url, - gateway_ip=gateway.ip_on_shared_network(), - gateway_ca_pem=gateway.ca_cert_pem(), + orchestrator_url=infra.control_plane_url, + gateway_ip=infra.gateway_ip, + gateway_ca_pem=service.ca_cert_pem(), network=service.network, ) diff --git a/bot_bottle/backend/macos_container/enumerate.py b/bot_bottle/backend/macos_container/enumerate.py index 7407b72..46dcb72 100644 --- a/bot_bottle/backend/macos_container/enumerate.py +++ b/bot_bottle/backend/macos_container/enumerate.py @@ -6,14 +6,13 @@ import subprocess from ...bottle_state import read_metadata from .. import ActiveAgent -from .gateway import GATEWAY_NAME -from .orchestrator_service import ORCHESTRATOR_NAME +from .infra import INFRA_NAME _PREFIX = "bot-bottle-" -# The shared per-host singletons carry the same prefix as agent containers but -# are infrastructure, not bottles — one gateway and one control plane serve -# every agent, so listing them as agents would invent one per host. -_INFRA_NAMES = frozenset({GATEWAY_NAME, ORCHESTRATOR_NAME}) +# The shared per-host infra container carries the same prefix as agent +# containers but is infrastructure, not a bottle — one control plane + gateway +# serves every agent, so listing it as an agent would invent one per host. +_INFRA_NAMES = frozenset({INFRA_NAME}) def enumerate_active() -> list[ActiveAgent]: diff --git a/bot_bottle/backend/macos_container/gateway.py b/bot_bottle/backend/macos_container/gateway.py index e4e2903..3d296ff 100644 --- a/bot_bottle/backend/macos_container/gateway.py +++ b/bot_bottle/backend/macos_container/gateway.py @@ -1,235 +1,46 @@ -"""The consolidated per-host gateway as an Apple container (PRD 0070). +"""Shared network/image constants for the macOS consolidated infra container. -The macOS counterpart of `orchestrator.gateway.DockerGateway`: one persistent -gateway per host, shared by every bottle, attributing each request to a bottle -by its source IP on the shared host-only network. - -Two Apple Container 1.0.0 constraints shape this and make it *not* a -transliteration of the docker gateway: - -- **No container DNS.** Containers cannot resolve each other by name (the - host-only network's resolver refuses the query), so the gateway reaches the - control plane by **IP**, not by name as the docker gateway does. The - orchestrator must therefore be started *before* the gateway — see - `orchestrator_service`. -- **Networks are fixed at `container run`.** There is no `network connect`, - so a network cannot be attached to a running container. The gateway must sit - on one shared, up-front network for the lifetime of the process; per-bottle - networks would mean restarting the gateway on every launch, which defeats the - consolidation. - -The gateway is dual-homed, **NAT network first**: Apple Container makes the -first `--network` the default route, so the egress network must lead or the -gateway has no route to the internet. +The gateway data plane no longer runs as its own Apple container — it shares a +single per-host **infra container** with the control plane (see `infra`), +because two Apple-Container guests writing one `bot-bottle.db` over virtiofs +would race incoherent `fcntl` locks. This module holds the pieces both the +infra service and the launch/provision glue need: the network names, the +gateway image, and the network-creation helper. """ from __future__ import annotations import os -import time -from pathlib import Path -from ...orchestrator.gateway import ( - GATEWAY_CA_CERT, - GATEWAY_DOCKERFILE, - MITMPROXY_HOME, - Gateway, - GatewayError, -) -from ...paths import host_db_dir -from ...supervise import DB_PATH_IN_CONTAINER +from ...orchestrator.gateway import GatewayError from . import util as container_mod -from .util import bind_mount_spec as _mount -# Distinct from the docker gateway's names so both backends' gateways can -# coexist on one host (a macOS host can run the docker backend too). -GATEWAY_NAME = "bot-bottle-mac-gateway" -# The shared host-only network the gateway and every agent bottle sit on. The -# agent's address here is the attribution key. +# The shared host-only network the infra container and every agent bottle sit +# on. The agent's address here is the attribution key. Distinct from the docker +# names so both backends can coexist on one host. GATEWAY_NETWORK = "bot-bottle-mac-gateway" -# The NAT network that gives the gateway (and only the gateway) a route out. +# The NAT network that gives the infra container (and only it) a route out. GATEWAY_EGRESS_NETWORK = "bot-bottle-mac-egress" GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest") -_REPO_ROOT = Path(__file__).resolve().parents[3] - -_SUPERVISE_DB_DIR_IN_CONTAINER = os.path.dirname(DB_PATH_IN_CONTAINER) - -# mitmproxy writes its CA a beat after start; reads poll rather than assume. -_CA_POLL_SECONDS = 0.5 DEFAULT_CA_TIMEOUT_SECONDS = 30.0 -def gateway_ca_dir() -> Path: - """Host dir bind-mounted as mitmproxy's home, keeping the gateway's - self-generated CA **stable across container recreation** — every agent - installs this one CA to trust the shared gateway's TLS interception, so it - must not rotate when the gateway restarts. - - The docker gateway uses a named volume for this; a plain host dir is the - same guarantee with fewer moving parts, and it lets `ca_cert_pem` read the - PEM straight off the host instead of shelling into the container.""" - path = host_db_dir() / "mac-gateway-ca" - path.mkdir(parents=True, exist_ok=True) - return path - - def ensure_networks( network: str = GATEWAY_NETWORK, egress_network: str = GATEWAY_EGRESS_NETWORK, ) -> None: - """Create the shared host-only network + the gateway's NAT network. - Idempotent — `create_network` tolerates 'already exists'. - - Module-level rather than a gateway method because the **orchestrator** - needs the shared network too, and it starts first (Apple has no container - DNS, so the gateway must be handed the control plane's IP). Both callers - ensure the networks; whoever runs first wins.""" + """Create the shared host-only network + the NAT egress network. Idempotent + — `create_network` tolerates 'already exists'.""" container_mod.create_network(egress_network) container_mod.create_network(network, internal=True) -class AppleGateway(Gateway): - """The consolidated gateway as a single, fixed-name Apple container.""" - - def __init__( - self, - image_ref: str = GATEWAY_IMAGE, - *, - name: str = GATEWAY_NAME, - network: str = GATEWAY_NETWORK, - egress_network: str = GATEWAY_EGRESS_NETWORK, - orchestrator_url: str = "", - build_context: Path | None = None, - dockerfile: str | None = GATEWAY_DOCKERFILE, - ) -> None: - self.image_ref = image_ref - self.name = name - self.network = network - self.egress_network = egress_network - # Reached by IP (no container DNS on Apple) — the caller resolves the - # orchestrator's address before constructing this. Empty → single-tenant. - self._orchestrator_url = orchestrator_url - self._build_context = build_context or _REPO_ROOT - self._dockerfile = dockerfile - - def ensure_built(self) -> None: - """Build the gateway data-plane image from its Dockerfile. Builds every - time (cache-aware, so it's cheap when nothing changed): a stale image - silently runs the OLD single-tenant daemons. Mirrors `DockerGateway`.""" - if self._dockerfile is None: - return - container_mod.build_image( - self.image_ref, str(self._build_context), dockerfile=self._dockerfile, - ) - - def is_running(self) -> bool: - return container_mod.container_is_running(self.name) - - def _running_image_is_current(self) -> bool: - """True iff the running gateway was created from the *current* - `image_ref`. `ensure_built` rebuilding the image is not enough on its - own — the running container still holds the OLD image, so this - mismatch check is what makes a rebuild take effect.""" - running = container_mod.container_image_digest(self.name) - current = container_mod.image_digest(self.image_ref) - if not running or not current: - return True # can't compare → don't churn a working container - return running == current - - def _running_control_plane_is_current(self) -> bool: - """True iff the running gateway points at the control plane we would - pass today. - - Docker gets this for free — it hands the gateway a container *name*, - which survives the orchestrator being recreated. Apple has no container - DNS, so the URL is an **IP baked into the gateway's env at run time**, - and a recreated orchestrator can come back on a different DHCP address. - Without this check the gateway would keep pointing at the old address - and every `/resolve` would fail — denying egress for *every* bottle on - the host until something else happened to recreate the gateway.""" - if not self._orchestrator_url: - return True - env = container_mod.container_env(self.name) - if not env: - return True # can't compare → don't churn a working container - return env.get("BOT_BOTTLE_ORCHESTRATOR_URL") == self._orchestrator_url - - def ensure_running(self) -> None: - if (self.is_running() - and self._running_image_is_current() - and self._running_control_plane_is_current()): - return - ensure_networks(self.network, self.egress_network) - container_mod.force_remove_container(self.name) - argv = [ - "container", "run", "--detach", - "--name", self.name, - "--label", "bot-bottle.backend=macos-container", - "--label", "bot-bottle-mac-gateway=1", - # NAT network FIRST: Apple Container takes the first --network as - # the default route, so this ordering is what gives the gateway a - # route out. Reversing it silently blackholes egress. - "--network", self.egress_network, - "--network", self.network, - # The NAT gateway routes but does not resolve, so DNS is explicit. - "--dns", container_mod.dns_server(), - "--mount", _mount(str(gateway_ca_dir()), MITMPROXY_HOME), - "--mount", _mount(str(host_db_dir()), _SUPERVISE_DB_DIR_IN_CONTAINER), - "--env", f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}", - ] - if self._orchestrator_url: - # Makes the data plane multi-tenant: each request resolves - # source-IP → policy against the control plane. - argv += ["--env", f"BOT_BOTTLE_ORCHESTRATOR_URL={self._orchestrator_url}"] - argv.append(self.image_ref) - result = container_mod.run_container_argv(argv) - if result.returncode != 0: - raise GatewayError( - f"gateway failed to start: " - f"{(result.stderr or '').strip() or ''}" - ) - - def ip_on_shared_network(self) -> str: - """The gateway's address on the shared host-only network — what agents - point their proxy / git-http / supervise URLs at. Polls: a freshly - run gateway can be up before vmnet's DHCP has assigned the address.""" - ip = container_mod.wait_container_ipv4_on_network(self.name, self.network) - if not ip: - raise GatewayError( - f"gateway {self.name} never got an address on {self.network}" - ) - return ip - - def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str: - """The gateway's CA certificate (PEM) that agents install to trust its - TLS interception. Polls: mitmproxy generates it a moment after start. - Read from the host bind-mount, so no exec into the container.""" - ca_path = gateway_ca_dir() / os.path.basename(GATEWAY_CA_CERT) - deadline = time.monotonic() + timeout - while True: - try: - pem = ca_path.read_text() - if pem.strip(): - return pem - except OSError: - pass - if time.monotonic() >= deadline: - raise GatewayError( - f"gateway CA cert not available at {ca_path} after {timeout:g}s" - ) - time.sleep(_CA_POLL_SECONDS) - - def stop(self) -> None: - container_mod.force_remove_container(self.name) - - __all__ = [ - "AppleGateway", - "GatewayError", - "GATEWAY_NAME", "GATEWAY_NETWORK", "GATEWAY_EGRESS_NETWORK", "GATEWAY_IMAGE", - "gateway_ca_dir", + "GatewayError", + "DEFAULT_CA_TIMEOUT_SECONDS", + "ensure_networks", ] diff --git a/bot_bottle/backend/macos_container/gateway_provision.py b/bot_bottle/backend/macos_container/gateway_provision.py index 0d4df9b..d58bac8 100644 --- a/bot_bottle/backend/macos_container/gateway_provision.py +++ b/bot_bottle/backend/macos_container/gateway_provision.py @@ -1,23 +1,23 @@ -"""`GatewayTransport` for the Apple gateway container (PRD 0070). +"""`GatewayTransport` for the Apple infra container (PRD 0070). The provisioning *logic* (per-bottle creds dirs, namespaced repo init) is backend-neutral and lives in `backend.docker.gateway_provision`; this is only the transport — how files and commands reach the running gateway. Docker uses `docker exec`/`docker cp` and Firecracker uses SSH; Apple uses the `container` -CLI's equivalents. +CLI's equivalents against the infra container that hosts the gateway daemons. """ from __future__ import annotations from ..docker.gateway_provision import GatewayProvisionError from . import util as container_mod -from .gateway import GATEWAY_NAME +from .infra import INFRA_NAME class AppleGatewayTransport: - """`GatewayTransport` for the gateway as an Apple container.""" + """`GatewayTransport` for the gateway daemons in the Apple infra container.""" - def __init__(self, gateway: str = GATEWAY_NAME) -> None: + def __init__(self, gateway: str = INFRA_NAME) -> None: self.gateway = gateway def exec(self, argv: list[str]) -> None: diff --git a/bot_bottle/backend/macos_container/infra.py b/bot_bottle/backend/macos_container/infra.py new file mode 100644 index 0000000..14b7bf3 --- /dev/null +++ b/bot_bottle/backend/macos_container/infra.py @@ -0,0 +1,292 @@ +"""The per-host infra container for the macOS backend (PRD 0070). + +A single persistent Apple container that runs BOTH the orchestrator control +plane and the gateway data plane — the macOS analogue of the Firecracker infra +VM (`backend/firecracker/infra_vm.py`), not the docker backend's two separate +containers. + +Why one container, not two: Apple Containers are lightweight VMs, each with its +own kernel. The docker backend runs the orchestrator and gateway as two +containers safely because they share the host kernel, so their concurrent +writes to the one `bot-bottle.db` (the orchestrator's registry + the gateway +supervise daemon's queue) are serialized by coherent `fcntl` locks. Across two +*guest* kernels sharing a virtiofs-mounted DB those locks are not coherent, and +concurrent writers can corrupt the file. Firecracker solved this by putting +both services in one guest with the DB on a device only that guest mounts; this +does the same with Apple primitives. + +Two consequences fall out of the single container, both simplifications: + +- **No DNS dance.** The control plane and the gateway daemons reach each other + over `127.0.0.1`, so nothing depends on Apple's (absent) container DNS and + there is no orchestrator-before-gateway ordering to get right. +- **The DB is never host-shared.** It lives on a container-only volume, so no + host process opens the live file. The host CLI reaches registry + supervise + state through the control-plane HTTP surface (`cli/supervise.py` already uses + `OrchestratorClient`), exactly as it does for firecracker. + +The control-plane source is bind-mounted (like the docker orchestrator), so a +code change takes effect on the next launch without an image rebuild; the +gateway daemons are baked in the gateway image and rebuild through its own +digest check. +""" + +from __future__ import annotations + +import time +import urllib.error +import urllib.request +from dataclasses import dataclass +from pathlib import Path + +from ... import log +from ...orchestrator.gateway import GATEWAY_CA_CERT +from ...orchestrator.lifecycle import ( + DEFAULT_PORT, + DEFAULT_STARTUP_TIMEOUT_SECONDS, + OrchestratorStartError, + source_hash, +) +from ...paths import HOST_DB_FILENAME +from . import util as container_mod +from .gateway import ( + DEFAULT_CA_TIMEOUT_SECONDS, + GATEWAY_EGRESS_NETWORK, + GATEWAY_IMAGE, + GATEWAY_NETWORK, + GatewayError, + ensure_networks, +) + +# The one per-host infra container: control plane + gateway data plane. +INFRA_NAME = "bot-bottle-mac-infra" +INFRA_LABEL = "bot-bottle-mac-infra=1" +# Container-only volume holding bot-bottle.db. No host bind-mount, so the DB is +# written by exactly one kernel (this container's). Survives recreation. +INFRA_DB_VOLUME = "bot-bottle-mac-db" + +# BOT_BOTTLE_ROOT inside the container; host_db_path() resolves the DB to +# /db/ and the supervise daemon writes the same file. +_DB_ROOT_IN_CONTAINER = "/var/lib/bot-bottle" +_DB_PATH_IN_CONTAINER = f"{_DB_ROOT_IN_CONTAINER}/db/{HOST_DB_FILENAME}" +_SRC_IN_CONTAINER = "/bot-bottle-src" + +_REPO_ROOT = Path(__file__).resolve().parents[3] + +_HEALTH_POLL_SECONDS = 0.25 +_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0 +_CA_POLL_SECONDS = 0.5 + +# The gateway subset the consolidated model runs (no per-bottle git:// daemon). +_GATEWAY_DAEMONS = "egress,git-http,supervise" + + +def _init_script(port: int) -> str: + """PID-1 init: start the control plane and the gateway daemons, both in + this container, reaching each other over loopback. Backgrounded so `wait` + reaps as PID 1. No `set -e` — a transient daemon failure must not kill the + whole container (gateway_init applies the same 'stay up' policy).""" + return ( + "export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\n" + f"mkdir -p $(dirname {_DB_PATH_IN_CONTAINER})\n" + # Control plane, from the bind-mounted source (stdlib-only package). + f"( cd {_SRC_IN_CONTAINER} && BOT_BOTTLE_ROOT={_DB_ROOT_IN_CONTAINER} " + f"python3 -m bot_bottle.orchestrator --host 0.0.0.0 --port {port} " + "--broker stub ) &\n" + # Gateway data plane, multi-tenant against the local control plane. + f"( cd /app && BOT_BOTTLE_GATEWAY_DAEMONS={_GATEWAY_DAEMONS} " + f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{port} " + f"SUPERVISE_DB_PATH={_DB_PATH_IN_CONTAINER} python3 /app/gateway_init.py ) &\n" + "while : ; do wait ; done\n" + ) + + +@dataclass(frozen=True) +class InfraEndpoint: + """How to reach the running infra container. The control plane and the + gateway are the same container, so one address serves both.""" + + control_plane_url: str # http://:8099 — host CLI + registration + gateway_ip: str # same container; agents' proxy / git-http / MCP target + + +class MacosInfraService: + """Manages the single per-host infra container. Callers use + `ensure_running()` (returns the endpoint) and `ca_cert_pem()`.""" + + def __init__( + self, + *, + port: int = DEFAULT_PORT, + network: str = GATEWAY_NETWORK, + egress_network: str = GATEWAY_EGRESS_NETWORK, + image: str = GATEWAY_IMAGE, + repo_root: Path = _REPO_ROOT, + name: str = INFRA_NAME, + db_volume: str = INFRA_DB_VOLUME, + ) -> None: + self.port = port + self.network = network + self.egress_network = egress_network + self.image = image + self._repo_root = repo_root + self._name = name + self._db_volume = db_volume + + def _resolve_url(self) -> str: + """The control-plane URL, or "" while the container has no address.""" + ip = container_mod.try_container_ipv4_on_network(self._name, self.network) + return f"http://{ip}:{self.port}" if ip else "" + + def is_healthy( + self, url: str, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS, + ) -> bool: + if not url: + return False + try: + with urllib.request.urlopen(f"{url}/health", timeout=timeout) as resp: + return resp.status == 200 + except (urllib.error.URLError, TimeoutError, OSError): + return False + + def _source_current(self, current_hash: str) -> bool: + """True iff the running infra container was created from the current + bind-mounted control-plane source. The control-plane process loads that + code at startup and won't reload it, so a stale container keeps serving + OLD code.""" + if not container_mod.container_is_running(self._name): + return False + env = container_mod.container_env(self._name) + if not env: + return True # can't compare → don't churn a working container + return env.get("BOT_BOTTLE_SOURCE_HASH") == current_hash + + def _running_healthy_endpoint(self, current_hash: str) -> InfraEndpoint | None: + """The endpoint if the running container is BOTH source-current and + answering /health, else None (→ recreate). Health, not just the source + label, is what lets a wedged-but-current container self-heal instead of + being polled to death forever.""" + if not self._source_current(current_hash): + return None + url = self._resolve_url() + if url and self.is_healthy(url): + return InfraEndpoint(control_plane_url=url, gateway_ip=_ip_of(url)) + return None + + def ensure_built(self) -> None: + """Ensure the gateway data-plane image exists. The control-plane source + is bind-mounted, not baked, so only the gateway image needs building.""" + container_mod.build_image( + self.image, str(self._repo_root), dockerfile="Dockerfile.gateway", + ) + + def ensure_running( + self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS, + ) -> InfraEndpoint: + """Ensure the single infra container is up; return how to reach it. + Idempotent per-host singleton — a healthy container on current source + is left untouched, so N launches share the one control plane + gateway. + Raises `OrchestratorStartError` on startup timeout.""" + current_hash = source_hash(self._repo_root) + endpoint = self._running_healthy_endpoint(current_hash) + if endpoint is not None: + return endpoint + self.ensure_built() + log.info("starting infra container", context={"name": self._name}) + self._run_container(current_hash) + return self._wait_healthy(startup_timeout) + + def _run_container(self, current_hash: str) -> None: + ensure_networks(self.network, self.egress_network) + container_mod.force_remove_container(self._name) + argv = [ + "container", "run", "--detach", + "--name", self._name, + "--label", "bot-bottle.backend=macos-container", + "--label", INFRA_LABEL, + # NAT network FIRST so the gateway's egress has a default route; + # the host-only network is where agents (and the host CLI) reach it. + "--network", self.egress_network, + "--network", self.network, + "--dns", container_mod.dns_server(), + # Container-only DB volume: one kernel writes bot-bottle.db, never + # shared with the host or another guest. + "--volume", f"{self._db_volume}:{_DB_ROOT_IN_CONTAINER}", + # Bind-mount the control-plane source (read-only); a code change + # takes effect on relaunch with no image rebuild. + "--mount", + container_mod.bind_mount_spec( + str(self._repo_root), _SRC_IN_CONTAINER, readonly=True), + # Baked onto the container so `_source_current` can detect a real + # control-plane code change and recreate. + "--env", f"BOT_BOTTLE_SOURCE_HASH={current_hash}", + "--entrypoint", "sh", + self.image, + "-c", _init_script(self.port), + ] + result = container_mod.run_container_argv(argv) + if result.returncode != 0: + raise OrchestratorStartError( + f"infra container failed to start: " + f"{(result.stderr or '').strip() or ''}" + ) + + def _wait_healthy(self, startup_timeout: float) -> InfraEndpoint: + deadline = time.monotonic() + startup_timeout + while True: + url = self._resolve_url() + if url and self.is_healthy(url): + log.info("infra container healthy", context={"url": url}) + return InfraEndpoint(control_plane_url=url, gateway_ip=_ip_of(url)) + if time.monotonic() >= deadline: + raise OrchestratorStartError( + f"infra container did not become healthy within " + f"{startup_timeout:g}s" + ) + time.sleep(_HEALTH_POLL_SECONDS) + + def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str: + """The gateway's mitmproxy CA (PEM) agents install to trust its TLS + interception. Read out of the container (the CA lives on a + container-internal path, not a host mount); polls because mitmproxy + writes it a beat after start.""" + deadline = time.monotonic() + timeout + while True: + result = container_mod.run_container_argv( + ["container", "exec", self._name, "cat", GATEWAY_CA_CERT]) + if result.returncode == 0 and result.stdout.strip(): + return result.stdout + if time.monotonic() >= deadline: + raise GatewayError( + f"gateway CA not available in {self._name} after {timeout:g}s: " + f"{(result.stderr or '').strip() or 'empty'}" + ) + time.sleep(_CA_POLL_SECONDS) + + def stop(self) -> None: + """Remove the infra container (idempotent). The DB volume persists.""" + container_mod.force_remove_container(self._name) + + +def _ip_of(url: str) -> str: + """The host from an http://host:port URL.""" + return url.split("://", 1)[-1].rsplit(":", 1)[0] + + +def probe_control_plane_url(port: int = DEFAULT_PORT) -> str: + """The running infra container's control-plane URL, or "" if it isn't up. + Used by host-side control-plane discovery (`discover_orchestrator_url`); + safe to call on any host — returns "" when the container or the `container` + CLI isn't present.""" + ip = container_mod.try_container_ipv4_on_network(INFRA_NAME, GATEWAY_NETWORK) + return f"http://{ip}:{port}" if ip else "" + + +__all__ = [ + "MacosInfraService", + "InfraEndpoint", + "OrchestratorStartError", + "GatewayError", + "INFRA_NAME", + "INFRA_DB_VOLUME", +] diff --git a/bot_bottle/backend/macos_container/orchestrator_service.py b/bot_bottle/backend/macos_container/orchestrator_service.py deleted file mode 100644 index 70b666d..0000000 --- a/bot_bottle/backend/macos_container/orchestrator_service.py +++ /dev/null @@ -1,242 +0,0 @@ -"""Orchestrator + gateway lifecycle for the macOS backend (PRD 0070). - -The macOS counterpart of `orchestrator.lifecycle.OrchestratorService`. Same -shape — an idempotent singleton that brings up the control plane and the shared -gateway and hands back the control-plane URL — but the startup **order is -reversed**, and the reason is a real Apple Container constraint rather than a -stylistic choice: - -- Docker starts the gateway first and lets it find the control plane by - *container name* over docker DNS. -- Apple Container 1.0.0 has **no container DNS** (see `gateway`), so the - gateway can only be handed an **IP**. That IP does not exist until the - orchestrator container is running — hence: orchestrator first, read its - address, then start the gateway pointed at it. - -The second difference: no published port. Apple Container puts every container -on a host-reachable address, and the host can reach the host-only network -directly, so the host CLI and the gateway use the **same** URL — the -orchestrator's address on the shared network. Docker needs a -`--publish 127.0.0.1:…` hop plus a separate `internal_url` for the same job. - -Like the docker service this runs the **register-only broker**: the backend -launches agent containers, so the control plane needs no privileged socket. -""" - -from __future__ import annotations - -import time -import urllib.error -import urllib.request -from pathlib import Path - -from ... import log -from ...orchestrator.lifecycle import ( - DEFAULT_PORT, - DEFAULT_STARTUP_TIMEOUT_SECONDS, - ORCHESTRATOR_DOCKERFILE, - ORCHESTRATOR_IMAGE, - ORCHESTRATOR_SOURCE_HASH_LABEL, - OrchestratorStartError, - source_hash, -) -from ...paths import bot_bottle_root -from . import util as container_mod -from .gateway import ( - GATEWAY_EGRESS_NETWORK, - GATEWAY_IMAGE, - GATEWAY_NETWORK, - AppleGateway, - ensure_networks, -) -from .util import bind_mount_spec as _mount - -# Distinct from the docker backend's container name so both can run on one host. -ORCHESTRATOR_NAME = "bot-bottle-mac-orchestrator" - -_REPO_ROOT = Path(__file__).resolve().parents[3] -_APP_DIR = "/app" -_ROOT_IN_CONTAINER = "/bot-bottle-root" - -_HEALTH_POLL_SECONDS = 0.25 -_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0 - - -class MacosOrchestratorService: - """Manages the orchestrator control-plane container + the shared Apple - gateway. Callers only need `ensure_running()`, which returns the URL.""" - - def __init__( - self, - *, - port: int = DEFAULT_PORT, - network: str = GATEWAY_NETWORK, - egress_network: str = GATEWAY_EGRESS_NETWORK, - image: str = ORCHESTRATOR_IMAGE, - gateway_image: str = GATEWAY_IMAGE, - repo_root: Path = _REPO_ROOT, - host_root: Path | None = None, - orchestrator_name: str = ORCHESTRATOR_NAME, - ) -> None: - self.port = port - self.network = network - self.egress_network = egress_network - self.image = image - self._gateway_image = gateway_image - self._repo_root = repo_root - self._host_root = host_root or bot_bottle_root() - self._orchestrator_name = orchestrator_name - - def _resolve_url(self) -> str: - """The control-plane URL, or "" while the container has no address. - Resolved fresh each time (there is no name to fall back on, and a - recreated orchestrator can come back on a different DHCP address), so - nothing caches a URL that could go stale.""" - ip = container_mod.try_container_ipv4_on_network( - self._orchestrator_name, self.network, - ) - return f"http://{ip}:{self.port}" if ip else "" - - def is_healthy( - self, url: str, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS, - ) -> bool: - if not url: - return False - try: - with urllib.request.urlopen(f"{url}/health", timeout=timeout) as resp: - return resp.status == 200 - except (urllib.error.URLError, TimeoutError, OSError): - return False - - def _ensure_orchestrator_image(self) -> None: - """Build the lean control-plane image when missing. Build-if-missing, - not build-every-time: the control plane bind-mounts its source, so a - code change is caught by the source-hash recreate, not a rebuild.""" - if container_mod.image_exists(self.image): - return - container_mod.build_image( - self.image, str(self._repo_root), dockerfile=ORCHESTRATOR_DOCKERFILE, - ) - - def _orchestrator_source_current(self, current_hash: str) -> bool: - """True iff the running orchestrator was created from the *current* - bind-mounted source. The process loaded that code at startup and won't - reload it, so a stale container would keep serving OLD control-plane - code.""" - if not container_mod.container_is_running(self._orchestrator_name): - return False - data = container_mod.inspect_container(self._orchestrator_name) - config = data.get("configuration") - labels = config.get("labels") if isinstance(config, dict) else None - if not isinstance(labels, dict): - return True # can't compare → don't churn a working container - return labels.get(ORCHESTRATOR_SOURCE_HASH_LABEL) == current_hash - - def _run_orchestrator_container(self, current_hash: str) -> None: - # The orchestrator is the first thing on the shared network, so it — - # not the gateway — is what has to bring the network into existence. - ensure_networks(self.network, self.egress_network) - container_mod.force_remove_container(self._orchestrator_name) - argv = [ - "container", "run", "--detach", - "--name", self._orchestrator_name, - "--label", "bot-bottle.backend=macos-container", - "--label", f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={current_hash}", - # Host-only network only: the control plane needs no route out, and - # the host reaches it here directly (no --publish needed). - "--network", self.network, - "--mount", _mount(str(self._repo_root), _APP_DIR, readonly=True), - "--workdir", _APP_DIR, - # Persist the registry DB on the host (sole-owner: only the - # orchestrator opens bot-bottle.db). - "--mount", _mount(str(self._host_root), _ROOT_IN_CONTAINER), - "--env", f"BOT_BOTTLE_ROOT={_ROOT_IN_CONTAINER}", - "--entrypoint", "python3", - self.image, - "-m", "bot_bottle.orchestrator", - "--host", "0.0.0.0", "--port", str(self.port), "--broker", "stub", - ] - result = container_mod.run_container_argv(argv) - if result.returncode != 0: - raise OrchestratorStartError( - f"orchestrator container failed to start: " - f"{(result.stderr or '').strip() or ''}" - ) - - def gateway(self, orchestrator_url: str) -> AppleGateway: - """The shared gateway, pointed at the control plane at - `orchestrator_url` (by IP — Apple has no container DNS).""" - return AppleGateway( - self._gateway_image, - network=self.network, - egress_network=self.egress_network, - orchestrator_url=orchestrator_url, - ) - - def ensure_running( - self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS, - ) -> str: - """Ensure the control plane + shared gateway are up; return the - control-plane URL. Idempotent — a healthy control plane running current - code and a running gateway are left untouched.""" - current_hash = source_hash(self._repo_root) - url = self._running_healthy_url(current_hash) - if not url: - self._ensure_orchestrator_image() - log.info( - "starting orchestrator container", - context={"name": self._orchestrator_name}, - ) - self._run_orchestrator_container(current_hash) - url = self._wait_healthy(startup_timeout) - - # Gateway second: it can only reach the control plane by IP, which does - # not exist until the orchestrator container is up (see the docstring). - gateway = self.gateway(url) - gateway.ensure_built() - gateway.ensure_running() - return url - - def _running_healthy_url(self, current_hash: str) -> str: - """The control-plane URL if the running orchestrator is BOTH current - and answering /health, else "" (→ recreate). - - Checking health, not just the source-hash label, is what lets the - service self-heal: a container that is running the current code but - whose HTTP server is wedged (bind failure, deadlock, OOM'd thread) - would otherwise be left alone and polled to death on every launch - forever. Mirrors the docker service's `is_healthy() and source_current` - gate.""" - if not self._orchestrator_source_current(current_hash): - return "" - url = self._resolve_url() - return url if url and self.is_healthy(url) else "" - - def _wait_healthy(self, startup_timeout: float) -> str: - """Poll until the control plane answers /health, resolving its address - each time: the container is up before it has an IP, and it has an IP - before the server binds.""" - deadline = time.monotonic() + startup_timeout - while True: - url = self._resolve_url() - if url and self.is_healthy(url): - log.info("orchestrator healthy", context={"url": url}) - return url - if time.monotonic() >= deadline: - raise OrchestratorStartError( - f"orchestrator did not become healthy within " - f"{startup_timeout:g}s" - ) - time.sleep(_HEALTH_POLL_SECONDS) - - def stop(self) -> None: - """Remove the orchestrator + gateway containers (idempotent).""" - container_mod.force_remove_container(self._orchestrator_name) - self.gateway("").stop() - - -__all__ = [ - "MacosOrchestratorService", - "OrchestratorStartError", - "ORCHESTRATOR_NAME", -] diff --git a/bot_bottle/orchestrator/client.py b/bot_bottle/orchestrator/client.py index b34f038..e094b21 100644 --- a/bot_bottle/orchestrator/client.py +++ b/bot_bottle/orchestrator/client.py @@ -186,6 +186,13 @@ def discover_orchestrator_url(*, timeout: float = 2.0) -> str: f"http://{netpool.orch_slot().guest_ip}:{CONTROL_PLANE_PORT}") except Exception: # noqa: BLE001 — backend optional / not firecracker pass + try: # macOS: infra container control plane on its host-only address + from ..backend.macos_container.infra import probe_control_plane_url + url = probe_control_plane_url() + if url: + candidates.append(url) + except Exception: # noqa: BLE001 — backend optional / not macOS + pass for url in candidates: if OrchestratorClient(url, timeout=timeout).health(): return url diff --git a/tests/unit/test_backend_selection.py b/tests/unit/test_backend_selection.py index 6edfe08..568a47a 100644 --- a/tests/unit/test_backend_selection.py +++ b/tests/unit/test_backend_selection.py @@ -247,7 +247,7 @@ class TestHasBackend(unittest.TestCase): class TestEnsureOrchestrator(unittest.TestCase): """The backend-agnostic orchestrator bring-up entry point. Docker starts the orchestrator + gateway containers; firecracker boots the infra VM; - backends without one (macos-container) die with a pointer.""" + macos-container starts the infra container.""" def test_docker_delegates_to_orchestrator_service(self): b = get_bottle_backend("docker") @@ -272,11 +272,16 @@ class TestEnsureOrchestrator(unittest.TestCase): url = b.ensure_orchestrator() self.assertEqual(url, "http://10.243.255.1:8099") - def test_macos_default_dies(self): - from bot_bottle.log import Die + def test_macos_delegates_to_infra_container(self): b = get_bottle_backend("macos-container") - with self.assertRaises(Die): - b.ensure_orchestrator() + with patch( + "bot_bottle.backend.macos_container.infra.MacosInfraService" + ) as service_cls: + service_cls.return_value.ensure_running.return_value.control_plane_url = ( + "http://192.168.128.2:8099" + ) + url = b.ensure_orchestrator() + self.assertEqual(url, "http://192.168.128.2:8099") if __name__ == "__main__": diff --git a/tests/unit/test_macos_consolidated_launch.py b/tests/unit/test_macos_consolidated_launch.py index 5ab4ef0..f3e395d 100644 --- a/tests/unit/test_macos_consolidated_launch.py +++ b/tests/unit/test_macos_consolidated_launch.py @@ -50,30 +50,35 @@ def _client() -> Mock: class TestEnsureGateway(unittest.TestCase): def _run(self, service: MagicMock) -> GatewayEndpoint: - with patch(f"{_MOD}.MacosOrchestratorService", return_value=service): + with patch(f"{_MOD}.MacosInfraService", return_value=service): return ensure_gateway() def _service(self) -> MagicMock: + from bot_bottle.backend.macos_container.infra import InfraEndpoint service = MagicMock() - service.ensure_running.return_value = "http://192.168.128.2:8099" + service.ensure_running.return_value = InfraEndpoint( + control_plane_url="http://192.168.128.2:8099", + gateway_ip="192.168.128.2", + ) service.network = "bot-bottle-mac-gateway" - service.gateway.return_value.ip_on_shared_network.return_value = "192.168.128.3" - service.gateway.return_value.ca_cert_pem.return_value = "PEM" + service.ca_cert_pem.return_value = "PEM" return service def test_reports_gateway_endpoint(self) -> None: endpoint = self._run(self._service()) self.assertEqual("http://192.168.128.2:8099", endpoint.orchestrator_url) - self.assertEqual("192.168.128.3", endpoint.gateway_ip) + self.assertEqual("192.168.128.2", endpoint.gateway_ip) self.assertEqual("PEM", endpoint.gateway_ca_pem) self.assertEqual("bot-bottle-mac-gateway", endpoint.network) - def test_gateway_is_pointed_at_the_resolved_control_plane(self) -> None: - """Apple has no container DNS, so the gateway must be handed the - control plane's *resolved URL* rather than a container name.""" - service = self._service() - self._run(service) - service.gateway.assert_called_with("http://192.168.128.2:8099") + def test_control_plane_and_gateway_share_one_address(self) -> None: + """One infra container hosts both, so the gateway IP and the + control-plane host are the same.""" + endpoint = self._run(self._service()) + self.assertEqual( + endpoint.gateway_ip, + endpoint.orchestrator_url.split("://")[1].split(":")[0], + ) class TestRegisterAgent(unittest.TestCase): diff --git a/tests/unit/test_macos_container_cleanup.py b/tests/unit/test_macos_container_cleanup.py index 4d5dd5e..990536e 100644 --- a/tests/unit/test_macos_container_cleanup.py +++ b/tests/unit/test_macos_container_cleanup.py @@ -60,13 +60,10 @@ class TestMacosContainerEnumerate(unittest.TestCase): self.assertEqual(["dev-abc"], [a.slug for a in agents]) self.assertEqual(["macos-container"], [a.backend_name for a in agents]) - def test_excludes_the_shared_singletons(self): - """The gateway and control plane share the bot-bottle- prefix but are - infrastructure — listing them would invent an agent per host.""" - agents = self._enumerate( - "bot-bottle-mac-gateway\nbot-bottle-mac-orchestrator\n" - "bot-bottle-dev-abc\n" - ) + def test_excludes_the_infra_singleton(self): + """The infra container shares the bot-bottle- prefix but is + infrastructure — listing it would invent an agent per host.""" + agents = self._enumerate("bot-bottle-mac-infra\nbot-bottle-dev-abc\n") self.assertEqual(["dev-abc"], [a.slug for a in agents]) def test_empty_when_the_cli_fails(self): diff --git a/tests/unit/test_macos_gateway.py b/tests/unit/test_macos_gateway.py deleted file mode 100644 index 359c344..0000000 --- a/tests/unit/test_macos_gateway.py +++ /dev/null @@ -1,297 +0,0 @@ -"""Unit: the Apple gateway + orchestrator lifecycle (PRD 0070).""" - -from __future__ import annotations - -import unittest -from pathlib import Path -from unittest.mock import Mock, patch - -from bot_bottle.backend.macos_container.gateway import AppleGateway, GatewayError -from bot_bottle.backend.macos_container.orchestrator_service import ( - MacosOrchestratorService, - OrchestratorStartError, -) - -_GW = "bot_bottle.backend.macos_container.gateway" -_ORCH = "bot_bottle.backend.macos_container.orchestrator_service" - - -def _ok(stdout: str = "") -> Mock: - return Mock(returncode=0, stdout=stdout, stderr="") - - -def _fail(stderr: str = "boom") -> Mock: - return Mock(returncode=1, stdout="", stderr=stderr) - - -class TestAppleGatewayRun(unittest.TestCase): - def _argv(self, run: Mock) -> list[str]: - return run.call_args.args[0] - - def _start(self, run: Mock) -> None: - with patch(f"{_GW}.container_mod") as mod: - mod.container_is_running.return_value = False - mod.dns_server.return_value = "1.1.1.1" - mod.run_container_argv = run - AppleGateway(orchestrator_url="http://192.168.128.2:8099").ensure_running() - - def test_nat_network_precedes_the_host_only_network(self) -> None: - """Apple Container makes the FIRST --network the default route, so the - NAT network must lead or the gateway has no route out.""" - run = Mock(return_value=_ok()) - self._start(run) - argv = self._argv(run) - networks = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"] - self.assertEqual(["bot-bottle-mac-egress", "bot-bottle-mac-gateway"], networks) - - def test_control_plane_url_is_passed_for_multi_tenancy(self) -> None: - run = Mock(return_value=_ok()) - self._start(run) - self.assertIn( - "BOT_BOTTLE_ORCHESTRATOR_URL=http://192.168.128.2:8099", self._argv(run), - ) - - def test_dns_is_explicit(self) -> None: - """The NAT gateway routes but does not resolve.""" - run = Mock(return_value=_ok()) - self._start(run) - argv = self._argv(run) - self.assertEqual("1.1.1.1", argv[argv.index("--dns") + 1]) - - def test_start_failure_raises(self) -> None: - with self.assertRaises(GatewayError): - self._start(Mock(return_value=_fail())) - - def test_running_current_gateway_is_left_alone(self) -> None: - """Idempotent singleton: N launches must not restart the gateway and - drop every other bottle's data plane.""" - run = Mock(return_value=_ok()) - with patch(f"{_GW}.container_mod") as mod: - mod.container_is_running.return_value = True - mod.container_image_digest.return_value = "abc" - mod.image_digest.return_value = "abc" - mod.run_container_argv = run - AppleGateway().ensure_running() - run.assert_not_called() - - def test_stale_image_forces_a_recreate(self) -> None: - """A rebuilt image only takes effect if the running container is - replaced — otherwise it keeps serving the OLD daemons.""" - run = Mock(return_value=_ok()) - with patch(f"{_GW}.container_mod") as mod: - mod.container_is_running.return_value = True - mod.container_image_digest.return_value = "old" - mod.image_digest.return_value = "new" - mod.dns_server.return_value = "1.1.1.1" - mod.run_container_argv = run - AppleGateway().ensure_running() - run.assert_called_once() - - def test_unreadable_digest_does_not_churn(self) -> None: - run = Mock(return_value=_ok()) - with patch(f"{_GW}.container_mod") as mod: - mod.container_is_running.return_value = True - mod.container_image_digest.return_value = "" - mod.image_digest.return_value = "" - mod.run_container_argv = run - AppleGateway().ensure_running() - run.assert_not_called() - - def _start_with_running_env(self, env: dict[str, str], url: str) -> Mock: - run = Mock(return_value=_ok()) - with patch(f"{_GW}.container_mod") as mod: - mod.container_is_running.return_value = True - mod.container_image_digest.return_value = "abc" - mod.image_digest.return_value = "abc" - mod.container_env.return_value = env - mod.dns_server.return_value = "1.1.1.1" - mod.run_container_argv = run - AppleGateway(orchestrator_url=url).ensure_running() - return run - - def test_moved_control_plane_forces_a_recreate(self) -> None: - """Docker hands the gateway a container *name*, stable across an - orchestrator recreate. Apple has no DNS, so the URL is an IP baked into - the gateway's env — if the orchestrator comes back on a new address and - the gateway isn't recreated, every /resolve fails and every bottle on - the host loses egress.""" - run = self._start_with_running_env( - {"BOT_BOTTLE_ORCHESTRATOR_URL": "http://192.168.128.2:8099"}, - "http://192.168.128.7:8099", - ) - run.assert_called_once() - self.assertIn( - "BOT_BOTTLE_ORCHESTRATOR_URL=http://192.168.128.7:8099", - run.call_args.args[0], - ) - - def test_unmoved_control_plane_does_not_churn(self) -> None: - run = self._start_with_running_env( - {"BOT_BOTTLE_ORCHESTRATOR_URL": "http://192.168.128.2:8099"}, - "http://192.168.128.2:8099", - ) - run.assert_not_called() - - def test_unreadable_env_does_not_churn(self) -> None: - run = self._start_with_running_env({}, "http://192.168.128.2:8099") - run.assert_not_called() - - -class TestMacosOrchestratorService(unittest.TestCase): - def test_orchestrator_starts_before_the_gateway(self) -> None: - """Apple has no container DNS, so the gateway can only be handed the - control plane's IP — which does not exist until it is running. This - ordering is the whole reason the macOS service diverges from docker's.""" - order: list[str] = [] - svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h")) - gateway = Mock() - gateway.ensure_running.side_effect = lambda: order.append("gateway") - - def _record_orchestrator(_hash: str) -> None: - order.append("orchestrator") - - with patch(f"{_ORCH}.container_mod") as mod, \ - patch(f"{_ORCH}.source_hash", return_value="h1"), \ - patch.object(svc, "_run_orchestrator_container", - side_effect=_record_orchestrator), \ - patch.object(svc, "gateway", return_value=gateway), \ - patch.object(svc, "is_healthy", return_value=True): - mod.container_is_running.return_value = False - mod.image_exists.return_value = True - mod.try_container_ipv4_on_network.return_value = "192.168.128.2" - url = svc.ensure_running() - self.assertEqual(["orchestrator", "gateway"], order) - self.assertEqual("http://192.168.128.2:8099", url) - - def test_gateway_is_handed_the_resolved_url(self) -> None: - svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h")) - with patch(f"{_ORCH}.container_mod") as mod, \ - patch(f"{_ORCH}.source_hash", return_value="h1"), \ - patch.object(svc, "_run_orchestrator_container"), \ - patch.object(svc, "gateway") as gw, \ - patch.object(svc, "is_healthy", return_value=True): - mod.container_is_running.return_value = False - mod.image_exists.return_value = True - mod.try_container_ipv4_on_network.return_value = "192.168.128.2" - svc.ensure_running() - gw.assert_called_with("http://192.168.128.2:8099") - - def test_current_source_leaves_a_healthy_orchestrator_alone(self) -> None: - """Recreating on every launch would drop every other live bottle's - in-memory egress tokens (#381).""" - svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h")) - run = Mock() - with patch(f"{_ORCH}.container_mod") as mod, \ - patch(f"{_ORCH}.source_hash", return_value="h1"), \ - patch.object(svc, "_run_orchestrator_container", run), \ - patch.object(svc, "gateway"), \ - patch.object(svc, "is_healthy", return_value=True): - mod.container_is_running.return_value = True - mod.inspect_container.return_value = { - "configuration": {"labels": {"bot-bottle-orchestrator-source-hash": "h1"}} - } - mod.try_container_ipv4_on_network.return_value = "192.168.128.2" - svc.ensure_running() - run.assert_not_called() - - def test_wedged_but_current_orchestrator_is_recreated(self) -> None: - """A container running the current code but whose HTTP server is dead - must be recreated, not left alone and polled to death forever. Checking - the source-hash label alone (without health) would strand the host.""" - svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h")) - run = Mock() - # Unhealthy on the pre-check, healthy once recreated + waited. - health = Mock(side_effect=[False, True]) - with patch(f"{_ORCH}.container_mod") as mod, \ - patch(f"{_ORCH}.source_hash", return_value="h1"), \ - patch.object(svc, "_run_orchestrator_container", run), \ - patch.object(svc, "_ensure_orchestrator_image"), \ - patch.object(svc, "gateway"), \ - patch.object(svc, "is_healthy", health): - mod.container_is_running.return_value = True - mod.inspect_container.return_value = { - "configuration": {"labels": {"bot-bottle-orchestrator-source-hash": "h1"}} - } - mod.try_container_ipv4_on_network.return_value = "192.168.128.2" - svc.ensure_running() - run.assert_called_once() - - def test_changed_source_recreates_the_orchestrator(self) -> None: - """The control-plane process loaded its bind-mounted source at startup - and won't reload it.""" - svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h")) - run = Mock() - with patch(f"{_ORCH}.container_mod") as mod, \ - patch(f"{_ORCH}.source_hash", return_value="h2"), \ - patch.object(svc, "_run_orchestrator_container", run), \ - patch.object(svc, "gateway"), \ - patch.object(svc, "is_healthy", return_value=True): - mod.container_is_running.return_value = True - mod.inspect_container.return_value = { - "configuration": {"labels": {"bot-bottle-orchestrator-source-hash": "h1"}} - } - mod.image_exists.return_value = True - mod.try_container_ipv4_on_network.return_value = "192.168.128.2" - svc.ensure_running() - run.assert_called_once() - - def test_never_healthy_raises(self) -> None: - svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h")) - with patch(f"{_ORCH}.container_mod") as mod, \ - patch(f"{_ORCH}.source_hash", return_value="h1"), \ - patch.object(svc, "_run_orchestrator_container"), \ - patch.object(svc, "is_healthy", return_value=False): - mod.container_is_running.return_value = False - mod.image_exists.return_value = True - mod.try_container_ipv4_on_network.return_value = "192.168.128.2" - with self.assertRaises(OrchestratorStartError): - svc.ensure_running(startup_timeout=0.01) - - def test_control_plane_needs_no_route_out(self) -> None: - """The orchestrator sits only on the host-only network: the host - reaches it there directly, so there is no --publish and no NAT leg.""" - svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h")) - run = Mock(return_value=_ok()) - # `ensure_networks` lives in the gateway module and shells out to the - # `container` CLI, which does not exist on the Linux CI host — patch it - # here, not gateway.container_mod, since it is called through this - # module's imported name. - with patch(f"{_ORCH}.container_mod") as mod, \ - patch(f"{_ORCH}.ensure_networks"): - mod.run_container_argv = run - svc._run_orchestrator_container("h1") - argv = run.call_args.args[0] - networks = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"] - self.assertEqual(["bot-bottle-mac-gateway"], networks) - self.assertNotIn("--publish", argv) - - def test_networks_exist_before_the_orchestrator_runs(self) -> None: - """The orchestrator is the first container on the shared network, so it - has to create it — the gateway that used to do so now starts second.""" - svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h")) - order: list[str] = [] - - def _networks(*_args: str) -> None: - order.append("networks") - - def _run(*_args: list[str]) -> Mock: - order.append("run") - return _ok() - - with patch(f"{_ORCH}.container_mod") as mod, \ - patch(f"{_ORCH}.ensure_networks", side_effect=_networks): - mod.run_container_argv = Mock(side_effect=_run) - svc._run_orchestrator_container("h1") - self.assertEqual(["networks", "run"], order) - - def test_orchestrator_start_failure_raises(self) -> None: - svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h")) - with patch(f"{_ORCH}.container_mod") as mod, \ - patch(f"{_ORCH}.ensure_networks"): - mod.run_container_argv = Mock(return_value=_fail()) - with self.assertRaises(OrchestratorStartError): - svc._run_orchestrator_container("h1") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/test_macos_infra.py b/tests/unit/test_macos_infra.py new file mode 100644 index 0000000..0971131 --- /dev/null +++ b/tests/unit/test_macos_infra.py @@ -0,0 +1,168 @@ +"""Unit: the single macOS infra container (control plane + gateway, PRD 0070).""" + +from __future__ import annotations + +import unittest +from pathlib import Path +from unittest.mock import Mock, patch + +from bot_bottle.backend.macos_container.infra import ( + INFRA_DB_VOLUME, + MacosInfraService, + OrchestratorStartError, + probe_control_plane_url, +) + +_INFRA = "bot_bottle.backend.macos_container.infra" + + +def _ok(stdout: str = "") -> Mock: + return Mock(returncode=0, stdout=stdout, stderr="") + + +def _fail(stderr: str = "boom") -> Mock: + return Mock(returncode=1, stdout="", stderr=stderr) + + +class TestInfraRun(unittest.TestCase): + def _run_container(self, svc: MacosInfraService) -> list[str]: + run = Mock(return_value=_ok()) + + def _spec(src: str, tgt: str, readonly: bool = False) -> str: + return f"type=bind,source={src},target={tgt}" + ( + ",readonly" if readonly else "") + + with patch(f"{_INFRA}.container_mod") as mod, \ + patch(f"{_INFRA}.ensure_networks"): + mod.dns_server.return_value = "1.1.1.1" + mod.bind_mount_spec.side_effect = _spec + mod.run_container_argv = run + svc._run_container("h1") + return run.call_args.args[0] + + def test_single_container_runs_both_processes(self) -> None: + """The whole point: one container starts the control plane AND the + gateway daemons, so one kernel owns the DB.""" + argv = self._run_container(MacosInfraService(repo_root=Path("/r"))) + script = argv[-1] + self.assertIn("bot_bottle.orchestrator", script) + self.assertIn("gateway_init.py", script) + self.assertIn("127.0.0.1", script) # they reach each other on loopback + + def test_db_is_a_container_only_volume(self) -> None: + """No host bind-mount of the DB — a named volume only this container + mounts, so the DB is never written by two kernels.""" + argv = self._run_container(MacosInfraService(repo_root=Path("/r"))) + vols = [argv[i + 1] for i, a in enumerate(argv) if a == "--volume"] + self.assertTrue(any(v.startswith(f"{INFRA_DB_VOLUME}:") for v in vols)) + # The repo source is bind-mounted read-only; the DB is not a bind mount. + mounts = [argv[i + 1] for i, a in enumerate(argv) if a == "--mount"] + self.assertTrue(all("bot-bottle.db" not in m for m in mounts)) + + def test_nat_network_precedes_the_host_only_network(self) -> None: + argv = self._run_container(MacosInfraService(repo_root=Path("/r"))) + nets = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"] + self.assertEqual(["bot-bottle-mac-egress", "bot-bottle-mac-gateway"], nets) + + def test_source_hash_is_labelled_for_recreate(self) -> None: + argv = self._run_container(MacosInfraService(repo_root=Path("/r"))) + self.assertIn("BOT_BOTTLE_SOURCE_HASH=h1", argv) + + def test_start_failure_raises(self) -> None: + svc = MacosInfraService(repo_root=Path("/r")) + with patch(f"{_INFRA}.container_mod") as mod, \ + patch(f"{_INFRA}.ensure_networks"): + mod.dns_server.return_value = "1.1.1.1" + mod.bind_mount_spec.return_value = "m" + mod.run_container_argv = Mock(return_value=_fail()) + with self.assertRaises(OrchestratorStartError): + svc._run_container("h1") + + +class TestInfraEnsureRunning(unittest.TestCase): + def test_current_healthy_container_is_left_alone(self) -> None: + """Idempotent singleton: N launches must not churn the infra container + and drop every live bottle's control plane.""" + svc = MacosInfraService(repo_root=Path("/r")) + run = Mock() + with patch(f"{_INFRA}.container_mod") as mod, \ + patch(f"{_INFRA}.source_hash", return_value="h1"), \ + patch.object(svc, "_run_container", run), \ + patch.object(svc, "is_healthy", return_value=True): + mod.container_is_running.return_value = True + mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"} + mod.try_container_ipv4_on_network.return_value = "192.168.128.2" + endpoint = svc.ensure_running() + run.assert_not_called() + self.assertEqual("http://192.168.128.2:8099", endpoint.control_plane_url) + self.assertEqual("192.168.128.2", endpoint.gateway_ip) + + def test_changed_source_recreates(self) -> None: + svc = MacosInfraService(repo_root=Path("/r")) + run = Mock() + with patch(f"{_INFRA}.container_mod") as mod, \ + patch(f"{_INFRA}.source_hash", return_value="h2"), \ + patch.object(svc, "ensure_built"), \ + patch.object(svc, "_run_container", run), \ + patch.object(svc, "is_healthy", return_value=True): + mod.container_is_running.return_value = True + mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"} + mod.try_container_ipv4_on_network.return_value = "192.168.128.2" + svc.ensure_running() + run.assert_called_once() + + def test_wedged_but_current_container_is_recreated(self) -> None: + """Current source but a dead HTTP server must be recreated, not polled + to death forever — health, not just the source label, gates reuse.""" + svc = MacosInfraService(repo_root=Path("/r")) + run = Mock() + health = Mock(side_effect=[False, True]) + with patch(f"{_INFRA}.container_mod") as mod, \ + patch(f"{_INFRA}.source_hash", return_value="h1"), \ + patch.object(svc, "ensure_built"), \ + patch.object(svc, "_run_container", run), \ + patch.object(svc, "is_healthy", health): + mod.container_is_running.return_value = True + mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"} + mod.try_container_ipv4_on_network.return_value = "192.168.128.2" + svc.ensure_running() + run.assert_called_once() + + def test_never_healthy_raises(self) -> None: + svc = MacosInfraService(repo_root=Path("/r")) + with patch(f"{_INFRA}.container_mod") as mod, \ + patch(f"{_INFRA}.source_hash", return_value="h1"), \ + patch.object(svc, "ensure_built"), \ + patch.object(svc, "_run_container"), \ + patch.object(svc, "is_healthy", return_value=False): + mod.container_is_running.return_value = False + mod.try_container_ipv4_on_network.return_value = "192.168.128.2" + with self.assertRaises(OrchestratorStartError): + svc.ensure_running(startup_timeout=0.01) + + +class TestCaCertPem(unittest.TestCase): + def test_reads_ca_out_of_the_container(self) -> None: + svc = MacosInfraService(repo_root=Path("/r")) + with patch(f"{_INFRA}.container_mod") as mod: + mod.run_container_argv.return_value = _ok("-----BEGIN CERTIFICATE-----\n") + pem = svc.ca_cert_pem() + self.assertTrue(pem.startswith("-----BEGIN CERTIFICATE-----")) + argv = mod.run_container_argv.call_args.args[0] + self.assertEqual(["container", "exec", "bot-bottle-mac-infra", "cat"], argv[:4]) + + +class TestProbeControlPlane(unittest.TestCase): + def test_returns_url_when_running(self) -> None: + with patch(f"{_INFRA}.container_mod") as mod: + mod.try_container_ipv4_on_network.return_value = "192.168.128.2" + self.assertEqual("http://192.168.128.2:8099", probe_control_plane_url()) + + def test_empty_when_absent(self) -> None: + with patch(f"{_INFRA}.container_mod") as mod: + mod.try_container_ipv4_on_network.return_value = "" + self.assertEqual("", probe_control_plane_url()) + + +if __name__ == "__main__": + unittest.main()