diff --git a/bot_bottle/backend/macos_container/bottle.py b/bot_bottle/backend/macos_container/bottle.py index 1a6f8c7..bc1180d 100644 --- a/bot_bottle/backend/macos_container/bottle.py +++ b/bot_bottle/backend/macos_container/bottle.py @@ -52,6 +52,7 @@ class MacosContainerBottle(Bottle): terminal_title: str = "", terminal_color: str = "", agent_workdir: str = "/home/node", + exec_env: dict[str, str] | None = None, ): self.name = container self._teardown = teardown @@ -62,6 +63,15 @@ class MacosContainerBottle(Bottle): self.terminal_color = terminal_color self.agent_provider_template = agent_provider_template self.agent_workdir = agent_workdir + # Env applied to the agent process at `container exec` time, on top of + # what the container was run with. This is how the identity token + # reaches the agent (PRD 0070): registration mints it *after* the + # container exists — its source IP is the registration key and Apple + # Container assigns that by DHCP — so it cannot be in the run-time env + # the way docker's compose spec does it. `container exec --env` wins + # over the run-time value, so the token-bearing proxy URL set here + # supersedes the token-less one baked in at launch. + self._exec_env = dict(exec_env or {}) self._closed = False def agent_argv(self, argv: list[str], *, tty: bool = True) -> list[str]: @@ -74,6 +84,12 @@ class MacosContainerBottle(Bottle): ) ) container_exec = ["container", "exec"] + # Bare env names, same rule as the terminal hints below: the value + # stays in the child env `exec_agent` builds and never reaches argv — + # the proxy URL here carries the identity token, which `ps` would + # otherwise expose to every process on the host. + for name in sorted(self._exec_env): + container_exec.extend(["--env", name]) if tty: container_exec.extend(["--interactive", "--tty"]) # Forward terminal capability hints so TUIs can enable modified-key @@ -94,21 +110,33 @@ class MacosContainerBottle(Bottle): def exec_agent(self, argv: list[str], *, tty: bool = True) -> int: agent_argv = self.agent_argv(argv, tty=tty) + # The values behind the bare `--env` names in `agent_argv`. `sh -lc` + # below is in this process tree, so the child env reaches `container + # exec` either way. + env = {**os.environ, **self._exec_env} if self._exec_env else None script = ( exec_shell_script(agent_argv, self.terminal_title, self.terminal_color) if tty else None ) if script is None: - return subprocess.run(agent_argv, check=False).returncode - return subprocess.run(["sh", "-lc", script], check=False).returncode + return subprocess.run(agent_argv, env=env, check=False).returncode + return subprocess.run(["sh", "-lc", script], env=env, check=False).returncode def exec(self, script: str, *, user: str = "node") -> ExecResult: + # Carry the same exec env the agent gets: provisioning steps run + # through here, and a provider whose provision step fetches anything + # would egress without the identity token and be denied by /resolve. + # Bare `--env NAME` again, so the token stays off argv. + argv = ["container", "exec", "--user", user, "--interactive"] + for name in sorted(self._exec_env): + argv.extend(["--env", name]) + argv.extend([self.name, "sh", "-s"]) result = subprocess.run( - ["container", "exec", "--user", user, "--interactive", - self.name, "sh", "-s"], + argv, input=script, capture_output=True, text=True, + env={**os.environ, **self._exec_env} if self._exec_env else None, check=False, ) return ExecResult( diff --git a/bot_bottle/backend/macos_container/consolidated_launch.py b/bot_bottle/backend/macos_container/consolidated_launch.py new file mode 100644 index 0000000..dedf4e8 --- /dev/null +++ b/bot_bottle/backend/macos_container/consolidated_launch.py @@ -0,0 +1,140 @@ +"""Consolidated bottle launch sequence for the macOS backend (PRD 0070). + +The docker backend allocates a free address, pins the agent to it with +`--ip`, registers it, *then* starts the agent — registration precedes launch +because the pinned address is known up front. + +**Apple Container 1.0.0 has no `--ip`.** The `--network` flag takes only +`[,mac=…][,mtu=…]`; the address is assigned by vmnet's DHCP and is +knowable only once the container is running. So the macOS order inverts: + + ensure_gateway() -> caller starts the agent -> register_agent(source_ip) + +That is why this module exposes two functions where docker has one — the +caller has to start the agent in between. `ensure_gateway` runs first because +the agent's proxy env needs the gateway's address at `container run` time; the +agent's *own* address (the attribution key) only exists afterwards. + +The consequence for the identity token: it is minted by registration, i.e. +*after* the agent container exists, so it cannot be baked into the run-time +env the way docker's compose spec does. It is delivered at `container exec` +time instead — see `bottle.MacosContainerBottle`. + +That delivery is load-bearing, not a nicety: `/resolve` requires a matching +`(source_ip, identity_token)` pair and fail-closes with no source-IP-only +fallback (#366). So egress that does not carry the token is denied — which is +the safe direction, and is why the agent's init process is a bare `sleep` and +every real command arrives through `container exec`. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from ...egress import EgressPlan +from ...git_gate import GitGatePlan +from ...orchestrator.client import OrchestratorClient +from ...orchestrator.registration import registration_inputs +from ..docker.gateway_provision import deprovision_git_gate, provision_git_gate +from .gateway import GATEWAY_NETWORK +from .gateway_provision import AppleGatewayTransport +from .orchestrator_service import MacosOrchestratorService, OrchestratorStartError + + +class ConsolidatedLaunchError(RuntimeError): + """The consolidated register/provision sequence could not complete.""" + + +@dataclass(frozen=True) +class GatewayEndpoint: + """What the agent `container run` needs to reach the shared gateway.""" + + orchestrator_url: str + gateway_ip: str # the gateway's address — the agent's proxy target + gateway_ca_pem: str # the shared CA the provisioner installs + network: str # the shared host-only network to attach to + + +@dataclass(frozen=True) +class LaunchContext: + """What the running agent needs once it has been registered.""" + + bottle_id: str + identity_token: str + source_ip: str # the agent's DHCP-assigned address (attribution key) + gateway_ip: str + network: str + orchestrator_url: str + + +def ensure_gateway( + *, service: MacosOrchestratorService | None = None, +) -> GatewayEndpoint: + """Ensure the orchestrator control plane + shared gateway are up, and + report how to reach them. Idempotent — both are per-host singletons, so N + bottle launches share the one pair. Call before starting the agent + container: the agent's proxy env needs `gateway_ip` at run time.""" + service = service or MacosOrchestratorService() + url = service.ensure_running() + gateway = service.gateway(url) + return GatewayEndpoint( + orchestrator_url=url, + gateway_ip=gateway.ip_on_shared_network(), + gateway_ca_pem=gateway.ca_cert_pem(), + network=service.network, + ) + + +def register_agent( + egress_plan: EgressPlan, + git_gate_plan: GitGatePlan, + *, + source_ip: str, + endpoint: GatewayEndpoint, + image_ref: str = "", + tokens: dict[str, str] | None = None, +) -> LaunchContext: + """Register the (already running) agent by its address and provision its + git-gate state into the gateway. `source_ip` must be read from the live + container — it is the attribution key the gateway resolves policy by. + Raises on failure; the caller tears down.""" + client = OrchestratorClient(endpoint.orchestrator_url) + inputs = registration_inputs(egress_plan) + reg = client.register_bottle( + source_ip, image_ref=image_ref, policy=inputs.policy, + metadata=inputs.metadata, tokens=tokens, + ) + try: + provision_git_gate(AppleGatewayTransport(), reg.bottle_id, git_gate_plan) + except Exception: + # Roll the registration back so a provisioning failure leaves no orphan. + client.teardown_bottle(reg.bottle_id) + raise + return LaunchContext( + bottle_id=reg.bottle_id, + identity_token=reg.identity_token, + source_ip=source_ip, + gateway_ip=endpoint.gateway_ip, + network=endpoint.network, + orchestrator_url=endpoint.orchestrator_url, + ) + + +def teardown_consolidated(bottle_id: str, *, orchestrator_url: str) -> None: + """Deregister the bottle and remove its git-gate state from the gateway. + Both steps are idempotent so this is safe from a cleanup trap. Does NOT + stop the gateway — it's a persistent per-host singleton.""" + OrchestratorClient(orchestrator_url).teardown_bottle(bottle_id) + deprovision_git_gate(AppleGatewayTransport(), bottle_id) + + +__all__ = [ + "GatewayEndpoint", + "LaunchContext", + "ensure_gateway", + "register_agent", + "teardown_consolidated", + "ConsolidatedLaunchError", + "OrchestratorStartError", + "GATEWAY_NETWORK", +] diff --git a/bot_bottle/backend/macos_container/egress_apply.py b/bot_bottle/backend/macos_container/egress_apply.py index c7e2cbb..5af5f78 100644 --- a/bot_bottle/backend/macos_container/egress_apply.py +++ b/bot_bottle/backend/macos_container/egress_apply.py @@ -1,9 +1,11 @@ """Host-side egress route-apply for the macos-container backend. The per-bottle companion container this used to signal (`container kill ---signal HUP `) was removed in the companion-container removal (#385), -along with the disabled macOS launch path. Fails closed until the macOS -backend grows the consolidated gateway. +--signal HUP `) was removed in the companion-container removal +(#385). In the consolidated model the shared gateway resolves egress policy +per-request against the orchestrator rather than reloading a per-bottle routes +file, so the live per-bottle reload is not supported here and fails closed +until the gateway-side apply lands — same posture as the docker backend. """ from __future__ import annotations @@ -16,8 +18,8 @@ class MacOSContainerEgressApplicator(EgressApplicator): del slug raise EgressApplyError( "live egress route-apply was removed with the per-bottle " - "companion container (#385); the macos-container backend is " - "disabled until it uses the consolidated gateway." + "companion container (#385); route changes will flow through " + "the consolidated gateway in a follow-up." ) diff --git a/bot_bottle/backend/macos_container/enumerate.py b/bot_bottle/backend/macos_container/enumerate.py index 7523ed5..7407b72 100644 --- a/bot_bottle/backend/macos_container/enumerate.py +++ b/bot_bottle/backend/macos_container/enumerate.py @@ -1,14 +1,43 @@ -"""Active-agent enumeration for the macOS Apple Container backend. - -The backend is disabled during the companion-container removal (#385) — it can't -launch bottles, so there are none to enumerate. Enumeration returns when -the backend grows the consolidated gateway. -""" +"""Active-agent enumeration for the macOS Apple Container backend.""" from __future__ import annotations +import subprocess + +from ...bottle_state import read_metadata from .. import ActiveAgent +from .gateway import GATEWAY_NAME +from .orchestrator_service import ORCHESTRATOR_NAME + +_PREFIX = "bot-bottle-" +# The shared per-host singletons carry the same prefix as agent containers but +# are infrastructure, not bottles — one gateway and one control plane serve +# every agent, so listing them as agents would invent one per host. +_INFRA_NAMES = frozenset({GATEWAY_NAME, ORCHESTRATOR_NAME}) def enumerate_active() -> list[ActiveAgent]: - return [] + result = subprocess.run( + ["container", "list", "--quiet"], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + return [] + out: list[ActiveAgent] = [] + for name in sorted(line.strip() for line in result.stdout.splitlines()): + if not name.startswith(_PREFIX) or name in _INFRA_NAMES: + continue + slug = name[len(_PREFIX):] + metadata = read_metadata(slug) + out.append(ActiveAgent( + backend_name="macos-container", + slug=slug, + agent_name=metadata.agent_name if metadata else "?", + started_at=metadata.started_at if metadata else "", + services=(), + label=metadata.label if metadata else "", + color=metadata.color if metadata else "", + )) + return out diff --git a/bot_bottle/backend/macos_container/gateway.py b/bot_bottle/backend/macos_container/gateway.py new file mode 100644 index 0000000..c04413b --- /dev/null +++ b/bot_bottle/backend/macos_container/gateway.py @@ -0,0 +1,241 @@ +"""The consolidated per-host gateway as an Apple container (PRD 0070). + +The macOS counterpart of `orchestrator.gateway.DockerGateway`: one persistent +gateway per host, shared by every bottle, attributing each request to a bottle +by its source IP on the shared host-only network. + +Two Apple Container 1.0.0 constraints shape this and make it *not* a +transliteration of the docker gateway: + +- **No container DNS.** Containers cannot resolve each other by name (the + host-only network's resolver refuses the query), so the gateway reaches the + control plane by **IP**, not by name as the docker gateway does. The + orchestrator must therefore be started *before* the gateway — see + `orchestrator_service`. +- **Networks are fixed at `container run`.** There is no `network connect`, + so a network cannot be attached to a running container. The gateway must sit + on one shared, up-front network for the lifetime of the process; per-bottle + networks would mean restarting the gateway on every launch, which defeats the + consolidation. + +The gateway is dual-homed, **NAT network first**: Apple Container makes the +first `--network` the default route, so the egress network must lead or the +gateway has no route to the internet. +""" + +from __future__ import annotations + +import os +import time +from pathlib import Path + +from ...orchestrator.gateway import ( + GATEWAY_CA_CERT, + GATEWAY_DOCKERFILE, + MITMPROXY_HOME, + Gateway, + GatewayError, +) +from ...paths import host_db_path +from ...supervise import DB_PATH_IN_CONTAINER +from . import util as container_mod + +# Distinct from the docker gateway's names so both backends' gateways can +# coexist on one host (a macOS host can run the docker backend too). +GATEWAY_NAME = "bot-bottle-mac-gateway" +# The shared host-only network the gateway and every agent bottle sit on. The +# agent's address here is the attribution key. +GATEWAY_NETWORK = "bot-bottle-mac-gateway" +# The NAT network that gives the gateway (and only the gateway) a route out. +GATEWAY_EGRESS_NETWORK = "bot-bottle-mac-egress" + +GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest") + +_REPO_ROOT = Path(__file__).resolve().parents[3] + +_SUPERVISE_DB_DIR_IN_CONTAINER = os.path.dirname(DB_PATH_IN_CONTAINER) + +# mitmproxy writes its CA a beat after start; reads poll rather than assume. +_CA_POLL_SECONDS = 0.5 +DEFAULT_CA_TIMEOUT_SECONDS = 30.0 + + +def gateway_ca_dir() -> Path: + """Host dir bind-mounted as mitmproxy's home, keeping the gateway's + self-generated CA **stable across container recreation** — every agent + installs this one CA to trust the shared gateway's TLS interception, so it + must not rotate when the gateway restarts. + + The docker gateway uses a named volume for this; a plain host dir is the + same guarantee with fewer moving parts, and it lets `ca_cert_pem` read the + PEM straight off the host instead of shelling into the container.""" + path = host_db_path().parent / "mac-gateway-ca" + path.mkdir(parents=True, exist_ok=True) + return path + + +def ensure_networks( + network: str = GATEWAY_NETWORK, egress_network: str = GATEWAY_EGRESS_NETWORK, +) -> None: + """Create the shared host-only network + the gateway's NAT network. + Idempotent — `create_network` tolerates 'already exists'. + + Module-level rather than a gateway method because the **orchestrator** + needs the shared network too, and it starts first (Apple has no container + DNS, so the gateway must be handed the control plane's IP). Both callers + ensure the networks; whoever runs first wins.""" + container_mod.create_network(egress_network) + container_mod.create_network(network, internal=True) + + +def _host_db_dir() -> str: + db_dir = host_db_path().parent + db_dir.mkdir(parents=True, exist_ok=True) + return str(db_dir) + + +def _mount(source: str, target: str, *, readonly: bool = False) -> str: + spec = f"type=bind,source={source},target={target}" + if readonly: + spec += ",readonly" + return spec + + +class AppleGateway(Gateway): + """The consolidated gateway as a single, fixed-name Apple container.""" + + def __init__( + self, + image_ref: str = GATEWAY_IMAGE, + *, + name: str = GATEWAY_NAME, + network: str = GATEWAY_NETWORK, + egress_network: str = GATEWAY_EGRESS_NETWORK, + orchestrator_url: str = "", + build_context: Path | None = None, + dockerfile: str | None = GATEWAY_DOCKERFILE, + ) -> None: + self.image_ref = image_ref + self.name = name + self.network = network + self.egress_network = egress_network + # Reached by IP (no container DNS on Apple) — the caller resolves the + # orchestrator's address before constructing this. Empty → single-tenant. + self._orchestrator_url = orchestrator_url + self._build_context = build_context or _REPO_ROOT + self._dockerfile = dockerfile + + def ensure_built(self) -> None: + """Build the gateway data-plane image from its Dockerfile. Builds every + time (cache-aware, so it's cheap when nothing changed): a stale image + silently runs the OLD single-tenant daemons. Mirrors `DockerGateway`.""" + if self._dockerfile is None: + return + container_mod.build_image( + self.image_ref, str(self._build_context), dockerfile=self._dockerfile, + ) + + def is_running(self) -> bool: + return container_mod.container_is_running(self.name) + + def _running_image_is_current(self) -> bool: + """True iff the running gateway was created from the *current* + `image_ref`. `ensure_built` rebuilding the image is not enough on its + own — the running container still holds the OLD image, so this + mismatch check is what makes a rebuild take effect.""" + running = container_mod.container_image_digest(self.name) + current = container_mod.image_digest(self.image_ref) + if not running or not current: + return True # can't compare → don't churn a working container + return running == current + + def _running_control_plane_is_current(self) -> bool: + """True iff the running gateway points at the control plane we would + pass today. + + Docker gets this for free — it hands the gateway a container *name*, + which survives the orchestrator being recreated. Apple has no container + DNS, so the URL is an **IP baked into the gateway's env at run time**, + and a recreated orchestrator can come back on a different DHCP address. + Without this check the gateway would keep pointing at the old address + and every `/resolve` would fail — denying egress for *every* bottle on + the host until something else happened to recreate the gateway.""" + if not self._orchestrator_url: + return True + env = container_mod.container_env(self.name) + if not env: + return True # can't compare → don't churn a working container + return env.get("BOT_BOTTLE_ORCHESTRATOR_URL") == self._orchestrator_url + + def ensure_running(self) -> None: + if (self.is_running() + and self._running_image_is_current() + and self._running_control_plane_is_current()): + return + ensure_networks(self.network, self.egress_network) + container_mod.force_remove_container(self.name) + argv = [ + "container", "run", "--detach", + "--name", self.name, + "--label", "bot-bottle.backend=macos-container", + "--label", "bot-bottle-mac-gateway=1", + # NAT network FIRST: Apple Container takes the first --network as + # the default route, so this ordering is what gives the gateway a + # route out. Reversing it silently blackholes egress. + "--network", self.egress_network, + "--network", self.network, + # The NAT gateway routes but does not resolve, so DNS is explicit. + "--dns", container_mod.dns_server(), + "--mount", _mount(str(gateway_ca_dir()), MITMPROXY_HOME), + "--mount", _mount(_host_db_dir(), _SUPERVISE_DB_DIR_IN_CONTAINER), + "--env", f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}", + ] + if self._orchestrator_url: + # Makes the data plane multi-tenant: each request resolves + # source-IP → policy against the control plane. + argv += ["--env", f"BOT_BOTTLE_ORCHESTRATOR_URL={self._orchestrator_url}"] + argv.append(self.image_ref) + result = container_mod.run_container_argv(argv) + if result.returncode != 0: + raise GatewayError( + f"gateway failed to start: " + f"{(result.stderr or '').strip() or ''}" + ) + + def ip_on_shared_network(self) -> str: + """The gateway's address on the shared host-only network — what agents + point their proxy / git-http / supervise URLs at.""" + return container_mod.container_ipv4_on_network(self.name, self.network) + + def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str: + """The gateway's CA certificate (PEM) that agents install to trust its + TLS interception. Polls: mitmproxy generates it a moment after start. + Read from the host bind-mount, so no exec into the container.""" + ca_path = gateway_ca_dir() / os.path.basename(GATEWAY_CA_CERT) + deadline = time.monotonic() + timeout + while True: + try: + pem = ca_path.read_text() + if pem.strip(): + return pem + except OSError: + pass + if time.monotonic() >= deadline: + raise GatewayError( + f"gateway CA cert not available at {ca_path} after {timeout:g}s" + ) + time.sleep(_CA_POLL_SECONDS) + + def stop(self) -> None: + container_mod.force_remove_container(self.name) + + +__all__ = [ + "AppleGateway", + "GatewayError", + "GATEWAY_NAME", + "GATEWAY_NETWORK", + "GATEWAY_EGRESS_NETWORK", + "GATEWAY_IMAGE", + "gateway_ca_dir", +] diff --git a/bot_bottle/backend/macos_container/gateway_provision.py b/bot_bottle/backend/macos_container/gateway_provision.py new file mode 100644 index 0000000..0d4df9b --- /dev/null +++ b/bot_bottle/backend/macos_container/gateway_provision.py @@ -0,0 +1,44 @@ +"""`GatewayTransport` for the Apple gateway container (PRD 0070). + +The provisioning *logic* (per-bottle creds dirs, namespaced repo init) is +backend-neutral and lives in `backend.docker.gateway_provision`; this is only +the transport — how files and commands reach the running gateway. Docker uses +`docker exec`/`docker cp` and Firecracker uses SSH; Apple uses the `container` +CLI's equivalents. +""" + +from __future__ import annotations + +from ..docker.gateway_provision import GatewayProvisionError +from . import util as container_mod +from .gateway import GATEWAY_NAME + + +class AppleGatewayTransport: + """`GatewayTransport` for the gateway as an Apple container.""" + + def __init__(self, gateway: str = GATEWAY_NAME) -> None: + self.gateway = gateway + + def exec(self, argv: list[str]) -> None: + result = container_mod.run_container_argv( + ["container", "exec", self.gateway, *argv] + ) + if result.returncode != 0: + raise GatewayProvisionError( + f"gateway exec {argv!r} failed: " + f"{(result.stderr or '').strip() or ''}" + ) + + def cp_into(self, src: str, dest: str) -> None: + result = container_mod.run_container_argv( + ["container", "cp", src, f"{self.gateway}:{dest}"] + ) + if result.returncode != 0: + raise GatewayProvisionError( + f"gateway cp {src} -> {dest} failed: " + f"{(result.stderr or '').strip() or ''}" + ) + + +__all__ = ["AppleGatewayTransport", "GatewayProvisionError"] diff --git a/bot_bottle/backend/macos_container/launch.py b/bot_bottle/backend/macos_container/launch.py index 31a9d93..a451761 100644 --- a/bot_bottle/backend/macos_container/launch.py +++ b/bot_bottle/backend/macos_container/launch.py @@ -1,24 +1,74 @@ -"""Launch flow for the macOS Apple Container backend — disabled (#385). +"""Launch flow for the macOS Apple Container backend (PRD 0070). -This backend launched a per-bottle companion container (the egress / -git-gate / supervise data plane) alongside the agent container, with the -agent's proxy env pointed at the companion's host-only IP. That -per-bottle-companion architecture was removed in the companion-container removal; -the macOS backend will be re-enabled once it grows the consolidated -per-host gateway the docker backend already uses. +The agent container attaches to the **shared host-only gateway network** and +proxies egress through the one per-host gateway, replacing the per-bottle +companion container removed in #385. -Until then, launching a macOS bottle fails closed. `prepare` / `status` -/ cleanup still work. +The order differs from docker's, forced by Apple Container 1.0.0 having no +`--ip` (see `consolidated_launch`): the agent is started *before* it is +registered, because its DHCP-assigned address — the attribution key — does not +exist until then. + + gateway up -> run agent -> read its IP -> register it -> provision + +Two things follow from that inversion: + +- The **identity token** is minted by registration and so cannot be in the + agent's run-time env; it rides the proxy URL applied at `container exec` + time (`bottle.MacosContainerBottle`). `/resolve` requires it (#366), so + egress without it is denied — hence the bare `sleep` init: every real agent + command goes through exec and therefore carries the token. +- The agent is run with `--cap-drop CAP_NET_RAW`. Apple Container grants + NET_RAW by default, which would let an agent open a raw socket and forge a + neighbour's source address on the shared segment. NET_ADMIN is already + absent (the agent cannot change its own address or route), so dropping + NET_RAW is what closes the source-address half of PRD 0070's invariant: + "a packet's source address, as seen by the orchestrator, provably identifies + the originating bottle." The identity token is the other half — an attacker + would need to forge the address *and* steal the token — but the invariant is + a stated precondition of consolidation, so it is enforced on its own terms + rather than left to the token. """ from __future__ import annotations -from contextlib import contextmanager +import dataclasses +import os +import subprocess +from contextlib import ExitStack, contextmanager +from pathlib import Path from typing import Callable, Generator -from ...log import die +from ...bottle_state import ( + egress_state_dir, + git_gate_state_dir, + read_committed_image, +) +from ...egress import ( + egress_agent_env_entries, + egress_resolve_token_values, +) +from ...git_gate import ( + provision_git_gate_dynamic_keys, + revoke_git_gate_provisioned_keys, +) +from ...log import die, info, warn +from ...supervise import SUPERVISE_PORT +from ..docker.egress import EGRESS_PORT +from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH +from . import util as container_mod from .bottle import MacosContainerBottle from .bottle_plan import MacosContainerBottlePlan +from .consolidated_launch import ( + GatewayEndpoint, + ensure_gateway, + register_agent, + teardown_consolidated, +) + +_REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent) +_AGENT_SLEEP_SECONDS = "2147483647" +_GIT_HTTP_PORT = 9420 @contextmanager @@ -27,13 +77,260 @@ def launch( *, provision: Callable[[MacosContainerBottlePlan, "MacosContainerBottle"], str | None], ) -> Generator[MacosContainerBottle, None, None]: - """Fail closed: the macOS backend is disabled until it grows the - consolidated per-host gateway (the companion-container path it used - was removed in #385).""" - del plan, provision - die( - "the macos-container backend is temporarily disabled during the " - "companion-container removal (#385); it will return once it uses " - "the consolidated gateway. Use --backend=docker for now." + """Build, run, register, provision, and yield an Apple Container bottle on + the shared per-host gateway.""" + stack = ExitStack() + bottle_for_revoke = plan.manifest.bottle + git_gate_dir_for_revoke = git_gate_state_dir(plan.slug) + + def teardown() -> None: + teardown_exc: BaseException | None = None + try: + stack.close() + except BaseException as exc: # noqa: W0718 - teardown must continue + teardown_exc = exc + warn(f"macos-container teardown failed: {exc!r}") + revoke_git_gate_provisioned_keys(bottle_for_revoke, git_gate_dir_for_revoke) + if teardown_exc is not None: + raise teardown_exc + + try: + plan = _build_images(plan) + + # Step 1: the per-host singletons. Must precede the agent run — its + # proxy env needs the gateway's address at `container run` time. + endpoint = ensure_gateway() + + # Step 2: mint this bottle's deploy keys, then point it at the SHARED + # gateway's CA + git-http/supervise ports. + plan = _provision_git_gate_keys(plan) + plan = _install_gateway_ca(plan, endpoint) + plan = _stamp_agent_urls(plan, endpoint) + + # Step 3: run the agent. It has no identity token yet — registration + # needs the address this run assigns. + container_mod.force_remove_container(plan.container_name) + _start_agent(plan, endpoint) + stack.callback(container_mod.force_remove_container, plan.container_name) + + # Step 4: read the assigned address and register by it. This is the + # attribution key; `--cap-drop CAP_NET_RAW` at run is what makes it + # unforgeable. + source_ip = container_mod.container_ipv4_on_network( + plan.container_name, endpoint.network, + ) + effective_env = {**os.environ, **plan.agent_provision.provisioned_env} + token_values = egress_resolve_token_values( + plan.egress_plan.token_env_map, effective_env, + ) + ctx = register_agent( + plan.egress_plan, + plan.git_gate_plan, + source_ip=source_ip, + endpoint=endpoint, + image_ref=plan.image, + tokens=token_values, + ) + stack.callback( + teardown_consolidated, ctx.bottle_id, + orchestrator_url=ctx.orchestrator_url, + ) + info( + f"agent {plan.container_name} registered " + f"(gateway {endpoint.gateway_ip}, ip {source_ip})" + ) + + bottle = MacosContainerBottle( + plan.container_name, + teardown, + None, + agent_command=plan.agent_command, + agent_prompt_mode=plan.agent_prompt_mode, + agent_provider_template=plan.agent_provider_template, + terminal_title=( + f"{plan.spec.label} ({plan.spec.agent_name})" + if plan.spec.label else plan.spec.agent_name + ), + terminal_color=plan.spec.color, + agent_workdir=plan.workspace_plan.workdir, + exec_env=_identity_proxy_env(endpoint, ctx.identity_token), + ) + bottle.prompt_path = provision(plan, bottle) + + yield bottle + finally: + teardown() + + +def _build_images(plan: MacosContainerBottlePlan) -> MacosContainerBottlePlan: + """Build the agent image. The gateway's own image is built by + `ensure_gateway` — it belongs to the shared singleton, not to a bottle.""" + committed = read_committed_image(plan.slug) + if committed and container_mod.image_exists(committed): + info(f"using committed image {committed!r}") + return dataclasses.replace( + plan, + agent_provision=dataclasses.replace( + plan.agent_provision, image=committed, + ), + ) + container_mod.build_image( + plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path, ) - yield # unreachable — `die` raises; keeps this a generator/contextmanager + return plan + + +def _provision_git_gate_keys( + plan: MacosContainerBottlePlan, +) -> MacosContainerBottlePlan: + if not plan.git_gate_plan.upstreams: + return plan + git_gate_plan = provision_git_gate_dynamic_keys( + plan.manifest.bottle, + plan.git_gate_plan, + git_gate_state_dir(plan.slug), + ) + return dataclasses.replace(plan, git_gate_plan=git_gate_plan) + + +def _install_gateway_ca( + plan: MacosContainerBottlePlan, endpoint: GatewayEndpoint, +) -> MacosContainerBottlePlan: + """Stage the SHARED gateway's CA for the provisioner to install, replacing + the per-bottle CA the companion container used to mint. Every bottle on + this host trusts this one CA.""" + ca_dir = egress_state_dir(plan.slug) / "gateway-ca" + ca_dir.mkdir(parents=True, exist_ok=True) + ca_file = ca_dir / "gateway-ca.pem" + ca_file.write_text(endpoint.gateway_ca_pem) + egress_plan = dataclasses.replace( + plan.egress_plan, + mitmproxy_ca_host_path=ca_file, + mitmproxy_ca_cert_only_host_path=ca_file, + ) + return dataclasses.replace(plan, egress_plan=egress_plan) + + +def _stamp_agent_urls( + plan: MacosContainerBottlePlan, endpoint: GatewayEndpoint, +) -> MacosContainerBottlePlan: + """Point the agent's git-gate insteadOf rewrites + supervise MCP at the + shared gateway's ports. Both bypass the egress proxy (NO_PROXY covers the + gateway address).""" + git_gate_url = ( + f"http://{endpoint.gateway_ip}:{_GIT_HTTP_PORT}" + if plan.git_gate_plan.upstreams else "" + ) + supervise_url = ( + f"http://{endpoint.gateway_ip}:{SUPERVISE_PORT}/" + if plan.supervise_plan is not None else "" + ) + return dataclasses.replace( + plan, + agent_proxy_url=f"http://{endpoint.gateway_ip}:{EGRESS_PORT}", + agent_git_gate_url=git_gate_url, + agent_supervise_url=supervise_url, + ) + + +def _proxy_url(gateway_ip: str, identity_token: str = "") -> str: + """The agent's egress proxy URL. The identity token rides as proxy + credentials — the gateway reads Proxy-Authorization, resolves the + (source_ip, token) pair against the control plane, and strips it before + upstream. Without a valid pair `/resolve` denies the request (#366).""" + cred = f"bottle:{identity_token}@" if identity_token else "" + return f"http://{cred}{gateway_ip}:{EGRESS_PORT}" + + +def _no_proxy(gateway_ip: str) -> str: + # git-http + supervise live on the gateway and must NOT go through the + # egress proxy — the agent reaches them directly by its address. + return f"localhost,127.0.0.1,{gateway_ip}" + + +def _identity_proxy_env( + endpoint: GatewayEndpoint, identity_token: str, +) -> dict[str, str]: + """The token-bearing proxy env applied at `container exec`. It supersedes + the token-less run-time value (exec `--env` wins), which is the only way to + get the token in: it does not exist until after the container runs.""" + if not identity_token: + return {} + url = _proxy_url(endpoint.gateway_ip, identity_token) + return { + "HTTPS_PROXY": url, "HTTP_PROXY": url, + "https_proxy": url, "http_proxy": url, + } + + +def _start_agent(plan: MacosContainerBottlePlan, endpoint: GatewayEndpoint) -> None: + argv = _agent_run_argv(plan, endpoint) + env = {**os.environ, **plan.forwarded_env} + info(f"container run agent {plan.container_name}") + result = subprocess.run( + argv, capture_output=True, text=True, env=env, check=False, + ) + if result.returncode != 0: + die( + f"container run for agent {plan.container_name} failed: " + f"{(result.stderr or '').strip() or ''}" + ) + + +def _agent_run_argv( + plan: MacosContainerBottlePlan, endpoint: GatewayEndpoint, +) -> list[str]: + argv = [ + "container", "run", + "--name", plan.container_name, + "--detach", + "--label", "bot-bottle.backend=macos-container", + "--network", endpoint.network, + # The attribution invariant: without NET_RAW the agent cannot open a + # raw socket, so it cannot source-IP-spoof its neighbours on the shared + # segment. NET_ADMIN is not granted by default, so its address and + # route are already fixed. See the module docstring. + "--cap-drop", "CAP_NET_RAW", + ] + for entry in _agent_env_entries(plan, endpoint): + argv += ["--env", entry] + # The init process is a no-op: every agent command arrives via + # `container exec`, which is also how the identity token gets in. + argv += [plan.image, "sleep", _AGENT_SLEEP_SECONDS] + return argv + + +def _agent_env_entries( + plan: MacosContainerBottlePlan, endpoint: GatewayEndpoint, +) -> tuple[str, ...]: + # Token-less at run time — the token does not exist yet (see + # `_identity_proxy_env`). Anything egressing before the exec-time override + # is denied by `/resolve`, which is the safe direction. + proxy_url = _proxy_url(endpoint.gateway_ip) + no_proxy = _no_proxy(endpoint.gateway_ip) + env = [ + f"HTTPS_PROXY={proxy_url}", + f"HTTP_PROXY={proxy_url}", + f"https_proxy={proxy_url}", + f"http_proxy={proxy_url}", + f"NO_PROXY={no_proxy}", + f"no_proxy={no_proxy}", + f"NODE_EXTRA_CA_CERTS={AGENT_CA_PATH}", + f"SSL_CERT_FILE={AGENT_CA_BUNDLE}", + f"REQUESTS_CA_BUNDLE={AGENT_CA_BUNDLE}", + ] + if plan.agent_git_gate_url: + env.append(f"GIT_GATE_URL={plan.agent_git_gate_url}") + if plan.agent_supervise_url: + env.append(f"MCP_SUPERVISE_URL={plan.agent_supervise_url}") + for name, value in sorted(plan.agent_provision.guest_env.items()): + env.append(f"{name}={value}") + # Forwarded vars: bare name → inherits from the `container run` process env + # so the secret value never lands on argv. + for name in sorted(plan.forwarded_env.keys()): + env.append(name) + env.extend(egress_agent_env_entries(plan.egress_plan)) + return tuple(env) + + +__all__ = ["launch"] diff --git a/bot_bottle/backend/macos_container/orchestrator_service.py b/bot_bottle/backend/macos_container/orchestrator_service.py new file mode 100644 index 0000000..09d8616 --- /dev/null +++ b/bot_bottle/backend/macos_container/orchestrator_service.py @@ -0,0 +1,241 @@ +"""Orchestrator + gateway lifecycle for the macOS backend (PRD 0070). + +The macOS counterpart of `orchestrator.lifecycle.OrchestratorService`. Same +shape — an idempotent singleton that brings up the control plane and the shared +gateway and hands back the control-plane URL — but the startup **order is +reversed**, and the reason is a real Apple Container constraint rather than a +stylistic choice: + +- Docker starts the gateway first and lets it find the control plane by + *container name* over docker DNS. +- Apple Container 1.0.0 has **no container DNS** (see `gateway`), so the + gateway can only be handed an **IP**. That IP does not exist until the + orchestrator container is running — hence: orchestrator first, read its + address, then start the gateway pointed at it. + +The second difference: no published port. Apple Container puts every container +on a host-reachable address, and the host can reach the host-only network +directly, so the host CLI and the gateway use the **same** URL — the +orchestrator's address on the shared network. Docker needs a +`--publish 127.0.0.1:…` hop plus a separate `internal_url` for the same job. + +Like the docker service this runs the **register-only broker**: the backend +launches agent containers, so the control plane needs no privileged socket. +""" + +from __future__ import annotations + +import time +import urllib.error +import urllib.request +from pathlib import Path + +from ... import log +from ...orchestrator.lifecycle import ( + DEFAULT_PORT, + DEFAULT_STARTUP_TIMEOUT_SECONDS, + ORCHESTRATOR_DOCKERFILE, + ORCHESTRATOR_IMAGE, + ORCHESTRATOR_SOURCE_HASH_LABEL, + OrchestratorStartError, + source_hash, +) +from ...paths import bot_bottle_root +from . import util as container_mod +from .gateway import ( + GATEWAY_EGRESS_NETWORK, + GATEWAY_IMAGE, + GATEWAY_NETWORK, + AppleGateway, + ensure_networks, +) + +# Distinct from the docker backend's container name so both can run on one host. +ORCHESTRATOR_NAME = "bot-bottle-mac-orchestrator" + +_REPO_ROOT = Path(__file__).resolve().parents[3] +_APP_DIR = "/app" +_ROOT_IN_CONTAINER = "/bot-bottle-root" + +_HEALTH_POLL_SECONDS = 0.25 +_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0 + + +def _mount(source: str, target: str, *, readonly: bool = False) -> str: + spec = f"type=bind,source={source},target={target}" + if readonly: + spec += ",readonly" + return spec + + +class MacosOrchestratorService: + """Manages the orchestrator control-plane container + the shared Apple + gateway. Callers only need `ensure_running()`, which returns the URL.""" + + def __init__( + self, + *, + port: int = DEFAULT_PORT, + network: str = GATEWAY_NETWORK, + egress_network: str = GATEWAY_EGRESS_NETWORK, + image: str = ORCHESTRATOR_IMAGE, + gateway_image: str = GATEWAY_IMAGE, + repo_root: Path = _REPO_ROOT, + host_root: Path | None = None, + orchestrator_name: str = ORCHESTRATOR_NAME, + ) -> None: + self.port = port + self.network = network + self.egress_network = egress_network + self.image = image + self._gateway_image = gateway_image + self._repo_root = repo_root + self._host_root = host_root or bot_bottle_root() + self._orchestrator_name = orchestrator_name + # Resolved once the container is up — there is no name to fall back on. + self._url = "" + + @property + def url(self) -> str: + """The control-plane URL, or "" before the container is up. One URL for + both the host CLI and the gateway (see the module docstring).""" + return self._url + + def _resolve_url(self) -> str: + """The control-plane URL, or "" while the container has no address.""" + ip = container_mod.try_container_ipv4_on_network( + self._orchestrator_name, self.network, + ) + return f"http://{ip}:{self.port}" if ip else "" + + def is_healthy( + self, url: str = "", *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS, + ) -> bool: + target = url or self._url + if not target: + return False + try: + with urllib.request.urlopen(f"{target}/health", timeout=timeout) as resp: + return resp.status == 200 + except (urllib.error.URLError, TimeoutError, OSError): + return False + + def _ensure_orchestrator_image(self) -> None: + """Build the lean control-plane image when missing. Build-if-missing, + not build-every-time: the control plane bind-mounts its source, so a + code change is caught by the source-hash recreate, not a rebuild.""" + if container_mod.image_exists(self.image): + return + container_mod.build_image( + self.image, str(self._repo_root), dockerfile=ORCHESTRATOR_DOCKERFILE, + ) + + def _orchestrator_source_current(self, current_hash: str) -> bool: + """True iff the running orchestrator was created from the *current* + bind-mounted source. The process loaded that code at startup and won't + reload it, so a stale container would keep serving OLD control-plane + code.""" + if not container_mod.container_is_running(self._orchestrator_name): + return False + data = container_mod.inspect_container(self._orchestrator_name) + config = data.get("configuration") + labels = config.get("labels") if isinstance(config, dict) else None + if not isinstance(labels, dict): + return True # can't compare → don't churn a working container + return labels.get(ORCHESTRATOR_SOURCE_HASH_LABEL) == current_hash + + def _run_orchestrator_container(self, current_hash: str) -> None: + # The orchestrator is the first thing on the shared network, so it — + # not the gateway — is what has to bring the network into existence. + ensure_networks(self.network, self.egress_network) + container_mod.force_remove_container(self._orchestrator_name) + argv = [ + "container", "run", "--detach", + "--name", self._orchestrator_name, + "--label", "bot-bottle.backend=macos-container", + "--label", f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={current_hash}", + # Host-only network only: the control plane needs no route out, and + # the host reaches it here directly (no --publish needed). + "--network", self.network, + "--mount", _mount(str(self._repo_root), _APP_DIR, readonly=True), + "--workdir", _APP_DIR, + # Persist the registry DB on the host (sole-owner: only the + # orchestrator opens bot-bottle.db). + "--mount", _mount(str(self._host_root), _ROOT_IN_CONTAINER), + "--env", f"BOT_BOTTLE_ROOT={_ROOT_IN_CONTAINER}", + "--entrypoint", "python3", + self.image, + "-m", "bot_bottle.orchestrator", + "--host", "0.0.0.0", "--port", str(self.port), "--broker", "stub", + ] + result = container_mod.run_container_argv(argv) + if result.returncode != 0: + raise OrchestratorStartError( + f"orchestrator container failed to start: " + f"{(result.stderr or '').strip() or ''}" + ) + + def gateway(self, orchestrator_url: str) -> AppleGateway: + """The shared gateway, pointed at the control plane at + `orchestrator_url` (by IP — Apple has no container DNS).""" + return AppleGateway( + self._gateway_image, + network=self.network, + egress_network=self.egress_network, + orchestrator_url=orchestrator_url, + ) + + def ensure_running( + self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS, + ) -> str: + """Ensure the control plane + shared gateway are up; return the + control-plane URL. Idempotent — a healthy control plane running current + code and a running gateway are left untouched.""" + current_hash = source_hash(self._repo_root) + if not self._orchestrator_source_current(current_hash): + self._ensure_orchestrator_image() + log.info( + "starting orchestrator container", + context={"name": self._orchestrator_name}, + ) + self._run_orchestrator_container(current_hash) + + url = self._wait_healthy(startup_timeout) + self._url = url + + # Gateway second: it can only reach the control plane by IP, which does + # not exist until the orchestrator container is up (see the docstring). + gateway = self.gateway(url) + gateway.ensure_built() + gateway.ensure_running() + return url + + def _wait_healthy(self, startup_timeout: float) -> str: + """Poll until the control plane answers /health, resolving its address + each time: the container is up before it has an IP, and it has an IP + before the server binds.""" + deadline = time.monotonic() + startup_timeout + while True: + url = self._resolve_url() + if url and self.is_healthy(url): + log.info("orchestrator healthy", context={"url": url}) + return url + if time.monotonic() >= deadline: + raise OrchestratorStartError( + f"orchestrator did not become healthy within " + f"{startup_timeout:g}s" + ) + time.sleep(_HEALTH_POLL_SECONDS) + + def stop(self) -> None: + """Remove the orchestrator + gateway containers (idempotent).""" + container_mod.force_remove_container(self._orchestrator_name) + self.gateway("").stop() + self._url = "" + + +__all__ = [ + "MacosOrchestratorService", + "OrchestratorStartError", + "ORCHESTRATOR_NAME", +] diff --git a/bot_bottle/backend/macos_container/util.py b/bot_bottle/backend/macos_container/util.py index 5853948..d400120 100644 --- a/bot_bottle/backend/macos_container/util.py +++ b/bot_bottle/backend/macos_container/util.py @@ -437,22 +437,131 @@ def inspect_container(name: str) -> dict[str, object]: def container_ipv4_on_network(name: str, network: str) -> str: - data = inspect_container(name) + """The container's IPv4 address on `network`. Fatal if absent — callers + that can tolerate "not yet" want `try_container_ipv4_on_network`.""" + ip = try_container_ipv4_on_network(name, network) + if not ip: + die(f"container {name} has no IPv4 address on {network}") + return ip + + +def run_container_argv(argv: list[str]) -> subprocess.CompletedProcess[str]: + """Run a `container` command, returning the result for the caller to + interpret. Unlike the `die`-on-failure helpers above, this lets callers + that raise their own typed errors (the gateway / orchestrator lifecycle) + keep control of the failure path.""" + return subprocess.run(argv, capture_output=True, text=True, check=False) + + +def _normalize_digest(value: str) -> str: + return value.split(":", 1)[1] if ":" in value else value + + +def image_digest(ref: str) -> str: + """The digest of image `ref`, or "" if it can't be read. Empty is a + 'don't know' signal — callers treat it as "don't churn a working + container" rather than as a mismatch.""" + result = run_container_argv([_CONTAINER, "image", "inspect", ref]) + if result.returncode != 0: + return "" + try: + data = json.loads(result.stdout or "{}") + except json.JSONDecodeError: + return "" + if isinstance(data, list) and data: + data = data[0] + if not isinstance(data, dict): + return "" + config = data.get("configuration") + if isinstance(config, dict): + descriptor = config.get("descriptor") + if isinstance(descriptor, dict) and descriptor.get("digest"): + return _normalize_digest(str(descriptor["digest"])) + value = data.get("id") + return _normalize_digest(str(value)) if value else "" + + +def container_image_digest(name: str) -> str: + """The digest of the image container `name` was created from, or "" if it + can't be read. Compare with `image_digest(ref)` to tell whether a running + container predates an image rebuild.""" + result = run_container_argv([_CONTAINER, "inspect", name]) + if result.returncode != 0: + return "" + try: + data = json.loads(result.stdout or "[]") + except json.JSONDecodeError: + return "" + if isinstance(data, list) and data: + data = data[0] + if not isinstance(data, dict): + return "" + config = data.get("configuration") + if not isinstance(config, dict): + return "" + image = config.get("image") + if not isinstance(image, dict): + return "" + descriptor = image.get("descriptor") + if isinstance(descriptor, dict) and descriptor.get("digest"): + return _normalize_digest(str(descriptor["digest"])) + return "" + + +def container_env(name: str) -> dict[str, str]: + """The env container `name` was started with, or {} if unreadable. Lets a + caller tell whether a running container's baked-in configuration still + matches what it would pass today.""" + result = run_container_argv([_CONTAINER, "inspect", name]) + if result.returncode != 0: + return {} + try: + data = json.loads(result.stdout or "[]") + except json.JSONDecodeError: + return {} + if isinstance(data, list) and data: + data = data[0] + if not isinstance(data, dict): + return {} + config = data.get("configuration") + init = config.get("initProcess") if isinstance(config, dict) else None + entries = init.get("environment") if isinstance(init, dict) else None + if not isinstance(entries, list): + return {} + env: dict[str, str] = {} + for entry in entries: + if isinstance(entry, str) and "=" in entry: + key, value = entry.split("=", 1) + env[key] = value + return env + + +def try_container_ipv4_on_network(name: str, network: str) -> str: + """`container_ipv4_on_network` without the fatal exit: "" when the address + isn't readable yet. For pollers — a container is created before it has an + address, so "not yet" is an expected state there, not an error.""" + result = run_container_argv([_CONTAINER, "inspect", name]) + if result.returncode != 0: + return "" + try: + data = json.loads(result.stdout or "[]") + except json.JSONDecodeError: + return "" + if isinstance(data, list) and data: + data = data[0] + if not isinstance(data, dict): + return "" status = data.get("status") networks = status.get("networks") if isinstance(status, dict) else None if not isinstance(networks, list): - die(f"container inspect {name} did not include status.networks") + return "" for entry in networks: - if not isinstance(entry, dict): - continue - if entry.get("network") != network: + if not isinstance(entry, dict) or entry.get("network") != network: continue raw = entry.get("ipv4Address") - if not isinstance(raw, str) or not raw: - die(f"container {name} has no IPv4 address on {network}") - return raw.split("/", 1)[0] - die(f"container {name} is not attached to network {network}") - raise AssertionError("unreachable") + if isinstance(raw, str) and raw: + return raw.split("/", 1)[0] + return "" def image_id(ref: str) -> str: diff --git a/bot_bottle/orchestrator/lifecycle.py b/bot_bottle/orchestrator/lifecycle.py index d6d94b2..b6cb62f 100644 --- a/bot_bottle/orchestrator/lifecycle.py +++ b/bot_bottle/orchestrator/lifecycle.py @@ -41,7 +41,7 @@ ORCHESTRATOR_IMAGE = os.environ.get( ORCHESTRATOR_DOCKERFILE = "Dockerfile.orchestrator" # Baked onto the container as a label so `ensure_running` can tell whether the # running process is executing the *current* bind-mounted source — see -# `_source_hash`. +# `source_hash`. ORCHESTRATOR_SOURCE_HASH_LABEL = "bot-bottle-orchestrator-source-hash" # The repo root is bind-mounted into the control-plane container so @@ -60,7 +60,7 @@ class OrchestratorStartError(RuntimeError): """The orchestrator container did not become healthy within the timeout.""" -def _source_hash(repo_root: Path) -> str: +def source_hash(repo_root: Path) -> str: """Content hash of the orchestrator's bind-mounted Python source (the `bot_bottle` package the control-plane process imports). This only changes when the code that would actually run inside the container @@ -134,17 +134,17 @@ class OrchestratorService: proc = run_docker(["docker", "ps", "--filter", f"name=^/{name}$", "--format", "{{.Names}}"]) return name in proc.stdout.split() - def _run_orchestrator_container(self, source_hash: str) -> None: + def _run_orchestrator_container(self, current_hash: str) -> None: """Start the control-plane container (idempotent: clears a stale fixed-name container first). Register-only broker → no docker socket. - Labels the container with `source_hash` so a later `ensure_running` - can detect a real code change (see `_source_hash`).""" + Labels the container with `current_hash` so a later `ensure_running` + can detect a real code change (see `source_hash`).""" run_docker(["docker", "rm", "--force", self._orchestrator_name]) proc = run_docker([ "docker", "run", "--detach", "--name", self._orchestrator_name, "--label", self._orchestrator_label, - "--label", f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={source_hash}", + "--label", f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={current_hash}", "--network", self.network, # Host CLI reaches the control plane here; bound to loopback so it # is not exposed on the host's external interfaces. @@ -224,7 +224,7 @@ class OrchestratorService: # launch (the prior behaviour) would drop every other active # bottle's in-memory egress tokens each time a new bottle starts, # since the orchestrator process holds them only in memory (#381). - current_hash = _source_hash(self._repo_root) + current_hash = source_hash(self._repo_root) if self.is_healthy() and self._orchestrator_source_current(current_hash): return self.url diff --git a/docs/research/apple-container-networking-spike.md b/docs/research/apple-container-networking-spike.md index e4fc405..edfd927 100644 --- a/docs/research/apple-container-networking-spike.md +++ b/docs/research/apple-container-networking-spike.md @@ -358,3 +358,111 @@ the Apple Container-specific constraints directly: Do not implement the backend as a direct clone of Docker Compose service aliases. That assumption failed in this run. + +## Addendum: consolidated-gateway findings (2026-07-17, PRD 0070) + +Re-tested on Apple Container 1.0.0 while porting the backend to the +per-host consolidated gateway (#351). The two-network shape above still +holds; these are the additional constraints that shaped the port, each +verified against the live CLI on this host. + +### No static IP for a container + +`container run --network` accepts only +`[,mac=XX:XX:XX:XX:XX:XX][,mtu=VALUE]`. There is no `--ip`. The +address comes from vmnet's DHCP and is knowable only after the container +is running: + +```console +$ container run --name a --network bb-net --detach alpine sleep 900 +$ container inspect a | jq -r '.[0].status.networks[0].ipv4Address' +192.168.128.3/24 +``` + +Consequence: the docker backend's "allocate a free IP -> pin it with +`--ip` -> register -> launch" order cannot be reproduced. macOS inverts +it to "launch -> read the assigned address -> register". The identity +token therefore cannot be in the agent's run-time env (registration mints +it after the container exists) and is delivered at `container exec` time. + +### Networks are fixed at run time + +There is no `container network connect`; `container network` exposes only +`create`, `delete`, `list`, `inspect`, `prune`. A network cannot be +attached to a running container, so a *persistent* shared gateway rules +out per-bottle networks — they would force a gateway restart per launch. +One shared host-only network, created up front, is the only shape that +keeps the gateway a singleton. + +### No container DNS + +Containers cannot resolve each other by name; the host-only network's +resolver refuses the query: + +```console +$ container exec agent nslookup gw +;; connection timed out; no servers could be reached +$ container exec agent cat /etc/resolv.conf +nameserver 192.168.128.1 +``` + +Consequence: the gateway is handed the control plane's **IP**, not a +container name as on docker. That forces the startup order +orchestrator -> read its address -> gateway. + +### The host can reach the host-only network directly + +```console +$ container inspect c | jq -r '.[0].status.networks[0].ipv4Address' +192.168.128.2/24 +$ curl -s http://192.168.128.2:8099/i +ok +``` + +So no `--publish` hop is needed: the host CLI and the gateway use the +same control-plane URL. Docker needs `--publish 127.0.0.1:...` plus a +separate internal URL for the same job. + +### CAP_NET_RAW is granted by default — and matters for attribution + +Apple grants NET_RAW but not NET_ADMIN. The agent therefore cannot change +its own address or route: + +```console +$ container exec agent ip addr add 192.168.128.99/24 dev eth0 +ip: RTNETLINK answers: Operation not permitted +$ container exec agent ip route replace default via 192.168.128.2 dev eth0 +ip: RTNETLINK answers: Operation not permitted +$ container exec agent grep CapEff /proc/self/status +CapEff: 00000000a80425fb # bit 13 (NET_RAW) set, bit 12 (NET_ADMIN) clear +``` + +But NET_RAW permits raw sockets, i.e. source-address forgery against +neighbours on the shared segment — directly against PRD 0070's invariant +("a packet's source address, as seen by the orchestrator, provably +identifies the originating bottle"). `--cap-drop CAP_NET_RAW` closes it: + +```console +$ container run --cap-drop CAP_NET_RAW ... alpine +$ container exec nr grep CapEff /proc/self/status +CapEff: 00000000a80405fb # bit 13 cleared +$ container exec nr ping -c1 192.168.128.2 +ping: permission denied (are you root?) +``` + +The agent is run with `--cap-drop CAP_NET_RAW` for this reason. + +### `container exec` inherits run-time env, and `--env` overrides it + +```console +$ container run --name e --env FOO=from_run --detach alpine sleep 120 +$ container exec e sh -c 'echo $FOO' +from_run +$ container exec --env FOO=from_exec e sh -c 'echo $FOO' +from_exec +``` + +This is what makes exec-time identity-token delivery work: the token-less +proxy URL baked in at launch is superseded by the token-bearing one at +exec. Bare `--env NAME` (inherit from the parent process) keeps the token +value off argv. diff --git a/tests/unit/test_macos_consolidated_launch.py b/tests/unit/test_macos_consolidated_launch.py new file mode 100644 index 0000000..5ab4ef0 --- /dev/null +++ b/tests/unit/test_macos_consolidated_launch.py @@ -0,0 +1,131 @@ +"""Unit: macOS consolidated launch — register-after-start (PRD 0070).""" + +from __future__ import annotations + +import unittest +from pathlib import Path +from unittest.mock import MagicMock, Mock, patch + +from bot_bottle.backend.macos_container.consolidated_launch import ( + GatewayEndpoint, + ensure_gateway, + register_agent, + teardown_consolidated, +) +from bot_bottle.egress import EgressPlan, EgressRoute +from bot_bottle.git_gate import GitGatePlan +from bot_bottle.orchestrator.client import RegisteredBottle + +_MOD = "bot_bottle.backend.macos_container.consolidated_launch" + + +def _egress_plan() -> EgressPlan: + return EgressPlan( + slug="demo", routes_path=Path("/x"), + routes=(EgressRoute(host="api.example.com"),), token_env_map={}, + ) + + +def _git_plan() -> GitGatePlan: + return GitGatePlan( + slug="demo", entrypoint_script=Path(), hook_script=Path(), + access_hook_script=Path(), upstreams=(), + ) + + +def _endpoint() -> GatewayEndpoint: + return GatewayEndpoint( + orchestrator_url="http://192.168.128.2:8099", + gateway_ip="192.168.128.3", + gateway_ca_pem="-----BEGIN CERTIFICATE-----\n", + network="bot-bottle-mac-gateway", + ) + + +def _client() -> Mock: + c = Mock() + c.register_bottle.return_value = RegisteredBottle("b1", "tok") + return c + + +class TestEnsureGateway(unittest.TestCase): + def _run(self, service: MagicMock) -> GatewayEndpoint: + with patch(f"{_MOD}.MacosOrchestratorService", return_value=service): + return ensure_gateway() + + def _service(self) -> MagicMock: + service = MagicMock() + service.ensure_running.return_value = "http://192.168.128.2:8099" + service.network = "bot-bottle-mac-gateway" + service.gateway.return_value.ip_on_shared_network.return_value = "192.168.128.3" + service.gateway.return_value.ca_cert_pem.return_value = "PEM" + return service + + def test_reports_gateway_endpoint(self) -> None: + endpoint = self._run(self._service()) + self.assertEqual("http://192.168.128.2:8099", endpoint.orchestrator_url) + self.assertEqual("192.168.128.3", endpoint.gateway_ip) + self.assertEqual("PEM", endpoint.gateway_ca_pem) + self.assertEqual("bot-bottle-mac-gateway", endpoint.network) + + def test_gateway_is_pointed_at_the_resolved_control_plane(self) -> None: + """Apple has no container DNS, so the gateway must be handed the + control plane's *resolved URL* rather than a container name.""" + service = self._service() + self._run(service) + service.gateway.assert_called_with("http://192.168.128.2:8099") + + +class TestRegisterAgent(unittest.TestCase): + def _run( + self, client: Mock, provision: Mock | None = None, + *, source_ip: str = "192.168.128.9", + ): + with patch(f"{_MOD}.OrchestratorClient", return_value=client), \ + patch(f"{_MOD}.provision_git_gate", provision or Mock()): + return register_agent( + _egress_plan(), _git_plan(), + source_ip=source_ip, endpoint=_endpoint(), image_ref="img:1", + ) + + def test_registers_by_the_address_read_from_the_live_container(self) -> None: + """The attribution key is the DHCP-assigned address the caller read + back — there is no --ip to pin it up front.""" + client = _client() + ctx = self._run(client, source_ip="192.168.128.9") + self.assertEqual("192.168.128.9", ctx.source_ip) + self.assertEqual("192.168.128.9", client.register_bottle.call_args.args[0]) + + def test_returns_identity_token_and_bottle_id(self) -> None: + ctx = self._run(_client()) + self.assertEqual("b1", ctx.bottle_id) + self.assertEqual("tok", ctx.identity_token) + + def test_provisions_git_gate_for_the_registered_bottle(self) -> None: + provision = Mock() + self._run(_client(), provision) + self.assertEqual("b1", provision.call_args.args[1]) + + def test_rolls_registration_back_when_provisioning_fails(self) -> None: + """A provisioning failure must not leave an orphan registration + holding the source IP.""" + client = _client() + provision = Mock(side_effect=RuntimeError("boom")) + with self.assertRaises(RuntimeError): + self._run(client, provision) + client.teardown_bottle.assert_called_once_with("b1") + + +class TestTeardown(unittest.TestCase): + def test_deregisters_and_deprovisions(self) -> None: + client = Mock() + deprovision = Mock() + with patch(f"{_MOD}.OrchestratorClient", return_value=client), \ + patch(f"{_MOD}.deprovision_git_gate", deprovision): + teardown_consolidated("b1", orchestrator_url="http://o:8099") + client.teardown_bottle.assert_called_once_with("b1") + self.assertEqual("b1", deprovision.call_args.args[1]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_macos_container_launch_wiring.py b/tests/unit/test_macos_container_launch_wiring.py new file mode 100644 index 0000000..fd8355a --- /dev/null +++ b/tests/unit/test_macos_container_launch_wiring.py @@ -0,0 +1,180 @@ +"""Unit: macOS agent run wiring — the attribution invariant + token delivery +(PRD 0070). + +Replaces the argv coverage from the per-bottle companion-container era +(`test_macos_container_launch.py`, removed with that architecture in #385). +""" + +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from typing import cast +from unittest.mock import patch + +from bot_bottle.backend.macos_container.bottle import MacosContainerBottle +from bot_bottle.backend.macos_container.bottle_plan import MacosContainerBottlePlan +from bot_bottle.backend.macos_container.consolidated_launch import GatewayEndpoint +from bot_bottle.backend.macos_container.launch import ( + _agent_run_argv, + _identity_proxy_env, + _proxy_url, +) +from bot_bottle.manifest import ManifestIndex + +_BOTTLE = "bot_bottle.backend.macos_container.bottle" + +_MANIFEST = ManifestIndex.from_json_obj({ + "bottles": {"dev": {}}, + "agents": {"demo": {"skills": [], "prompt": "", "bottle": "dev"}}, +}).load_for_agent("demo") + + +def _endpoint() -> GatewayEndpoint: + return GatewayEndpoint( + orchestrator_url="http://192.168.128.2:8099", + gateway_ip="192.168.128.3", + gateway_ca_pem="PEM", + network="bot-bottle-mac-gateway", + ) + + +def _plan( + stage_dir: Path, + *, + agent_git_gate_url: str = "", + agent_supervise_url: str = "", +) -> MacosContainerBottlePlan: + routes_path = stage_dir / "routes.yaml" + routes_path.write_text("routes: []\n", encoding="utf-8") + ca_path = stage_dir / "gateway-ca.pem" + ca_path.write_text("ca\n", encoding="utf-8") + egress_plan = SimpleNamespace( + mitmproxy_ca_host_path=ca_path, + routes_path=routes_path, + routes=("route",), + token_env_map={"EGRESS_TOKEN_0": "HOST_TOKEN"}, + canary="", + canary_env="", + ) + return cast(MacosContainerBottlePlan, SimpleNamespace( + spec=SimpleNamespace(), + manifest=_MANIFEST, + stage_dir=stage_dir, + slug="dev-abc", + container_name="bot-bottle-dev-abc", + image="bot-bottle-agent:latest", + forwarded_env={"OAUTH_TOKEN": "host-value"}, + egress_plan=egress_plan, + git_gate_plan=SimpleNamespace(upstreams=()), + supervise_plan=None, + agent_provision=SimpleNamespace( + guest_env={"LITERAL": "value"}, + provisioned_env={}, + ), + agent_git_gate_url=agent_git_gate_url, + agent_supervise_url=agent_supervise_url, + )) + + +class TestAgentRunArgv(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.argv = _agent_run_argv(_plan(Path(self._tmp.name)), _endpoint()) + + def tearDown(self) -> None: + self._tmp.cleanup() + + def test_drops_net_raw(self) -> None: + """The attribution invariant: Apple grants CAP_NET_RAW by default, + which would let an agent forge a neighbour's source address with a raw + socket and be attributed as that bottle.""" + self.assertIn("--cap-drop", self.argv) + self.assertEqual("CAP_NET_RAW", self.argv[self.argv.index("--cap-drop") + 1]) + + def test_attaches_to_the_shared_gateway_network(self) -> None: + self.assertEqual( + "bot-bottle-mac-gateway", self.argv[self.argv.index("--network") + 1], + ) + + def test_never_pins_an_ip(self) -> None: + """Apple Container 1.0.0 has no --ip: the address is DHCP-assigned and + read back after start.""" + self.assertNotIn("--ip", self.argv) + + def test_run_time_proxy_carries_no_identity_token(self) -> None: + """The token is minted by registration, which happens after this run — + so it cannot be here. `/resolve` denies the token-less pair (#366), + which is the safe direction; the real value arrives at exec time.""" + joined = " ".join(self.argv) + self.assertIn(f"HTTP_PROXY={_proxy_url('192.168.128.3')}", joined) + self.assertNotIn("bottle:", joined) + + def test_gateway_bypasses_the_proxy(self) -> None: + """git-http + supervise live on the gateway and must be reached + directly, not through its own egress proxy.""" + entry = next(a for a in self.argv if a.startswith("NO_PROXY=")) + self.assertIn("192.168.128.3", entry) + + def test_forwarded_secrets_stay_off_argv(self) -> None: + """Bare name → inherited from the run process env, so the value never + lands on the command line.""" + self.assertIn("OAUTH_TOKEN", self.argv) + self.assertNotIn("host-value", " ".join(self.argv)) + + def test_agent_init_is_a_no_op(self) -> None: + """Every agent command arrives via `container exec`; the init process + just holds the container open.""" + self.assertEqual("sleep", self.argv[-2]) + + +class TestIdentityTokenDelivery(unittest.TestCase): + def test_exec_env_carries_the_token_as_proxy_credentials(self) -> None: + env = _identity_proxy_env(_endpoint(), "s3cret") + self.assertEqual( + "http://bottle:s3cret@192.168.128.3:9099", env["HTTP_PROXY"], + ) + self.assertEqual(env["HTTP_PROXY"], env["https_proxy"]) + + def test_no_token_means_no_override(self) -> None: + self.assertEqual({}, _identity_proxy_env(_endpoint(), "")) + + def test_token_value_never_reaches_argv(self) -> None: + """`ps` is world-readable: the token rides the child env behind a bare + `--env` name, never the command line.""" + bottle = MacosContainerBottle( + "bot-bottle-demo", lambda: None, None, + exec_env=_identity_proxy_env(_endpoint(), "s3cret"), + ) + argv = bottle.agent_argv(["--help"], tty=False) + self.assertNotIn("s3cret", " ".join(argv)) + self.assertIn("HTTP_PROXY", argv) + self.assertEqual("--env", argv[argv.index("HTTP_PROXY") - 1]) + + def test_bottle_without_exec_env_is_unchanged(self) -> None: + bottle = MacosContainerBottle("bot-bottle-demo", lambda: None, None) + argv = bottle.agent_argv(["--help"], tty=False) + self.assertNotIn("--env", argv) + + def test_provisioning_exec_also_carries_the_token(self) -> None: + """`provision` runs through `exec`; a provider whose provision step + fetches anything would otherwise egress token-less and be denied.""" + bottle = MacosContainerBottle( + "bot-bottle-demo", lambda: None, None, + exec_env=_identity_proxy_env(_endpoint(), "s3cret"), + ) + with patch(f"{_BOTTLE}.subprocess.run") as run: + run.return_value = SimpleNamespace(returncode=0, stdout="", stderr="") + bottle.exec("echo hi") + argv, kwargs = run.call_args.args[0], run.call_args.kwargs + self.assertIn("HTTP_PROXY", argv) + self.assertNotIn("s3cret", " ".join(argv)) + self.assertEqual( + "http://bottle:s3cret@192.168.128.3:9099", kwargs["env"]["HTTP_PROXY"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_macos_gateway.py b/tests/unit/test_macos_gateway.py new file mode 100644 index 0000000..cf9d9c0 --- /dev/null +++ b/tests/unit/test_macos_gateway.py @@ -0,0 +1,250 @@ +"""Unit: the Apple gateway + orchestrator lifecycle (PRD 0070).""" + +from __future__ import annotations + +import unittest +from pathlib import Path +from unittest.mock import Mock, patch + +from bot_bottle.backend.macos_container.gateway import AppleGateway, GatewayError +from bot_bottle.backend.macos_container.orchestrator_service import ( + MacosOrchestratorService, + OrchestratorStartError, +) + +_GW = "bot_bottle.backend.macos_container.gateway" +_ORCH = "bot_bottle.backend.macos_container.orchestrator_service" + + +def _ok(stdout: str = "") -> Mock: + return Mock(returncode=0, stdout=stdout, stderr="") + + +def _fail(stderr: str = "boom") -> Mock: + return Mock(returncode=1, stdout="", stderr=stderr) + + +class TestAppleGatewayRun(unittest.TestCase): + def _argv(self, run: Mock) -> list[str]: + return run.call_args.args[0] + + def _start(self, run: Mock) -> None: + with patch(f"{_GW}.container_mod") as mod: + mod.container_is_running.return_value = False + mod.dns_server.return_value = "1.1.1.1" + mod.run_container_argv = run + AppleGateway(orchestrator_url="http://192.168.128.2:8099").ensure_running() + + def test_nat_network_precedes_the_host_only_network(self) -> None: + """Apple Container makes the FIRST --network the default route, so the + NAT network must lead or the gateway has no route out.""" + run = Mock(return_value=_ok()) + self._start(run) + argv = self._argv(run) + networks = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"] + self.assertEqual(["bot-bottle-mac-egress", "bot-bottle-mac-gateway"], networks) + + def test_control_plane_url_is_passed_for_multi_tenancy(self) -> None: + run = Mock(return_value=_ok()) + self._start(run) + self.assertIn( + "BOT_BOTTLE_ORCHESTRATOR_URL=http://192.168.128.2:8099", self._argv(run), + ) + + def test_dns_is_explicit(self) -> None: + """The NAT gateway routes but does not resolve.""" + run = Mock(return_value=_ok()) + self._start(run) + argv = self._argv(run) + self.assertEqual("1.1.1.1", argv[argv.index("--dns") + 1]) + + def test_start_failure_raises(self) -> None: + with self.assertRaises(GatewayError): + self._start(Mock(return_value=_fail())) + + def test_running_current_gateway_is_left_alone(self) -> None: + """Idempotent singleton: N launches must not restart the gateway and + drop every other bottle's data plane.""" + run = Mock(return_value=_ok()) + with patch(f"{_GW}.container_mod") as mod: + mod.container_is_running.return_value = True + mod.container_image_digest.return_value = "abc" + mod.image_digest.return_value = "abc" + mod.run_container_argv = run + AppleGateway().ensure_running() + run.assert_not_called() + + def test_stale_image_forces_a_recreate(self) -> None: + """A rebuilt image only takes effect if the running container is + replaced — otherwise it keeps serving the OLD daemons.""" + run = Mock(return_value=_ok()) + with patch(f"{_GW}.container_mod") as mod: + mod.container_is_running.return_value = True + mod.container_image_digest.return_value = "old" + mod.image_digest.return_value = "new" + mod.dns_server.return_value = "1.1.1.1" + mod.run_container_argv = run + AppleGateway().ensure_running() + run.assert_called_once() + + def test_unreadable_digest_does_not_churn(self) -> None: + run = Mock(return_value=_ok()) + with patch(f"{_GW}.container_mod") as mod: + mod.container_is_running.return_value = True + mod.container_image_digest.return_value = "" + mod.image_digest.return_value = "" + mod.run_container_argv = run + AppleGateway().ensure_running() + run.assert_not_called() + + def _start_with_running_env(self, env: dict[str, str], url: str) -> Mock: + run = Mock(return_value=_ok()) + with patch(f"{_GW}.container_mod") as mod: + mod.container_is_running.return_value = True + mod.container_image_digest.return_value = "abc" + mod.image_digest.return_value = "abc" + mod.container_env.return_value = env + mod.dns_server.return_value = "1.1.1.1" + mod.run_container_argv = run + AppleGateway(orchestrator_url=url).ensure_running() + return run + + def test_moved_control_plane_forces_a_recreate(self) -> None: + """Docker hands the gateway a container *name*, stable across an + orchestrator recreate. Apple has no DNS, so the URL is an IP baked into + the gateway's env — if the orchestrator comes back on a new address and + the gateway isn't recreated, every /resolve fails and every bottle on + the host loses egress.""" + run = self._start_with_running_env( + {"BOT_BOTTLE_ORCHESTRATOR_URL": "http://192.168.128.2:8099"}, + "http://192.168.128.7:8099", + ) + run.assert_called_once() + self.assertIn( + "BOT_BOTTLE_ORCHESTRATOR_URL=http://192.168.128.7:8099", + run.call_args.args[0], + ) + + def test_unmoved_control_plane_does_not_churn(self) -> None: + run = self._start_with_running_env( + {"BOT_BOTTLE_ORCHESTRATOR_URL": "http://192.168.128.2:8099"}, + "http://192.168.128.2:8099", + ) + run.assert_not_called() + + def test_unreadable_env_does_not_churn(self) -> None: + run = self._start_with_running_env({}, "http://192.168.128.2:8099") + run.assert_not_called() + + +class TestMacosOrchestratorService(unittest.TestCase): + def test_orchestrator_starts_before_the_gateway(self) -> None: + """Apple has no container DNS, so the gateway can only be handed the + control plane's IP — which does not exist until it is running. This + ordering is the whole reason the macOS service diverges from docker's.""" + order: list[str] = [] + svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h")) + gateway = Mock() + gateway.ensure_running.side_effect = lambda: order.append("gateway") + + def _record_orchestrator(_hash: str) -> None: + order.append("orchestrator") + + with patch(f"{_ORCH}.container_mod") as mod, \ + patch(f"{_ORCH}.source_hash", return_value="h1"), \ + patch.object(svc, "_run_orchestrator_container", + side_effect=_record_orchestrator), \ + patch.object(svc, "gateway", return_value=gateway), \ + patch.object(svc, "is_healthy", return_value=True): + mod.container_is_running.return_value = False + mod.image_exists.return_value = True + mod.try_container_ipv4_on_network.return_value = "192.168.128.2" + url = svc.ensure_running() + self.assertEqual(["orchestrator", "gateway"], order) + self.assertEqual("http://192.168.128.2:8099", url) + + def test_gateway_is_handed_the_resolved_url(self) -> None: + svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h")) + with patch(f"{_ORCH}.container_mod") as mod, \ + patch(f"{_ORCH}.source_hash", return_value="h1"), \ + patch.object(svc, "_run_orchestrator_container"), \ + patch.object(svc, "gateway") as gw, \ + patch.object(svc, "is_healthy", return_value=True): + mod.container_is_running.return_value = False + mod.image_exists.return_value = True + mod.try_container_ipv4_on_network.return_value = "192.168.128.2" + svc.ensure_running() + gw.assert_called_with("http://192.168.128.2:8099") + + def test_current_source_leaves_a_healthy_orchestrator_alone(self) -> None: + """Recreating on every launch would drop every other live bottle's + in-memory egress tokens (#381).""" + svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h")) + run = Mock() + with patch(f"{_ORCH}.container_mod") as mod, \ + patch(f"{_ORCH}.source_hash", return_value="h1"), \ + patch.object(svc, "_run_orchestrator_container", run), \ + patch.object(svc, "gateway"), \ + patch.object(svc, "is_healthy", return_value=True): + mod.container_is_running.return_value = True + mod.inspect_container.return_value = { + "configuration": {"labels": {"bot-bottle-orchestrator-source-hash": "h1"}} + } + mod.try_container_ipv4_on_network.return_value = "192.168.128.2" + svc.ensure_running() + run.assert_not_called() + + def test_changed_source_recreates_the_orchestrator(self) -> None: + """The control-plane process loaded its bind-mounted source at startup + and won't reload it.""" + svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h")) + run = Mock() + with patch(f"{_ORCH}.container_mod") as mod, \ + patch(f"{_ORCH}.source_hash", return_value="h2"), \ + patch.object(svc, "_run_orchestrator_container", run), \ + patch.object(svc, "gateway"), \ + patch.object(svc, "is_healthy", return_value=True): + mod.container_is_running.return_value = True + mod.inspect_container.return_value = { + "configuration": {"labels": {"bot-bottle-orchestrator-source-hash": "h1"}} + } + mod.image_exists.return_value = True + mod.try_container_ipv4_on_network.return_value = "192.168.128.2" + svc.ensure_running() + run.assert_called_once() + + def test_never_healthy_raises(self) -> None: + svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h")) + with patch(f"{_ORCH}.container_mod") as mod, \ + patch(f"{_ORCH}.source_hash", return_value="h1"), \ + patch.object(svc, "_run_orchestrator_container"), \ + patch.object(svc, "is_healthy", return_value=False): + mod.container_is_running.return_value = False + mod.image_exists.return_value = True + mod.try_container_ipv4_on_network.return_value = "192.168.128.2" + with self.assertRaises(OrchestratorStartError): + svc.ensure_running(startup_timeout=0.01) + + def test_control_plane_needs_no_route_out(self) -> None: + """The orchestrator sits only on the host-only network: the host + reaches it there directly, so there is no --publish and no NAT leg.""" + svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h")) + run = Mock(return_value=_ok()) + with patch(f"{_ORCH}.container_mod") as mod: + mod.run_container_argv = run + svc._run_orchestrator_container("h1") + argv = run.call_args.args[0] + networks = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"] + self.assertEqual(["bot-bottle-mac-gateway"], networks) + self.assertNotIn("--publish", argv) + + def test_orchestrator_start_failure_raises(self) -> None: + svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h")) + with patch(f"{_ORCH}.container_mod") as mod: + mod.run_container_argv = Mock(return_value=_fail()) + with self.assertRaises(OrchestratorStartError): + svc._run_orchestrator_container("h1") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_orchestrator_lifecycle.py b/tests/unit/test_orchestrator_lifecycle.py index abc27c5..81a2e23 100644 --- a/tests/unit/test_orchestrator_lifecycle.py +++ b/tests/unit/test_orchestrator_lifecycle.py @@ -14,7 +14,7 @@ from bot_bottle.orchestrator.lifecycle import ( ORCHESTRATOR_SOURCE_HASH_LABEL, OrchestratorService, OrchestratorStartError, - _source_hash, + source_hash, ) from tests.unit import use_bottle_root @@ -57,7 +57,7 @@ class TestOrchestratorService(unittest.TestCase): # A healthy control plane already running the *current* bind-mounted # source is left alone — recreating it on every launch would drop # every other active bottle's in-memory egress tokens (#381). - current = _source_hash(self.svc._repo_root) + current = source_hash(self.svc._repo_root) calls: list[list[str]] = [] def fake(argv: list[str]) -> Mock: @@ -98,7 +98,7 @@ class TestOrchestratorService(unittest.TestCase): self.assertEqual(1, len(runs)) self.assertIn(ORCHESTRATOR_NAME, runs[0]) # the fresh container is labeled with the current hash, not the stale one - current = _source_hash(self.svc._repo_root) + current = source_hash(self.svc._repo_root) self.assertIn(f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={current}", runs[0]) def test_ensure_running_starts_orchestrator_container_when_absent(self) -> None: