feat(macos): split orchestrator and gateway into separate containers (PRD 0070)
test / integration-docker (pull_request) Successful in 18s
tracker-policy-pr / check-pr (pull_request) Successful in 22s
test / unit (pull_request) Failing after 42s
lint / lint (push) Successful in 57s
test / integration-firecracker (pull_request) Successful in 3m22s
test / coverage (pull_request) Has been skipped
test / publish-infra (pull_request) Has been skipped

The single Apple container existed only because two guests writing one
bot-bottle.db over virtiofs would race incoherent fcntl locks; #469 removed
that (the data plane no longer opens the DB), so the macOS backend now runs
the planes as two containers like docker:

  * bot-bottle-mac-orchestrator — lean control plane on the host-only
    `bot-bottle-mac-control` network only (image Dockerfile.orchestrator,
    `-m bot_bottle.orchestrator`). Sole mounter of the container-only DB
    volume; holds the signing key. The CLI + gateway reach it at its
    control-network address.
  * bot-bottle-mac-infra — the gateway, triple-homed on the NAT egress net,
    the host-only agent net, and the control net. Resolves the orchestrator by
    IP (Apple has no container DNS) via BOT_BOTTLE_ORCHESTRATOR_URL; holds the
    CA + gateway JWT.

Agents sit on the agent network only, so they have no route to the control
plane. ensure_networks gains the control network; MacosInfraService brings up
the orchestrator then the gateway; probe_orchestrator_url + ca_cert_pem target
the right containers. pyright 0 errors; unit suite green (2274). Needs a real
Apple-container host to validate the networking.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-25 02:39:37 -04:00
parent d62d19a2e7
commit b676ce3156
3 changed files with 251 additions and 233 deletions
+17 -4
View File
@@ -19,27 +19,40 @@ from . import util as container_mod
# on. The agent's address here is the attribution key. Distinct from the docker # on. The agent's address here is the attribution key. Distinct from the docker
# names so both backends can coexist on one host. # names so both backends can coexist on one host.
GATEWAY_NETWORK = "bot-bottle-mac-gateway" GATEWAY_NETWORK = "bot-bottle-mac-gateway"
# The NAT network that gives the infra container (and only it) a route out. # The NAT network that gives the gateway (and only it) a route out.
GATEWAY_EGRESS_NETWORK = "bot-bottle-mac-egress" GATEWAY_EGRESS_NETWORK = "bot-bottle-mac-egress"
# The control network the gateway reaches the orchestrator over (host-only).
# Only the orchestrator + gateway join it; agents never do, so agents have no
# route to the control plane (PRD 0070 "Separating the planes").
CONTROL_NETWORK = "bot-bottle-mac-control"
GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest") GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest")
ORCHESTRATOR_IMAGE = os.environ.get(
"BOT_BOTTLE_ORCHESTRATOR_IMAGE", "bot-bottle-orchestrator:latest"
)
DEFAULT_CA_TIMEOUT_SECONDS = 30.0 DEFAULT_CA_TIMEOUT_SECONDS = 30.0
def ensure_networks( def ensure_networks(
network: str = GATEWAY_NETWORK, egress_network: str = GATEWAY_EGRESS_NETWORK, network: str = GATEWAY_NETWORK,
egress_network: str = GATEWAY_EGRESS_NETWORK,
control_network: str = CONTROL_NETWORK,
) -> None: ) -> None:
"""Create the shared host-only network + the NAT egress network. Idempotent """Create the shared host-only agent network, the NAT egress network, and
— `create_network` tolerates 'already exists'.""" the host-only control network. Idempotent — `create_network` tolerates
'already exists'."""
container_mod.create_network(egress_network) container_mod.create_network(egress_network)
container_mod.create_network(network, internal=True) container_mod.create_network(network, internal=True)
container_mod.create_network(control_network, internal=True)
__all__ = [ __all__ = [
"GATEWAY_NETWORK", "GATEWAY_NETWORK",
"GATEWAY_EGRESS_NETWORK", "GATEWAY_EGRESS_NETWORK",
"CONTROL_NETWORK",
"GATEWAY_IMAGE", "GATEWAY_IMAGE",
"ORCHESTRATOR_IMAGE",
"GatewayError", "GatewayError",
"DEFAULT_CA_TIMEOUT_SECONDS", "DEFAULT_CA_TIMEOUT_SECONDS",
"ensure_networks", "ensure_networks",
+165 -168
View File
@@ -1,34 +1,22 @@
"""The per-host infra container for the macOS backend (PRD 0070). """The per-host control plane + gateway for the macOS backend (PRD 0070).
A single persistent Apple container that runs BOTH the orchestrator control Two Apple containers — the orchestrator (control plane) and the gateway (data
plane and the gateway data plane — the macOS analogue of the Firecracker infra plane) — split now that #469 got the DB off the data plane. The single-container
VM (`backend/firecracker/infra_vm.py`), not the docker backend's two separate model existed only because two Apple-Container guests writing one `bot-bottle.db`
containers. over virtiofs would race incoherent `fcntl` locks; with the data plane no longer
opening the DB at all, only the orchestrator does, so the split is safe.
Why one container, not two: Apple Containers are lightweight VMs, each with its * `bot-bottle-mac-orchestrator` — the lean control plane. Joins the host-only
own kernel. The docker backend runs the orchestrator and gateway as two **control network** (`bot-bottle-mac-control`) only. Sole opener of the
containers safely because they share the host kernel, so their concurrent container-only DB volume; holds the signing key. The host CLI reaches it at
writes to the one `bot-bottle.db` (the orchestrator's registry + the gateway its control-network address; the gateway reaches it there too.
supervise daemon's queue) are serialized by coherent `fcntl` locks. Across two * `bot-bottle-mac-infra` — the gateway data plane. Triple-homed: the NAT
*guest* kernels sharing a virtiofs-mounted DB those locks are not coherent, and egress network (route out), the host-only agent network (agents + CLI reach
concurrent writers can corrupt the file. Firecracker solved this by putting the gateway), and the control network (reach the orchestrator by IP — Apple
both services in one guest with the DB on a device only that guest mounts; this has no container DNS). Holds the mitmproxy CA + the `gateway` JWT.
does the same with Apple primitives.
Two consequences fall out of the single container, both simplifications: Agents sit on the agent network only, never the control network, so they have no
route to the control plane (the L3 block, not just the JWT).
- **No DNS dance.** The control plane and the gateway daemons reach each other
over `127.0.0.1`, so nothing depends on Apple's (absent) container DNS and
there is no orchestrator-before-gateway ordering to get right.
- **The DB is never host-shared.** It lives on a container-only volume, so no
host process opens the live file. The host CLI reaches registry + supervise
state through the control-plane HTTP surface (`cli/supervise.py` already uses
`OrchestratorClient`), exactly as it does for firecracker.
The control-plane source is bind-mounted (like the docker orchestrator), so a
code change takes effect on the next launch without an image rebuild; the
gateway daemons are baked in the gateway image and rebuild through its own
digest check.
""" """
from __future__ import annotations from __future__ import annotations
@@ -59,23 +47,29 @@ from ...paths import (
from .. import util as backend_util from .. import util as backend_util
from . import util as container_mod from . import util as container_mod
from .gateway import ( from .gateway import (
CONTROL_NETWORK,
DEFAULT_CA_TIMEOUT_SECONDS, DEFAULT_CA_TIMEOUT_SECONDS,
GATEWAY_EGRESS_NETWORK, GATEWAY_EGRESS_NETWORK,
GATEWAY_IMAGE, GATEWAY_IMAGE,
GATEWAY_NETWORK, GATEWAY_NETWORK,
GatewayError, GatewayError,
ORCHESTRATOR_IMAGE,
ensure_networks, ensure_networks,
) )
# The one per-host infra container: control plane + gateway data plane. # The orchestrator (control plane) container + the gateway (data plane)
INFRA_NAME = "bot-bottle-mac-infra" # container. `INFRA_NAME` is kept — now the gateway container — for callers that
# still import it (probe / reprovision attribute against the gateway).
ORCHESTRATOR_NAME = "bot-bottle-mac-orchestrator"
ORCHESTRATOR_LABEL = "bot-bottle-mac-orchestrator=1"
INFRA_NAME = "bot-bottle-mac-infra" # the gateway container
INFRA_LABEL = "bot-bottle-mac-infra=1" INFRA_LABEL = "bot-bottle-mac-infra=1"
# Container-only volume holding bot-bottle.db. No host bind-mount, so the DB is # Container-only volume holding bot-bottle.db, mounted ONLY into the
# written by exactly one kernel (this container's). Survives recreation. # orchestrator. One kernel writes it (never host-shared or cross-guest).
INFRA_DB_VOLUME = "bot-bottle-mac-db" INFRA_DB_VOLUME = "bot-bottle-mac-db"
# BOT_BOTTLE_ROOT inside the container; host_db_path() resolves the DB to # BOT_BOTTLE_ROOT inside the orchestrator; host_db_path() resolves the DB to
# <root>/db/<filename> and the supervise daemon writes the same file. # <root>/db/<filename>.
_DB_ROOT_IN_CONTAINER = "/var/lib/bot-bottle" _DB_ROOT_IN_CONTAINER = "/var/lib/bot-bottle"
_DB_PATH_IN_CONTAINER = f"{_DB_ROOT_IN_CONTAINER}/db/{HOST_DB_FILENAME}" _DB_PATH_IN_CONTAINER = f"{_DB_ROOT_IN_CONTAINER}/db/{HOST_DB_FILENAME}"
_SRC_IN_CONTAINER = "/bot-bottle-src" _SRC_IN_CONTAINER = "/bot-bottle-src"
@@ -84,45 +78,23 @@ _REPO_ROOT = Path(__file__).resolve().parents[3]
_HEALTH_POLL_SECONDS = 0.25 _HEALTH_POLL_SECONDS = 0.25
_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0 _HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0
_CA_POLL_SECONDS = 0.5
# The gateway subset the consolidated model runs (no per-bottle git:// daemon). # The gateway subset the consolidated model runs (no per-bottle git:// daemon).
_GATEWAY_DAEMONS = "egress,git-http,supervise" _GATEWAY_DAEMONS = "egress,git-http,supervise"
def _init_script(port: int) -> str:
"""PID-1 init: start the control plane and the gateway daemons, both in
this container, reaching each other over loopback. Backgrounded so `wait`
reaps as PID 1. No `set -e` — a transient daemon failure must not kill the
whole container (gateway_init applies the same 'stay up' policy)."""
return (
"export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\n"
f"mkdir -p $(dirname {_DB_PATH_IN_CONTAINER})\n"
# Control plane, from the bind-mounted source (stdlib-only package).
f"( cd {_SRC_IN_CONTAINER} && BOT_BOTTLE_ROOT={_DB_ROOT_IN_CONTAINER} "
f"python3 -m bot_bottle.orchestrator --host 0.0.0.0 --port {port} "
"--broker stub ) &\n"
# Gateway data plane, multi-tenant against the local control plane. No
# SUPERVISE_DB_PATH: the data plane reaches the supervise queue over the
# control-plane RPC and never opens bot-bottle.db (PRD 0070 / #469).
f"( cd /app && BOT_BOTTLE_GATEWAY_DAEMONS={_GATEWAY_DAEMONS} "
f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{port} "
f"python3 -m bot_bottle.gateway.bootstrap ) &\n"
"while : ; do wait ; done\n"
)
@dataclass(frozen=True) @dataclass(frozen=True)
class InfraEndpoint: class InfraEndpoint:
"""How to reach the running infra container. The control plane and the """How to reach the running pair. `orchestrator_url` is the orchestrator's
gateway are the same container, so one address serves both.""" control-network address (host CLI + registration); `gateway_ip` is the
gateway's agent-network address (proxy / git-http / MCP target)."""
orchestrator_url: str # http://<infra ip>:8099 — host CLI + registration orchestrator_url: str
gateway_ip: str # same container; agents' proxy / git-http / MCP target gateway_ip: str
class MacosInfraService: class MacosInfraService:
"""Manages the single per-host infra container. Callers use """Manages the per-host orchestrator + gateway containers. Callers use
`ensure_running()` (returns the endpoint) and `ca_cert_pem()`.""" `ensure_running()` (returns the endpoint) and `ca_cert_pem()`."""
def __init__( def __init__(
@@ -131,24 +103,41 @@ class MacosInfraService:
port: int = DEFAULT_PORT, port: int = DEFAULT_PORT,
network: str = GATEWAY_NETWORK, network: str = GATEWAY_NETWORK,
egress_network: str = GATEWAY_EGRESS_NETWORK, egress_network: str = GATEWAY_EGRESS_NETWORK,
image: str = GATEWAY_IMAGE, control_network: str = CONTROL_NETWORK,
gateway_image: str = GATEWAY_IMAGE,
orchestrator_image: str = ORCHESTRATOR_IMAGE,
repo_root: Path = _REPO_ROOT, repo_root: Path = _REPO_ROOT,
name: str = INFRA_NAME, orchestrator_name: str = ORCHESTRATOR_NAME,
gateway_name: str = INFRA_NAME,
db_volume: str = INFRA_DB_VOLUME, db_volume: str = INFRA_DB_VOLUME,
) -> None: ) -> None:
self.port = port self.port = port
self.network = network self.network = network
self.egress_network = egress_network self.egress_network = egress_network
self.image = image self.control_network = control_network
self.gateway_image = gateway_image
self.orchestrator_image = orchestrator_image
self._repo_root = repo_root self._repo_root = repo_root
self._name = name self._orchestrator_name = orchestrator_name
self._gateway_name = gateway_name
self._db_volume = db_volume self._db_volume = db_volume
def _resolve_url(self) -> str: @property
"""The control-plane URL, or "" while the container has no address.""" def gateway_name(self) -> str:
ip = container_mod.try_container_ipv4_on_network(self._name, self.network) return self._gateway_name
def _resolve_orchestrator_url(self) -> str:
"""The control-plane URL (orchestrator's control-network address), or ""
while it has no address."""
ip = container_mod.try_container_ipv4_on_network(
self._orchestrator_name, self.control_network)
return f"http://{ip}:{self.port}" if ip else "" return f"http://{ip}:{self.port}" if ip else ""
def _resolve_gateway_ip(self) -> str:
"""The gateway's agent-network address, or "" while it has none."""
return container_mod.try_container_ipv4_on_network(
self._gateway_name, self.network)
def is_healthy( def is_healthy(
self, url: str, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS, self, url: str, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS,
) -> bool: ) -> bool:
@@ -160,158 +149,165 @@ class MacosInfraService:
except (urllib.error.URLError, TimeoutError, OSError): except (urllib.error.URLError, TimeoutError, OSError):
return False return False
def _source_current(self, current_hash: str) -> bool: def _orchestrator_source_current(self, current_hash: str) -> bool:
"""True iff the running infra container was created from the current """True iff the running orchestrator was created from the current
bind-mounted control-plane source. The control-plane process loads that bind-mounted control-plane source (it loads that code at startup and
code at startup and won't reload it, so a stale container keeps serving won't reload it)."""
OLD code.""" if not container_mod.container_is_running(self._orchestrator_name):
if not container_mod.container_is_running(self._name):
return False return False
env = container_mod.container_env(self._name) env = container_mod.container_env(self._orchestrator_name)
if not env: if not env:
return True # can't compare → don't churn a working container return True # can't compare → don't churn a working container
return env.get("BOT_BOTTLE_SOURCE_HASH") == current_hash return env.get("BOT_BOTTLE_SOURCE_HASH") == current_hash
def _running_healthy_endpoint(self, current_hash: str) -> InfraEndpoint | None:
"""The endpoint if the running container is BOTH source-current and
answering /health, else None (→ recreate). Health, not just the source
label, is what lets a wedged-but-current container self-heal instead of
being polled to death forever."""
if not self._source_current(current_hash):
return None
url = self._resolve_url()
if url and self.is_healthy(url):
return InfraEndpoint(orchestrator_url=url, gateway_ip=_ip_of(url))
return None
def ensure_built(self) -> None: def ensure_built(self) -> None:
"""Ensure the gateway data-plane image exists. The control-plane source """Ensure the gateway + orchestrator images exist. The control-plane
is bind-mounted, not baked, so only the gateway image needs building.""" source is bind-mounted, so a code change takes effect without a rebuild;
the images still carry the package for their entrypoints."""
container_mod.build_image( container_mod.build_image(
self.image, str(self._repo_root), dockerfile="Dockerfile.gateway", self.gateway_image, str(self._repo_root), dockerfile="Dockerfile.gateway")
) container_mod.build_image(
self.orchestrator_image, str(self._repo_root),
dockerfile="Dockerfile.orchestrator")
def ensure_running( def ensure_running(
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS, self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
) -> InfraEndpoint: ) -> InfraEndpoint:
"""Ensure the single infra container is up; return how to reach it. """Ensure the orchestrator + gateway containers are up; return how to
Idempotent per-host singleton — a healthy container on current source reach them. Idempotent per-host singleton — a healthy orchestrator on
is left untouched, so N launches share the one control plane + gateway. current source is left untouched. Raises `OrchestratorStartError` on
Raises `OrchestratorStartError` on startup timeout.""" control-plane startup timeout."""
current_hash = source_hash(self._repo_root)
endpoint = self._running_healthy_endpoint(current_hash)
if endpoint is not None:
return endpoint
self.ensure_built() self.ensure_built()
log.info("starting infra container", context={"name": self._name}) ensure_networks(self.network, self.egress_network, self.control_network)
self._run_container(current_hash)
return self._wait_healthy(startup_timeout)
def _run_container(self, current_hash: str) -> None: current_hash = source_hash(self._repo_root)
ensure_networks(self.network, self.egress_network) url = self._resolve_orchestrator_url()
container_mod.force_remove_container(self._name) if not (self._orchestrator_source_current(current_hash)
and url and self.is_healthy(url)):
log.info("starting orchestrator container",
context={"name": self._orchestrator_name})
self._run_orchestrator_container(current_hash)
url = self._wait_healthy(startup_timeout)
# (Re)ensure the gateway once the control plane it resolves against is
# healthy — it needs the orchestrator's control-network address.
self._ensure_gateway_container(url)
return InfraEndpoint(orchestrator_url=url, gateway_ip=self._resolve_gateway_ip())
def _run_orchestrator_container(self, current_hash: str) -> None:
container_mod.force_remove_container(self._orchestrator_name)
_signing_key = host_orchestrator_token()
argv = [ argv = [
"container", "run", "--detach", "container", "run", "--detach",
"--name", self._name, "--name", self._orchestrator_name,
"--label", "bot-bottle.backend=macos-container", "--label", "bot-bottle.backend=macos-container",
"--label", INFRA_LABEL, "--label", ORCHESTRATOR_LABEL,
# NAT network FIRST so the gateway's egress has a default route; # Control network only — agents are never on it (L3-isolated).
# the host-only network is where agents (and the host CLI) reach it. "--network", self.control_network,
"--network", self.egress_network,
"--network", self.network,
"--dns", container_mod.dns_server(), "--dns", container_mod.dns_server(),
# Container-only DB volume: one kernel writes bot-bottle.db, never # Container-only DB volume: exactly one kernel writes bot-bottle.db.
# shared with the host or another guest.
"--volume", f"{self._db_volume}:{_DB_ROOT_IN_CONTAINER}", "--volume", f"{self._db_volume}:{_DB_ROOT_IN_CONTAINER}",
# The DB needs a container-only ext4 volume for coherent SQLite # Live control-plane source (a code change takes effect on relaunch).
# locking, but the CA has no such constraint. Keep it in the host
# app-data root so infra-container recreation and Apple Container
# volume pruning cannot silently rotate every bottle's trust
# anchor (issue #450).
"--mount",
container_mod.bind_mount_spec(
str(host_gateway_ca_dir()), MITMPROXY_HOME),
# Bind-mount the control-plane source (read-only); a code change
# takes effect on relaunch with no image rebuild.
"--mount", "--mount",
container_mod.bind_mount_spec( container_mod.bind_mount_spec(
str(self._repo_root), _SRC_IN_CONTAINER, readonly=True), str(self._repo_root), _SRC_IN_CONTAINER, readonly=True),
# Baked onto the container so `_source_current` can detect a real "--env", f"PYTHONPATH={_SRC_IN_CONTAINER}",
# control-plane code change and recreate. "--env", f"BOT_BOTTLE_ROOT={_DB_ROOT_IN_CONTAINER}",
# Detect a real control-plane code change and recreate.
"--env", f"BOT_BOTTLE_SOURCE_HASH={current_hash}", "--env", f"BOT_BOTTLE_SOURCE_HASH={current_hash}",
# The control-plane signing key (control plane: verifies tokens) and # The signing key — held ONLY by the orchestrator (issue #469). Bare
# the pre-minted `gateway` JWT (the gateway's PolicyResolver: presents # `--env NAME` keeps the value off argv / `container inspect`.
# it) — they share this one container, and gateway_init scopes each to
# its process so a compromised data-plane daemon never sees the key
# (issue #469 review). Bare `--env NAME` inherits the value from the
# run process below, so neither lands on argv or in `container
# inspect`'s command line. The agent runs in a SEPARATE container that
# is never given these vars, which is the whole point.
"--env", ORCHESTRATOR_TOKEN_ENV, "--env", ORCHESTRATOR_TOKEN_ENV,
"--env", ORCHESTRATOR_AUTH_JWT_ENV, self.orchestrator_image,
"--entrypoint", "sh", # Dockerfile.orchestrator ENTRYPOINT is `-m bot_bottle.orchestrator`.
self.image, "--host", "0.0.0.0", "--port", str(self.port), "--broker", "stub",
"-c", _init_script(self.port),
] ]
_signing_key = host_orchestrator_token() result = container_mod.run_container_argv(
run_env = { argv, env={**os.environ, ORCHESTRATOR_TOKEN_ENV: _signing_key})
**os.environ,
ORCHESTRATOR_TOKEN_ENV: _signing_key,
ORCHESTRATOR_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key),
}
result = container_mod.run_container_argv(argv, env=run_env)
if result.returncode != 0: if result.returncode != 0:
raise OrchestratorStartError( raise OrchestratorStartError(
f"infra container failed to start: " f"orchestrator container failed to start: "
f"{(result.stderr or '').strip() or '<no stderr>'}" f"{(result.stderr or '').strip() or '<no stderr>'}"
) )
def _wait_healthy(self, startup_timeout: float) -> InfraEndpoint: def _ensure_gateway_container(self, orchestrator_url: str) -> None:
"""Start (recreate) the gateway container, dual-homed on the agent +
control networks, resolving policy against `orchestrator_url` (the
orchestrator's control-network address — Apple has no DNS)."""
container_mod.force_remove_container(self._gateway_name)
_signing_key = host_orchestrator_token()
argv = [
"container", "run", "--detach",
"--name", self._gateway_name,
"--label", "bot-bottle.backend=macos-container",
"--label", INFRA_LABEL,
# NAT egress FIRST (default route out); the host-only agent network
# is where agents reach the gateway; the control network reaches the
# orchestrator.
"--network", self.egress_network,
"--network", self.network,
"--network", self.control_network,
"--dns", container_mod.dns_server(),
# The mitmproxy CA on a host bind-mount (survives recreation +
# volume pruning — issue #450). No DB mount: the data plane never
# opens bot-bottle.db (#469).
"--mount",
container_mod.bind_mount_spec(str(host_gateway_ca_dir()), MITMPROXY_HOME),
"--env", f"BOT_BOTTLE_GATEWAY_DAEMONS={_GATEWAY_DAEMONS}",
"--env", f"BOT_BOTTLE_ORCHESTRATOR_URL={orchestrator_url}",
# The pre-minted `gateway` JWT (never the signing key). Bare
# `--env NAME` inherits the value from run_env below.
"--env", ORCHESTRATOR_AUTH_JWT_ENV,
self.gateway_image,
]
run_env = {**os.environ, ORCHESTRATOR_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key)}
result = container_mod.run_container_argv(argv, env=run_env)
if result.returncode != 0:
raise GatewayError(
f"gateway container failed to start: "
f"{(result.stderr or '').strip() or '<no stderr>'}"
)
def _wait_healthy(self, startup_timeout: float) -> str:
deadline = time.monotonic() + startup_timeout deadline = time.monotonic() + startup_timeout
while True: while True:
url = self._resolve_url() url = self._resolve_orchestrator_url()
if url and self.is_healthy(url): if url and self.is_healthy(url):
log.info("infra container healthy", context={"url": url}) log.info("orchestrator healthy", context={"url": url})
return InfraEndpoint(orchestrator_url=url, gateway_ip=_ip_of(url)) return url
if time.monotonic() >= deadline: if time.monotonic() >= deadline:
raise OrchestratorStartError( raise OrchestratorStartError(
f"infra container did not become healthy within " f"orchestrator did not become healthy within "
f"{startup_timeout:g}s" f"{startup_timeout:g}s"
) )
time.sleep(_HEALTH_POLL_SECONDS) time.sleep(_HEALTH_POLL_SECONDS)
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str: def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
"""The gateway's mitmproxy CA (PEM) agents install to trust its TLS """The gateway's mitmproxy CA (PEM) agents install to trust its TLS
interception. Read through the container path backed by the persistent interception. Read from the gateway container; polls because mitmproxy
host CA directory; polls because mitmproxy writes it a beat after writes it a beat after start."""
start."""
def _fetch() -> str | None: def _fetch() -> str | None:
result = container_mod.run_container_argv( result = container_mod.run_container_argv(
["container", "exec", self._name, "cat", GATEWAY_CA_CERT]) ["container", "exec", self._gateway_name, "cat", GATEWAY_CA_CERT])
return result.stdout if result.returncode == 0 and result.stdout.strip() else None return result.stdout if result.returncode == 0 and result.stdout.strip() else None
try: try:
return backend_util.poll_ca_cert(_fetch, timeout=timeout) return backend_util.poll_ca_cert(_fetch, timeout=timeout)
except TimeoutError as exc: except TimeoutError as exc:
raise GatewayError( raise GatewayError(
f"gateway CA not available in {self._name} after {timeout:g}s" f"gateway CA not available in {self._gateway_name} after {timeout:g}s"
) from exc ) from exc
def stop(self) -> None: def stop(self) -> None:
"""Remove the infra container (idempotent). The DB volume persists.""" """Remove both containers (idempotent). The DB volume persists."""
container_mod.force_remove_container(self._name) container_mod.force_remove_container(self._gateway_name)
container_mod.force_remove_container(self._orchestrator_name)
def _ip_of(url: str) -> str:
"""The host from an http://host:port URL."""
return url.split("://", 1)[-1].rsplit(":", 1)[0]
def probe_orchestrator_url(port: int = DEFAULT_PORT) -> str: def probe_orchestrator_url(port: int = DEFAULT_PORT) -> str:
"""The running infra container's control-plane URL, or "" if it isn't up. """The running orchestrator's control-plane URL, or "" if it isn't up. Used
Used by host-side control-plane discovery (`discover_orchestrator_url`); by host-side control-plane discovery; safe on any host (returns "" when the
safe to call on any host — returns "" when the container or the `container` container or the `container` CLI isn't present)."""
CLI isn't present.""" ip = container_mod.try_container_ipv4_on_network(ORCHESTRATOR_NAME, CONTROL_NETWORK)
ip = container_mod.try_container_ipv4_on_network(INFRA_NAME, GATEWAY_NETWORK)
return f"http://{ip}:{port}" if ip else "" return f"http://{ip}:{port}" if ip else ""
@@ -320,6 +316,7 @@ __all__ = [
"InfraEndpoint", "InfraEndpoint",
"OrchestratorStartError", "OrchestratorStartError",
"GatewayError", "GatewayError",
"ORCHESTRATOR_NAME",
"INFRA_NAME", "INFRA_NAME",
"INFRA_DB_VOLUME", "INFRA_DB_VOLUME",
] ]
+69 -61
View File
@@ -1,4 +1,4 @@
"""Unit: the single macOS infra container (control plane + gateway, PRD 0070).""" """Unit: macOS orchestrator + gateway containers (PRD 0070 plane split)."""
from __future__ import annotations from __future__ import annotations
@@ -24,86 +24,94 @@ def _fail(stderr: str = "boom") -> Mock:
return Mock(returncode=1, stdout="", stderr=stderr) return Mock(returncode=1, stdout="", stderr=stderr)
class TestInfraRun(unittest.TestCase): def _spec(src: str, tgt: str, readonly: bool = False) -> str:
def _run_container(self, svc: MacosInfraService) -> list[str]: return f"type=bind,source={src},target={tgt}" + (",readonly" if readonly else "")
class TestOrchestratorRun(unittest.TestCase):
def _run(self, svc: MacosInfraService) -> list[str]:
run = Mock(return_value=_ok()) run = Mock(return_value=_ok())
def _spec(src: str, tgt: str, readonly: bool = False) -> str:
return f"type=bind,source={src},target={tgt}" + (
",readonly" if readonly else "")
with patch(f"{_INFRA}.container_mod") as mod, \ with patch(f"{_INFRA}.container_mod") as mod, \
patch(f"{_INFRA}.ensure_networks"): patch(f"{_INFRA}.host_orchestrator_token", return_value="k"):
mod.dns_server.return_value = "1.1.1.1" mod.dns_server.return_value = "1.1.1.1"
mod.bind_mount_spec.side_effect = _spec mod.bind_mount_spec.side_effect = _spec
mod.run_container_argv = run mod.run_container_argv = run
svc._run_container("h1") svc._run_orchestrator_container("h1")
return run.call_args.args[0] return run.call_args.args[0]
def test_single_container_runs_both_processes(self) -> None: def test_runs_the_orchestrator_on_the_control_network_only(self) -> None:
"""The whole point: one container starts the control plane AND the argv = self._run(MacosInfraService(repo_root=Path("/r")))
gateway daemons, so one kernel owns the DB.""" nets = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
argv = self._run_container(MacosInfraService(repo_root=Path("/r"))) self.assertEqual(["bot-bottle-mac-control"], nets)
script = argv[-1] # Image ENTRYPOINT is `-m bot_bottle.orchestrator`; these are its args.
self.assertIn("bot_bottle.orchestrator", script) self.assertIn("--broker", argv)
# Gateway launches via the installed package (there is no self.assertIn("stub", argv)
# /app/gateway_init.py file since the daemons moved into bot_bottle). self.assertIn("bot-bottle-orchestrator:latest", argv)
self.assertIn("bot_bottle.gateway.bootstrap", script)
self.assertIn("127.0.0.1", script) # they reach each other on loopback
def test_db_is_a_container_only_volume(self) -> None: def test_db_is_a_container_only_volume(self) -> None:
"""No host bind-mount of the DB — a named volume only this container argv = self._run(MacosInfraService(repo_root=Path("/r")))
mounts, so the DB is never written by two kernels."""
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
vols = [argv[i + 1] for i, a in enumerate(argv) if a == "--volume"] vols = [argv[i + 1] for i, a in enumerate(argv) if a == "--volume"]
self.assertTrue(any(v.startswith(f"{INFRA_DB_VOLUME}:") for v in vols)) self.assertTrue(any(v.startswith(f"{INFRA_DB_VOLUME}:") for v in vols))
# The repo source is bind-mounted read-only; the DB is not a bind mount.
mounts = [argv[i + 1] for i, a in enumerate(argv) if a == "--mount"]
self.assertTrue(all("bot-bottle.db" not in m for m in mounts))
def test_ca_is_persisted_on_the_host_not_the_container_volume(self) -> None: def test_orchestrator_has_no_ca_mount(self) -> None:
"""The CA survives infra recreation and cannot be removed by Apple # The CA lives with the gateway, not the control plane.
Container's volume-prune command.""" argv = self._run(MacosInfraService(repo_root=Path("/r")))
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
mounts = [argv[i + 1] for i, a in enumerate(argv) if a == "--mount"] mounts = [argv[i + 1] for i, a in enumerate(argv) if a == "--mount"]
ca_mounts = [ self.assertFalse([m for m in mounts if "/home/mitmproxy" in m])
m for m in mounts
if "target=/home/mitmproxy/.mitmproxy" in m
]
self.assertEqual(1, len(ca_mounts))
self.assertIn("source=", ca_mounts[0])
self.assertIn("/gateway-ca", ca_mounts[0])
self.assertNotIn(",readonly", ca_mounts[0])
def test_nat_network_precedes_the_host_only_network(self) -> None:
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
nets = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
self.assertEqual(["bot-bottle-mac-egress", "bot-bottle-mac-gateway"], nets)
def test_source_hash_is_labelled_for_recreate(self) -> None: def test_source_hash_is_labelled_for_recreate(self) -> None:
argv = self._run_container(MacosInfraService(repo_root=Path("/r"))) argv = self._run(MacosInfraService(repo_root=Path("/r")))
self.assertIn("BOT_BOTTLE_SOURCE_HASH=h1", argv) self.assertIn("BOT_BOTTLE_SOURCE_HASH=h1", argv)
def test_start_failure_raises(self) -> None: def test_start_failure_raises(self) -> None:
svc = MacosInfraService(repo_root=Path("/r")) svc = MacosInfraService(repo_root=Path("/r"))
with patch(f"{_INFRA}.container_mod") as mod, \ with patch(f"{_INFRA}.container_mod") as mod, \
patch(f"{_INFRA}.ensure_networks"): patch(f"{_INFRA}.host_orchestrator_token", return_value="k"):
mod.dns_server.return_value = "1.1.1.1" mod.dns_server.return_value = "1.1.1.1"
mod.bind_mount_spec.return_value = "m"
mod.run_container_argv = Mock(return_value=_fail()) mod.run_container_argv = Mock(return_value=_fail())
with self.assertRaises(OrchestratorStartError): with self.assertRaises(OrchestratorStartError):
svc._run_container("h1") svc._run_orchestrator_container("h1")
class TestGatewayRun(unittest.TestCase):
def _run(self, svc: MacosInfraService, url: str = "http://10.0.0.5:8099") -> list[str]:
run = Mock(return_value=_ok())
with patch(f"{_INFRA}.container_mod") as mod, \
patch(f"{_INFRA}.host_orchestrator_token", return_value="k"), \
patch(f"{_INFRA}.mint", return_value="jwt"):
mod.dns_server.return_value = "1.1.1.1"
mod.bind_mount_spec.side_effect = _spec
mod.run_container_argv = run
svc._ensure_gateway_container(url)
return run.call_args.args[0]
def test_gateway_is_triple_homed(self) -> None:
argv = self._run(MacosInfraService(repo_root=Path("/r")))
nets = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
self.assertEqual(
["bot-bottle-mac-egress", "bot-bottle-mac-gateway", "bot-bottle-mac-control"],
nets,
)
def test_gateway_resolves_orchestrator_by_url_and_holds_ca(self) -> None:
argv = self._run(MacosInfraService(repo_root=Path("/r")), "http://10.0.0.5:8099")
self.assertIn("BOT_BOTTLE_ORCHESTRATOR_URL=http://10.0.0.5:8099", argv)
mounts = [argv[i + 1] for i, a in enumerate(argv) if a == "--mount"]
self.assertTrue([m for m in mounts if "target=/home/mitmproxy/.mitmproxy" in m])
self.assertIn("bot-bottle-gateway:latest", argv)
self.assertIn(
"BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise", argv)
class TestInfraEnsureRunning(unittest.TestCase): class TestInfraEnsureRunning(unittest.TestCase):
def test_current_healthy_container_is_left_alone(self) -> None: def test_current_healthy_orchestrator_left_alone(self) -> None:
"""Idempotent singleton: N launches must not churn the infra container
and drop every live bottle's control plane."""
svc = MacosInfraService(repo_root=Path("/r")) svc = MacosInfraService(repo_root=Path("/r"))
run = Mock() run = Mock()
with patch(f"{_INFRA}.container_mod") as mod, \ with patch(f"{_INFRA}.container_mod") as mod, \
patch(f"{_INFRA}.source_hash", return_value="h1"), \ patch(f"{_INFRA}.source_hash", return_value="h1"), \
patch.object(svc, "_run_container", run), \ patch.object(svc, "ensure_built"), \
patch.object(svc, "_run_orchestrator_container", run), \
patch.object(svc, "_ensure_gateway_container"), \
patch.object(svc, "is_healthy", return_value=True): patch.object(svc, "is_healthy", return_value=True):
mod.container_is_running.return_value = True mod.container_is_running.return_value = True
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"} mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
@@ -113,13 +121,14 @@ class TestInfraEnsureRunning(unittest.TestCase):
self.assertEqual("http://192.168.128.2:8099", endpoint.orchestrator_url) self.assertEqual("http://192.168.128.2:8099", endpoint.orchestrator_url)
self.assertEqual("192.168.128.2", endpoint.gateway_ip) self.assertEqual("192.168.128.2", endpoint.gateway_ip)
def test_changed_source_recreates(self) -> None: def test_changed_source_recreates_orchestrator(self) -> None:
svc = MacosInfraService(repo_root=Path("/r")) svc = MacosInfraService(repo_root=Path("/r"))
run = Mock() run = Mock()
with patch(f"{_INFRA}.container_mod") as mod, \ with patch(f"{_INFRA}.container_mod") as mod, \
patch(f"{_INFRA}.source_hash", return_value="h2"), \ patch(f"{_INFRA}.source_hash", return_value="h2"), \
patch.object(svc, "ensure_built"), \ patch.object(svc, "ensure_built"), \
patch.object(svc, "_run_container", run), \ patch.object(svc, "_run_orchestrator_container", run), \
patch.object(svc, "_ensure_gateway_container"), \
patch.object(svc, "is_healthy", return_value=True): patch.object(svc, "is_healthy", return_value=True):
mod.container_is_running.return_value = True mod.container_is_running.return_value = True
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"} mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
@@ -127,17 +136,15 @@ class TestInfraEnsureRunning(unittest.TestCase):
svc.ensure_running() svc.ensure_running()
run.assert_called_once() run.assert_called_once()
def test_wedged_but_current_container_is_recreated(self) -> None: def test_wedged_but_current_orchestrator_recreated(self) -> None:
"""Current source but a dead HTTP server must be recreated, not polled
to death forever — health, not just the source label, gates reuse."""
svc = MacosInfraService(repo_root=Path("/r")) svc = MacosInfraService(repo_root=Path("/r"))
run = Mock() run = Mock()
health = Mock(side_effect=[False, True])
with patch(f"{_INFRA}.container_mod") as mod, \ with patch(f"{_INFRA}.container_mod") as mod, \
patch(f"{_INFRA}.source_hash", return_value="h1"), \ patch(f"{_INFRA}.source_hash", return_value="h1"), \
patch.object(svc, "ensure_built"), \ patch.object(svc, "ensure_built"), \
patch.object(svc, "_run_container", run), \ patch.object(svc, "_run_orchestrator_container", run), \
patch.object(svc, "is_healthy", health): patch.object(svc, "_ensure_gateway_container"), \
patch.object(svc, "is_healthy", Mock(side_effect=[False, True])):
mod.container_is_running.return_value = True mod.container_is_running.return_value = True
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"} mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
mod.try_container_ipv4_on_network.return_value = "192.168.128.2" mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
@@ -149,7 +156,8 @@ class TestInfraEnsureRunning(unittest.TestCase):
with patch(f"{_INFRA}.container_mod") as mod, \ with patch(f"{_INFRA}.container_mod") as mod, \
patch(f"{_INFRA}.source_hash", return_value="h1"), \ patch(f"{_INFRA}.source_hash", return_value="h1"), \
patch.object(svc, "ensure_built"), \ patch.object(svc, "ensure_built"), \
patch.object(svc, "_run_container"), \ patch.object(svc, "_run_orchestrator_container"), \
patch.object(svc, "_ensure_gateway_container"), \
patch.object(svc, "is_healthy", return_value=False): patch.object(svc, "is_healthy", return_value=False):
mod.container_is_running.return_value = False mod.container_is_running.return_value = False
mod.try_container_ipv4_on_network.return_value = "192.168.128.2" mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
@@ -158,7 +166,7 @@ class TestInfraEnsureRunning(unittest.TestCase):
class TestCaCertPem(unittest.TestCase): class TestCaCertPem(unittest.TestCase):
def test_reads_ca_out_of_the_container(self) -> None: def test_reads_ca_out_of_the_gateway_container(self) -> None:
svc = MacosInfraService(repo_root=Path("/r")) svc = MacosInfraService(repo_root=Path("/r"))
with patch(f"{_INFRA}.container_mod") as mod: with patch(f"{_INFRA}.container_mod") as mod:
mod.run_container_argv.return_value = _ok("-----BEGIN CERTIFICATE-----\n") mod.run_container_argv.return_value = _ok("-----BEGIN CERTIFICATE-----\n")