refactor(macos): one infra container (control plane + gateway), fixes shared-DB races
lint / lint (push) Successful in 2m15s
test / unit (pull_request) Successful in 1m16s
test / integration (pull_request) Successful in 23s
test / coverage (pull_request) Successful in 1m17s

Adopts the firecracker infra-VM pattern for macOS: the orchestrator control
plane and the gateway data plane now run in a SINGLE Apple container instead of
two. Apple Containers are lightweight VMs with separate kernels, so the prior
two-container design had both guests writing one bot-bottle.db over virtiofs,
where fcntl locks are not coherent across kernels — concurrent writes (the
orchestrator's registry vs the gateway supervise daemon's queue) could corrupt
it. One container = one kernel = coherent locking.

The DB moves onto a container-only Apple volume (bot-bottle-mac-db), never
bind-mounted from the host, so no host process opens the live file either. The
host CLI already reaches registry + supervise state over the control-plane HTTP
surface (cli/supervise.py uses OrchestratorClient), exactly as firecracker's
VM-only DB requires.

Two simplifications fall out of the single container:
- No DNS dance: the control plane and gateway daemons reach each other over
  127.0.0.1, so the orchestrator-before-gateway ordering (a workaround for
  Apple having no container DNS) is gone, along with the moved-IP recreate
  logic it needed.
- Net -243 lines.

Mechanics: the infra container runs from the gateway image with the
control-plane source bind-mounted read-only (like the docker orchestrator, so a
code change needs no rebuild) and a small sh -c init that starts both processes
(mirrors firecracker's _infra_init). Also implements the macOS backend's
ensure_orchestrator() and adds it to discover_orchestrator_url, so operator
tools (supervise) can bring up / find the control plane on demand — previously
the macOS backend died with "no orchestrator control plane".

Verified end-to-end on real Apple Container 1.0.0: the single infra container
comes up healthy (one address for control plane + gateway), both processes run,
the DB is written on the container-only volume, host-side supervise works over
HTTP, and a registered agent gets 200 for an allowed host / 403 for a denied
one. 1824 unit tests pass with `container` absent (CI parity), pyright clean,
pylint 9.89.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-17 04:14:14 -04:00
parent e24b62b6b9
commit 4a607ad098
13 changed files with 549 additions and 792 deletions
@@ -89,6 +89,14 @@ class MacosContainerBottleBackend(
with _launch.launch(plan, provision=self.provision) as bottle: with _launch.launch(plan, provision=self.provision) as bottle:
yield bottle yield bottle
def ensure_orchestrator(self) -> str:
"""Bring up the per-host infra container (control plane + gateway) and
return its control-plane URL — the on-demand entry point operator tools
(`supervise`) call when no control plane is running yet. Mirrors
firecracker's infra-VM bring-up."""
from .infra import MacosInfraService
return MacosInfraService().ensure_running().control_plane_url
def prepare_cleanup(self) -> MacosContainerBottleCleanupPlan: def prepare_cleanup(self) -> MacosContainerBottleCleanupPlan:
return _cleanup.prepare_cleanup() return _cleanup.prepare_cleanup()
@@ -15,6 +15,9 @@ caller has to start the agent in between. `ensure_gateway` runs first because
the agent's proxy env needs the gateway's address at `container run` time; the the agent's proxy env needs the gateway's address at `container run` time; the
agent's *own* address (the attribution key) only exists afterwards. agent's *own* address (the attribution key) only exists afterwards.
The control plane and the gateway are one **infra container** here (see
`infra`), so `gateway_ip` and the control-plane host are the same address.
The consequence for the identity token: it is minted by registration, i.e. The consequence for the identity token: it is minted by registration, i.e.
*after* the agent container exists, so it cannot be baked into the run-time *after* the agent container exists, so it cannot be baked into the run-time
env the way docker's compose spec does. It is delivered at `container exec` env the way docker's compose spec does. It is delivered at `container exec`
@@ -38,7 +41,7 @@ from ...orchestrator.registration import registration_inputs
from ..docker.gateway_provision import deprovision_git_gate, provision_git_gate from ..docker.gateway_provision import deprovision_git_gate, provision_git_gate
from .gateway import GATEWAY_NETWORK from .gateway import GATEWAY_NETWORK
from .gateway_provision import AppleGatewayTransport from .gateway_provision import AppleGatewayTransport
from .orchestrator_service import MacosOrchestratorService, OrchestratorStartError from .infra import MacosInfraService, OrchestratorStartError
class ConsolidatedLaunchError(RuntimeError): class ConsolidatedLaunchError(RuntimeError):
@@ -47,7 +50,9 @@ class ConsolidatedLaunchError(RuntimeError):
@dataclass(frozen=True) @dataclass(frozen=True)
class GatewayEndpoint: class GatewayEndpoint:
"""What the agent `container run` needs to reach the shared gateway.""" """What the agent `container run` needs to reach the shared gateway (the
infra container). `gateway_ip` is that container's host-only address, the
same host the control-plane URL points at."""
orchestrator_url: str orchestrator_url: str
gateway_ip: str # the gateway's address — the agent's proxy target gateway_ip: str # the gateway's address — the agent's proxy target
@@ -68,19 +73,18 @@ class LaunchContext:
def ensure_gateway( def ensure_gateway(
*, service: MacosOrchestratorService | None = None, *, service: MacosInfraService | None = None,
) -> GatewayEndpoint: ) -> GatewayEndpoint:
"""Ensure the orchestrator control plane + shared gateway are up, and """Ensure the per-host infra container (control plane + gateway) is up and
report how to reach them. Idempotent — both are per-host singletons, so N report how to reach it. Idempotent — one singleton, so N bottle launches
bottle launches share the one pair. Call before starting the agent share it. Call before starting the agent container: the agent's proxy env
container: the agent's proxy env needs `gateway_ip` at run time.""" needs `gateway_ip` at run time."""
service = service or MacosOrchestratorService() service = service or MacosInfraService()
url = service.ensure_running() infra = service.ensure_running()
gateway = service.gateway(url)
return GatewayEndpoint( return GatewayEndpoint(
orchestrator_url=url, orchestrator_url=infra.control_plane_url,
gateway_ip=gateway.ip_on_shared_network(), gateway_ip=infra.gateway_ip,
gateway_ca_pem=gateway.ca_cert_pem(), gateway_ca_pem=service.ca_cert_pem(),
network=service.network, network=service.network,
) )
@@ -6,14 +6,13 @@ import subprocess
from ...bottle_state import read_metadata from ...bottle_state import read_metadata
from .. import ActiveAgent from .. import ActiveAgent
from .gateway import GATEWAY_NAME from .infra import INFRA_NAME
from .orchestrator_service import ORCHESTRATOR_NAME
_PREFIX = "bot-bottle-" _PREFIX = "bot-bottle-"
# The shared per-host singletons carry the same prefix as agent containers but # The shared per-host infra container carries the same prefix as agent
# are infrastructure, not bottles one gateway and one control plane serve # containers but is infrastructure, not a bottle — one control plane + gateway
# every agent, so listing them as agents would invent one per host. # serves every agent, so listing it as an agent would invent one per host.
_INFRA_NAMES = frozenset({GATEWAY_NAME, ORCHESTRATOR_NAME}) _INFRA_NAMES = frozenset({INFRA_NAME})
def enumerate_active() -> list[ActiveAgent]: def enumerate_active() -> list[ActiveAgent]:
+17 -206
View File
@@ -1,235 +1,46 @@
"""The consolidated per-host gateway as an Apple container (PRD 0070). """Shared network/image constants for the macOS consolidated infra container.
The macOS counterpart of `orchestrator.gateway.DockerGateway`: one persistent The gateway data plane no longer runs as its own Apple container — it shares a
gateway per host, shared by every bottle, attributing each request to a bottle single per-host **infra container** with the control plane (see `infra`),
by its source IP on the shared host-only network. because two Apple-Container guests writing one `bot-bottle.db` over virtiofs
would race incoherent `fcntl` locks. This module holds the pieces both the
Two Apple Container 1.0.0 constraints shape this and make it *not* a infra service and the launch/provision glue need: the network names, the
transliteration of the docker gateway: gateway image, and the network-creation helper.
- **No container DNS.** Containers cannot resolve each other by name (the
host-only network's resolver refuses the query), so the gateway reaches the
control plane by **IP**, not by name as the docker gateway does. The
orchestrator must therefore be started *before* the gateway — see
`orchestrator_service`.
- **Networks are fixed at `container run`.** There is no `network connect`,
so a network cannot be attached to a running container. The gateway must sit
on one shared, up-front network for the lifetime of the process; per-bottle
networks would mean restarting the gateway on every launch, which defeats the
consolidation.
The gateway is dual-homed, **NAT network first**: Apple Container makes the
first `--network` the default route, so the egress network must lead or the
gateway has no route to the internet.
""" """
from __future__ import annotations from __future__ import annotations
import os import os
import time
from pathlib import Path
from ...orchestrator.gateway import ( from ...orchestrator.gateway import GatewayError
GATEWAY_CA_CERT,
GATEWAY_DOCKERFILE,
MITMPROXY_HOME,
Gateway,
GatewayError,
)
from ...paths import host_db_dir
from ...supervise import DB_PATH_IN_CONTAINER
from . import util as container_mod from . import util as container_mod
from .util import bind_mount_spec as _mount
# Distinct from the docker gateway's names so both backends' gateways can # The shared host-only network the infra container and every agent bottle sit
# coexist on one host (a macOS host can run the docker backend too). # on. The agent's address here is the attribution key. Distinct from the docker
GATEWAY_NAME = "bot-bottle-mac-gateway" # names so both backends can coexist on one host.
# The shared host-only network the gateway and every agent bottle sit on. The
# agent's address here is the attribution key.
GATEWAY_NETWORK = "bot-bottle-mac-gateway" GATEWAY_NETWORK = "bot-bottle-mac-gateway"
# The NAT network that gives the gateway (and only the gateway) a route out. # The NAT network that gives the infra container (and only it) a route out.
GATEWAY_EGRESS_NETWORK = "bot-bottle-mac-egress" GATEWAY_EGRESS_NETWORK = "bot-bottle-mac-egress"
GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest") GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest")
_REPO_ROOT = Path(__file__).resolve().parents[3]
_SUPERVISE_DB_DIR_IN_CONTAINER = os.path.dirname(DB_PATH_IN_CONTAINER)
# mitmproxy writes its CA a beat after start; reads poll rather than assume.
_CA_POLL_SECONDS = 0.5
DEFAULT_CA_TIMEOUT_SECONDS = 30.0 DEFAULT_CA_TIMEOUT_SECONDS = 30.0
def gateway_ca_dir() -> Path:
"""Host dir bind-mounted as mitmproxy's home, keeping the gateway's
self-generated CA **stable across container recreation** — every agent
installs this one CA to trust the shared gateway's TLS interception, so it
must not rotate when the gateway restarts.
The docker gateway uses a named volume for this; a plain host dir is the
same guarantee with fewer moving parts, and it lets `ca_cert_pem` read the
PEM straight off the host instead of shelling into the container."""
path = host_db_dir() / "mac-gateway-ca"
path.mkdir(parents=True, exist_ok=True)
return path
def ensure_networks( def ensure_networks(
network: str = GATEWAY_NETWORK, egress_network: str = GATEWAY_EGRESS_NETWORK, network: str = GATEWAY_NETWORK, egress_network: str = GATEWAY_EGRESS_NETWORK,
) -> None: ) -> None:
"""Create the shared host-only network + the gateway's NAT network. """Create the shared host-only network + the NAT egress network. Idempotent
Idempotent — `create_network` tolerates 'already exists'. — `create_network` tolerates 'already exists'."""
Module-level rather than a gateway method because the **orchestrator**
needs the shared network too, and it starts first (Apple has no container
DNS, so the gateway must be handed the control plane's IP). Both callers
ensure the networks; whoever runs first wins."""
container_mod.create_network(egress_network) container_mod.create_network(egress_network)
container_mod.create_network(network, internal=True) container_mod.create_network(network, internal=True)
class AppleGateway(Gateway):
"""The consolidated gateway as a single, fixed-name Apple container."""
def __init__(
self,
image_ref: str = GATEWAY_IMAGE,
*,
name: str = GATEWAY_NAME,
network: str = GATEWAY_NETWORK,
egress_network: str = GATEWAY_EGRESS_NETWORK,
orchestrator_url: str = "",
build_context: Path | None = None,
dockerfile: str | None = GATEWAY_DOCKERFILE,
) -> None:
self.image_ref = image_ref
self.name = name
self.network = network
self.egress_network = egress_network
# Reached by IP (no container DNS on Apple) — the caller resolves the
# orchestrator's address before constructing this. Empty → single-tenant.
self._orchestrator_url = orchestrator_url
self._build_context = build_context or _REPO_ROOT
self._dockerfile = dockerfile
def ensure_built(self) -> None:
"""Build the gateway data-plane image from its Dockerfile. Builds every
time (cache-aware, so it's cheap when nothing changed): a stale image
silently runs the OLD single-tenant daemons. Mirrors `DockerGateway`."""
if self._dockerfile is None:
return
container_mod.build_image(
self.image_ref, str(self._build_context), dockerfile=self._dockerfile,
)
def is_running(self) -> bool:
return container_mod.container_is_running(self.name)
def _running_image_is_current(self) -> bool:
"""True iff the running gateway was created from the *current*
`image_ref`. `ensure_built` rebuilding the image is not enough on its
own — the running container still holds the OLD image, so this
mismatch check is what makes a rebuild take effect."""
running = container_mod.container_image_digest(self.name)
current = container_mod.image_digest(self.image_ref)
if not running or not current:
return True # can't compare → don't churn a working container
return running == current
def _running_control_plane_is_current(self) -> bool:
"""True iff the running gateway points at the control plane we would
pass today.
Docker gets this for free — it hands the gateway a container *name*,
which survives the orchestrator being recreated. Apple has no container
DNS, so the URL is an **IP baked into the gateway's env at run time**,
and a recreated orchestrator can come back on a different DHCP address.
Without this check the gateway would keep pointing at the old address
and every `/resolve` would fail — denying egress for *every* bottle on
the host until something else happened to recreate the gateway."""
if not self._orchestrator_url:
return True
env = container_mod.container_env(self.name)
if not env:
return True # can't compare → don't churn a working container
return env.get("BOT_BOTTLE_ORCHESTRATOR_URL") == self._orchestrator_url
def ensure_running(self) -> None:
if (self.is_running()
and self._running_image_is_current()
and self._running_control_plane_is_current()):
return
ensure_networks(self.network, self.egress_network)
container_mod.force_remove_container(self.name)
argv = [
"container", "run", "--detach",
"--name", self.name,
"--label", "bot-bottle.backend=macos-container",
"--label", "bot-bottle-mac-gateway=1",
# NAT network FIRST: Apple Container takes the first --network as
# the default route, so this ordering is what gives the gateway a
# route out. Reversing it silently blackholes egress.
"--network", self.egress_network,
"--network", self.network,
# The NAT gateway routes but does not resolve, so DNS is explicit.
"--dns", container_mod.dns_server(),
"--mount", _mount(str(gateway_ca_dir()), MITMPROXY_HOME),
"--mount", _mount(str(host_db_dir()), _SUPERVISE_DB_DIR_IN_CONTAINER),
"--env", f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}",
]
if self._orchestrator_url:
# Makes the data plane multi-tenant: each request resolves
# source-IP → policy against the control plane.
argv += ["--env", f"BOT_BOTTLE_ORCHESTRATOR_URL={self._orchestrator_url}"]
argv.append(self.image_ref)
result = container_mod.run_container_argv(argv)
if result.returncode != 0:
raise GatewayError(
f"gateway failed to start: "
f"{(result.stderr or '').strip() or '<no stderr>'}"
)
def ip_on_shared_network(self) -> str:
"""The gateway's address on the shared host-only network — what agents
point their proxy / git-http / supervise URLs at. Polls: a freshly
run gateway can be up before vmnet's DHCP has assigned the address."""
ip = container_mod.wait_container_ipv4_on_network(self.name, self.network)
if not ip:
raise GatewayError(
f"gateway {self.name} never got an address on {self.network}"
)
return ip
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
"""The gateway's CA certificate (PEM) that agents install to trust its
TLS interception. Polls: mitmproxy generates it a moment after start.
Read from the host bind-mount, so no exec into the container."""
ca_path = gateway_ca_dir() / os.path.basename(GATEWAY_CA_CERT)
deadline = time.monotonic() + timeout
while True:
try:
pem = ca_path.read_text()
if pem.strip():
return pem
except OSError:
pass
if time.monotonic() >= deadline:
raise GatewayError(
f"gateway CA cert not available at {ca_path} after {timeout:g}s"
)
time.sleep(_CA_POLL_SECONDS)
def stop(self) -> None:
container_mod.force_remove_container(self.name)
__all__ = [ __all__ = [
"AppleGateway",
"GatewayError",
"GATEWAY_NAME",
"GATEWAY_NETWORK", "GATEWAY_NETWORK",
"GATEWAY_EGRESS_NETWORK", "GATEWAY_EGRESS_NETWORK",
"GATEWAY_IMAGE", "GATEWAY_IMAGE",
"gateway_ca_dir", "GatewayError",
"DEFAULT_CA_TIMEOUT_SECONDS",
"ensure_networks",
] ]
@@ -1,23 +1,23 @@
"""`GatewayTransport` for the Apple gateway container (PRD 0070). """`GatewayTransport` for the Apple infra container (PRD 0070).
The provisioning *logic* (per-bottle creds dirs, namespaced repo init) is The provisioning *logic* (per-bottle creds dirs, namespaced repo init) is
backend-neutral and lives in `backend.docker.gateway_provision`; this is only backend-neutral and lives in `backend.docker.gateway_provision`; this is only
the transport — how files and commands reach the running gateway. Docker uses the transport — how files and commands reach the running gateway. Docker uses
`docker exec`/`docker cp` and Firecracker uses SSH; Apple uses the `container` `docker exec`/`docker cp` and Firecracker uses SSH; Apple uses the `container`
CLI's equivalents. CLI's equivalents against the infra container that hosts the gateway daemons.
""" """
from __future__ import annotations from __future__ import annotations
from ..docker.gateway_provision import GatewayProvisionError from ..docker.gateway_provision import GatewayProvisionError
from . import util as container_mod from . import util as container_mod
from .gateway import GATEWAY_NAME from .infra import INFRA_NAME
class AppleGatewayTransport: class AppleGatewayTransport:
"""`GatewayTransport` for the gateway as an Apple container.""" """`GatewayTransport` for the gateway daemons in the Apple infra container."""
def __init__(self, gateway: str = GATEWAY_NAME) -> None: def __init__(self, gateway: str = INFRA_NAME) -> None:
self.gateway = gateway self.gateway = gateway
def exec(self, argv: list[str]) -> None: def exec(self, argv: list[str]) -> None:
+292
View File
@@ -0,0 +1,292 @@
"""The per-host infra container for the macOS backend (PRD 0070).
A single persistent Apple container that runs BOTH the orchestrator control
plane and the gateway data plane — the macOS analogue of the Firecracker infra
VM (`backend/firecracker/infra_vm.py`), not the docker backend's two separate
containers.
Why one container, not two: Apple Containers are lightweight VMs, each with its
own kernel. The docker backend runs the orchestrator and gateway as two
containers safely because they share the host kernel, so their concurrent
writes to the one `bot-bottle.db` (the orchestrator's registry + the gateway
supervise daemon's queue) are serialized by coherent `fcntl` locks. Across two
*guest* kernels sharing a virtiofs-mounted DB those locks are not coherent, and
concurrent writers can corrupt the file. Firecracker solved this by putting
both services in one guest with the DB on a device only that guest mounts; this
does the same with Apple primitives.
Two consequences fall out of the single container, both simplifications:
- **No DNS dance.** The control plane and the gateway daemons reach each other
over `127.0.0.1`, so nothing depends on Apple's (absent) container DNS and
there is no orchestrator-before-gateway ordering to get right.
- **The DB is never host-shared.** It lives on a container-only volume, so no
host process opens the live file. The host CLI reaches registry + supervise
state through the control-plane HTTP surface (`cli/supervise.py` already uses
`OrchestratorClient`), exactly as it does for firecracker.
The control-plane source is bind-mounted (like the docker orchestrator), so a
code change takes effect on the next launch without an image rebuild; the
gateway daemons are baked in the gateway image and rebuild through its own
digest check.
"""
from __future__ import annotations
import time
import urllib.error
import urllib.request
from dataclasses import dataclass
from pathlib import Path
from ... import log
from ...orchestrator.gateway import GATEWAY_CA_CERT
from ...orchestrator.lifecycle import (
DEFAULT_PORT,
DEFAULT_STARTUP_TIMEOUT_SECONDS,
OrchestratorStartError,
source_hash,
)
from ...paths import HOST_DB_FILENAME
from . import util as container_mod
from .gateway import (
DEFAULT_CA_TIMEOUT_SECONDS,
GATEWAY_EGRESS_NETWORK,
GATEWAY_IMAGE,
GATEWAY_NETWORK,
GatewayError,
ensure_networks,
)
# The one per-host infra container: control plane + gateway data plane.
INFRA_NAME = "bot-bottle-mac-infra"
INFRA_LABEL = "bot-bottle-mac-infra=1"
# Container-only volume holding bot-bottle.db. No host bind-mount, so the DB is
# written by exactly one kernel (this container's). Survives recreation.
INFRA_DB_VOLUME = "bot-bottle-mac-db"
# BOT_BOTTLE_ROOT inside the container; host_db_path() resolves the DB to
# <root>/db/<filename> and the supervise daemon writes the same file.
_DB_ROOT_IN_CONTAINER = "/var/lib/bot-bottle"
_DB_PATH_IN_CONTAINER = f"{_DB_ROOT_IN_CONTAINER}/db/{HOST_DB_FILENAME}"
_SRC_IN_CONTAINER = "/bot-bottle-src"
_REPO_ROOT = Path(__file__).resolve().parents[3]
_HEALTH_POLL_SECONDS = 0.25
_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0
_CA_POLL_SECONDS = 0.5
# The gateway subset the consolidated model runs (no per-bottle git:// daemon).
_GATEWAY_DAEMONS = "egress,git-http,supervise"
def _init_script(port: int) -> str:
"""PID-1 init: start the control plane and the gateway daemons, both in
this container, reaching each other over loopback. Backgrounded so `wait`
reaps as PID 1. No `set -e` — a transient daemon failure must not kill the
whole container (gateway_init applies the same 'stay up' policy)."""
return (
"export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\n"
f"mkdir -p $(dirname {_DB_PATH_IN_CONTAINER})\n"
# Control plane, from the bind-mounted source (stdlib-only package).
f"( cd {_SRC_IN_CONTAINER} && BOT_BOTTLE_ROOT={_DB_ROOT_IN_CONTAINER} "
f"python3 -m bot_bottle.orchestrator --host 0.0.0.0 --port {port} "
"--broker stub ) &\n"
# Gateway data plane, multi-tenant against the local control plane.
f"( cd /app && BOT_BOTTLE_GATEWAY_DAEMONS={_GATEWAY_DAEMONS} "
f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{port} "
f"SUPERVISE_DB_PATH={_DB_PATH_IN_CONTAINER} python3 /app/gateway_init.py ) &\n"
"while : ; do wait ; done\n"
)
@dataclass(frozen=True)
class InfraEndpoint:
"""How to reach the running infra container. The control plane and the
gateway are the same container, so one address serves both."""
control_plane_url: str # http://<infra ip>:8099 — host CLI + registration
gateway_ip: str # same container; agents' proxy / git-http / MCP target
class MacosInfraService:
"""Manages the single per-host infra container. Callers use
`ensure_running()` (returns the endpoint) and `ca_cert_pem()`."""
def __init__(
self,
*,
port: int = DEFAULT_PORT,
network: str = GATEWAY_NETWORK,
egress_network: str = GATEWAY_EGRESS_NETWORK,
image: str = GATEWAY_IMAGE,
repo_root: Path = _REPO_ROOT,
name: str = INFRA_NAME,
db_volume: str = INFRA_DB_VOLUME,
) -> None:
self.port = port
self.network = network
self.egress_network = egress_network
self.image = image
self._repo_root = repo_root
self._name = name
self._db_volume = db_volume
def _resolve_url(self) -> str:
"""The control-plane URL, or "" while the container has no address."""
ip = container_mod.try_container_ipv4_on_network(self._name, self.network)
return f"http://{ip}:{self.port}" if ip else ""
def is_healthy(
self, url: str, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS,
) -> bool:
if not url:
return False
try:
with urllib.request.urlopen(f"{url}/health", timeout=timeout) as resp:
return resp.status == 200
except (urllib.error.URLError, TimeoutError, OSError):
return False
def _source_current(self, current_hash: str) -> bool:
"""True iff the running infra container was created from the current
bind-mounted control-plane source. The control-plane process loads that
code at startup and won't reload it, so a stale container keeps serving
OLD code."""
if not container_mod.container_is_running(self._name):
return False
env = container_mod.container_env(self._name)
if not env:
return True # can't compare → don't churn a working container
return env.get("BOT_BOTTLE_SOURCE_HASH") == current_hash
def _running_healthy_endpoint(self, current_hash: str) -> InfraEndpoint | None:
"""The endpoint if the running container is BOTH source-current and
answering /health, else None (→ recreate). Health, not just the source
label, is what lets a wedged-but-current container self-heal instead of
being polled to death forever."""
if not self._source_current(current_hash):
return None
url = self._resolve_url()
if url and self.is_healthy(url):
return InfraEndpoint(control_plane_url=url, gateway_ip=_ip_of(url))
return None
def ensure_built(self) -> None:
"""Ensure the gateway data-plane image exists. The control-plane source
is bind-mounted, not baked, so only the gateway image needs building."""
container_mod.build_image(
self.image, str(self._repo_root), dockerfile="Dockerfile.gateway",
)
def ensure_running(
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
) -> InfraEndpoint:
"""Ensure the single infra container is up; return how to reach it.
Idempotent per-host singleton — a healthy container on current source
is left untouched, so N launches share the one control plane + gateway.
Raises `OrchestratorStartError` on startup timeout."""
current_hash = source_hash(self._repo_root)
endpoint = self._running_healthy_endpoint(current_hash)
if endpoint is not None:
return endpoint
self.ensure_built()
log.info("starting infra container", context={"name": self._name})
self._run_container(current_hash)
return self._wait_healthy(startup_timeout)
def _run_container(self, current_hash: str) -> None:
ensure_networks(self.network, self.egress_network)
container_mod.force_remove_container(self._name)
argv = [
"container", "run", "--detach",
"--name", self._name,
"--label", "bot-bottle.backend=macos-container",
"--label", INFRA_LABEL,
# NAT network FIRST so the gateway's egress has a default route;
# the host-only network is where agents (and the host CLI) reach it.
"--network", self.egress_network,
"--network", self.network,
"--dns", container_mod.dns_server(),
# Container-only DB volume: one kernel writes bot-bottle.db, never
# shared with the host or another guest.
"--volume", f"{self._db_volume}:{_DB_ROOT_IN_CONTAINER}",
# Bind-mount the control-plane source (read-only); a code change
# takes effect on relaunch with no image rebuild.
"--mount",
container_mod.bind_mount_spec(
str(self._repo_root), _SRC_IN_CONTAINER, readonly=True),
# Baked onto the container so `_source_current` can detect a real
# control-plane code change and recreate.
"--env", f"BOT_BOTTLE_SOURCE_HASH={current_hash}",
"--entrypoint", "sh",
self.image,
"-c", _init_script(self.port),
]
result = container_mod.run_container_argv(argv)
if result.returncode != 0:
raise OrchestratorStartError(
f"infra container failed to start: "
f"{(result.stderr or '').strip() or '<no stderr>'}"
)
def _wait_healthy(self, startup_timeout: float) -> InfraEndpoint:
deadline = time.monotonic() + startup_timeout
while True:
url = self._resolve_url()
if url and self.is_healthy(url):
log.info("infra container healthy", context={"url": url})
return InfraEndpoint(control_plane_url=url, gateway_ip=_ip_of(url))
if time.monotonic() >= deadline:
raise OrchestratorStartError(
f"infra container did not become healthy within "
f"{startup_timeout:g}s"
)
time.sleep(_HEALTH_POLL_SECONDS)
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
"""The gateway's mitmproxy CA (PEM) agents install to trust its TLS
interception. Read out of the container (the CA lives on a
container-internal path, not a host mount); polls because mitmproxy
writes it a beat after start."""
deadline = time.monotonic() + timeout
while True:
result = container_mod.run_container_argv(
["container", "exec", self._name, "cat", GATEWAY_CA_CERT])
if result.returncode == 0 and result.stdout.strip():
return result.stdout
if time.monotonic() >= deadline:
raise GatewayError(
f"gateway CA not available in {self._name} after {timeout:g}s: "
f"{(result.stderr or '').strip() or 'empty'}"
)
time.sleep(_CA_POLL_SECONDS)
def stop(self) -> None:
"""Remove the infra container (idempotent). The DB volume persists."""
container_mod.force_remove_container(self._name)
def _ip_of(url: str) -> str:
"""The host from an http://host:port URL."""
return url.split("://", 1)[-1].rsplit(":", 1)[0]
def probe_control_plane_url(port: int = DEFAULT_PORT) -> str:
"""The running infra container's control-plane URL, or "" if it isn't up.
Used by host-side control-plane discovery (`discover_orchestrator_url`);
safe to call on any host — returns "" when the container or the `container`
CLI isn't present."""
ip = container_mod.try_container_ipv4_on_network(INFRA_NAME, GATEWAY_NETWORK)
return f"http://{ip}:{port}" if ip else ""
__all__ = [
"MacosInfraService",
"InfraEndpoint",
"OrchestratorStartError",
"GatewayError",
"INFRA_NAME",
"INFRA_DB_VOLUME",
]
@@ -1,242 +0,0 @@
"""Orchestrator + gateway lifecycle for the macOS backend (PRD 0070).
The macOS counterpart of `orchestrator.lifecycle.OrchestratorService`. Same
shape — an idempotent singleton that brings up the control plane and the shared
gateway and hands back the control-plane URL — but the startup **order is
reversed**, and the reason is a real Apple Container constraint rather than a
stylistic choice:
- Docker starts the gateway first and lets it find the control plane by
*container name* over docker DNS.
- Apple Container 1.0.0 has **no container DNS** (see `gateway`), so the
gateway can only be handed an **IP**. That IP does not exist until the
orchestrator container is running — hence: orchestrator first, read its
address, then start the gateway pointed at it.
The second difference: no published port. Apple Container puts every container
on a host-reachable address, and the host can reach the host-only network
directly, so the host CLI and the gateway use the **same** URL — the
orchestrator's address on the shared network. Docker needs a
`--publish 127.0.0.1:…` hop plus a separate `internal_url` for the same job.
Like the docker service this runs the **register-only broker**: the backend
launches agent containers, so the control plane needs no privileged socket.
"""
from __future__ import annotations
import time
import urllib.error
import urllib.request
from pathlib import Path
from ... import log
from ...orchestrator.lifecycle import (
DEFAULT_PORT,
DEFAULT_STARTUP_TIMEOUT_SECONDS,
ORCHESTRATOR_DOCKERFILE,
ORCHESTRATOR_IMAGE,
ORCHESTRATOR_SOURCE_HASH_LABEL,
OrchestratorStartError,
source_hash,
)
from ...paths import bot_bottle_root
from . import util as container_mod
from .gateway import (
GATEWAY_EGRESS_NETWORK,
GATEWAY_IMAGE,
GATEWAY_NETWORK,
AppleGateway,
ensure_networks,
)
from .util import bind_mount_spec as _mount
# Distinct from the docker backend's container name so both can run on one host.
ORCHESTRATOR_NAME = "bot-bottle-mac-orchestrator"
_REPO_ROOT = Path(__file__).resolve().parents[3]
_APP_DIR = "/app"
_ROOT_IN_CONTAINER = "/bot-bottle-root"
_HEALTH_POLL_SECONDS = 0.25
_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0
class MacosOrchestratorService:
"""Manages the orchestrator control-plane container + the shared Apple
gateway. Callers only need `ensure_running()`, which returns the URL."""
def __init__(
self,
*,
port: int = DEFAULT_PORT,
network: str = GATEWAY_NETWORK,
egress_network: str = GATEWAY_EGRESS_NETWORK,
image: str = ORCHESTRATOR_IMAGE,
gateway_image: str = GATEWAY_IMAGE,
repo_root: Path = _REPO_ROOT,
host_root: Path | None = None,
orchestrator_name: str = ORCHESTRATOR_NAME,
) -> None:
self.port = port
self.network = network
self.egress_network = egress_network
self.image = image
self._gateway_image = gateway_image
self._repo_root = repo_root
self._host_root = host_root or bot_bottle_root()
self._orchestrator_name = orchestrator_name
def _resolve_url(self) -> str:
"""The control-plane URL, or "" while the container has no address.
Resolved fresh each time (there is no name to fall back on, and a
recreated orchestrator can come back on a different DHCP address), so
nothing caches a URL that could go stale."""
ip = container_mod.try_container_ipv4_on_network(
self._orchestrator_name, self.network,
)
return f"http://{ip}:{self.port}" if ip else ""
def is_healthy(
self, url: str, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS,
) -> bool:
if not url:
return False
try:
with urllib.request.urlopen(f"{url}/health", timeout=timeout) as resp:
return resp.status == 200
except (urllib.error.URLError, TimeoutError, OSError):
return False
def _ensure_orchestrator_image(self) -> None:
"""Build the lean control-plane image when missing. Build-if-missing,
not build-every-time: the control plane bind-mounts its source, so a
code change is caught by the source-hash recreate, not a rebuild."""
if container_mod.image_exists(self.image):
return
container_mod.build_image(
self.image, str(self._repo_root), dockerfile=ORCHESTRATOR_DOCKERFILE,
)
def _orchestrator_source_current(self, current_hash: str) -> bool:
"""True iff the running orchestrator was created from the *current*
bind-mounted source. The process loaded that code at startup and won't
reload it, so a stale container would keep serving OLD control-plane
code."""
if not container_mod.container_is_running(self._orchestrator_name):
return False
data = container_mod.inspect_container(self._orchestrator_name)
config = data.get("configuration")
labels = config.get("labels") if isinstance(config, dict) else None
if not isinstance(labels, dict):
return True # can't compare → don't churn a working container
return labels.get(ORCHESTRATOR_SOURCE_HASH_LABEL) == current_hash
def _run_orchestrator_container(self, current_hash: str) -> None:
# The orchestrator is the first thing on the shared network, so it —
# not the gateway — is what has to bring the network into existence.
ensure_networks(self.network, self.egress_network)
container_mod.force_remove_container(self._orchestrator_name)
argv = [
"container", "run", "--detach",
"--name", self._orchestrator_name,
"--label", "bot-bottle.backend=macos-container",
"--label", f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={current_hash}",
# Host-only network only: the control plane needs no route out, and
# the host reaches it here directly (no --publish needed).
"--network", self.network,
"--mount", _mount(str(self._repo_root), _APP_DIR, readonly=True),
"--workdir", _APP_DIR,
# Persist the registry DB on the host (sole-owner: only the
# orchestrator opens bot-bottle.db).
"--mount", _mount(str(self._host_root), _ROOT_IN_CONTAINER),
"--env", f"BOT_BOTTLE_ROOT={_ROOT_IN_CONTAINER}",
"--entrypoint", "python3",
self.image,
"-m", "bot_bottle.orchestrator",
"--host", "0.0.0.0", "--port", str(self.port), "--broker", "stub",
]
result = container_mod.run_container_argv(argv)
if result.returncode != 0:
raise OrchestratorStartError(
f"orchestrator container failed to start: "
f"{(result.stderr or '').strip() or '<no stderr>'}"
)
def gateway(self, orchestrator_url: str) -> AppleGateway:
"""The shared gateway, pointed at the control plane at
`orchestrator_url` (by IP — Apple has no container DNS)."""
return AppleGateway(
self._gateway_image,
network=self.network,
egress_network=self.egress_network,
orchestrator_url=orchestrator_url,
)
def ensure_running(
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
) -> str:
"""Ensure the control plane + shared gateway are up; return the
control-plane URL. Idempotent — a healthy control plane running current
code and a running gateway are left untouched."""
current_hash = source_hash(self._repo_root)
url = self._running_healthy_url(current_hash)
if not url:
self._ensure_orchestrator_image()
log.info(
"starting orchestrator container",
context={"name": self._orchestrator_name},
)
self._run_orchestrator_container(current_hash)
url = self._wait_healthy(startup_timeout)
# Gateway second: it can only reach the control plane by IP, which does
# not exist until the orchestrator container is up (see the docstring).
gateway = self.gateway(url)
gateway.ensure_built()
gateway.ensure_running()
return url
def _running_healthy_url(self, current_hash: str) -> str:
"""The control-plane URL if the running orchestrator is BOTH current
and answering /health, else "" (→ recreate).
Checking health, not just the source-hash label, is what lets the
service self-heal: a container that is running the current code but
whose HTTP server is wedged (bind failure, deadlock, OOM'd thread)
would otherwise be left alone and polled to death on every launch
forever. Mirrors the docker service's `is_healthy() and source_current`
gate."""
if not self._orchestrator_source_current(current_hash):
return ""
url = self._resolve_url()
return url if url and self.is_healthy(url) else ""
def _wait_healthy(self, startup_timeout: float) -> str:
"""Poll until the control plane answers /health, resolving its address
each time: the container is up before it has an IP, and it has an IP
before the server binds."""
deadline = time.monotonic() + startup_timeout
while True:
url = self._resolve_url()
if url and self.is_healthy(url):
log.info("orchestrator healthy", context={"url": url})
return url
if time.monotonic() >= deadline:
raise OrchestratorStartError(
f"orchestrator did not become healthy within "
f"{startup_timeout:g}s"
)
time.sleep(_HEALTH_POLL_SECONDS)
def stop(self) -> None:
"""Remove the orchestrator + gateway containers (idempotent)."""
container_mod.force_remove_container(self._orchestrator_name)
self.gateway("").stop()
__all__ = [
"MacosOrchestratorService",
"OrchestratorStartError",
"ORCHESTRATOR_NAME",
]
+7
View File
@@ -186,6 +186,13 @@ def discover_orchestrator_url(*, timeout: float = 2.0) -> str:
f"http://{netpool.orch_slot().guest_ip}:{CONTROL_PLANE_PORT}") f"http://{netpool.orch_slot().guest_ip}:{CONTROL_PLANE_PORT}")
except Exception: # noqa: BLE001 — backend optional / not firecracker except Exception: # noqa: BLE001 — backend optional / not firecracker
pass pass
try: # macOS: infra container control plane on its host-only address
from ..backend.macos_container.infra import probe_control_plane_url
url = probe_control_plane_url()
if url:
candidates.append(url)
except Exception: # noqa: BLE001 — backend optional / not macOS
pass
for url in candidates: for url in candidates:
if OrchestratorClient(url, timeout=timeout).health(): if OrchestratorClient(url, timeout=timeout).health():
return url return url
+10 -5
View File
@@ -247,7 +247,7 @@ class TestHasBackend(unittest.TestCase):
class TestEnsureOrchestrator(unittest.TestCase): class TestEnsureOrchestrator(unittest.TestCase):
"""The backend-agnostic orchestrator bring-up entry point. Docker starts """The backend-agnostic orchestrator bring-up entry point. Docker starts
the orchestrator + gateway containers; firecracker boots the infra VM; the orchestrator + gateway containers; firecracker boots the infra VM;
backends without one (macos-container) die with a pointer.""" macos-container starts the infra container."""
def test_docker_delegates_to_orchestrator_service(self): def test_docker_delegates_to_orchestrator_service(self):
b = get_bottle_backend("docker") b = get_bottle_backend("docker")
@@ -272,11 +272,16 @@ class TestEnsureOrchestrator(unittest.TestCase):
url = b.ensure_orchestrator() url = b.ensure_orchestrator()
self.assertEqual(url, "http://10.243.255.1:8099") self.assertEqual(url, "http://10.243.255.1:8099")
def test_macos_default_dies(self): def test_macos_delegates_to_infra_container(self):
from bot_bottle.log import Die
b = get_bottle_backend("macos-container") b = get_bottle_backend("macos-container")
with self.assertRaises(Die): with patch(
b.ensure_orchestrator() "bot_bottle.backend.macos_container.infra.MacosInfraService"
) as service_cls:
service_cls.return_value.ensure_running.return_value.control_plane_url = (
"http://192.168.128.2:8099"
)
url = b.ensure_orchestrator()
self.assertEqual(url, "http://192.168.128.2:8099")
if __name__ == "__main__": if __name__ == "__main__":
+16 -11
View File
@@ -50,30 +50,35 @@ def _client() -> Mock:
class TestEnsureGateway(unittest.TestCase): class TestEnsureGateway(unittest.TestCase):
def _run(self, service: MagicMock) -> GatewayEndpoint: def _run(self, service: MagicMock) -> GatewayEndpoint:
with patch(f"{_MOD}.MacosOrchestratorService", return_value=service): with patch(f"{_MOD}.MacosInfraService", return_value=service):
return ensure_gateway() return ensure_gateway()
def _service(self) -> MagicMock: def _service(self) -> MagicMock:
from bot_bottle.backend.macos_container.infra import InfraEndpoint
service = MagicMock() service = MagicMock()
service.ensure_running.return_value = "http://192.168.128.2:8099" service.ensure_running.return_value = InfraEndpoint(
control_plane_url="http://192.168.128.2:8099",
gateway_ip="192.168.128.2",
)
service.network = "bot-bottle-mac-gateway" service.network = "bot-bottle-mac-gateway"
service.gateway.return_value.ip_on_shared_network.return_value = "192.168.128.3" service.ca_cert_pem.return_value = "PEM"
service.gateway.return_value.ca_cert_pem.return_value = "PEM"
return service return service
def test_reports_gateway_endpoint(self) -> None: def test_reports_gateway_endpoint(self) -> None:
endpoint = self._run(self._service()) endpoint = self._run(self._service())
self.assertEqual("http://192.168.128.2:8099", endpoint.orchestrator_url) self.assertEqual("http://192.168.128.2:8099", endpoint.orchestrator_url)
self.assertEqual("192.168.128.3", endpoint.gateway_ip) self.assertEqual("192.168.128.2", endpoint.gateway_ip)
self.assertEqual("PEM", endpoint.gateway_ca_pem) self.assertEqual("PEM", endpoint.gateway_ca_pem)
self.assertEqual("bot-bottle-mac-gateway", endpoint.network) self.assertEqual("bot-bottle-mac-gateway", endpoint.network)
def test_gateway_is_pointed_at_the_resolved_control_plane(self) -> None: def test_control_plane_and_gateway_share_one_address(self) -> None:
"""Apple has no container DNS, so the gateway must be handed the """One infra container hosts both, so the gateway IP and the
control plane's *resolved URL* rather than a container name.""" control-plane host are the same."""
service = self._service() endpoint = self._run(self._service())
self._run(service) self.assertEqual(
service.gateway.assert_called_with("http://192.168.128.2:8099") endpoint.gateway_ip,
endpoint.orchestrator_url.split("://")[1].split(":")[0],
)
class TestRegisterAgent(unittest.TestCase): class TestRegisterAgent(unittest.TestCase):
+4 -7
View File
@@ -60,13 +60,10 @@ class TestMacosContainerEnumerate(unittest.TestCase):
self.assertEqual(["dev-abc"], [a.slug for a in agents]) self.assertEqual(["dev-abc"], [a.slug for a in agents])
self.assertEqual(["macos-container"], [a.backend_name for a in agents]) self.assertEqual(["macos-container"], [a.backend_name for a in agents])
def test_excludes_the_shared_singletons(self): def test_excludes_the_infra_singleton(self):
"""The gateway and control plane share the bot-bottle- prefix but are """The infra container shares the bot-bottle- prefix but is
infrastructure listing them would invent an agent per host.""" infrastructure listing it would invent an agent per host."""
agents = self._enumerate( agents = self._enumerate("bot-bottle-mac-infra\nbot-bottle-dev-abc\n")
"bot-bottle-mac-gateway\nbot-bottle-mac-orchestrator\n"
"bot-bottle-dev-abc\n"
)
self.assertEqual(["dev-abc"], [a.slug for a in agents]) self.assertEqual(["dev-abc"], [a.slug for a in agents])
def test_empty_when_the_cli_fails(self): def test_empty_when_the_cli_fails(self):
-297
View File
@@ -1,297 +0,0 @@
"""Unit: the Apple gateway + orchestrator lifecycle (PRD 0070)."""
from __future__ import annotations
import unittest
from pathlib import Path
from unittest.mock import Mock, patch
from bot_bottle.backend.macos_container.gateway import AppleGateway, GatewayError
from bot_bottle.backend.macos_container.orchestrator_service import (
MacosOrchestratorService,
OrchestratorStartError,
)
_GW = "bot_bottle.backend.macos_container.gateway"
_ORCH = "bot_bottle.backend.macos_container.orchestrator_service"
def _ok(stdout: str = "") -> Mock:
return Mock(returncode=0, stdout=stdout, stderr="")
def _fail(stderr: str = "boom") -> Mock:
return Mock(returncode=1, stdout="", stderr=stderr)
class TestAppleGatewayRun(unittest.TestCase):
def _argv(self, run: Mock) -> list[str]:
return run.call_args.args[0]
def _start(self, run: Mock) -> None:
with patch(f"{_GW}.container_mod") as mod:
mod.container_is_running.return_value = False
mod.dns_server.return_value = "1.1.1.1"
mod.run_container_argv = run
AppleGateway(orchestrator_url="http://192.168.128.2:8099").ensure_running()
def test_nat_network_precedes_the_host_only_network(self) -> None:
"""Apple Container makes the FIRST --network the default route, so the
NAT network must lead or the gateway has no route out."""
run = Mock(return_value=_ok())
self._start(run)
argv = self._argv(run)
networks = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
self.assertEqual(["bot-bottle-mac-egress", "bot-bottle-mac-gateway"], networks)
def test_control_plane_url_is_passed_for_multi_tenancy(self) -> None:
run = Mock(return_value=_ok())
self._start(run)
self.assertIn(
"BOT_BOTTLE_ORCHESTRATOR_URL=http://192.168.128.2:8099", self._argv(run),
)
def test_dns_is_explicit(self) -> None:
"""The NAT gateway routes but does not resolve."""
run = Mock(return_value=_ok())
self._start(run)
argv = self._argv(run)
self.assertEqual("1.1.1.1", argv[argv.index("--dns") + 1])
def test_start_failure_raises(self) -> None:
with self.assertRaises(GatewayError):
self._start(Mock(return_value=_fail()))
def test_running_current_gateway_is_left_alone(self) -> None:
"""Idempotent singleton: N launches must not restart the gateway and
drop every other bottle's data plane."""
run = Mock(return_value=_ok())
with patch(f"{_GW}.container_mod") as mod:
mod.container_is_running.return_value = True
mod.container_image_digest.return_value = "abc"
mod.image_digest.return_value = "abc"
mod.run_container_argv = run
AppleGateway().ensure_running()
run.assert_not_called()
def test_stale_image_forces_a_recreate(self) -> None:
"""A rebuilt image only takes effect if the running container is
replaced otherwise it keeps serving the OLD daemons."""
run = Mock(return_value=_ok())
with patch(f"{_GW}.container_mod") as mod:
mod.container_is_running.return_value = True
mod.container_image_digest.return_value = "old"
mod.image_digest.return_value = "new"
mod.dns_server.return_value = "1.1.1.1"
mod.run_container_argv = run
AppleGateway().ensure_running()
run.assert_called_once()
def test_unreadable_digest_does_not_churn(self) -> None:
run = Mock(return_value=_ok())
with patch(f"{_GW}.container_mod") as mod:
mod.container_is_running.return_value = True
mod.container_image_digest.return_value = ""
mod.image_digest.return_value = ""
mod.run_container_argv = run
AppleGateway().ensure_running()
run.assert_not_called()
def _start_with_running_env(self, env: dict[str, str], url: str) -> Mock:
run = Mock(return_value=_ok())
with patch(f"{_GW}.container_mod") as mod:
mod.container_is_running.return_value = True
mod.container_image_digest.return_value = "abc"
mod.image_digest.return_value = "abc"
mod.container_env.return_value = env
mod.dns_server.return_value = "1.1.1.1"
mod.run_container_argv = run
AppleGateway(orchestrator_url=url).ensure_running()
return run
def test_moved_control_plane_forces_a_recreate(self) -> None:
"""Docker hands the gateway a container *name*, stable across an
orchestrator recreate. Apple has no DNS, so the URL is an IP baked into
the gateway's env — if the orchestrator comes back on a new address and
the gateway isn't recreated, every /resolve fails and every bottle on
the host loses egress."""
run = self._start_with_running_env(
{"BOT_BOTTLE_ORCHESTRATOR_URL": "http://192.168.128.2:8099"},
"http://192.168.128.7:8099",
)
run.assert_called_once()
self.assertIn(
"BOT_BOTTLE_ORCHESTRATOR_URL=http://192.168.128.7:8099",
run.call_args.args[0],
)
def test_unmoved_control_plane_does_not_churn(self) -> None:
run = self._start_with_running_env(
{"BOT_BOTTLE_ORCHESTRATOR_URL": "http://192.168.128.2:8099"},
"http://192.168.128.2:8099",
)
run.assert_not_called()
def test_unreadable_env_does_not_churn(self) -> None:
run = self._start_with_running_env({}, "http://192.168.128.2:8099")
run.assert_not_called()
class TestMacosOrchestratorService(unittest.TestCase):
def test_orchestrator_starts_before_the_gateway(self) -> None:
"""Apple has no container DNS, so the gateway can only be handed the
control plane's IP — which does not exist until it is running. This
ordering is the whole reason the macOS service diverges from docker's."""
order: list[str] = []
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
gateway = Mock()
gateway.ensure_running.side_effect = lambda: order.append("gateway")
def _record_orchestrator(_hash: str) -> None:
order.append("orchestrator")
with patch(f"{_ORCH}.container_mod") as mod, \
patch(f"{_ORCH}.source_hash", return_value="h1"), \
patch.object(svc, "_run_orchestrator_container",
side_effect=_record_orchestrator), \
patch.object(svc, "gateway", return_value=gateway), \
patch.object(svc, "is_healthy", return_value=True):
mod.container_is_running.return_value = False
mod.image_exists.return_value = True
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
url = svc.ensure_running()
self.assertEqual(["orchestrator", "gateway"], order)
self.assertEqual("http://192.168.128.2:8099", url)
def test_gateway_is_handed_the_resolved_url(self) -> None:
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
with patch(f"{_ORCH}.container_mod") as mod, \
patch(f"{_ORCH}.source_hash", return_value="h1"), \
patch.object(svc, "_run_orchestrator_container"), \
patch.object(svc, "gateway") as gw, \
patch.object(svc, "is_healthy", return_value=True):
mod.container_is_running.return_value = False
mod.image_exists.return_value = True
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
svc.ensure_running()
gw.assert_called_with("http://192.168.128.2:8099")
def test_current_source_leaves_a_healthy_orchestrator_alone(self) -> None:
"""Recreating on every launch would drop every other live bottle's
in-memory egress tokens (#381)."""
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
run = Mock()
with patch(f"{_ORCH}.container_mod") as mod, \
patch(f"{_ORCH}.source_hash", return_value="h1"), \
patch.object(svc, "_run_orchestrator_container", run), \
patch.object(svc, "gateway"), \
patch.object(svc, "is_healthy", return_value=True):
mod.container_is_running.return_value = True
mod.inspect_container.return_value = {
"configuration": {"labels": {"bot-bottle-orchestrator-source-hash": "h1"}}
}
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
svc.ensure_running()
run.assert_not_called()
def test_wedged_but_current_orchestrator_is_recreated(self) -> None:
"""A container running the current code but whose HTTP server is dead
must be recreated, not left alone and polled to death forever. Checking
the source-hash label alone (without health) would strand the host."""
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
run = Mock()
# Unhealthy on the pre-check, healthy once recreated + waited.
health = Mock(side_effect=[False, True])
with patch(f"{_ORCH}.container_mod") as mod, \
patch(f"{_ORCH}.source_hash", return_value="h1"), \
patch.object(svc, "_run_orchestrator_container", run), \
patch.object(svc, "_ensure_orchestrator_image"), \
patch.object(svc, "gateway"), \
patch.object(svc, "is_healthy", health):
mod.container_is_running.return_value = True
mod.inspect_container.return_value = {
"configuration": {"labels": {"bot-bottle-orchestrator-source-hash": "h1"}}
}
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
svc.ensure_running()
run.assert_called_once()
def test_changed_source_recreates_the_orchestrator(self) -> None:
"""The control-plane process loaded its bind-mounted source at startup
and won't reload it."""
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
run = Mock()
with patch(f"{_ORCH}.container_mod") as mod, \
patch(f"{_ORCH}.source_hash", return_value="h2"), \
patch.object(svc, "_run_orchestrator_container", run), \
patch.object(svc, "gateway"), \
patch.object(svc, "is_healthy", return_value=True):
mod.container_is_running.return_value = True
mod.inspect_container.return_value = {
"configuration": {"labels": {"bot-bottle-orchestrator-source-hash": "h1"}}
}
mod.image_exists.return_value = True
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
svc.ensure_running()
run.assert_called_once()
def test_never_healthy_raises(self) -> None:
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
with patch(f"{_ORCH}.container_mod") as mod, \
patch(f"{_ORCH}.source_hash", return_value="h1"), \
patch.object(svc, "_run_orchestrator_container"), \
patch.object(svc, "is_healthy", return_value=False):
mod.container_is_running.return_value = False
mod.image_exists.return_value = True
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
with self.assertRaises(OrchestratorStartError):
svc.ensure_running(startup_timeout=0.01)
def test_control_plane_needs_no_route_out(self) -> None:
"""The orchestrator sits only on the host-only network: the host
reaches it there directly, so there is no --publish and no NAT leg."""
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
run = Mock(return_value=_ok())
# `ensure_networks` lives in the gateway module and shells out to the
# `container` CLI, which does not exist on the Linux CI host — patch it
# here, not gateway.container_mod, since it is called through this
# module's imported name.
with patch(f"{_ORCH}.container_mod") as mod, \
patch(f"{_ORCH}.ensure_networks"):
mod.run_container_argv = run
svc._run_orchestrator_container("h1")
argv = run.call_args.args[0]
networks = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
self.assertEqual(["bot-bottle-mac-gateway"], networks)
self.assertNotIn("--publish", argv)
def test_networks_exist_before_the_orchestrator_runs(self) -> None:
"""The orchestrator is the first container on the shared network, so it
has to create it the gateway that used to do so now starts second."""
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
order: list[str] = []
def _networks(*_args: str) -> None:
order.append("networks")
def _run(*_args: list[str]) -> Mock:
order.append("run")
return _ok()
with patch(f"{_ORCH}.container_mod") as mod, \
patch(f"{_ORCH}.ensure_networks", side_effect=_networks):
mod.run_container_argv = Mock(side_effect=_run)
svc._run_orchestrator_container("h1")
self.assertEqual(["networks", "run"], order)
def test_orchestrator_start_failure_raises(self) -> None:
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
with patch(f"{_ORCH}.container_mod") as mod, \
patch(f"{_ORCH}.ensure_networks"):
mod.run_container_argv = Mock(return_value=_fail())
with self.assertRaises(OrchestratorStartError):
svc._run_orchestrator_container("h1")
if __name__ == "__main__":
unittest.main()
+168
View File
@@ -0,0 +1,168 @@
"""Unit: the single macOS infra container (control plane + gateway, PRD 0070)."""
from __future__ import annotations
import unittest
from pathlib import Path
from unittest.mock import Mock, patch
from bot_bottle.backend.macos_container.infra import (
INFRA_DB_VOLUME,
MacosInfraService,
OrchestratorStartError,
probe_control_plane_url,
)
_INFRA = "bot_bottle.backend.macos_container.infra"
def _ok(stdout: str = "") -> Mock:
return Mock(returncode=0, stdout=stdout, stderr="")
def _fail(stderr: str = "boom") -> Mock:
return Mock(returncode=1, stdout="", stderr=stderr)
class TestInfraRun(unittest.TestCase):
def _run_container(self, svc: MacosInfraService) -> list[str]:
run = Mock(return_value=_ok())
def _spec(src: str, tgt: str, readonly: bool = False) -> str:
return f"type=bind,source={src},target={tgt}" + (
",readonly" if readonly else "")
with patch(f"{_INFRA}.container_mod") as mod, \
patch(f"{_INFRA}.ensure_networks"):
mod.dns_server.return_value = "1.1.1.1"
mod.bind_mount_spec.side_effect = _spec
mod.run_container_argv = run
svc._run_container("h1")
return run.call_args.args[0]
def test_single_container_runs_both_processes(self) -> None:
"""The whole point: one container starts the control plane AND the
gateway daemons, so one kernel owns the DB."""
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
script = argv[-1]
self.assertIn("bot_bottle.orchestrator", script)
self.assertIn("gateway_init.py", script)
self.assertIn("127.0.0.1", script) # they reach each other on loopback
def test_db_is_a_container_only_volume(self) -> None:
"""No host bind-mount of the DB — a named volume only this container
mounts, so the DB is never written by two kernels."""
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
vols = [argv[i + 1] for i, a in enumerate(argv) if a == "--volume"]
self.assertTrue(any(v.startswith(f"{INFRA_DB_VOLUME}:") for v in vols))
# The repo source is bind-mounted read-only; the DB is not a bind mount.
mounts = [argv[i + 1] for i, a in enumerate(argv) if a == "--mount"]
self.assertTrue(all("bot-bottle.db" not in m for m in mounts))
def test_nat_network_precedes_the_host_only_network(self) -> None:
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
nets = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
self.assertEqual(["bot-bottle-mac-egress", "bot-bottle-mac-gateway"], nets)
def test_source_hash_is_labelled_for_recreate(self) -> None:
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
self.assertIn("BOT_BOTTLE_SOURCE_HASH=h1", argv)
def test_start_failure_raises(self) -> None:
svc = MacosInfraService(repo_root=Path("/r"))
with patch(f"{_INFRA}.container_mod") as mod, \
patch(f"{_INFRA}.ensure_networks"):
mod.dns_server.return_value = "1.1.1.1"
mod.bind_mount_spec.return_value = "m"
mod.run_container_argv = Mock(return_value=_fail())
with self.assertRaises(OrchestratorStartError):
svc._run_container("h1")
class TestInfraEnsureRunning(unittest.TestCase):
def test_current_healthy_container_is_left_alone(self) -> None:
"""Idempotent singleton: N launches must not churn the infra container
and drop every live bottle's control plane."""
svc = MacosInfraService(repo_root=Path("/r"))
run = Mock()
with patch(f"{_INFRA}.container_mod") as mod, \
patch(f"{_INFRA}.source_hash", return_value="h1"), \
patch.object(svc, "_run_container", run), \
patch.object(svc, "is_healthy", return_value=True):
mod.container_is_running.return_value = True
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
endpoint = svc.ensure_running()
run.assert_not_called()
self.assertEqual("http://192.168.128.2:8099", endpoint.control_plane_url)
self.assertEqual("192.168.128.2", endpoint.gateway_ip)
def test_changed_source_recreates(self) -> None:
svc = MacosInfraService(repo_root=Path("/r"))
run = Mock()
with patch(f"{_INFRA}.container_mod") as mod, \
patch(f"{_INFRA}.source_hash", return_value="h2"), \
patch.object(svc, "ensure_built"), \
patch.object(svc, "_run_container", run), \
patch.object(svc, "is_healthy", return_value=True):
mod.container_is_running.return_value = True
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
svc.ensure_running()
run.assert_called_once()
def test_wedged_but_current_container_is_recreated(self) -> None:
"""Current source but a dead HTTP server must be recreated, not polled
to death forever health, not just the source label, gates reuse."""
svc = MacosInfraService(repo_root=Path("/r"))
run = Mock()
health = Mock(side_effect=[False, True])
with patch(f"{_INFRA}.container_mod") as mod, \
patch(f"{_INFRA}.source_hash", return_value="h1"), \
patch.object(svc, "ensure_built"), \
patch.object(svc, "_run_container", run), \
patch.object(svc, "is_healthy", health):
mod.container_is_running.return_value = True
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
svc.ensure_running()
run.assert_called_once()
def test_never_healthy_raises(self) -> None:
svc = MacosInfraService(repo_root=Path("/r"))
with patch(f"{_INFRA}.container_mod") as mod, \
patch(f"{_INFRA}.source_hash", return_value="h1"), \
patch.object(svc, "ensure_built"), \
patch.object(svc, "_run_container"), \
patch.object(svc, "is_healthy", return_value=False):
mod.container_is_running.return_value = False
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
with self.assertRaises(OrchestratorStartError):
svc.ensure_running(startup_timeout=0.01)
class TestCaCertPem(unittest.TestCase):
def test_reads_ca_out_of_the_container(self) -> None:
svc = MacosInfraService(repo_root=Path("/r"))
with patch(f"{_INFRA}.container_mod") as mod:
mod.run_container_argv.return_value = _ok("-----BEGIN CERTIFICATE-----\n")
pem = svc.ca_cert_pem()
self.assertTrue(pem.startswith("-----BEGIN CERTIFICATE-----"))
argv = mod.run_container_argv.call_args.args[0]
self.assertEqual(["container", "exec", "bot-bottle-mac-infra", "cat"], argv[:4])
class TestProbeControlPlane(unittest.TestCase):
def test_returns_url_when_running(self) -> None:
with patch(f"{_INFRA}.container_mod") as mod:
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
self.assertEqual("http://192.168.128.2:8099", probe_control_plane_url())
def test_empty_when_absent(self) -> None:
with patch(f"{_INFRA}.container_mod") as mod:
mod.try_container_ipv4_on_network.return_value = ""
self.assertEqual("", probe_control_plane_url())
if __name__ == "__main__":
unittest.main()