refactor(macos): one infra container (control plane + gateway), fixes shared-DB races
Adopts the firecracker infra-VM pattern for macOS: the orchestrator control plane and the gateway data plane now run in a SINGLE Apple container instead of two. Apple Containers are lightweight VMs with separate kernels, so the prior two-container design had both guests writing one bot-bottle.db over virtiofs, where fcntl locks are not coherent across kernels — concurrent writes (the orchestrator's registry vs the gateway supervise daemon's queue) could corrupt it. One container = one kernel = coherent locking. The DB moves onto a container-only Apple volume (bot-bottle-mac-db), never bind-mounted from the host, so no host process opens the live file either. The host CLI already reaches registry + supervise state over the control-plane HTTP surface (cli/supervise.py uses OrchestratorClient), exactly as firecracker's VM-only DB requires. Two simplifications fall out of the single container: - No DNS dance: the control plane and gateway daemons reach each other over 127.0.0.1, so the orchestrator-before-gateway ordering (a workaround for Apple having no container DNS) is gone, along with the moved-IP recreate logic it needed. - Net -243 lines. Mechanics: the infra container runs from the gateway image with the control-plane source bind-mounted read-only (like the docker orchestrator, so a code change needs no rebuild) and a small sh -c init that starts both processes (mirrors firecracker's _infra_init). Also implements the macOS backend's ensure_orchestrator() and adds it to discover_orchestrator_url, so operator tools (supervise) can bring up / find the control plane on demand — previously the macOS backend died with "no orchestrator control plane". Verified end-to-end on real Apple Container 1.0.0: the single infra container comes up healthy (one address for control plane + gateway), both processes run, the DB is written on the container-only volume, host-side supervise works over HTTP, and a registered agent gets 200 for an allowed host / 403 for a denied one. 1824 unit tests pass with `container` absent (CI parity), pyright clean, pylint 9.89. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -89,6 +89,14 @@ class MacosContainerBottleBackend(
|
|||||||
with _launch.launch(plan, provision=self.provision) as bottle:
|
with _launch.launch(plan, provision=self.provision) as bottle:
|
||||||
yield bottle
|
yield bottle
|
||||||
|
|
||||||
|
def ensure_orchestrator(self) -> str:
|
||||||
|
"""Bring up the per-host infra container (control plane + gateway) and
|
||||||
|
return its control-plane URL — the on-demand entry point operator tools
|
||||||
|
(`supervise`) call when no control plane is running yet. Mirrors
|
||||||
|
firecracker's infra-VM bring-up."""
|
||||||
|
from .infra import MacosInfraService
|
||||||
|
return MacosInfraService().ensure_running().control_plane_url
|
||||||
|
|
||||||
def prepare_cleanup(self) -> MacosContainerBottleCleanupPlan:
|
def prepare_cleanup(self) -> MacosContainerBottleCleanupPlan:
|
||||||
return _cleanup.prepare_cleanup()
|
return _cleanup.prepare_cleanup()
|
||||||
|
|
||||||
|
|||||||
@@ -15,6 +15,9 @@ caller has to start the agent in between. `ensure_gateway` runs first because
|
|||||||
the agent's proxy env needs the gateway's address at `container run` time; the
|
the agent's proxy env needs the gateway's address at `container run` time; the
|
||||||
agent's *own* address (the attribution key) only exists afterwards.
|
agent's *own* address (the attribution key) only exists afterwards.
|
||||||
|
|
||||||
|
The control plane and the gateway are one **infra container** here (see
|
||||||
|
`infra`), so `gateway_ip` and the control-plane host are the same address.
|
||||||
|
|
||||||
The consequence for the identity token: it is minted by registration, i.e.
|
The consequence for the identity token: it is minted by registration, i.e.
|
||||||
*after* the agent container exists, so it cannot be baked into the run-time
|
*after* the agent container exists, so it cannot be baked into the run-time
|
||||||
env the way docker's compose spec does. It is delivered at `container exec`
|
env the way docker's compose spec does. It is delivered at `container exec`
|
||||||
@@ -38,7 +41,7 @@ from ...orchestrator.registration import registration_inputs
|
|||||||
from ..docker.gateway_provision import deprovision_git_gate, provision_git_gate
|
from ..docker.gateway_provision import deprovision_git_gate, provision_git_gate
|
||||||
from .gateway import GATEWAY_NETWORK
|
from .gateway import GATEWAY_NETWORK
|
||||||
from .gateway_provision import AppleGatewayTransport
|
from .gateway_provision import AppleGatewayTransport
|
||||||
from .orchestrator_service import MacosOrchestratorService, OrchestratorStartError
|
from .infra import MacosInfraService, OrchestratorStartError
|
||||||
|
|
||||||
|
|
||||||
class ConsolidatedLaunchError(RuntimeError):
|
class ConsolidatedLaunchError(RuntimeError):
|
||||||
@@ -47,7 +50,9 @@ class ConsolidatedLaunchError(RuntimeError):
|
|||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class GatewayEndpoint:
|
class GatewayEndpoint:
|
||||||
"""What the agent `container run` needs to reach the shared gateway."""
|
"""What the agent `container run` needs to reach the shared gateway (the
|
||||||
|
infra container). `gateway_ip` is that container's host-only address, the
|
||||||
|
same host the control-plane URL points at."""
|
||||||
|
|
||||||
orchestrator_url: str
|
orchestrator_url: str
|
||||||
gateway_ip: str # the gateway's address — the agent's proxy target
|
gateway_ip: str # the gateway's address — the agent's proxy target
|
||||||
@@ -68,19 +73,18 @@ class LaunchContext:
|
|||||||
|
|
||||||
|
|
||||||
def ensure_gateway(
|
def ensure_gateway(
|
||||||
*, service: MacosOrchestratorService | None = None,
|
*, service: MacosInfraService | None = None,
|
||||||
) -> GatewayEndpoint:
|
) -> GatewayEndpoint:
|
||||||
"""Ensure the orchestrator control plane + shared gateway are up, and
|
"""Ensure the per-host infra container (control plane + gateway) is up and
|
||||||
report how to reach them. Idempotent — both are per-host singletons, so N
|
report how to reach it. Idempotent — one singleton, so N bottle launches
|
||||||
bottle launches share the one pair. Call before starting the agent
|
share it. Call before starting the agent container: the agent's proxy env
|
||||||
container: the agent's proxy env needs `gateway_ip` at run time."""
|
needs `gateway_ip` at run time."""
|
||||||
service = service or MacosOrchestratorService()
|
service = service or MacosInfraService()
|
||||||
url = service.ensure_running()
|
infra = service.ensure_running()
|
||||||
gateway = service.gateway(url)
|
|
||||||
return GatewayEndpoint(
|
return GatewayEndpoint(
|
||||||
orchestrator_url=url,
|
orchestrator_url=infra.control_plane_url,
|
||||||
gateway_ip=gateway.ip_on_shared_network(),
|
gateway_ip=infra.gateway_ip,
|
||||||
gateway_ca_pem=gateway.ca_cert_pem(),
|
gateway_ca_pem=service.ca_cert_pem(),
|
||||||
network=service.network,
|
network=service.network,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -6,14 +6,13 @@ import subprocess
|
|||||||
|
|
||||||
from ...bottle_state import read_metadata
|
from ...bottle_state import read_metadata
|
||||||
from .. import ActiveAgent
|
from .. import ActiveAgent
|
||||||
from .gateway import GATEWAY_NAME
|
from .infra import INFRA_NAME
|
||||||
from .orchestrator_service import ORCHESTRATOR_NAME
|
|
||||||
|
|
||||||
_PREFIX = "bot-bottle-"
|
_PREFIX = "bot-bottle-"
|
||||||
# The shared per-host singletons carry the same prefix as agent containers but
|
# The shared per-host infra container carries the same prefix as agent
|
||||||
# are infrastructure, not bottles — one gateway and one control plane serve
|
# containers but is infrastructure, not a bottle — one control plane + gateway
|
||||||
# every agent, so listing them as agents would invent one per host.
|
# serves every agent, so listing it as an agent would invent one per host.
|
||||||
_INFRA_NAMES = frozenset({GATEWAY_NAME, ORCHESTRATOR_NAME})
|
_INFRA_NAMES = frozenset({INFRA_NAME})
|
||||||
|
|
||||||
|
|
||||||
def enumerate_active() -> list[ActiveAgent]:
|
def enumerate_active() -> list[ActiveAgent]:
|
||||||
|
|||||||
@@ -1,235 +1,46 @@
|
|||||||
"""The consolidated per-host gateway as an Apple container (PRD 0070).
|
"""Shared network/image constants for the macOS consolidated infra container.
|
||||||
|
|
||||||
The macOS counterpart of `orchestrator.gateway.DockerGateway`: one persistent
|
The gateway data plane no longer runs as its own Apple container — it shares a
|
||||||
gateway per host, shared by every bottle, attributing each request to a bottle
|
single per-host **infra container** with the control plane (see `infra`),
|
||||||
by its source IP on the shared host-only network.
|
because two Apple-Container guests writing one `bot-bottle.db` over virtiofs
|
||||||
|
would race incoherent `fcntl` locks. This module holds the pieces both the
|
||||||
Two Apple Container 1.0.0 constraints shape this and make it *not* a
|
infra service and the launch/provision glue need: the network names, the
|
||||||
transliteration of the docker gateway:
|
gateway image, and the network-creation helper.
|
||||||
|
|
||||||
- **No container DNS.** Containers cannot resolve each other by name (the
|
|
||||||
host-only network's resolver refuses the query), so the gateway reaches the
|
|
||||||
control plane by **IP**, not by name as the docker gateway does. The
|
|
||||||
orchestrator must therefore be started *before* the gateway — see
|
|
||||||
`orchestrator_service`.
|
|
||||||
- **Networks are fixed at `container run`.** There is no `network connect`,
|
|
||||||
so a network cannot be attached to a running container. The gateway must sit
|
|
||||||
on one shared, up-front network for the lifetime of the process; per-bottle
|
|
||||||
networks would mean restarting the gateway on every launch, which defeats the
|
|
||||||
consolidation.
|
|
||||||
|
|
||||||
The gateway is dual-homed, **NAT network first**: Apple Container makes the
|
|
||||||
first `--network` the default route, so the egress network must lead or the
|
|
||||||
gateway has no route to the internet.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import time
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from ...orchestrator.gateway import (
|
from ...orchestrator.gateway import GatewayError
|
||||||
GATEWAY_CA_CERT,
|
|
||||||
GATEWAY_DOCKERFILE,
|
|
||||||
MITMPROXY_HOME,
|
|
||||||
Gateway,
|
|
||||||
GatewayError,
|
|
||||||
)
|
|
||||||
from ...paths import host_db_dir
|
|
||||||
from ...supervise import DB_PATH_IN_CONTAINER
|
|
||||||
from . import util as container_mod
|
from . import util as container_mod
|
||||||
from .util import bind_mount_spec as _mount
|
|
||||||
|
|
||||||
# Distinct from the docker gateway's names so both backends' gateways can
|
# The shared host-only network the infra container and every agent bottle sit
|
||||||
# coexist on one host (a macOS host can run the docker backend too).
|
# on. The agent's address here is the attribution key. Distinct from the docker
|
||||||
GATEWAY_NAME = "bot-bottle-mac-gateway"
|
# names so both backends can coexist on one host.
|
||||||
# The shared host-only network the gateway and every agent bottle sit on. The
|
|
||||||
# agent's address here is the attribution key.
|
|
||||||
GATEWAY_NETWORK = "bot-bottle-mac-gateway"
|
GATEWAY_NETWORK = "bot-bottle-mac-gateway"
|
||||||
# The NAT network that gives the gateway (and only the gateway) a route out.
|
# The NAT network that gives the infra container (and only it) a route out.
|
||||||
GATEWAY_EGRESS_NETWORK = "bot-bottle-mac-egress"
|
GATEWAY_EGRESS_NETWORK = "bot-bottle-mac-egress"
|
||||||
|
|
||||||
GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest")
|
GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest")
|
||||||
|
|
||||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
||||||
|
|
||||||
_SUPERVISE_DB_DIR_IN_CONTAINER = os.path.dirname(DB_PATH_IN_CONTAINER)
|
|
||||||
|
|
||||||
# mitmproxy writes its CA a beat after start; reads poll rather than assume.
|
|
||||||
_CA_POLL_SECONDS = 0.5
|
|
||||||
DEFAULT_CA_TIMEOUT_SECONDS = 30.0
|
DEFAULT_CA_TIMEOUT_SECONDS = 30.0
|
||||||
|
|
||||||
|
|
||||||
def gateway_ca_dir() -> Path:
|
|
||||||
"""Host dir bind-mounted as mitmproxy's home, keeping the gateway's
|
|
||||||
self-generated CA **stable across container recreation** — every agent
|
|
||||||
installs this one CA to trust the shared gateway's TLS interception, so it
|
|
||||||
must not rotate when the gateway restarts.
|
|
||||||
|
|
||||||
The docker gateway uses a named volume for this; a plain host dir is the
|
|
||||||
same guarantee with fewer moving parts, and it lets `ca_cert_pem` read the
|
|
||||||
PEM straight off the host instead of shelling into the container."""
|
|
||||||
path = host_db_dir() / "mac-gateway-ca"
|
|
||||||
path.mkdir(parents=True, exist_ok=True)
|
|
||||||
return path
|
|
||||||
|
|
||||||
|
|
||||||
def ensure_networks(
|
def ensure_networks(
|
||||||
network: str = GATEWAY_NETWORK, egress_network: str = GATEWAY_EGRESS_NETWORK,
|
network: str = GATEWAY_NETWORK, egress_network: str = GATEWAY_EGRESS_NETWORK,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Create the shared host-only network + the gateway's NAT network.
|
"""Create the shared host-only network + the NAT egress network. Idempotent
|
||||||
Idempotent — `create_network` tolerates 'already exists'.
|
— `create_network` tolerates 'already exists'."""
|
||||||
|
|
||||||
Module-level rather than a gateway method because the **orchestrator**
|
|
||||||
needs the shared network too, and it starts first (Apple has no container
|
|
||||||
DNS, so the gateway must be handed the control plane's IP). Both callers
|
|
||||||
ensure the networks; whoever runs first wins."""
|
|
||||||
container_mod.create_network(egress_network)
|
container_mod.create_network(egress_network)
|
||||||
container_mod.create_network(network, internal=True)
|
container_mod.create_network(network, internal=True)
|
||||||
|
|
||||||
|
|
||||||
class AppleGateway(Gateway):
|
|
||||||
"""The consolidated gateway as a single, fixed-name Apple container."""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
image_ref: str = GATEWAY_IMAGE,
|
|
||||||
*,
|
|
||||||
name: str = GATEWAY_NAME,
|
|
||||||
network: str = GATEWAY_NETWORK,
|
|
||||||
egress_network: str = GATEWAY_EGRESS_NETWORK,
|
|
||||||
orchestrator_url: str = "",
|
|
||||||
build_context: Path | None = None,
|
|
||||||
dockerfile: str | None = GATEWAY_DOCKERFILE,
|
|
||||||
) -> None:
|
|
||||||
self.image_ref = image_ref
|
|
||||||
self.name = name
|
|
||||||
self.network = network
|
|
||||||
self.egress_network = egress_network
|
|
||||||
# Reached by IP (no container DNS on Apple) — the caller resolves the
|
|
||||||
# orchestrator's address before constructing this. Empty → single-tenant.
|
|
||||||
self._orchestrator_url = orchestrator_url
|
|
||||||
self._build_context = build_context or _REPO_ROOT
|
|
||||||
self._dockerfile = dockerfile
|
|
||||||
|
|
||||||
def ensure_built(self) -> None:
|
|
||||||
"""Build the gateway data-plane image from its Dockerfile. Builds every
|
|
||||||
time (cache-aware, so it's cheap when nothing changed): a stale image
|
|
||||||
silently runs the OLD single-tenant daemons. Mirrors `DockerGateway`."""
|
|
||||||
if self._dockerfile is None:
|
|
||||||
return
|
|
||||||
container_mod.build_image(
|
|
||||||
self.image_ref, str(self._build_context), dockerfile=self._dockerfile,
|
|
||||||
)
|
|
||||||
|
|
||||||
def is_running(self) -> bool:
|
|
||||||
return container_mod.container_is_running(self.name)
|
|
||||||
|
|
||||||
def _running_image_is_current(self) -> bool:
|
|
||||||
"""True iff the running gateway was created from the *current*
|
|
||||||
`image_ref`. `ensure_built` rebuilding the image is not enough on its
|
|
||||||
own — the running container still holds the OLD image, so this
|
|
||||||
mismatch check is what makes a rebuild take effect."""
|
|
||||||
running = container_mod.container_image_digest(self.name)
|
|
||||||
current = container_mod.image_digest(self.image_ref)
|
|
||||||
if not running or not current:
|
|
||||||
return True # can't compare → don't churn a working container
|
|
||||||
return running == current
|
|
||||||
|
|
||||||
def _running_control_plane_is_current(self) -> bool:
|
|
||||||
"""True iff the running gateway points at the control plane we would
|
|
||||||
pass today.
|
|
||||||
|
|
||||||
Docker gets this for free — it hands the gateway a container *name*,
|
|
||||||
which survives the orchestrator being recreated. Apple has no container
|
|
||||||
DNS, so the URL is an **IP baked into the gateway's env at run time**,
|
|
||||||
and a recreated orchestrator can come back on a different DHCP address.
|
|
||||||
Without this check the gateway would keep pointing at the old address
|
|
||||||
and every `/resolve` would fail — denying egress for *every* bottle on
|
|
||||||
the host until something else happened to recreate the gateway."""
|
|
||||||
if not self._orchestrator_url:
|
|
||||||
return True
|
|
||||||
env = container_mod.container_env(self.name)
|
|
||||||
if not env:
|
|
||||||
return True # can't compare → don't churn a working container
|
|
||||||
return env.get("BOT_BOTTLE_ORCHESTRATOR_URL") == self._orchestrator_url
|
|
||||||
|
|
||||||
def ensure_running(self) -> None:
|
|
||||||
if (self.is_running()
|
|
||||||
and self._running_image_is_current()
|
|
||||||
and self._running_control_plane_is_current()):
|
|
||||||
return
|
|
||||||
ensure_networks(self.network, self.egress_network)
|
|
||||||
container_mod.force_remove_container(self.name)
|
|
||||||
argv = [
|
|
||||||
"container", "run", "--detach",
|
|
||||||
"--name", self.name,
|
|
||||||
"--label", "bot-bottle.backend=macos-container",
|
|
||||||
"--label", "bot-bottle-mac-gateway=1",
|
|
||||||
# NAT network FIRST: Apple Container takes the first --network as
|
|
||||||
# the default route, so this ordering is what gives the gateway a
|
|
||||||
# route out. Reversing it silently blackholes egress.
|
|
||||||
"--network", self.egress_network,
|
|
||||||
"--network", self.network,
|
|
||||||
# The NAT gateway routes but does not resolve, so DNS is explicit.
|
|
||||||
"--dns", container_mod.dns_server(),
|
|
||||||
"--mount", _mount(str(gateway_ca_dir()), MITMPROXY_HOME),
|
|
||||||
"--mount", _mount(str(host_db_dir()), _SUPERVISE_DB_DIR_IN_CONTAINER),
|
|
||||||
"--env", f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}",
|
|
||||||
]
|
|
||||||
if self._orchestrator_url:
|
|
||||||
# Makes the data plane multi-tenant: each request resolves
|
|
||||||
# source-IP → policy against the control plane.
|
|
||||||
argv += ["--env", f"BOT_BOTTLE_ORCHESTRATOR_URL={self._orchestrator_url}"]
|
|
||||||
argv.append(self.image_ref)
|
|
||||||
result = container_mod.run_container_argv(argv)
|
|
||||||
if result.returncode != 0:
|
|
||||||
raise GatewayError(
|
|
||||||
f"gateway failed to start: "
|
|
||||||
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
|
||||||
)
|
|
||||||
|
|
||||||
def ip_on_shared_network(self) -> str:
|
|
||||||
"""The gateway's address on the shared host-only network — what agents
|
|
||||||
point their proxy / git-http / supervise URLs at. Polls: a freshly
|
|
||||||
run gateway can be up before vmnet's DHCP has assigned the address."""
|
|
||||||
ip = container_mod.wait_container_ipv4_on_network(self.name, self.network)
|
|
||||||
if not ip:
|
|
||||||
raise GatewayError(
|
|
||||||
f"gateway {self.name} never got an address on {self.network}"
|
|
||||||
)
|
|
||||||
return ip
|
|
||||||
|
|
||||||
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
|
|
||||||
"""The gateway's CA certificate (PEM) that agents install to trust its
|
|
||||||
TLS interception. Polls: mitmproxy generates it a moment after start.
|
|
||||||
Read from the host bind-mount, so no exec into the container."""
|
|
||||||
ca_path = gateway_ca_dir() / os.path.basename(GATEWAY_CA_CERT)
|
|
||||||
deadline = time.monotonic() + timeout
|
|
||||||
while True:
|
|
||||||
try:
|
|
||||||
pem = ca_path.read_text()
|
|
||||||
if pem.strip():
|
|
||||||
return pem
|
|
||||||
except OSError:
|
|
||||||
pass
|
|
||||||
if time.monotonic() >= deadline:
|
|
||||||
raise GatewayError(
|
|
||||||
f"gateway CA cert not available at {ca_path} after {timeout:g}s"
|
|
||||||
)
|
|
||||||
time.sleep(_CA_POLL_SECONDS)
|
|
||||||
|
|
||||||
def stop(self) -> None:
|
|
||||||
container_mod.force_remove_container(self.name)
|
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"AppleGateway",
|
|
||||||
"GatewayError",
|
|
||||||
"GATEWAY_NAME",
|
|
||||||
"GATEWAY_NETWORK",
|
"GATEWAY_NETWORK",
|
||||||
"GATEWAY_EGRESS_NETWORK",
|
"GATEWAY_EGRESS_NETWORK",
|
||||||
"GATEWAY_IMAGE",
|
"GATEWAY_IMAGE",
|
||||||
"gateway_ca_dir",
|
"GatewayError",
|
||||||
|
"DEFAULT_CA_TIMEOUT_SECONDS",
|
||||||
|
"ensure_networks",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -1,23 +1,23 @@
|
|||||||
"""`GatewayTransport` for the Apple gateway container (PRD 0070).
|
"""`GatewayTransport` for the Apple infra container (PRD 0070).
|
||||||
|
|
||||||
The provisioning *logic* (per-bottle creds dirs, namespaced repo init) is
|
The provisioning *logic* (per-bottle creds dirs, namespaced repo init) is
|
||||||
backend-neutral and lives in `backend.docker.gateway_provision`; this is only
|
backend-neutral and lives in `backend.docker.gateway_provision`; this is only
|
||||||
the transport — how files and commands reach the running gateway. Docker uses
|
the transport — how files and commands reach the running gateway. Docker uses
|
||||||
`docker exec`/`docker cp` and Firecracker uses SSH; Apple uses the `container`
|
`docker exec`/`docker cp` and Firecracker uses SSH; Apple uses the `container`
|
||||||
CLI's equivalents.
|
CLI's equivalents against the infra container that hosts the gateway daemons.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from ..docker.gateway_provision import GatewayProvisionError
|
from ..docker.gateway_provision import GatewayProvisionError
|
||||||
from . import util as container_mod
|
from . import util as container_mod
|
||||||
from .gateway import GATEWAY_NAME
|
from .infra import INFRA_NAME
|
||||||
|
|
||||||
|
|
||||||
class AppleGatewayTransport:
|
class AppleGatewayTransport:
|
||||||
"""`GatewayTransport` for the gateway as an Apple container."""
|
"""`GatewayTransport` for the gateway daemons in the Apple infra container."""
|
||||||
|
|
||||||
def __init__(self, gateway: str = GATEWAY_NAME) -> None:
|
def __init__(self, gateway: str = INFRA_NAME) -> None:
|
||||||
self.gateway = gateway
|
self.gateway = gateway
|
||||||
|
|
||||||
def exec(self, argv: list[str]) -> None:
|
def exec(self, argv: list[str]) -> None:
|
||||||
|
|||||||
@@ -0,0 +1,292 @@
|
|||||||
|
"""The per-host infra container for the macOS backend (PRD 0070).
|
||||||
|
|
||||||
|
A single persistent Apple container that runs BOTH the orchestrator control
|
||||||
|
plane and the gateway data plane — the macOS analogue of the Firecracker infra
|
||||||
|
VM (`backend/firecracker/infra_vm.py`), not the docker backend's two separate
|
||||||
|
containers.
|
||||||
|
|
||||||
|
Why one container, not two: Apple Containers are lightweight VMs, each with its
|
||||||
|
own kernel. The docker backend runs the orchestrator and gateway as two
|
||||||
|
containers safely because they share the host kernel, so their concurrent
|
||||||
|
writes to the one `bot-bottle.db` (the orchestrator's registry + the gateway
|
||||||
|
supervise daemon's queue) are serialized by coherent `fcntl` locks. Across two
|
||||||
|
*guest* kernels sharing a virtiofs-mounted DB those locks are not coherent, and
|
||||||
|
concurrent writers can corrupt the file. Firecracker solved this by putting
|
||||||
|
both services in one guest with the DB on a device only that guest mounts; this
|
||||||
|
does the same with Apple primitives.
|
||||||
|
|
||||||
|
Two consequences fall out of the single container, both simplifications:
|
||||||
|
|
||||||
|
- **No DNS dance.** The control plane and the gateway daemons reach each other
|
||||||
|
over `127.0.0.1`, so nothing depends on Apple's (absent) container DNS and
|
||||||
|
there is no orchestrator-before-gateway ordering to get right.
|
||||||
|
- **The DB is never host-shared.** It lives on a container-only volume, so no
|
||||||
|
host process opens the live file. The host CLI reaches registry + supervise
|
||||||
|
state through the control-plane HTTP surface (`cli/supervise.py` already uses
|
||||||
|
`OrchestratorClient`), exactly as it does for firecracker.
|
||||||
|
|
||||||
|
The control-plane source is bind-mounted (like the docker orchestrator), so a
|
||||||
|
code change takes effect on the next launch without an image rebuild; the
|
||||||
|
gateway daemons are baked in the gateway image and rebuild through its own
|
||||||
|
digest check.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ... import log
|
||||||
|
from ...orchestrator.gateway import GATEWAY_CA_CERT
|
||||||
|
from ...orchestrator.lifecycle import (
|
||||||
|
DEFAULT_PORT,
|
||||||
|
DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||||
|
OrchestratorStartError,
|
||||||
|
source_hash,
|
||||||
|
)
|
||||||
|
from ...paths import HOST_DB_FILENAME
|
||||||
|
from . import util as container_mod
|
||||||
|
from .gateway import (
|
||||||
|
DEFAULT_CA_TIMEOUT_SECONDS,
|
||||||
|
GATEWAY_EGRESS_NETWORK,
|
||||||
|
GATEWAY_IMAGE,
|
||||||
|
GATEWAY_NETWORK,
|
||||||
|
GatewayError,
|
||||||
|
ensure_networks,
|
||||||
|
)
|
||||||
|
|
||||||
|
# The one per-host infra container: control plane + gateway data plane.
|
||||||
|
INFRA_NAME = "bot-bottle-mac-infra"
|
||||||
|
INFRA_LABEL = "bot-bottle-mac-infra=1"
|
||||||
|
# Container-only volume holding bot-bottle.db. No host bind-mount, so the DB is
|
||||||
|
# written by exactly one kernel (this container's). Survives recreation.
|
||||||
|
INFRA_DB_VOLUME = "bot-bottle-mac-db"
|
||||||
|
|
||||||
|
# BOT_BOTTLE_ROOT inside the container; host_db_path() resolves the DB to
|
||||||
|
# <root>/db/<filename> and the supervise daemon writes the same file.
|
||||||
|
_DB_ROOT_IN_CONTAINER = "/var/lib/bot-bottle"
|
||||||
|
_DB_PATH_IN_CONTAINER = f"{_DB_ROOT_IN_CONTAINER}/db/{HOST_DB_FILENAME}"
|
||||||
|
_SRC_IN_CONTAINER = "/bot-bottle-src"
|
||||||
|
|
||||||
|
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||||
|
|
||||||
|
_HEALTH_POLL_SECONDS = 0.25
|
||||||
|
_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0
|
||||||
|
_CA_POLL_SECONDS = 0.5
|
||||||
|
|
||||||
|
# The gateway subset the consolidated model runs (no per-bottle git:// daemon).
|
||||||
|
_GATEWAY_DAEMONS = "egress,git-http,supervise"
|
||||||
|
|
||||||
|
|
||||||
|
def _init_script(port: int) -> str:
|
||||||
|
"""PID-1 init: start the control plane and the gateway daemons, both in
|
||||||
|
this container, reaching each other over loopback. Backgrounded so `wait`
|
||||||
|
reaps as PID 1. No `set -e` — a transient daemon failure must not kill the
|
||||||
|
whole container (gateway_init applies the same 'stay up' policy)."""
|
||||||
|
return (
|
||||||
|
"export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\n"
|
||||||
|
f"mkdir -p $(dirname {_DB_PATH_IN_CONTAINER})\n"
|
||||||
|
# Control plane, from the bind-mounted source (stdlib-only package).
|
||||||
|
f"( cd {_SRC_IN_CONTAINER} && BOT_BOTTLE_ROOT={_DB_ROOT_IN_CONTAINER} "
|
||||||
|
f"python3 -m bot_bottle.orchestrator --host 0.0.0.0 --port {port} "
|
||||||
|
"--broker stub ) &\n"
|
||||||
|
# Gateway data plane, multi-tenant against the local control plane.
|
||||||
|
f"( cd /app && BOT_BOTTLE_GATEWAY_DAEMONS={_GATEWAY_DAEMONS} "
|
||||||
|
f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{port} "
|
||||||
|
f"SUPERVISE_DB_PATH={_DB_PATH_IN_CONTAINER} python3 /app/gateway_init.py ) &\n"
|
||||||
|
"while : ; do wait ; done\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class InfraEndpoint:
|
||||||
|
"""How to reach the running infra container. The control plane and the
|
||||||
|
gateway are the same container, so one address serves both."""
|
||||||
|
|
||||||
|
control_plane_url: str # http://<infra ip>:8099 — host CLI + registration
|
||||||
|
gateway_ip: str # same container; agents' proxy / git-http / MCP target
|
||||||
|
|
||||||
|
|
||||||
|
class MacosInfraService:
|
||||||
|
"""Manages the single per-host infra container. Callers use
|
||||||
|
`ensure_running()` (returns the endpoint) and `ca_cert_pem()`."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
port: int = DEFAULT_PORT,
|
||||||
|
network: str = GATEWAY_NETWORK,
|
||||||
|
egress_network: str = GATEWAY_EGRESS_NETWORK,
|
||||||
|
image: str = GATEWAY_IMAGE,
|
||||||
|
repo_root: Path = _REPO_ROOT,
|
||||||
|
name: str = INFRA_NAME,
|
||||||
|
db_volume: str = INFRA_DB_VOLUME,
|
||||||
|
) -> None:
|
||||||
|
self.port = port
|
||||||
|
self.network = network
|
||||||
|
self.egress_network = egress_network
|
||||||
|
self.image = image
|
||||||
|
self._repo_root = repo_root
|
||||||
|
self._name = name
|
||||||
|
self._db_volume = db_volume
|
||||||
|
|
||||||
|
def _resolve_url(self) -> str:
|
||||||
|
"""The control-plane URL, or "" while the container has no address."""
|
||||||
|
ip = container_mod.try_container_ipv4_on_network(self._name, self.network)
|
||||||
|
return f"http://{ip}:{self.port}" if ip else ""
|
||||||
|
|
||||||
|
def is_healthy(
|
||||||
|
self, url: str, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS,
|
||||||
|
) -> bool:
|
||||||
|
if not url:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(f"{url}/health", timeout=timeout) as resp:
|
||||||
|
return resp.status == 200
|
||||||
|
except (urllib.error.URLError, TimeoutError, OSError):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _source_current(self, current_hash: str) -> bool:
|
||||||
|
"""True iff the running infra container was created from the current
|
||||||
|
bind-mounted control-plane source. The control-plane process loads that
|
||||||
|
code at startup and won't reload it, so a stale container keeps serving
|
||||||
|
OLD code."""
|
||||||
|
if not container_mod.container_is_running(self._name):
|
||||||
|
return False
|
||||||
|
env = container_mod.container_env(self._name)
|
||||||
|
if not env:
|
||||||
|
return True # can't compare → don't churn a working container
|
||||||
|
return env.get("BOT_BOTTLE_SOURCE_HASH") == current_hash
|
||||||
|
|
||||||
|
def _running_healthy_endpoint(self, current_hash: str) -> InfraEndpoint | None:
|
||||||
|
"""The endpoint if the running container is BOTH source-current and
|
||||||
|
answering /health, else None (→ recreate). Health, not just the source
|
||||||
|
label, is what lets a wedged-but-current container self-heal instead of
|
||||||
|
being polled to death forever."""
|
||||||
|
if not self._source_current(current_hash):
|
||||||
|
return None
|
||||||
|
url = self._resolve_url()
|
||||||
|
if url and self.is_healthy(url):
|
||||||
|
return InfraEndpoint(control_plane_url=url, gateway_ip=_ip_of(url))
|
||||||
|
return None
|
||||||
|
|
||||||
|
def ensure_built(self) -> None:
|
||||||
|
"""Ensure the gateway data-plane image exists. The control-plane source
|
||||||
|
is bind-mounted, not baked, so only the gateway image needs building."""
|
||||||
|
container_mod.build_image(
|
||||||
|
self.image, str(self._repo_root), dockerfile="Dockerfile.gateway",
|
||||||
|
)
|
||||||
|
|
||||||
|
def ensure_running(
|
||||||
|
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||||
|
) -> InfraEndpoint:
|
||||||
|
"""Ensure the single infra container is up; return how to reach it.
|
||||||
|
Idempotent per-host singleton — a healthy container on current source
|
||||||
|
is left untouched, so N launches share the one control plane + gateway.
|
||||||
|
Raises `OrchestratorStartError` on startup timeout."""
|
||||||
|
current_hash = source_hash(self._repo_root)
|
||||||
|
endpoint = self._running_healthy_endpoint(current_hash)
|
||||||
|
if endpoint is not None:
|
||||||
|
return endpoint
|
||||||
|
self.ensure_built()
|
||||||
|
log.info("starting infra container", context={"name": self._name})
|
||||||
|
self._run_container(current_hash)
|
||||||
|
return self._wait_healthy(startup_timeout)
|
||||||
|
|
||||||
|
def _run_container(self, current_hash: str) -> None:
|
||||||
|
ensure_networks(self.network, self.egress_network)
|
||||||
|
container_mod.force_remove_container(self._name)
|
||||||
|
argv = [
|
||||||
|
"container", "run", "--detach",
|
||||||
|
"--name", self._name,
|
||||||
|
"--label", "bot-bottle.backend=macos-container",
|
||||||
|
"--label", INFRA_LABEL,
|
||||||
|
# NAT network FIRST so the gateway's egress has a default route;
|
||||||
|
# the host-only network is where agents (and the host CLI) reach it.
|
||||||
|
"--network", self.egress_network,
|
||||||
|
"--network", self.network,
|
||||||
|
"--dns", container_mod.dns_server(),
|
||||||
|
# Container-only DB volume: one kernel writes bot-bottle.db, never
|
||||||
|
# shared with the host or another guest.
|
||||||
|
"--volume", f"{self._db_volume}:{_DB_ROOT_IN_CONTAINER}",
|
||||||
|
# Bind-mount the control-plane source (read-only); a code change
|
||||||
|
# takes effect on relaunch with no image rebuild.
|
||||||
|
"--mount",
|
||||||
|
container_mod.bind_mount_spec(
|
||||||
|
str(self._repo_root), _SRC_IN_CONTAINER, readonly=True),
|
||||||
|
# Baked onto the container so `_source_current` can detect a real
|
||||||
|
# control-plane code change and recreate.
|
||||||
|
"--env", f"BOT_BOTTLE_SOURCE_HASH={current_hash}",
|
||||||
|
"--entrypoint", "sh",
|
||||||
|
self.image,
|
||||||
|
"-c", _init_script(self.port),
|
||||||
|
]
|
||||||
|
result = container_mod.run_container_argv(argv)
|
||||||
|
if result.returncode != 0:
|
||||||
|
raise OrchestratorStartError(
|
||||||
|
f"infra container failed to start: "
|
||||||
|
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _wait_healthy(self, startup_timeout: float) -> InfraEndpoint:
|
||||||
|
deadline = time.monotonic() + startup_timeout
|
||||||
|
while True:
|
||||||
|
url = self._resolve_url()
|
||||||
|
if url and self.is_healthy(url):
|
||||||
|
log.info("infra container healthy", context={"url": url})
|
||||||
|
return InfraEndpoint(control_plane_url=url, gateway_ip=_ip_of(url))
|
||||||
|
if time.monotonic() >= deadline:
|
||||||
|
raise OrchestratorStartError(
|
||||||
|
f"infra container did not become healthy within "
|
||||||
|
f"{startup_timeout:g}s"
|
||||||
|
)
|
||||||
|
time.sleep(_HEALTH_POLL_SECONDS)
|
||||||
|
|
||||||
|
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
|
||||||
|
"""The gateway's mitmproxy CA (PEM) agents install to trust its TLS
|
||||||
|
interception. Read out of the container (the CA lives on a
|
||||||
|
container-internal path, not a host mount); polls because mitmproxy
|
||||||
|
writes it a beat after start."""
|
||||||
|
deadline = time.monotonic() + timeout
|
||||||
|
while True:
|
||||||
|
result = container_mod.run_container_argv(
|
||||||
|
["container", "exec", self._name, "cat", GATEWAY_CA_CERT])
|
||||||
|
if result.returncode == 0 and result.stdout.strip():
|
||||||
|
return result.stdout
|
||||||
|
if time.monotonic() >= deadline:
|
||||||
|
raise GatewayError(
|
||||||
|
f"gateway CA not available in {self._name} after {timeout:g}s: "
|
||||||
|
f"{(result.stderr or '').strip() or 'empty'}"
|
||||||
|
)
|
||||||
|
time.sleep(_CA_POLL_SECONDS)
|
||||||
|
|
||||||
|
def stop(self) -> None:
|
||||||
|
"""Remove the infra container (idempotent). The DB volume persists."""
|
||||||
|
container_mod.force_remove_container(self._name)
|
||||||
|
|
||||||
|
|
||||||
|
def _ip_of(url: str) -> str:
|
||||||
|
"""The host from an http://host:port URL."""
|
||||||
|
return url.split("://", 1)[-1].rsplit(":", 1)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def probe_control_plane_url(port: int = DEFAULT_PORT) -> str:
|
||||||
|
"""The running infra container's control-plane URL, or "" if it isn't up.
|
||||||
|
Used by host-side control-plane discovery (`discover_orchestrator_url`);
|
||||||
|
safe to call on any host — returns "" when the container or the `container`
|
||||||
|
CLI isn't present."""
|
||||||
|
ip = container_mod.try_container_ipv4_on_network(INFRA_NAME, GATEWAY_NETWORK)
|
||||||
|
return f"http://{ip}:{port}" if ip else ""
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"MacosInfraService",
|
||||||
|
"InfraEndpoint",
|
||||||
|
"OrchestratorStartError",
|
||||||
|
"GatewayError",
|
||||||
|
"INFRA_NAME",
|
||||||
|
"INFRA_DB_VOLUME",
|
||||||
|
]
|
||||||
@@ -1,242 +0,0 @@
|
|||||||
"""Orchestrator + gateway lifecycle for the macOS backend (PRD 0070).
|
|
||||||
|
|
||||||
The macOS counterpart of `orchestrator.lifecycle.OrchestratorService`. Same
|
|
||||||
shape — an idempotent singleton that brings up the control plane and the shared
|
|
||||||
gateway and hands back the control-plane URL — but the startup **order is
|
|
||||||
reversed**, and the reason is a real Apple Container constraint rather than a
|
|
||||||
stylistic choice:
|
|
||||||
|
|
||||||
- Docker starts the gateway first and lets it find the control plane by
|
|
||||||
*container name* over docker DNS.
|
|
||||||
- Apple Container 1.0.0 has **no container DNS** (see `gateway`), so the
|
|
||||||
gateway can only be handed an **IP**. That IP does not exist until the
|
|
||||||
orchestrator container is running — hence: orchestrator first, read its
|
|
||||||
address, then start the gateway pointed at it.
|
|
||||||
|
|
||||||
The second difference: no published port. Apple Container puts every container
|
|
||||||
on a host-reachable address, and the host can reach the host-only network
|
|
||||||
directly, so the host CLI and the gateway use the **same** URL — the
|
|
||||||
orchestrator's address on the shared network. Docker needs a
|
|
||||||
`--publish 127.0.0.1:…` hop plus a separate `internal_url` for the same job.
|
|
||||||
|
|
||||||
Like the docker service this runs the **register-only broker**: the backend
|
|
||||||
launches agent containers, so the control plane needs no privileged socket.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import time
|
|
||||||
import urllib.error
|
|
||||||
import urllib.request
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from ... import log
|
|
||||||
from ...orchestrator.lifecycle import (
|
|
||||||
DEFAULT_PORT,
|
|
||||||
DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
|
||||||
ORCHESTRATOR_DOCKERFILE,
|
|
||||||
ORCHESTRATOR_IMAGE,
|
|
||||||
ORCHESTRATOR_SOURCE_HASH_LABEL,
|
|
||||||
OrchestratorStartError,
|
|
||||||
source_hash,
|
|
||||||
)
|
|
||||||
from ...paths import bot_bottle_root
|
|
||||||
from . import util as container_mod
|
|
||||||
from .gateway import (
|
|
||||||
GATEWAY_EGRESS_NETWORK,
|
|
||||||
GATEWAY_IMAGE,
|
|
||||||
GATEWAY_NETWORK,
|
|
||||||
AppleGateway,
|
|
||||||
ensure_networks,
|
|
||||||
)
|
|
||||||
from .util import bind_mount_spec as _mount
|
|
||||||
|
|
||||||
# Distinct from the docker backend's container name so both can run on one host.
|
|
||||||
ORCHESTRATOR_NAME = "bot-bottle-mac-orchestrator"
|
|
||||||
|
|
||||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
||||||
_APP_DIR = "/app"
|
|
||||||
_ROOT_IN_CONTAINER = "/bot-bottle-root"
|
|
||||||
|
|
||||||
_HEALTH_POLL_SECONDS = 0.25
|
|
||||||
_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0
|
|
||||||
|
|
||||||
|
|
||||||
class MacosOrchestratorService:
|
|
||||||
"""Manages the orchestrator control-plane container + the shared Apple
|
|
||||||
gateway. Callers only need `ensure_running()`, which returns the URL."""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
port: int = DEFAULT_PORT,
|
|
||||||
network: str = GATEWAY_NETWORK,
|
|
||||||
egress_network: str = GATEWAY_EGRESS_NETWORK,
|
|
||||||
image: str = ORCHESTRATOR_IMAGE,
|
|
||||||
gateway_image: str = GATEWAY_IMAGE,
|
|
||||||
repo_root: Path = _REPO_ROOT,
|
|
||||||
host_root: Path | None = None,
|
|
||||||
orchestrator_name: str = ORCHESTRATOR_NAME,
|
|
||||||
) -> None:
|
|
||||||
self.port = port
|
|
||||||
self.network = network
|
|
||||||
self.egress_network = egress_network
|
|
||||||
self.image = image
|
|
||||||
self._gateway_image = gateway_image
|
|
||||||
self._repo_root = repo_root
|
|
||||||
self._host_root = host_root or bot_bottle_root()
|
|
||||||
self._orchestrator_name = orchestrator_name
|
|
||||||
|
|
||||||
def _resolve_url(self) -> str:
|
|
||||||
"""The control-plane URL, or "" while the container has no address.
|
|
||||||
Resolved fresh each time (there is no name to fall back on, and a
|
|
||||||
recreated orchestrator can come back on a different DHCP address), so
|
|
||||||
nothing caches a URL that could go stale."""
|
|
||||||
ip = container_mod.try_container_ipv4_on_network(
|
|
||||||
self._orchestrator_name, self.network,
|
|
||||||
)
|
|
||||||
return f"http://{ip}:{self.port}" if ip else ""
|
|
||||||
|
|
||||||
def is_healthy(
|
|
||||||
self, url: str, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS,
|
|
||||||
) -> bool:
|
|
||||||
if not url:
|
|
||||||
return False
|
|
||||||
try:
|
|
||||||
with urllib.request.urlopen(f"{url}/health", timeout=timeout) as resp:
|
|
||||||
return resp.status == 200
|
|
||||||
except (urllib.error.URLError, TimeoutError, OSError):
|
|
||||||
return False
|
|
||||||
|
|
||||||
def _ensure_orchestrator_image(self) -> None:
|
|
||||||
"""Build the lean control-plane image when missing. Build-if-missing,
|
|
||||||
not build-every-time: the control plane bind-mounts its source, so a
|
|
||||||
code change is caught by the source-hash recreate, not a rebuild."""
|
|
||||||
if container_mod.image_exists(self.image):
|
|
||||||
return
|
|
||||||
container_mod.build_image(
|
|
||||||
self.image, str(self._repo_root), dockerfile=ORCHESTRATOR_DOCKERFILE,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _orchestrator_source_current(self, current_hash: str) -> bool:
|
|
||||||
"""True iff the running orchestrator was created from the *current*
|
|
||||||
bind-mounted source. The process loaded that code at startup and won't
|
|
||||||
reload it, so a stale container would keep serving OLD control-plane
|
|
||||||
code."""
|
|
||||||
if not container_mod.container_is_running(self._orchestrator_name):
|
|
||||||
return False
|
|
||||||
data = container_mod.inspect_container(self._orchestrator_name)
|
|
||||||
config = data.get("configuration")
|
|
||||||
labels = config.get("labels") if isinstance(config, dict) else None
|
|
||||||
if not isinstance(labels, dict):
|
|
||||||
return True # can't compare → don't churn a working container
|
|
||||||
return labels.get(ORCHESTRATOR_SOURCE_HASH_LABEL) == current_hash
|
|
||||||
|
|
||||||
def _run_orchestrator_container(self, current_hash: str) -> None:
|
|
||||||
# The orchestrator is the first thing on the shared network, so it —
|
|
||||||
# not the gateway — is what has to bring the network into existence.
|
|
||||||
ensure_networks(self.network, self.egress_network)
|
|
||||||
container_mod.force_remove_container(self._orchestrator_name)
|
|
||||||
argv = [
|
|
||||||
"container", "run", "--detach",
|
|
||||||
"--name", self._orchestrator_name,
|
|
||||||
"--label", "bot-bottle.backend=macos-container",
|
|
||||||
"--label", f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={current_hash}",
|
|
||||||
# Host-only network only: the control plane needs no route out, and
|
|
||||||
# the host reaches it here directly (no --publish needed).
|
|
||||||
"--network", self.network,
|
|
||||||
"--mount", _mount(str(self._repo_root), _APP_DIR, readonly=True),
|
|
||||||
"--workdir", _APP_DIR,
|
|
||||||
# Persist the registry DB on the host (sole-owner: only the
|
|
||||||
# orchestrator opens bot-bottle.db).
|
|
||||||
"--mount", _mount(str(self._host_root), _ROOT_IN_CONTAINER),
|
|
||||||
"--env", f"BOT_BOTTLE_ROOT={_ROOT_IN_CONTAINER}",
|
|
||||||
"--entrypoint", "python3",
|
|
||||||
self.image,
|
|
||||||
"-m", "bot_bottle.orchestrator",
|
|
||||||
"--host", "0.0.0.0", "--port", str(self.port), "--broker", "stub",
|
|
||||||
]
|
|
||||||
result = container_mod.run_container_argv(argv)
|
|
||||||
if result.returncode != 0:
|
|
||||||
raise OrchestratorStartError(
|
|
||||||
f"orchestrator container failed to start: "
|
|
||||||
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
|
||||||
)
|
|
||||||
|
|
||||||
def gateway(self, orchestrator_url: str) -> AppleGateway:
|
|
||||||
"""The shared gateway, pointed at the control plane at
|
|
||||||
`orchestrator_url` (by IP — Apple has no container DNS)."""
|
|
||||||
return AppleGateway(
|
|
||||||
self._gateway_image,
|
|
||||||
network=self.network,
|
|
||||||
egress_network=self.egress_network,
|
|
||||||
orchestrator_url=orchestrator_url,
|
|
||||||
)
|
|
||||||
|
|
||||||
def ensure_running(
|
|
||||||
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
|
||||||
) -> str:
|
|
||||||
"""Ensure the control plane + shared gateway are up; return the
|
|
||||||
control-plane URL. Idempotent — a healthy control plane running current
|
|
||||||
code and a running gateway are left untouched."""
|
|
||||||
current_hash = source_hash(self._repo_root)
|
|
||||||
url = self._running_healthy_url(current_hash)
|
|
||||||
if not url:
|
|
||||||
self._ensure_orchestrator_image()
|
|
||||||
log.info(
|
|
||||||
"starting orchestrator container",
|
|
||||||
context={"name": self._orchestrator_name},
|
|
||||||
)
|
|
||||||
self._run_orchestrator_container(current_hash)
|
|
||||||
url = self._wait_healthy(startup_timeout)
|
|
||||||
|
|
||||||
# Gateway second: it can only reach the control plane by IP, which does
|
|
||||||
# not exist until the orchestrator container is up (see the docstring).
|
|
||||||
gateway = self.gateway(url)
|
|
||||||
gateway.ensure_built()
|
|
||||||
gateway.ensure_running()
|
|
||||||
return url
|
|
||||||
|
|
||||||
def _running_healthy_url(self, current_hash: str) -> str:
|
|
||||||
"""The control-plane URL if the running orchestrator is BOTH current
|
|
||||||
and answering /health, else "" (→ recreate).
|
|
||||||
|
|
||||||
Checking health, not just the source-hash label, is what lets the
|
|
||||||
service self-heal: a container that is running the current code but
|
|
||||||
whose HTTP server is wedged (bind failure, deadlock, OOM'd thread)
|
|
||||||
would otherwise be left alone and polled to death on every launch
|
|
||||||
forever. Mirrors the docker service's `is_healthy() and source_current`
|
|
||||||
gate."""
|
|
||||||
if not self._orchestrator_source_current(current_hash):
|
|
||||||
return ""
|
|
||||||
url = self._resolve_url()
|
|
||||||
return url if url and self.is_healthy(url) else ""
|
|
||||||
|
|
||||||
def _wait_healthy(self, startup_timeout: float) -> str:
|
|
||||||
"""Poll until the control plane answers /health, resolving its address
|
|
||||||
each time: the container is up before it has an IP, and it has an IP
|
|
||||||
before the server binds."""
|
|
||||||
deadline = time.monotonic() + startup_timeout
|
|
||||||
while True:
|
|
||||||
url = self._resolve_url()
|
|
||||||
if url and self.is_healthy(url):
|
|
||||||
log.info("orchestrator healthy", context={"url": url})
|
|
||||||
return url
|
|
||||||
if time.monotonic() >= deadline:
|
|
||||||
raise OrchestratorStartError(
|
|
||||||
f"orchestrator did not become healthy within "
|
|
||||||
f"{startup_timeout:g}s"
|
|
||||||
)
|
|
||||||
time.sleep(_HEALTH_POLL_SECONDS)
|
|
||||||
|
|
||||||
def stop(self) -> None:
|
|
||||||
"""Remove the orchestrator + gateway containers (idempotent)."""
|
|
||||||
container_mod.force_remove_container(self._orchestrator_name)
|
|
||||||
self.gateway("").stop()
|
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
|
||||||
"MacosOrchestratorService",
|
|
||||||
"OrchestratorStartError",
|
|
||||||
"ORCHESTRATOR_NAME",
|
|
||||||
]
|
|
||||||
@@ -186,6 +186,13 @@ def discover_orchestrator_url(*, timeout: float = 2.0) -> str:
|
|||||||
f"http://{netpool.orch_slot().guest_ip}:{CONTROL_PLANE_PORT}")
|
f"http://{netpool.orch_slot().guest_ip}:{CONTROL_PLANE_PORT}")
|
||||||
except Exception: # noqa: BLE001 — backend optional / not firecracker
|
except Exception: # noqa: BLE001 — backend optional / not firecracker
|
||||||
pass
|
pass
|
||||||
|
try: # macOS: infra container control plane on its host-only address
|
||||||
|
from ..backend.macos_container.infra import probe_control_plane_url
|
||||||
|
url = probe_control_plane_url()
|
||||||
|
if url:
|
||||||
|
candidates.append(url)
|
||||||
|
except Exception: # noqa: BLE001 — backend optional / not macOS
|
||||||
|
pass
|
||||||
for url in candidates:
|
for url in candidates:
|
||||||
if OrchestratorClient(url, timeout=timeout).health():
|
if OrchestratorClient(url, timeout=timeout).health():
|
||||||
return url
|
return url
|
||||||
|
|||||||
@@ -247,7 +247,7 @@ class TestHasBackend(unittest.TestCase):
|
|||||||
class TestEnsureOrchestrator(unittest.TestCase):
|
class TestEnsureOrchestrator(unittest.TestCase):
|
||||||
"""The backend-agnostic orchestrator bring-up entry point. Docker starts
|
"""The backend-agnostic orchestrator bring-up entry point. Docker starts
|
||||||
the orchestrator + gateway containers; firecracker boots the infra VM;
|
the orchestrator + gateway containers; firecracker boots the infra VM;
|
||||||
backends without one (macos-container) die with a pointer."""
|
macos-container starts the infra container."""
|
||||||
|
|
||||||
def test_docker_delegates_to_orchestrator_service(self):
|
def test_docker_delegates_to_orchestrator_service(self):
|
||||||
b = get_bottle_backend("docker")
|
b = get_bottle_backend("docker")
|
||||||
@@ -272,11 +272,16 @@ class TestEnsureOrchestrator(unittest.TestCase):
|
|||||||
url = b.ensure_orchestrator()
|
url = b.ensure_orchestrator()
|
||||||
self.assertEqual(url, "http://10.243.255.1:8099")
|
self.assertEqual(url, "http://10.243.255.1:8099")
|
||||||
|
|
||||||
def test_macos_default_dies(self):
|
def test_macos_delegates_to_infra_container(self):
|
||||||
from bot_bottle.log import Die
|
|
||||||
b = get_bottle_backend("macos-container")
|
b = get_bottle_backend("macos-container")
|
||||||
with self.assertRaises(Die):
|
with patch(
|
||||||
b.ensure_orchestrator()
|
"bot_bottle.backend.macos_container.infra.MacosInfraService"
|
||||||
|
) as service_cls:
|
||||||
|
service_cls.return_value.ensure_running.return_value.control_plane_url = (
|
||||||
|
"http://192.168.128.2:8099"
|
||||||
|
)
|
||||||
|
url = b.ensure_orchestrator()
|
||||||
|
self.assertEqual(url, "http://192.168.128.2:8099")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -50,30 +50,35 @@ def _client() -> Mock:
|
|||||||
|
|
||||||
class TestEnsureGateway(unittest.TestCase):
|
class TestEnsureGateway(unittest.TestCase):
|
||||||
def _run(self, service: MagicMock) -> GatewayEndpoint:
|
def _run(self, service: MagicMock) -> GatewayEndpoint:
|
||||||
with patch(f"{_MOD}.MacosOrchestratorService", return_value=service):
|
with patch(f"{_MOD}.MacosInfraService", return_value=service):
|
||||||
return ensure_gateway()
|
return ensure_gateway()
|
||||||
|
|
||||||
def _service(self) -> MagicMock:
|
def _service(self) -> MagicMock:
|
||||||
|
from bot_bottle.backend.macos_container.infra import InfraEndpoint
|
||||||
service = MagicMock()
|
service = MagicMock()
|
||||||
service.ensure_running.return_value = "http://192.168.128.2:8099"
|
service.ensure_running.return_value = InfraEndpoint(
|
||||||
|
control_plane_url="http://192.168.128.2:8099",
|
||||||
|
gateway_ip="192.168.128.2",
|
||||||
|
)
|
||||||
service.network = "bot-bottle-mac-gateway"
|
service.network = "bot-bottle-mac-gateway"
|
||||||
service.gateway.return_value.ip_on_shared_network.return_value = "192.168.128.3"
|
service.ca_cert_pem.return_value = "PEM"
|
||||||
service.gateway.return_value.ca_cert_pem.return_value = "PEM"
|
|
||||||
return service
|
return service
|
||||||
|
|
||||||
def test_reports_gateway_endpoint(self) -> None:
|
def test_reports_gateway_endpoint(self) -> None:
|
||||||
endpoint = self._run(self._service())
|
endpoint = self._run(self._service())
|
||||||
self.assertEqual("http://192.168.128.2:8099", endpoint.orchestrator_url)
|
self.assertEqual("http://192.168.128.2:8099", endpoint.orchestrator_url)
|
||||||
self.assertEqual("192.168.128.3", endpoint.gateway_ip)
|
self.assertEqual("192.168.128.2", endpoint.gateway_ip)
|
||||||
self.assertEqual("PEM", endpoint.gateway_ca_pem)
|
self.assertEqual("PEM", endpoint.gateway_ca_pem)
|
||||||
self.assertEqual("bot-bottle-mac-gateway", endpoint.network)
|
self.assertEqual("bot-bottle-mac-gateway", endpoint.network)
|
||||||
|
|
||||||
def test_gateway_is_pointed_at_the_resolved_control_plane(self) -> None:
|
def test_control_plane_and_gateway_share_one_address(self) -> None:
|
||||||
"""Apple has no container DNS, so the gateway must be handed the
|
"""One infra container hosts both, so the gateway IP and the
|
||||||
control plane's *resolved URL* rather than a container name."""
|
control-plane host are the same."""
|
||||||
service = self._service()
|
endpoint = self._run(self._service())
|
||||||
self._run(service)
|
self.assertEqual(
|
||||||
service.gateway.assert_called_with("http://192.168.128.2:8099")
|
endpoint.gateway_ip,
|
||||||
|
endpoint.orchestrator_url.split("://")[1].split(":")[0],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestRegisterAgent(unittest.TestCase):
|
class TestRegisterAgent(unittest.TestCase):
|
||||||
|
|||||||
@@ -60,13 +60,10 @@ class TestMacosContainerEnumerate(unittest.TestCase):
|
|||||||
self.assertEqual(["dev-abc"], [a.slug for a in agents])
|
self.assertEqual(["dev-abc"], [a.slug for a in agents])
|
||||||
self.assertEqual(["macos-container"], [a.backend_name for a in agents])
|
self.assertEqual(["macos-container"], [a.backend_name for a in agents])
|
||||||
|
|
||||||
def test_excludes_the_shared_singletons(self):
|
def test_excludes_the_infra_singleton(self):
|
||||||
"""The gateway and control plane share the bot-bottle- prefix but are
|
"""The infra container shares the bot-bottle- prefix but is
|
||||||
infrastructure — listing them would invent an agent per host."""
|
infrastructure — listing it would invent an agent per host."""
|
||||||
agents = self._enumerate(
|
agents = self._enumerate("bot-bottle-mac-infra\nbot-bottle-dev-abc\n")
|
||||||
"bot-bottle-mac-gateway\nbot-bottle-mac-orchestrator\n"
|
|
||||||
"bot-bottle-dev-abc\n"
|
|
||||||
)
|
|
||||||
self.assertEqual(["dev-abc"], [a.slug for a in agents])
|
self.assertEqual(["dev-abc"], [a.slug for a in agents])
|
||||||
|
|
||||||
def test_empty_when_the_cli_fails(self):
|
def test_empty_when_the_cli_fails(self):
|
||||||
|
|||||||
@@ -1,297 +0,0 @@
|
|||||||
"""Unit: the Apple gateway + orchestrator lifecycle (PRD 0070)."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import unittest
|
|
||||||
from pathlib import Path
|
|
||||||
from unittest.mock import Mock, patch
|
|
||||||
|
|
||||||
from bot_bottle.backend.macos_container.gateway import AppleGateway, GatewayError
|
|
||||||
from bot_bottle.backend.macos_container.orchestrator_service import (
|
|
||||||
MacosOrchestratorService,
|
|
||||||
OrchestratorStartError,
|
|
||||||
)
|
|
||||||
|
|
||||||
_GW = "bot_bottle.backend.macos_container.gateway"
|
|
||||||
_ORCH = "bot_bottle.backend.macos_container.orchestrator_service"
|
|
||||||
|
|
||||||
|
|
||||||
def _ok(stdout: str = "") -> Mock:
|
|
||||||
return Mock(returncode=0, stdout=stdout, stderr="")
|
|
||||||
|
|
||||||
|
|
||||||
def _fail(stderr: str = "boom") -> Mock:
|
|
||||||
return Mock(returncode=1, stdout="", stderr=stderr)
|
|
||||||
|
|
||||||
|
|
||||||
class TestAppleGatewayRun(unittest.TestCase):
|
|
||||||
def _argv(self, run: Mock) -> list[str]:
|
|
||||||
return run.call_args.args[0]
|
|
||||||
|
|
||||||
def _start(self, run: Mock) -> None:
|
|
||||||
with patch(f"{_GW}.container_mod") as mod:
|
|
||||||
mod.container_is_running.return_value = False
|
|
||||||
mod.dns_server.return_value = "1.1.1.1"
|
|
||||||
mod.run_container_argv = run
|
|
||||||
AppleGateway(orchestrator_url="http://192.168.128.2:8099").ensure_running()
|
|
||||||
|
|
||||||
def test_nat_network_precedes_the_host_only_network(self) -> None:
|
|
||||||
"""Apple Container makes the FIRST --network the default route, so the
|
|
||||||
NAT network must lead or the gateway has no route out."""
|
|
||||||
run = Mock(return_value=_ok())
|
|
||||||
self._start(run)
|
|
||||||
argv = self._argv(run)
|
|
||||||
networks = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
|
|
||||||
self.assertEqual(["bot-bottle-mac-egress", "bot-bottle-mac-gateway"], networks)
|
|
||||||
|
|
||||||
def test_control_plane_url_is_passed_for_multi_tenancy(self) -> None:
|
|
||||||
run = Mock(return_value=_ok())
|
|
||||||
self._start(run)
|
|
||||||
self.assertIn(
|
|
||||||
"BOT_BOTTLE_ORCHESTRATOR_URL=http://192.168.128.2:8099", self._argv(run),
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_dns_is_explicit(self) -> None:
|
|
||||||
"""The NAT gateway routes but does not resolve."""
|
|
||||||
run = Mock(return_value=_ok())
|
|
||||||
self._start(run)
|
|
||||||
argv = self._argv(run)
|
|
||||||
self.assertEqual("1.1.1.1", argv[argv.index("--dns") + 1])
|
|
||||||
|
|
||||||
def test_start_failure_raises(self) -> None:
|
|
||||||
with self.assertRaises(GatewayError):
|
|
||||||
self._start(Mock(return_value=_fail()))
|
|
||||||
|
|
||||||
def test_running_current_gateway_is_left_alone(self) -> None:
|
|
||||||
"""Idempotent singleton: N launches must not restart the gateway and
|
|
||||||
drop every other bottle's data plane."""
|
|
||||||
run = Mock(return_value=_ok())
|
|
||||||
with patch(f"{_GW}.container_mod") as mod:
|
|
||||||
mod.container_is_running.return_value = True
|
|
||||||
mod.container_image_digest.return_value = "abc"
|
|
||||||
mod.image_digest.return_value = "abc"
|
|
||||||
mod.run_container_argv = run
|
|
||||||
AppleGateway().ensure_running()
|
|
||||||
run.assert_not_called()
|
|
||||||
|
|
||||||
def test_stale_image_forces_a_recreate(self) -> None:
|
|
||||||
"""A rebuilt image only takes effect if the running container is
|
|
||||||
replaced — otherwise it keeps serving the OLD daemons."""
|
|
||||||
run = Mock(return_value=_ok())
|
|
||||||
with patch(f"{_GW}.container_mod") as mod:
|
|
||||||
mod.container_is_running.return_value = True
|
|
||||||
mod.container_image_digest.return_value = "old"
|
|
||||||
mod.image_digest.return_value = "new"
|
|
||||||
mod.dns_server.return_value = "1.1.1.1"
|
|
||||||
mod.run_container_argv = run
|
|
||||||
AppleGateway().ensure_running()
|
|
||||||
run.assert_called_once()
|
|
||||||
|
|
||||||
def test_unreadable_digest_does_not_churn(self) -> None:
|
|
||||||
run = Mock(return_value=_ok())
|
|
||||||
with patch(f"{_GW}.container_mod") as mod:
|
|
||||||
mod.container_is_running.return_value = True
|
|
||||||
mod.container_image_digest.return_value = ""
|
|
||||||
mod.image_digest.return_value = ""
|
|
||||||
mod.run_container_argv = run
|
|
||||||
AppleGateway().ensure_running()
|
|
||||||
run.assert_not_called()
|
|
||||||
|
|
||||||
def _start_with_running_env(self, env: dict[str, str], url: str) -> Mock:
|
|
||||||
run = Mock(return_value=_ok())
|
|
||||||
with patch(f"{_GW}.container_mod") as mod:
|
|
||||||
mod.container_is_running.return_value = True
|
|
||||||
mod.container_image_digest.return_value = "abc"
|
|
||||||
mod.image_digest.return_value = "abc"
|
|
||||||
mod.container_env.return_value = env
|
|
||||||
mod.dns_server.return_value = "1.1.1.1"
|
|
||||||
mod.run_container_argv = run
|
|
||||||
AppleGateway(orchestrator_url=url).ensure_running()
|
|
||||||
return run
|
|
||||||
|
|
||||||
def test_moved_control_plane_forces_a_recreate(self) -> None:
|
|
||||||
"""Docker hands the gateway a container *name*, stable across an
|
|
||||||
orchestrator recreate. Apple has no DNS, so the URL is an IP baked into
|
|
||||||
the gateway's env — if the orchestrator comes back on a new address and
|
|
||||||
the gateway isn't recreated, every /resolve fails and every bottle on
|
|
||||||
the host loses egress."""
|
|
||||||
run = self._start_with_running_env(
|
|
||||||
{"BOT_BOTTLE_ORCHESTRATOR_URL": "http://192.168.128.2:8099"},
|
|
||||||
"http://192.168.128.7:8099",
|
|
||||||
)
|
|
||||||
run.assert_called_once()
|
|
||||||
self.assertIn(
|
|
||||||
"BOT_BOTTLE_ORCHESTRATOR_URL=http://192.168.128.7:8099",
|
|
||||||
run.call_args.args[0],
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_unmoved_control_plane_does_not_churn(self) -> None:
|
|
||||||
run = self._start_with_running_env(
|
|
||||||
{"BOT_BOTTLE_ORCHESTRATOR_URL": "http://192.168.128.2:8099"},
|
|
||||||
"http://192.168.128.2:8099",
|
|
||||||
)
|
|
||||||
run.assert_not_called()
|
|
||||||
|
|
||||||
def test_unreadable_env_does_not_churn(self) -> None:
|
|
||||||
run = self._start_with_running_env({}, "http://192.168.128.2:8099")
|
|
||||||
run.assert_not_called()
|
|
||||||
|
|
||||||
|
|
||||||
class TestMacosOrchestratorService(unittest.TestCase):
|
|
||||||
def test_orchestrator_starts_before_the_gateway(self) -> None:
|
|
||||||
"""Apple has no container DNS, so the gateway can only be handed the
|
|
||||||
control plane's IP — which does not exist until it is running. This
|
|
||||||
ordering is the whole reason the macOS service diverges from docker's."""
|
|
||||||
order: list[str] = []
|
|
||||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
|
||||||
gateway = Mock()
|
|
||||||
gateway.ensure_running.side_effect = lambda: order.append("gateway")
|
|
||||||
|
|
||||||
def _record_orchestrator(_hash: str) -> None:
|
|
||||||
order.append("orchestrator")
|
|
||||||
|
|
||||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
|
||||||
patch(f"{_ORCH}.source_hash", return_value="h1"), \
|
|
||||||
patch.object(svc, "_run_orchestrator_container",
|
|
||||||
side_effect=_record_orchestrator), \
|
|
||||||
patch.object(svc, "gateway", return_value=gateway), \
|
|
||||||
patch.object(svc, "is_healthy", return_value=True):
|
|
||||||
mod.container_is_running.return_value = False
|
|
||||||
mod.image_exists.return_value = True
|
|
||||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
|
||||||
url = svc.ensure_running()
|
|
||||||
self.assertEqual(["orchestrator", "gateway"], order)
|
|
||||||
self.assertEqual("http://192.168.128.2:8099", url)
|
|
||||||
|
|
||||||
def test_gateway_is_handed_the_resolved_url(self) -> None:
|
|
||||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
|
||||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
|
||||||
patch(f"{_ORCH}.source_hash", return_value="h1"), \
|
|
||||||
patch.object(svc, "_run_orchestrator_container"), \
|
|
||||||
patch.object(svc, "gateway") as gw, \
|
|
||||||
patch.object(svc, "is_healthy", return_value=True):
|
|
||||||
mod.container_is_running.return_value = False
|
|
||||||
mod.image_exists.return_value = True
|
|
||||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
|
||||||
svc.ensure_running()
|
|
||||||
gw.assert_called_with("http://192.168.128.2:8099")
|
|
||||||
|
|
||||||
def test_current_source_leaves_a_healthy_orchestrator_alone(self) -> None:
|
|
||||||
"""Recreating on every launch would drop every other live bottle's
|
|
||||||
in-memory egress tokens (#381)."""
|
|
||||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
|
||||||
run = Mock()
|
|
||||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
|
||||||
patch(f"{_ORCH}.source_hash", return_value="h1"), \
|
|
||||||
patch.object(svc, "_run_orchestrator_container", run), \
|
|
||||||
patch.object(svc, "gateway"), \
|
|
||||||
patch.object(svc, "is_healthy", return_value=True):
|
|
||||||
mod.container_is_running.return_value = True
|
|
||||||
mod.inspect_container.return_value = {
|
|
||||||
"configuration": {"labels": {"bot-bottle-orchestrator-source-hash": "h1"}}
|
|
||||||
}
|
|
||||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
|
||||||
svc.ensure_running()
|
|
||||||
run.assert_not_called()
|
|
||||||
|
|
||||||
def test_wedged_but_current_orchestrator_is_recreated(self) -> None:
|
|
||||||
"""A container running the current code but whose HTTP server is dead
|
|
||||||
must be recreated, not left alone and polled to death forever. Checking
|
|
||||||
the source-hash label alone (without health) would strand the host."""
|
|
||||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
|
||||||
run = Mock()
|
|
||||||
# Unhealthy on the pre-check, healthy once recreated + waited.
|
|
||||||
health = Mock(side_effect=[False, True])
|
|
||||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
|
||||||
patch(f"{_ORCH}.source_hash", return_value="h1"), \
|
|
||||||
patch.object(svc, "_run_orchestrator_container", run), \
|
|
||||||
patch.object(svc, "_ensure_orchestrator_image"), \
|
|
||||||
patch.object(svc, "gateway"), \
|
|
||||||
patch.object(svc, "is_healthy", health):
|
|
||||||
mod.container_is_running.return_value = True
|
|
||||||
mod.inspect_container.return_value = {
|
|
||||||
"configuration": {"labels": {"bot-bottle-orchestrator-source-hash": "h1"}}
|
|
||||||
}
|
|
||||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
|
||||||
svc.ensure_running()
|
|
||||||
run.assert_called_once()
|
|
||||||
|
|
||||||
def test_changed_source_recreates_the_orchestrator(self) -> None:
|
|
||||||
"""The control-plane process loaded its bind-mounted source at startup
|
|
||||||
and won't reload it."""
|
|
||||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
|
||||||
run = Mock()
|
|
||||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
|
||||||
patch(f"{_ORCH}.source_hash", return_value="h2"), \
|
|
||||||
patch.object(svc, "_run_orchestrator_container", run), \
|
|
||||||
patch.object(svc, "gateway"), \
|
|
||||||
patch.object(svc, "is_healthy", return_value=True):
|
|
||||||
mod.container_is_running.return_value = True
|
|
||||||
mod.inspect_container.return_value = {
|
|
||||||
"configuration": {"labels": {"bot-bottle-orchestrator-source-hash": "h1"}}
|
|
||||||
}
|
|
||||||
mod.image_exists.return_value = True
|
|
||||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
|
||||||
svc.ensure_running()
|
|
||||||
run.assert_called_once()
|
|
||||||
|
|
||||||
def test_never_healthy_raises(self) -> None:
|
|
||||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
|
||||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
|
||||||
patch(f"{_ORCH}.source_hash", return_value="h1"), \
|
|
||||||
patch.object(svc, "_run_orchestrator_container"), \
|
|
||||||
patch.object(svc, "is_healthy", return_value=False):
|
|
||||||
mod.container_is_running.return_value = False
|
|
||||||
mod.image_exists.return_value = True
|
|
||||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
|
||||||
with self.assertRaises(OrchestratorStartError):
|
|
||||||
svc.ensure_running(startup_timeout=0.01)
|
|
||||||
|
|
||||||
def test_control_plane_needs_no_route_out(self) -> None:
|
|
||||||
"""The orchestrator sits only on the host-only network: the host
|
|
||||||
reaches it there directly, so there is no --publish and no NAT leg."""
|
|
||||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
|
||||||
run = Mock(return_value=_ok())
|
|
||||||
# `ensure_networks` lives in the gateway module and shells out to the
|
|
||||||
# `container` CLI, which does not exist on the Linux CI host — patch it
|
|
||||||
# here, not gateway.container_mod, since it is called through this
|
|
||||||
# module's imported name.
|
|
||||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
|
||||||
patch(f"{_ORCH}.ensure_networks"):
|
|
||||||
mod.run_container_argv = run
|
|
||||||
svc._run_orchestrator_container("h1")
|
|
||||||
argv = run.call_args.args[0]
|
|
||||||
networks = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
|
|
||||||
self.assertEqual(["bot-bottle-mac-gateway"], networks)
|
|
||||||
self.assertNotIn("--publish", argv)
|
|
||||||
|
|
||||||
def test_networks_exist_before_the_orchestrator_runs(self) -> None:
|
|
||||||
"""The orchestrator is the first container on the shared network, so it
|
|
||||||
has to create it — the gateway that used to do so now starts second."""
|
|
||||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
|
||||||
order: list[str] = []
|
|
||||||
|
|
||||||
def _networks(*_args: str) -> None:
|
|
||||||
order.append("networks")
|
|
||||||
|
|
||||||
def _run(*_args: list[str]) -> Mock:
|
|
||||||
order.append("run")
|
|
||||||
return _ok()
|
|
||||||
|
|
||||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
|
||||||
patch(f"{_ORCH}.ensure_networks", side_effect=_networks):
|
|
||||||
mod.run_container_argv = Mock(side_effect=_run)
|
|
||||||
svc._run_orchestrator_container("h1")
|
|
||||||
self.assertEqual(["networks", "run"], order)
|
|
||||||
|
|
||||||
def test_orchestrator_start_failure_raises(self) -> None:
|
|
||||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
|
||||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
|
||||||
patch(f"{_ORCH}.ensure_networks"):
|
|
||||||
mod.run_container_argv = Mock(return_value=_fail())
|
|
||||||
with self.assertRaises(OrchestratorStartError):
|
|
||||||
svc._run_orchestrator_container("h1")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
unittest.main()
|
|
||||||
@@ -0,0 +1,168 @@
|
|||||||
|
"""Unit: the single macOS infra container (control plane + gateway, PRD 0070)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
|
from bot_bottle.backend.macos_container.infra import (
|
||||||
|
INFRA_DB_VOLUME,
|
||||||
|
MacosInfraService,
|
||||||
|
OrchestratorStartError,
|
||||||
|
probe_control_plane_url,
|
||||||
|
)
|
||||||
|
|
||||||
|
_INFRA = "bot_bottle.backend.macos_container.infra"
|
||||||
|
|
||||||
|
|
||||||
|
def _ok(stdout: str = "") -> Mock:
|
||||||
|
return Mock(returncode=0, stdout=stdout, stderr="")
|
||||||
|
|
||||||
|
|
||||||
|
def _fail(stderr: str = "boom") -> Mock:
|
||||||
|
return Mock(returncode=1, stdout="", stderr=stderr)
|
||||||
|
|
||||||
|
|
||||||
|
class TestInfraRun(unittest.TestCase):
|
||||||
|
def _run_container(self, svc: MacosInfraService) -> list[str]:
|
||||||
|
run = Mock(return_value=_ok())
|
||||||
|
|
||||||
|
def _spec(src: str, tgt: str, readonly: bool = False) -> str:
|
||||||
|
return f"type=bind,source={src},target={tgt}" + (
|
||||||
|
",readonly" if readonly else "")
|
||||||
|
|
||||||
|
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||||
|
patch(f"{_INFRA}.ensure_networks"):
|
||||||
|
mod.dns_server.return_value = "1.1.1.1"
|
||||||
|
mod.bind_mount_spec.side_effect = _spec
|
||||||
|
mod.run_container_argv = run
|
||||||
|
svc._run_container("h1")
|
||||||
|
return run.call_args.args[0]
|
||||||
|
|
||||||
|
def test_single_container_runs_both_processes(self) -> None:
|
||||||
|
"""The whole point: one container starts the control plane AND the
|
||||||
|
gateway daemons, so one kernel owns the DB."""
|
||||||
|
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
|
||||||
|
script = argv[-1]
|
||||||
|
self.assertIn("bot_bottle.orchestrator", script)
|
||||||
|
self.assertIn("gateway_init.py", script)
|
||||||
|
self.assertIn("127.0.0.1", script) # they reach each other on loopback
|
||||||
|
|
||||||
|
def test_db_is_a_container_only_volume(self) -> None:
|
||||||
|
"""No host bind-mount of the DB — a named volume only this container
|
||||||
|
mounts, so the DB is never written by two kernels."""
|
||||||
|
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
|
||||||
|
vols = [argv[i + 1] for i, a in enumerate(argv) if a == "--volume"]
|
||||||
|
self.assertTrue(any(v.startswith(f"{INFRA_DB_VOLUME}:") for v in vols))
|
||||||
|
# The repo source is bind-mounted read-only; the DB is not a bind mount.
|
||||||
|
mounts = [argv[i + 1] for i, a in enumerate(argv) if a == "--mount"]
|
||||||
|
self.assertTrue(all("bot-bottle.db" not in m for m in mounts))
|
||||||
|
|
||||||
|
def test_nat_network_precedes_the_host_only_network(self) -> None:
|
||||||
|
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
|
||||||
|
nets = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
|
||||||
|
self.assertEqual(["bot-bottle-mac-egress", "bot-bottle-mac-gateway"], nets)
|
||||||
|
|
||||||
|
def test_source_hash_is_labelled_for_recreate(self) -> None:
|
||||||
|
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
|
||||||
|
self.assertIn("BOT_BOTTLE_SOURCE_HASH=h1", argv)
|
||||||
|
|
||||||
|
def test_start_failure_raises(self) -> None:
|
||||||
|
svc = MacosInfraService(repo_root=Path("/r"))
|
||||||
|
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||||
|
patch(f"{_INFRA}.ensure_networks"):
|
||||||
|
mod.dns_server.return_value = "1.1.1.1"
|
||||||
|
mod.bind_mount_spec.return_value = "m"
|
||||||
|
mod.run_container_argv = Mock(return_value=_fail())
|
||||||
|
with self.assertRaises(OrchestratorStartError):
|
||||||
|
svc._run_container("h1")
|
||||||
|
|
||||||
|
|
||||||
|
class TestInfraEnsureRunning(unittest.TestCase):
|
||||||
|
def test_current_healthy_container_is_left_alone(self) -> None:
|
||||||
|
"""Idempotent singleton: N launches must not churn the infra container
|
||||||
|
and drop every live bottle's control plane."""
|
||||||
|
svc = MacosInfraService(repo_root=Path("/r"))
|
||||||
|
run = Mock()
|
||||||
|
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||||
|
patch(f"{_INFRA}.source_hash", return_value="h1"), \
|
||||||
|
patch.object(svc, "_run_container", run), \
|
||||||
|
patch.object(svc, "is_healthy", return_value=True):
|
||||||
|
mod.container_is_running.return_value = True
|
||||||
|
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
|
||||||
|
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||||
|
endpoint = svc.ensure_running()
|
||||||
|
run.assert_not_called()
|
||||||
|
self.assertEqual("http://192.168.128.2:8099", endpoint.control_plane_url)
|
||||||
|
self.assertEqual("192.168.128.2", endpoint.gateway_ip)
|
||||||
|
|
||||||
|
def test_changed_source_recreates(self) -> None:
|
||||||
|
svc = MacosInfraService(repo_root=Path("/r"))
|
||||||
|
run = Mock()
|
||||||
|
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||||
|
patch(f"{_INFRA}.source_hash", return_value="h2"), \
|
||||||
|
patch.object(svc, "ensure_built"), \
|
||||||
|
patch.object(svc, "_run_container", run), \
|
||||||
|
patch.object(svc, "is_healthy", return_value=True):
|
||||||
|
mod.container_is_running.return_value = True
|
||||||
|
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
|
||||||
|
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||||
|
svc.ensure_running()
|
||||||
|
run.assert_called_once()
|
||||||
|
|
||||||
|
def test_wedged_but_current_container_is_recreated(self) -> None:
|
||||||
|
"""Current source but a dead HTTP server must be recreated, not polled
|
||||||
|
to death forever — health, not just the source label, gates reuse."""
|
||||||
|
svc = MacosInfraService(repo_root=Path("/r"))
|
||||||
|
run = Mock()
|
||||||
|
health = Mock(side_effect=[False, True])
|
||||||
|
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||||
|
patch(f"{_INFRA}.source_hash", return_value="h1"), \
|
||||||
|
patch.object(svc, "ensure_built"), \
|
||||||
|
patch.object(svc, "_run_container", run), \
|
||||||
|
patch.object(svc, "is_healthy", health):
|
||||||
|
mod.container_is_running.return_value = True
|
||||||
|
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
|
||||||
|
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||||
|
svc.ensure_running()
|
||||||
|
run.assert_called_once()
|
||||||
|
|
||||||
|
def test_never_healthy_raises(self) -> None:
|
||||||
|
svc = MacosInfraService(repo_root=Path("/r"))
|
||||||
|
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||||
|
patch(f"{_INFRA}.source_hash", return_value="h1"), \
|
||||||
|
patch.object(svc, "ensure_built"), \
|
||||||
|
patch.object(svc, "_run_container"), \
|
||||||
|
patch.object(svc, "is_healthy", return_value=False):
|
||||||
|
mod.container_is_running.return_value = False
|
||||||
|
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||||
|
with self.assertRaises(OrchestratorStartError):
|
||||||
|
svc.ensure_running(startup_timeout=0.01)
|
||||||
|
|
||||||
|
|
||||||
|
class TestCaCertPem(unittest.TestCase):
|
||||||
|
def test_reads_ca_out_of_the_container(self) -> None:
|
||||||
|
svc = MacosInfraService(repo_root=Path("/r"))
|
||||||
|
with patch(f"{_INFRA}.container_mod") as mod:
|
||||||
|
mod.run_container_argv.return_value = _ok("-----BEGIN CERTIFICATE-----\n")
|
||||||
|
pem = svc.ca_cert_pem()
|
||||||
|
self.assertTrue(pem.startswith("-----BEGIN CERTIFICATE-----"))
|
||||||
|
argv = mod.run_container_argv.call_args.args[0]
|
||||||
|
self.assertEqual(["container", "exec", "bot-bottle-mac-infra", "cat"], argv[:4])
|
||||||
|
|
||||||
|
|
||||||
|
class TestProbeControlPlane(unittest.TestCase):
|
||||||
|
def test_returns_url_when_running(self) -> None:
|
||||||
|
with patch(f"{_INFRA}.container_mod") as mod:
|
||||||
|
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||||
|
self.assertEqual("http://192.168.128.2:8099", probe_control_plane_url())
|
||||||
|
|
||||||
|
def test_empty_when_absent(self) -> None:
|
||||||
|
with patch(f"{_INFRA}.container_mod") as mod:
|
||||||
|
mod.try_container_ipv4_on_network.return_value = ""
|
||||||
|
self.assertEqual("", probe_control_plane_url())
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user