feat(macos): split orchestrator and gateway into separate containers (PRD 0070)
test / integration-docker (pull_request) Successful in 18s
tracker-policy-pr / check-pr (pull_request) Successful in 22s
test / unit (pull_request) Failing after 42s
lint / lint (push) Successful in 57s
test / integration-firecracker (pull_request) Successful in 3m22s
test / coverage (pull_request) Has been skipped
test / publish-infra (pull_request) Has been skipped
test / integration-docker (pull_request) Successful in 18s
tracker-policy-pr / check-pr (pull_request) Successful in 22s
test / unit (pull_request) Failing after 42s
lint / lint (push) Successful in 57s
test / integration-firecracker (pull_request) Successful in 3m22s
test / coverage (pull_request) Has been skipped
test / publish-infra (pull_request) Has been skipped
The single Apple container existed only because two guests writing one bot-bottle.db over virtiofs would race incoherent fcntl locks; #469 removed that (the data plane no longer opens the DB), so the macOS backend now runs the planes as two containers like docker: * bot-bottle-mac-orchestrator — lean control plane on the host-only `bot-bottle-mac-control` network only (image Dockerfile.orchestrator, `-m bot_bottle.orchestrator`). Sole mounter of the container-only DB volume; holds the signing key. The CLI + gateway reach it at its control-network address. * bot-bottle-mac-infra — the gateway, triple-homed on the NAT egress net, the host-only agent net, and the control net. Resolves the orchestrator by IP (Apple has no container DNS) via BOT_BOTTLE_ORCHESTRATOR_URL; holds the CA + gateway JWT. Agents sit on the agent network only, so they have no route to the control plane. ensure_networks gains the control network; MacosInfraService brings up the orchestrator then the gateway; probe_orchestrator_url + ca_cert_pem target the right containers. pyright 0 errors; unit suite green (2274). Needs a real Apple-container host to validate the networking. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -19,27 +19,40 @@ from . import util as container_mod
|
|||||||
# on. The agent's address here is the attribution key. Distinct from the docker
|
# on. The agent's address here is the attribution key. Distinct from the docker
|
||||||
# names so both backends can coexist on one host.
|
# names so both backends can coexist on one host.
|
||||||
GATEWAY_NETWORK = "bot-bottle-mac-gateway"
|
GATEWAY_NETWORK = "bot-bottle-mac-gateway"
|
||||||
# The NAT network that gives the infra container (and only it) a route out.
|
# The NAT network that gives the gateway (and only it) a route out.
|
||||||
GATEWAY_EGRESS_NETWORK = "bot-bottle-mac-egress"
|
GATEWAY_EGRESS_NETWORK = "bot-bottle-mac-egress"
|
||||||
|
# The control network the gateway reaches the orchestrator over (host-only).
|
||||||
|
# Only the orchestrator + gateway join it; agents never do, so agents have no
|
||||||
|
# route to the control plane (PRD 0070 "Separating the planes").
|
||||||
|
CONTROL_NETWORK = "bot-bottle-mac-control"
|
||||||
|
|
||||||
GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest")
|
GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest")
|
||||||
|
ORCHESTRATOR_IMAGE = os.environ.get(
|
||||||
|
"BOT_BOTTLE_ORCHESTRATOR_IMAGE", "bot-bottle-orchestrator:latest"
|
||||||
|
)
|
||||||
|
|
||||||
DEFAULT_CA_TIMEOUT_SECONDS = 30.0
|
DEFAULT_CA_TIMEOUT_SECONDS = 30.0
|
||||||
|
|
||||||
|
|
||||||
def ensure_networks(
|
def ensure_networks(
|
||||||
network: str = GATEWAY_NETWORK, egress_network: str = GATEWAY_EGRESS_NETWORK,
|
network: str = GATEWAY_NETWORK,
|
||||||
|
egress_network: str = GATEWAY_EGRESS_NETWORK,
|
||||||
|
control_network: str = CONTROL_NETWORK,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Create the shared host-only network + the NAT egress network. Idempotent
|
"""Create the shared host-only agent network, the NAT egress network, and
|
||||||
— `create_network` tolerates 'already exists'."""
|
the host-only control network. Idempotent — `create_network` tolerates
|
||||||
|
'already exists'."""
|
||||||
container_mod.create_network(egress_network)
|
container_mod.create_network(egress_network)
|
||||||
container_mod.create_network(network, internal=True)
|
container_mod.create_network(network, internal=True)
|
||||||
|
container_mod.create_network(control_network, internal=True)
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"GATEWAY_NETWORK",
|
"GATEWAY_NETWORK",
|
||||||
"GATEWAY_EGRESS_NETWORK",
|
"GATEWAY_EGRESS_NETWORK",
|
||||||
|
"CONTROL_NETWORK",
|
||||||
"GATEWAY_IMAGE",
|
"GATEWAY_IMAGE",
|
||||||
|
"ORCHESTRATOR_IMAGE",
|
||||||
"GatewayError",
|
"GatewayError",
|
||||||
"DEFAULT_CA_TIMEOUT_SECONDS",
|
"DEFAULT_CA_TIMEOUT_SECONDS",
|
||||||
"ensure_networks",
|
"ensure_networks",
|
||||||
|
|||||||
@@ -1,34 +1,22 @@
|
|||||||
"""The per-host infra container for the macOS backend (PRD 0070).
|
"""The per-host control plane + gateway for the macOS backend (PRD 0070).
|
||||||
|
|
||||||
A single persistent Apple container that runs BOTH the orchestrator control
|
Two Apple containers — the orchestrator (control plane) and the gateway (data
|
||||||
plane and the gateway data plane — the macOS analogue of the Firecracker infra
|
plane) — split now that #469 got the DB off the data plane. The single-container
|
||||||
VM (`backend/firecracker/infra_vm.py`), not the docker backend's two separate
|
model existed only because two Apple-Container guests writing one `bot-bottle.db`
|
||||||
containers.
|
over virtiofs would race incoherent `fcntl` locks; with the data plane no longer
|
||||||
|
opening the DB at all, only the orchestrator does, so the split is safe.
|
||||||
|
|
||||||
Why one container, not two: Apple Containers are lightweight VMs, each with its
|
* `bot-bottle-mac-orchestrator` — the lean control plane. Joins the host-only
|
||||||
own kernel. The docker backend runs the orchestrator and gateway as two
|
**control network** (`bot-bottle-mac-control`) only. Sole opener of the
|
||||||
containers safely because they share the host kernel, so their concurrent
|
container-only DB volume; holds the signing key. The host CLI reaches it at
|
||||||
writes to the one `bot-bottle.db` (the orchestrator's registry + the gateway
|
its control-network address; the gateway reaches it there too.
|
||||||
supervise daemon's queue) are serialized by coherent `fcntl` locks. Across two
|
* `bot-bottle-mac-infra` — the gateway data plane. Triple-homed: the NAT
|
||||||
*guest* kernels sharing a virtiofs-mounted DB those locks are not coherent, and
|
egress network (route out), the host-only agent network (agents + CLI reach
|
||||||
concurrent writers can corrupt the file. Firecracker solved this by putting
|
the gateway), and the control network (reach the orchestrator by IP — Apple
|
||||||
both services in one guest with the DB on a device only that guest mounts; this
|
has no container DNS). Holds the mitmproxy CA + the `gateway` JWT.
|
||||||
does the same with Apple primitives.
|
|
||||||
|
|
||||||
Two consequences fall out of the single container, both simplifications:
|
Agents sit on the agent network only, never the control network, so they have no
|
||||||
|
route to the control plane (the L3 block, not just the JWT).
|
||||||
- **No DNS dance.** The control plane and the gateway daemons reach each other
|
|
||||||
over `127.0.0.1`, so nothing depends on Apple's (absent) container DNS and
|
|
||||||
there is no orchestrator-before-gateway ordering to get right.
|
|
||||||
- **The DB is never host-shared.** It lives on a container-only volume, so no
|
|
||||||
host process opens the live file. The host CLI reaches registry + supervise
|
|
||||||
state through the control-plane HTTP surface (`cli/supervise.py` already uses
|
|
||||||
`OrchestratorClient`), exactly as it does for firecracker.
|
|
||||||
|
|
||||||
The control-plane source is bind-mounted (like the docker orchestrator), so a
|
|
||||||
code change takes effect on the next launch without an image rebuild; the
|
|
||||||
gateway daemons are baked in the gateway image and rebuild through its own
|
|
||||||
digest check.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -59,23 +47,29 @@ from ...paths import (
|
|||||||
from .. import util as backend_util
|
from .. import util as backend_util
|
||||||
from . import util as container_mod
|
from . import util as container_mod
|
||||||
from .gateway import (
|
from .gateway import (
|
||||||
|
CONTROL_NETWORK,
|
||||||
DEFAULT_CA_TIMEOUT_SECONDS,
|
DEFAULT_CA_TIMEOUT_SECONDS,
|
||||||
GATEWAY_EGRESS_NETWORK,
|
GATEWAY_EGRESS_NETWORK,
|
||||||
GATEWAY_IMAGE,
|
GATEWAY_IMAGE,
|
||||||
GATEWAY_NETWORK,
|
GATEWAY_NETWORK,
|
||||||
GatewayError,
|
GatewayError,
|
||||||
|
ORCHESTRATOR_IMAGE,
|
||||||
ensure_networks,
|
ensure_networks,
|
||||||
)
|
)
|
||||||
|
|
||||||
# The one per-host infra container: control plane + gateway data plane.
|
# The orchestrator (control plane) container + the gateway (data plane)
|
||||||
INFRA_NAME = "bot-bottle-mac-infra"
|
# container. `INFRA_NAME` is kept — now the gateway container — for callers that
|
||||||
|
# still import it (probe / reprovision attribute against the gateway).
|
||||||
|
ORCHESTRATOR_NAME = "bot-bottle-mac-orchestrator"
|
||||||
|
ORCHESTRATOR_LABEL = "bot-bottle-mac-orchestrator=1"
|
||||||
|
INFRA_NAME = "bot-bottle-mac-infra" # the gateway container
|
||||||
INFRA_LABEL = "bot-bottle-mac-infra=1"
|
INFRA_LABEL = "bot-bottle-mac-infra=1"
|
||||||
# Container-only volume holding bot-bottle.db. No host bind-mount, so the DB is
|
# Container-only volume holding bot-bottle.db, mounted ONLY into the
|
||||||
# written by exactly one kernel (this container's). Survives recreation.
|
# orchestrator. One kernel writes it (never host-shared or cross-guest).
|
||||||
INFRA_DB_VOLUME = "bot-bottle-mac-db"
|
INFRA_DB_VOLUME = "bot-bottle-mac-db"
|
||||||
|
|
||||||
# BOT_BOTTLE_ROOT inside the container; host_db_path() resolves the DB to
|
# BOT_BOTTLE_ROOT inside the orchestrator; host_db_path() resolves the DB to
|
||||||
# <root>/db/<filename> and the supervise daemon writes the same file.
|
# <root>/db/<filename>.
|
||||||
_DB_ROOT_IN_CONTAINER = "/var/lib/bot-bottle"
|
_DB_ROOT_IN_CONTAINER = "/var/lib/bot-bottle"
|
||||||
_DB_PATH_IN_CONTAINER = f"{_DB_ROOT_IN_CONTAINER}/db/{HOST_DB_FILENAME}"
|
_DB_PATH_IN_CONTAINER = f"{_DB_ROOT_IN_CONTAINER}/db/{HOST_DB_FILENAME}"
|
||||||
_SRC_IN_CONTAINER = "/bot-bottle-src"
|
_SRC_IN_CONTAINER = "/bot-bottle-src"
|
||||||
@@ -84,45 +78,23 @@ _REPO_ROOT = Path(__file__).resolve().parents[3]
|
|||||||
|
|
||||||
_HEALTH_POLL_SECONDS = 0.25
|
_HEALTH_POLL_SECONDS = 0.25
|
||||||
_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0
|
_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0
|
||||||
_CA_POLL_SECONDS = 0.5
|
|
||||||
|
|
||||||
# The gateway subset the consolidated model runs (no per-bottle git:// daemon).
|
# The gateway subset the consolidated model runs (no per-bottle git:// daemon).
|
||||||
_GATEWAY_DAEMONS = "egress,git-http,supervise"
|
_GATEWAY_DAEMONS = "egress,git-http,supervise"
|
||||||
|
|
||||||
|
|
||||||
def _init_script(port: int) -> str:
|
|
||||||
"""PID-1 init: start the control plane and the gateway daemons, both in
|
|
||||||
this container, reaching each other over loopback. Backgrounded so `wait`
|
|
||||||
reaps as PID 1. No `set -e` — a transient daemon failure must not kill the
|
|
||||||
whole container (gateway_init applies the same 'stay up' policy)."""
|
|
||||||
return (
|
|
||||||
"export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\n"
|
|
||||||
f"mkdir -p $(dirname {_DB_PATH_IN_CONTAINER})\n"
|
|
||||||
# Control plane, from the bind-mounted source (stdlib-only package).
|
|
||||||
f"( cd {_SRC_IN_CONTAINER} && BOT_BOTTLE_ROOT={_DB_ROOT_IN_CONTAINER} "
|
|
||||||
f"python3 -m bot_bottle.orchestrator --host 0.0.0.0 --port {port} "
|
|
||||||
"--broker stub ) &\n"
|
|
||||||
# Gateway data plane, multi-tenant against the local control plane. No
|
|
||||||
# SUPERVISE_DB_PATH: the data plane reaches the supervise queue over the
|
|
||||||
# control-plane RPC and never opens bot-bottle.db (PRD 0070 / #469).
|
|
||||||
f"( cd /app && BOT_BOTTLE_GATEWAY_DAEMONS={_GATEWAY_DAEMONS} "
|
|
||||||
f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{port} "
|
|
||||||
f"python3 -m bot_bottle.gateway.bootstrap ) &\n"
|
|
||||||
"while : ; do wait ; done\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class InfraEndpoint:
|
class InfraEndpoint:
|
||||||
"""How to reach the running infra container. The control plane and the
|
"""How to reach the running pair. `orchestrator_url` is the orchestrator's
|
||||||
gateway are the same container, so one address serves both."""
|
control-network address (host CLI + registration); `gateway_ip` is the
|
||||||
|
gateway's agent-network address (proxy / git-http / MCP target)."""
|
||||||
|
|
||||||
orchestrator_url: str # http://<infra ip>:8099 — host CLI + registration
|
orchestrator_url: str
|
||||||
gateway_ip: str # same container; agents' proxy / git-http / MCP target
|
gateway_ip: str
|
||||||
|
|
||||||
|
|
||||||
class MacosInfraService:
|
class MacosInfraService:
|
||||||
"""Manages the single per-host infra container. Callers use
|
"""Manages the per-host orchestrator + gateway containers. Callers use
|
||||||
`ensure_running()` (returns the endpoint) and `ca_cert_pem()`."""
|
`ensure_running()` (returns the endpoint) and `ca_cert_pem()`."""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -131,24 +103,41 @@ class MacosInfraService:
|
|||||||
port: int = DEFAULT_PORT,
|
port: int = DEFAULT_PORT,
|
||||||
network: str = GATEWAY_NETWORK,
|
network: str = GATEWAY_NETWORK,
|
||||||
egress_network: str = GATEWAY_EGRESS_NETWORK,
|
egress_network: str = GATEWAY_EGRESS_NETWORK,
|
||||||
image: str = GATEWAY_IMAGE,
|
control_network: str = CONTROL_NETWORK,
|
||||||
|
gateway_image: str = GATEWAY_IMAGE,
|
||||||
|
orchestrator_image: str = ORCHESTRATOR_IMAGE,
|
||||||
repo_root: Path = _REPO_ROOT,
|
repo_root: Path = _REPO_ROOT,
|
||||||
name: str = INFRA_NAME,
|
orchestrator_name: str = ORCHESTRATOR_NAME,
|
||||||
|
gateway_name: str = INFRA_NAME,
|
||||||
db_volume: str = INFRA_DB_VOLUME,
|
db_volume: str = INFRA_DB_VOLUME,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.port = port
|
self.port = port
|
||||||
self.network = network
|
self.network = network
|
||||||
self.egress_network = egress_network
|
self.egress_network = egress_network
|
||||||
self.image = image
|
self.control_network = control_network
|
||||||
|
self.gateway_image = gateway_image
|
||||||
|
self.orchestrator_image = orchestrator_image
|
||||||
self._repo_root = repo_root
|
self._repo_root = repo_root
|
||||||
self._name = name
|
self._orchestrator_name = orchestrator_name
|
||||||
|
self._gateway_name = gateway_name
|
||||||
self._db_volume = db_volume
|
self._db_volume = db_volume
|
||||||
|
|
||||||
def _resolve_url(self) -> str:
|
@property
|
||||||
"""The control-plane URL, or "" while the container has no address."""
|
def gateway_name(self) -> str:
|
||||||
ip = container_mod.try_container_ipv4_on_network(self._name, self.network)
|
return self._gateway_name
|
||||||
|
|
||||||
|
def _resolve_orchestrator_url(self) -> str:
|
||||||
|
"""The control-plane URL (orchestrator's control-network address), or ""
|
||||||
|
while it has no address."""
|
||||||
|
ip = container_mod.try_container_ipv4_on_network(
|
||||||
|
self._orchestrator_name, self.control_network)
|
||||||
return f"http://{ip}:{self.port}" if ip else ""
|
return f"http://{ip}:{self.port}" if ip else ""
|
||||||
|
|
||||||
|
def _resolve_gateway_ip(self) -> str:
|
||||||
|
"""The gateway's agent-network address, or "" while it has none."""
|
||||||
|
return container_mod.try_container_ipv4_on_network(
|
||||||
|
self._gateway_name, self.network)
|
||||||
|
|
||||||
def is_healthy(
|
def is_healthy(
|
||||||
self, url: str, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS,
|
self, url: str, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
@@ -160,158 +149,165 @@ class MacosInfraService:
|
|||||||
except (urllib.error.URLError, TimeoutError, OSError):
|
except (urllib.error.URLError, TimeoutError, OSError):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def _source_current(self, current_hash: str) -> bool:
|
def _orchestrator_source_current(self, current_hash: str) -> bool:
|
||||||
"""True iff the running infra container was created from the current
|
"""True iff the running orchestrator was created from the current
|
||||||
bind-mounted control-plane source. The control-plane process loads that
|
bind-mounted control-plane source (it loads that code at startup and
|
||||||
code at startup and won't reload it, so a stale container keeps serving
|
won't reload it)."""
|
||||||
OLD code."""
|
if not container_mod.container_is_running(self._orchestrator_name):
|
||||||
if not container_mod.container_is_running(self._name):
|
|
||||||
return False
|
return False
|
||||||
env = container_mod.container_env(self._name)
|
env = container_mod.container_env(self._orchestrator_name)
|
||||||
if not env:
|
if not env:
|
||||||
return True # can't compare → don't churn a working container
|
return True # can't compare → don't churn a working container
|
||||||
return env.get("BOT_BOTTLE_SOURCE_HASH") == current_hash
|
return env.get("BOT_BOTTLE_SOURCE_HASH") == current_hash
|
||||||
|
|
||||||
def _running_healthy_endpoint(self, current_hash: str) -> InfraEndpoint | None:
|
|
||||||
"""The endpoint if the running container is BOTH source-current and
|
|
||||||
answering /health, else None (→ recreate). Health, not just the source
|
|
||||||
label, is what lets a wedged-but-current container self-heal instead of
|
|
||||||
being polled to death forever."""
|
|
||||||
if not self._source_current(current_hash):
|
|
||||||
return None
|
|
||||||
url = self._resolve_url()
|
|
||||||
if url and self.is_healthy(url):
|
|
||||||
return InfraEndpoint(orchestrator_url=url, gateway_ip=_ip_of(url))
|
|
||||||
return None
|
|
||||||
|
|
||||||
def ensure_built(self) -> None:
|
def ensure_built(self) -> None:
|
||||||
"""Ensure the gateway data-plane image exists. The control-plane source
|
"""Ensure the gateway + orchestrator images exist. The control-plane
|
||||||
is bind-mounted, not baked, so only the gateway image needs building."""
|
source is bind-mounted, so a code change takes effect without a rebuild;
|
||||||
|
the images still carry the package for their entrypoints."""
|
||||||
container_mod.build_image(
|
container_mod.build_image(
|
||||||
self.image, str(self._repo_root), dockerfile="Dockerfile.gateway",
|
self.gateway_image, str(self._repo_root), dockerfile="Dockerfile.gateway")
|
||||||
)
|
container_mod.build_image(
|
||||||
|
self.orchestrator_image, str(self._repo_root),
|
||||||
|
dockerfile="Dockerfile.orchestrator")
|
||||||
|
|
||||||
def ensure_running(
|
def ensure_running(
|
||||||
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||||
) -> InfraEndpoint:
|
) -> InfraEndpoint:
|
||||||
"""Ensure the single infra container is up; return how to reach it.
|
"""Ensure the orchestrator + gateway containers are up; return how to
|
||||||
Idempotent per-host singleton — a healthy container on current source
|
reach them. Idempotent per-host singleton — a healthy orchestrator on
|
||||||
is left untouched, so N launches share the one control plane + gateway.
|
current source is left untouched. Raises `OrchestratorStartError` on
|
||||||
Raises `OrchestratorStartError` on startup timeout."""
|
control-plane startup timeout."""
|
||||||
current_hash = source_hash(self._repo_root)
|
|
||||||
endpoint = self._running_healthy_endpoint(current_hash)
|
|
||||||
if endpoint is not None:
|
|
||||||
return endpoint
|
|
||||||
self.ensure_built()
|
self.ensure_built()
|
||||||
log.info("starting infra container", context={"name": self._name})
|
ensure_networks(self.network, self.egress_network, self.control_network)
|
||||||
self._run_container(current_hash)
|
|
||||||
return self._wait_healthy(startup_timeout)
|
|
||||||
|
|
||||||
def _run_container(self, current_hash: str) -> None:
|
current_hash = source_hash(self._repo_root)
|
||||||
ensure_networks(self.network, self.egress_network)
|
url = self._resolve_orchestrator_url()
|
||||||
container_mod.force_remove_container(self._name)
|
if not (self._orchestrator_source_current(current_hash)
|
||||||
|
and url and self.is_healthy(url)):
|
||||||
|
log.info("starting orchestrator container",
|
||||||
|
context={"name": self._orchestrator_name})
|
||||||
|
self._run_orchestrator_container(current_hash)
|
||||||
|
url = self._wait_healthy(startup_timeout)
|
||||||
|
|
||||||
|
# (Re)ensure the gateway once the control plane it resolves against is
|
||||||
|
# healthy — it needs the orchestrator's control-network address.
|
||||||
|
self._ensure_gateway_container(url)
|
||||||
|
return InfraEndpoint(orchestrator_url=url, gateway_ip=self._resolve_gateway_ip())
|
||||||
|
|
||||||
|
def _run_orchestrator_container(self, current_hash: str) -> None:
|
||||||
|
container_mod.force_remove_container(self._orchestrator_name)
|
||||||
|
_signing_key = host_orchestrator_token()
|
||||||
argv = [
|
argv = [
|
||||||
"container", "run", "--detach",
|
"container", "run", "--detach",
|
||||||
"--name", self._name,
|
"--name", self._orchestrator_name,
|
||||||
"--label", "bot-bottle.backend=macos-container",
|
"--label", "bot-bottle.backend=macos-container",
|
||||||
"--label", INFRA_LABEL,
|
"--label", ORCHESTRATOR_LABEL,
|
||||||
# NAT network FIRST so the gateway's egress has a default route;
|
# Control network only — agents are never on it (L3-isolated).
|
||||||
# the host-only network is where agents (and the host CLI) reach it.
|
"--network", self.control_network,
|
||||||
"--network", self.egress_network,
|
|
||||||
"--network", self.network,
|
|
||||||
"--dns", container_mod.dns_server(),
|
"--dns", container_mod.dns_server(),
|
||||||
# Container-only DB volume: one kernel writes bot-bottle.db, never
|
# Container-only DB volume: exactly one kernel writes bot-bottle.db.
|
||||||
# shared with the host or another guest.
|
|
||||||
"--volume", f"{self._db_volume}:{_DB_ROOT_IN_CONTAINER}",
|
"--volume", f"{self._db_volume}:{_DB_ROOT_IN_CONTAINER}",
|
||||||
# The DB needs a container-only ext4 volume for coherent SQLite
|
# Live control-plane source (a code change takes effect on relaunch).
|
||||||
# locking, but the CA has no such constraint. Keep it in the host
|
|
||||||
# app-data root so infra-container recreation and Apple Container
|
|
||||||
# volume pruning cannot silently rotate every bottle's trust
|
|
||||||
# anchor (issue #450).
|
|
||||||
"--mount",
|
|
||||||
container_mod.bind_mount_spec(
|
|
||||||
str(host_gateway_ca_dir()), MITMPROXY_HOME),
|
|
||||||
# Bind-mount the control-plane source (read-only); a code change
|
|
||||||
# takes effect on relaunch with no image rebuild.
|
|
||||||
"--mount",
|
"--mount",
|
||||||
container_mod.bind_mount_spec(
|
container_mod.bind_mount_spec(
|
||||||
str(self._repo_root), _SRC_IN_CONTAINER, readonly=True),
|
str(self._repo_root), _SRC_IN_CONTAINER, readonly=True),
|
||||||
# Baked onto the container so `_source_current` can detect a real
|
"--env", f"PYTHONPATH={_SRC_IN_CONTAINER}",
|
||||||
# control-plane code change and recreate.
|
"--env", f"BOT_BOTTLE_ROOT={_DB_ROOT_IN_CONTAINER}",
|
||||||
|
# Detect a real control-plane code change and recreate.
|
||||||
"--env", f"BOT_BOTTLE_SOURCE_HASH={current_hash}",
|
"--env", f"BOT_BOTTLE_SOURCE_HASH={current_hash}",
|
||||||
# The control-plane signing key (control plane: verifies tokens) and
|
# The signing key — held ONLY by the orchestrator (issue #469). Bare
|
||||||
# the pre-minted `gateway` JWT (the gateway's PolicyResolver: presents
|
# `--env NAME` keeps the value off argv / `container inspect`.
|
||||||
# it) — they share this one container, and gateway_init scopes each to
|
|
||||||
# its process so a compromised data-plane daemon never sees the key
|
|
||||||
# (issue #469 review). Bare `--env NAME` inherits the value from the
|
|
||||||
# run process below, so neither lands on argv or in `container
|
|
||||||
# inspect`'s command line. The agent runs in a SEPARATE container that
|
|
||||||
# is never given these vars, which is the whole point.
|
|
||||||
"--env", ORCHESTRATOR_TOKEN_ENV,
|
"--env", ORCHESTRATOR_TOKEN_ENV,
|
||||||
"--env", ORCHESTRATOR_AUTH_JWT_ENV,
|
self.orchestrator_image,
|
||||||
"--entrypoint", "sh",
|
# Dockerfile.orchestrator ENTRYPOINT is `-m bot_bottle.orchestrator`.
|
||||||
self.image,
|
"--host", "0.0.0.0", "--port", str(self.port), "--broker", "stub",
|
||||||
"-c", _init_script(self.port),
|
|
||||||
]
|
]
|
||||||
_signing_key = host_orchestrator_token()
|
result = container_mod.run_container_argv(
|
||||||
run_env = {
|
argv, env={**os.environ, ORCHESTRATOR_TOKEN_ENV: _signing_key})
|
||||||
**os.environ,
|
|
||||||
ORCHESTRATOR_TOKEN_ENV: _signing_key,
|
|
||||||
ORCHESTRATOR_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key),
|
|
||||||
}
|
|
||||||
result = container_mod.run_container_argv(argv, env=run_env)
|
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
raise OrchestratorStartError(
|
raise OrchestratorStartError(
|
||||||
f"infra container failed to start: "
|
f"orchestrator container failed to start: "
|
||||||
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
||||||
)
|
)
|
||||||
|
|
||||||
def _wait_healthy(self, startup_timeout: float) -> InfraEndpoint:
|
def _ensure_gateway_container(self, orchestrator_url: str) -> None:
|
||||||
|
"""Start (recreate) the gateway container, dual-homed on the agent +
|
||||||
|
control networks, resolving policy against `orchestrator_url` (the
|
||||||
|
orchestrator's control-network address — Apple has no DNS)."""
|
||||||
|
container_mod.force_remove_container(self._gateway_name)
|
||||||
|
_signing_key = host_orchestrator_token()
|
||||||
|
argv = [
|
||||||
|
"container", "run", "--detach",
|
||||||
|
"--name", self._gateway_name,
|
||||||
|
"--label", "bot-bottle.backend=macos-container",
|
||||||
|
"--label", INFRA_LABEL,
|
||||||
|
# NAT egress FIRST (default route out); the host-only agent network
|
||||||
|
# is where agents reach the gateway; the control network reaches the
|
||||||
|
# orchestrator.
|
||||||
|
"--network", self.egress_network,
|
||||||
|
"--network", self.network,
|
||||||
|
"--network", self.control_network,
|
||||||
|
"--dns", container_mod.dns_server(),
|
||||||
|
# The mitmproxy CA on a host bind-mount (survives recreation +
|
||||||
|
# volume pruning — issue #450). No DB mount: the data plane never
|
||||||
|
# opens bot-bottle.db (#469).
|
||||||
|
"--mount",
|
||||||
|
container_mod.bind_mount_spec(str(host_gateway_ca_dir()), MITMPROXY_HOME),
|
||||||
|
"--env", f"BOT_BOTTLE_GATEWAY_DAEMONS={_GATEWAY_DAEMONS}",
|
||||||
|
"--env", f"BOT_BOTTLE_ORCHESTRATOR_URL={orchestrator_url}",
|
||||||
|
# The pre-minted `gateway` JWT (never the signing key). Bare
|
||||||
|
# `--env NAME` inherits the value from run_env below.
|
||||||
|
"--env", ORCHESTRATOR_AUTH_JWT_ENV,
|
||||||
|
self.gateway_image,
|
||||||
|
]
|
||||||
|
run_env = {**os.environ, ORCHESTRATOR_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key)}
|
||||||
|
result = container_mod.run_container_argv(argv, env=run_env)
|
||||||
|
if result.returncode != 0:
|
||||||
|
raise GatewayError(
|
||||||
|
f"gateway container failed to start: "
|
||||||
|
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _wait_healthy(self, startup_timeout: float) -> str:
|
||||||
deadline = time.monotonic() + startup_timeout
|
deadline = time.monotonic() + startup_timeout
|
||||||
while True:
|
while True:
|
||||||
url = self._resolve_url()
|
url = self._resolve_orchestrator_url()
|
||||||
if url and self.is_healthy(url):
|
if url and self.is_healthy(url):
|
||||||
log.info("infra container healthy", context={"url": url})
|
log.info("orchestrator healthy", context={"url": url})
|
||||||
return InfraEndpoint(orchestrator_url=url, gateway_ip=_ip_of(url))
|
return url
|
||||||
if time.monotonic() >= deadline:
|
if time.monotonic() >= deadline:
|
||||||
raise OrchestratorStartError(
|
raise OrchestratorStartError(
|
||||||
f"infra container did not become healthy within "
|
f"orchestrator did not become healthy within "
|
||||||
f"{startup_timeout:g}s"
|
f"{startup_timeout:g}s"
|
||||||
)
|
)
|
||||||
time.sleep(_HEALTH_POLL_SECONDS)
|
time.sleep(_HEALTH_POLL_SECONDS)
|
||||||
|
|
||||||
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
|
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
|
||||||
"""The gateway's mitmproxy CA (PEM) agents install to trust its TLS
|
"""The gateway's mitmproxy CA (PEM) agents install to trust its TLS
|
||||||
interception. Read through the container path backed by the persistent
|
interception. Read from the gateway container; polls because mitmproxy
|
||||||
host CA directory; polls because mitmproxy writes it a beat after
|
writes it a beat after start."""
|
||||||
start."""
|
|
||||||
def _fetch() -> str | None:
|
def _fetch() -> str | None:
|
||||||
result = container_mod.run_container_argv(
|
result = container_mod.run_container_argv(
|
||||||
["container", "exec", self._name, "cat", GATEWAY_CA_CERT])
|
["container", "exec", self._gateway_name, "cat", GATEWAY_CA_CERT])
|
||||||
return result.stdout if result.returncode == 0 and result.stdout.strip() else None
|
return result.stdout if result.returncode == 0 and result.stdout.strip() else None
|
||||||
try:
|
try:
|
||||||
return backend_util.poll_ca_cert(_fetch, timeout=timeout)
|
return backend_util.poll_ca_cert(_fetch, timeout=timeout)
|
||||||
except TimeoutError as exc:
|
except TimeoutError as exc:
|
||||||
raise GatewayError(
|
raise GatewayError(
|
||||||
f"gateway CA not available in {self._name} after {timeout:g}s"
|
f"gateway CA not available in {self._gateway_name} after {timeout:g}s"
|
||||||
) from exc
|
) from exc
|
||||||
|
|
||||||
def stop(self) -> None:
|
def stop(self) -> None:
|
||||||
"""Remove the infra container (idempotent). The DB volume persists."""
|
"""Remove both containers (idempotent). The DB volume persists."""
|
||||||
container_mod.force_remove_container(self._name)
|
container_mod.force_remove_container(self._gateway_name)
|
||||||
|
container_mod.force_remove_container(self._orchestrator_name)
|
||||||
|
|
||||||
def _ip_of(url: str) -> str:
|
|
||||||
"""The host from an http://host:port URL."""
|
|
||||||
return url.split("://", 1)[-1].rsplit(":", 1)[0]
|
|
||||||
|
|
||||||
|
|
||||||
def probe_orchestrator_url(port: int = DEFAULT_PORT) -> str:
|
def probe_orchestrator_url(port: int = DEFAULT_PORT) -> str:
|
||||||
"""The running infra container's control-plane URL, or "" if it isn't up.
|
"""The running orchestrator's control-plane URL, or "" if it isn't up. Used
|
||||||
Used by host-side control-plane discovery (`discover_orchestrator_url`);
|
by host-side control-plane discovery; safe on any host (returns "" when the
|
||||||
safe to call on any host — returns "" when the container or the `container`
|
container or the `container` CLI isn't present)."""
|
||||||
CLI isn't present."""
|
ip = container_mod.try_container_ipv4_on_network(ORCHESTRATOR_NAME, CONTROL_NETWORK)
|
||||||
ip = container_mod.try_container_ipv4_on_network(INFRA_NAME, GATEWAY_NETWORK)
|
|
||||||
return f"http://{ip}:{port}" if ip else ""
|
return f"http://{ip}:{port}" if ip else ""
|
||||||
|
|
||||||
|
|
||||||
@@ -320,6 +316,7 @@ __all__ = [
|
|||||||
"InfraEndpoint",
|
"InfraEndpoint",
|
||||||
"OrchestratorStartError",
|
"OrchestratorStartError",
|
||||||
"GatewayError",
|
"GatewayError",
|
||||||
|
"ORCHESTRATOR_NAME",
|
||||||
"INFRA_NAME",
|
"INFRA_NAME",
|
||||||
"INFRA_DB_VOLUME",
|
"INFRA_DB_VOLUME",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
"""Unit: the single macOS infra container (control plane + gateway, PRD 0070)."""
|
"""Unit: macOS orchestrator + gateway containers (PRD 0070 plane split)."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -24,86 +24,94 @@ def _fail(stderr: str = "boom") -> Mock:
|
|||||||
return Mock(returncode=1, stdout="", stderr=stderr)
|
return Mock(returncode=1, stdout="", stderr=stderr)
|
||||||
|
|
||||||
|
|
||||||
class TestInfraRun(unittest.TestCase):
|
def _spec(src: str, tgt: str, readonly: bool = False) -> str:
|
||||||
def _run_container(self, svc: MacosInfraService) -> list[str]:
|
return f"type=bind,source={src},target={tgt}" + (",readonly" if readonly else "")
|
||||||
|
|
||||||
|
|
||||||
|
class TestOrchestratorRun(unittest.TestCase):
|
||||||
|
def _run(self, svc: MacosInfraService) -> list[str]:
|
||||||
run = Mock(return_value=_ok())
|
run = Mock(return_value=_ok())
|
||||||
|
|
||||||
def _spec(src: str, tgt: str, readonly: bool = False) -> str:
|
|
||||||
return f"type=bind,source={src},target={tgt}" + (
|
|
||||||
",readonly" if readonly else "")
|
|
||||||
|
|
||||||
with patch(f"{_INFRA}.container_mod") as mod, \
|
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||||
patch(f"{_INFRA}.ensure_networks"):
|
patch(f"{_INFRA}.host_orchestrator_token", return_value="k"):
|
||||||
mod.dns_server.return_value = "1.1.1.1"
|
mod.dns_server.return_value = "1.1.1.1"
|
||||||
mod.bind_mount_spec.side_effect = _spec
|
mod.bind_mount_spec.side_effect = _spec
|
||||||
mod.run_container_argv = run
|
mod.run_container_argv = run
|
||||||
svc._run_container("h1")
|
svc._run_orchestrator_container("h1")
|
||||||
return run.call_args.args[0]
|
return run.call_args.args[0]
|
||||||
|
|
||||||
def test_single_container_runs_both_processes(self) -> None:
|
def test_runs_the_orchestrator_on_the_control_network_only(self) -> None:
|
||||||
"""The whole point: one container starts the control plane AND the
|
argv = self._run(MacosInfraService(repo_root=Path("/r")))
|
||||||
gateway daemons, so one kernel owns the DB."""
|
nets = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
|
||||||
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
|
self.assertEqual(["bot-bottle-mac-control"], nets)
|
||||||
script = argv[-1]
|
# Image ENTRYPOINT is `-m bot_bottle.orchestrator`; these are its args.
|
||||||
self.assertIn("bot_bottle.orchestrator", script)
|
self.assertIn("--broker", argv)
|
||||||
# Gateway launches via the installed package (there is no
|
self.assertIn("stub", argv)
|
||||||
# /app/gateway_init.py file since the daemons moved into bot_bottle).
|
self.assertIn("bot-bottle-orchestrator:latest", argv)
|
||||||
self.assertIn("bot_bottle.gateway.bootstrap", script)
|
|
||||||
self.assertIn("127.0.0.1", script) # they reach each other on loopback
|
|
||||||
|
|
||||||
def test_db_is_a_container_only_volume(self) -> None:
|
def test_db_is_a_container_only_volume(self) -> None:
|
||||||
"""No host bind-mount of the DB — a named volume only this container
|
argv = self._run(MacosInfraService(repo_root=Path("/r")))
|
||||||
mounts, so the DB is never written by two kernels."""
|
|
||||||
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
|
|
||||||
vols = [argv[i + 1] for i, a in enumerate(argv) if a == "--volume"]
|
vols = [argv[i + 1] for i, a in enumerate(argv) if a == "--volume"]
|
||||||
self.assertTrue(any(v.startswith(f"{INFRA_DB_VOLUME}:") for v in vols))
|
self.assertTrue(any(v.startswith(f"{INFRA_DB_VOLUME}:") for v in vols))
|
||||||
# The repo source is bind-mounted read-only; the DB is not a bind mount.
|
|
||||||
mounts = [argv[i + 1] for i, a in enumerate(argv) if a == "--mount"]
|
|
||||||
self.assertTrue(all("bot-bottle.db" not in m for m in mounts))
|
|
||||||
|
|
||||||
def test_ca_is_persisted_on_the_host_not_the_container_volume(self) -> None:
|
def test_orchestrator_has_no_ca_mount(self) -> None:
|
||||||
"""The CA survives infra recreation and cannot be removed by Apple
|
# The CA lives with the gateway, not the control plane.
|
||||||
Container's volume-prune command."""
|
argv = self._run(MacosInfraService(repo_root=Path("/r")))
|
||||||
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
|
|
||||||
mounts = [argv[i + 1] for i, a in enumerate(argv) if a == "--mount"]
|
mounts = [argv[i + 1] for i, a in enumerate(argv) if a == "--mount"]
|
||||||
ca_mounts = [
|
self.assertFalse([m for m in mounts if "/home/mitmproxy" in m])
|
||||||
m for m in mounts
|
|
||||||
if "target=/home/mitmproxy/.mitmproxy" in m
|
|
||||||
]
|
|
||||||
self.assertEqual(1, len(ca_mounts))
|
|
||||||
self.assertIn("source=", ca_mounts[0])
|
|
||||||
self.assertIn("/gateway-ca", ca_mounts[0])
|
|
||||||
self.assertNotIn(",readonly", ca_mounts[0])
|
|
||||||
|
|
||||||
def test_nat_network_precedes_the_host_only_network(self) -> None:
|
|
||||||
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
|
|
||||||
nets = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
|
|
||||||
self.assertEqual(["bot-bottle-mac-egress", "bot-bottle-mac-gateway"], nets)
|
|
||||||
|
|
||||||
def test_source_hash_is_labelled_for_recreate(self) -> None:
|
def test_source_hash_is_labelled_for_recreate(self) -> None:
|
||||||
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
|
argv = self._run(MacosInfraService(repo_root=Path("/r")))
|
||||||
self.assertIn("BOT_BOTTLE_SOURCE_HASH=h1", argv)
|
self.assertIn("BOT_BOTTLE_SOURCE_HASH=h1", argv)
|
||||||
|
|
||||||
def test_start_failure_raises(self) -> None:
|
def test_start_failure_raises(self) -> None:
|
||||||
svc = MacosInfraService(repo_root=Path("/r"))
|
svc = MacosInfraService(repo_root=Path("/r"))
|
||||||
with patch(f"{_INFRA}.container_mod") as mod, \
|
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||||
patch(f"{_INFRA}.ensure_networks"):
|
patch(f"{_INFRA}.host_orchestrator_token", return_value="k"):
|
||||||
mod.dns_server.return_value = "1.1.1.1"
|
mod.dns_server.return_value = "1.1.1.1"
|
||||||
mod.bind_mount_spec.return_value = "m"
|
|
||||||
mod.run_container_argv = Mock(return_value=_fail())
|
mod.run_container_argv = Mock(return_value=_fail())
|
||||||
with self.assertRaises(OrchestratorStartError):
|
with self.assertRaises(OrchestratorStartError):
|
||||||
svc._run_container("h1")
|
svc._run_orchestrator_container("h1")
|
||||||
|
|
||||||
|
|
||||||
|
class TestGatewayRun(unittest.TestCase):
|
||||||
|
def _run(self, svc: MacosInfraService, url: str = "http://10.0.0.5:8099") -> list[str]:
|
||||||
|
run = Mock(return_value=_ok())
|
||||||
|
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||||
|
patch(f"{_INFRA}.host_orchestrator_token", return_value="k"), \
|
||||||
|
patch(f"{_INFRA}.mint", return_value="jwt"):
|
||||||
|
mod.dns_server.return_value = "1.1.1.1"
|
||||||
|
mod.bind_mount_spec.side_effect = _spec
|
||||||
|
mod.run_container_argv = run
|
||||||
|
svc._ensure_gateway_container(url)
|
||||||
|
return run.call_args.args[0]
|
||||||
|
|
||||||
|
def test_gateway_is_triple_homed(self) -> None:
|
||||||
|
argv = self._run(MacosInfraService(repo_root=Path("/r")))
|
||||||
|
nets = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
|
||||||
|
self.assertEqual(
|
||||||
|
["bot-bottle-mac-egress", "bot-bottle-mac-gateway", "bot-bottle-mac-control"],
|
||||||
|
nets,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_gateway_resolves_orchestrator_by_url_and_holds_ca(self) -> None:
|
||||||
|
argv = self._run(MacosInfraService(repo_root=Path("/r")), "http://10.0.0.5:8099")
|
||||||
|
self.assertIn("BOT_BOTTLE_ORCHESTRATOR_URL=http://10.0.0.5:8099", argv)
|
||||||
|
mounts = [argv[i + 1] for i, a in enumerate(argv) if a == "--mount"]
|
||||||
|
self.assertTrue([m for m in mounts if "target=/home/mitmproxy/.mitmproxy" in m])
|
||||||
|
self.assertIn("bot-bottle-gateway:latest", argv)
|
||||||
|
self.assertIn(
|
||||||
|
"BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise", argv)
|
||||||
|
|
||||||
|
|
||||||
class TestInfraEnsureRunning(unittest.TestCase):
|
class TestInfraEnsureRunning(unittest.TestCase):
|
||||||
def test_current_healthy_container_is_left_alone(self) -> None:
|
def test_current_healthy_orchestrator_left_alone(self) -> None:
|
||||||
"""Idempotent singleton: N launches must not churn the infra container
|
|
||||||
and drop every live bottle's control plane."""
|
|
||||||
svc = MacosInfraService(repo_root=Path("/r"))
|
svc = MacosInfraService(repo_root=Path("/r"))
|
||||||
run = Mock()
|
run = Mock()
|
||||||
with patch(f"{_INFRA}.container_mod") as mod, \
|
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||||
patch(f"{_INFRA}.source_hash", return_value="h1"), \
|
patch(f"{_INFRA}.source_hash", return_value="h1"), \
|
||||||
patch.object(svc, "_run_container", run), \
|
patch.object(svc, "ensure_built"), \
|
||||||
|
patch.object(svc, "_run_orchestrator_container", run), \
|
||||||
|
patch.object(svc, "_ensure_gateway_container"), \
|
||||||
patch.object(svc, "is_healthy", return_value=True):
|
patch.object(svc, "is_healthy", return_value=True):
|
||||||
mod.container_is_running.return_value = True
|
mod.container_is_running.return_value = True
|
||||||
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
|
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
|
||||||
@@ -113,13 +121,14 @@ class TestInfraEnsureRunning(unittest.TestCase):
|
|||||||
self.assertEqual("http://192.168.128.2:8099", endpoint.orchestrator_url)
|
self.assertEqual("http://192.168.128.2:8099", endpoint.orchestrator_url)
|
||||||
self.assertEqual("192.168.128.2", endpoint.gateway_ip)
|
self.assertEqual("192.168.128.2", endpoint.gateway_ip)
|
||||||
|
|
||||||
def test_changed_source_recreates(self) -> None:
|
def test_changed_source_recreates_orchestrator(self) -> None:
|
||||||
svc = MacosInfraService(repo_root=Path("/r"))
|
svc = MacosInfraService(repo_root=Path("/r"))
|
||||||
run = Mock()
|
run = Mock()
|
||||||
with patch(f"{_INFRA}.container_mod") as mod, \
|
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||||
patch(f"{_INFRA}.source_hash", return_value="h2"), \
|
patch(f"{_INFRA}.source_hash", return_value="h2"), \
|
||||||
patch.object(svc, "ensure_built"), \
|
patch.object(svc, "ensure_built"), \
|
||||||
patch.object(svc, "_run_container", run), \
|
patch.object(svc, "_run_orchestrator_container", run), \
|
||||||
|
patch.object(svc, "_ensure_gateway_container"), \
|
||||||
patch.object(svc, "is_healthy", return_value=True):
|
patch.object(svc, "is_healthy", return_value=True):
|
||||||
mod.container_is_running.return_value = True
|
mod.container_is_running.return_value = True
|
||||||
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
|
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
|
||||||
@@ -127,17 +136,15 @@ class TestInfraEnsureRunning(unittest.TestCase):
|
|||||||
svc.ensure_running()
|
svc.ensure_running()
|
||||||
run.assert_called_once()
|
run.assert_called_once()
|
||||||
|
|
||||||
def test_wedged_but_current_container_is_recreated(self) -> None:
|
def test_wedged_but_current_orchestrator_recreated(self) -> None:
|
||||||
"""Current source but a dead HTTP server must be recreated, not polled
|
|
||||||
to death forever — health, not just the source label, gates reuse."""
|
|
||||||
svc = MacosInfraService(repo_root=Path("/r"))
|
svc = MacosInfraService(repo_root=Path("/r"))
|
||||||
run = Mock()
|
run = Mock()
|
||||||
health = Mock(side_effect=[False, True])
|
|
||||||
with patch(f"{_INFRA}.container_mod") as mod, \
|
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||||
patch(f"{_INFRA}.source_hash", return_value="h1"), \
|
patch(f"{_INFRA}.source_hash", return_value="h1"), \
|
||||||
patch.object(svc, "ensure_built"), \
|
patch.object(svc, "ensure_built"), \
|
||||||
patch.object(svc, "_run_container", run), \
|
patch.object(svc, "_run_orchestrator_container", run), \
|
||||||
patch.object(svc, "is_healthy", health):
|
patch.object(svc, "_ensure_gateway_container"), \
|
||||||
|
patch.object(svc, "is_healthy", Mock(side_effect=[False, True])):
|
||||||
mod.container_is_running.return_value = True
|
mod.container_is_running.return_value = True
|
||||||
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
|
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
|
||||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||||
@@ -149,7 +156,8 @@ class TestInfraEnsureRunning(unittest.TestCase):
|
|||||||
with patch(f"{_INFRA}.container_mod") as mod, \
|
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||||
patch(f"{_INFRA}.source_hash", return_value="h1"), \
|
patch(f"{_INFRA}.source_hash", return_value="h1"), \
|
||||||
patch.object(svc, "ensure_built"), \
|
patch.object(svc, "ensure_built"), \
|
||||||
patch.object(svc, "_run_container"), \
|
patch.object(svc, "_run_orchestrator_container"), \
|
||||||
|
patch.object(svc, "_ensure_gateway_container"), \
|
||||||
patch.object(svc, "is_healthy", return_value=False):
|
patch.object(svc, "is_healthy", return_value=False):
|
||||||
mod.container_is_running.return_value = False
|
mod.container_is_running.return_value = False
|
||||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||||
@@ -158,7 +166,7 @@ class TestInfraEnsureRunning(unittest.TestCase):
|
|||||||
|
|
||||||
|
|
||||||
class TestCaCertPem(unittest.TestCase):
|
class TestCaCertPem(unittest.TestCase):
|
||||||
def test_reads_ca_out_of_the_container(self) -> None:
|
def test_reads_ca_out_of_the_gateway_container(self) -> None:
|
||||||
svc = MacosInfraService(repo_root=Path("/r"))
|
svc = MacosInfraService(repo_root=Path("/r"))
|
||||||
with patch(f"{_INFRA}.container_mod") as mod:
|
with patch(f"{_INFRA}.container_mod") as mod:
|
||||||
mod.run_container_argv.return_value = _ok("-----BEGIN CERTIFICATE-----\n")
|
mod.run_container_argv.return_value = _ok("-----BEGIN CERTIFICATE-----\n")
|
||||||
|
|||||||
Reference in New Issue
Block a user