refactor(orchestrator): introduce the Orchestrator service ABC (docker impl)
Mirror the Gateway work for the control plane. Add the backend-neutral `Orchestrator` service ABC in orchestrator/lifecycle.py — build the image/rootfs, `ensure_running` (start + health-gate), `is_running`/`is_healthy`/`stop`, `url()` (host-facing) vs `gateway_url()` (what the gateway resolves against), and a concrete `mint_gateway_token()` (the orchestrator holds the signing key, so it issues the gateway's token — #469). `DockerOrchestrator` (backend/docker/orchestrator.py) is the first impl, extracted out of `DockerInfraService`: the control-plane container run, the `--internal` control network, the source-hash recreate gate, health polling, and the orchestrator constants (name/label/network/image/dockerfile/hash-label). `DockerInfraService` now composes `orchestrator()` + `gateway()` — each builds its own image via `ensure_built`, the orchestrator comes up first, then the gateway is connected with the orchestrator-minted token. Its `url`/`is_healthy` delegate to the orchestrator service. Split the container-lifecycle tests into test_docker_orchestrator; test_docker_infra now covers the composition. macOS + firecracker follow. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -3,10 +3,10 @@
|
||||
Runs the orchestrator (control plane) and the gateway (data plane) as **two
|
||||
separate containers**, split now that #469 got the DB off the data plane:
|
||||
|
||||
* `bot-bottle-orchestrator` — the lean control-plane container. Joins the
|
||||
`bot-bottle-orchestrator` **control network** (`--internal`) only, plus a
|
||||
host-loopback publish for the CLI. Sole opener of `bot-bottle.db`; holds the
|
||||
signing key.
|
||||
* `bot-bottle-orchestrator` — the lean control-plane container
|
||||
(`DockerOrchestrator`). Joins the `bot-bottle-orchestrator` **control
|
||||
network** (`--internal`) only, plus a host-loopback publish for the CLI.
|
||||
Sole opener of `bot-bottle.db`; holds the signing key.
|
||||
* `bot-bottle-gateway` — the data-plane container (`DockerGateway`).
|
||||
**Dual-homed** on the agent-facing `bot-bottle-gateway` network *and* the
|
||||
control network, so it reaches the orchestrator by name over docker DNS
|
||||
@@ -14,83 +14,47 @@ separate containers**, split now that #469 got the DB off the data plane:
|
||||
the control network — cannot reach the control plane at all (the L3 block
|
||||
Firecracker's nft already has). Holds the `gateway` JWT + the mitmproxy CA.
|
||||
|
||||
`DockerInfraService` brings both up as an idempotent per-host pair. Callers use
|
||||
`ensure_running()` (returns the host control-plane URL) + `gateway_name` (the
|
||||
container the launch flow attributes agents against).
|
||||
`DockerInfraService` composes the two services (`orchestrator()` + `gateway()`)
|
||||
and brings them up as an idempotent per-host pair. Callers use `ensure_running()`
|
||||
(returns the host control-plane URL) + `gateway_name` (the container the launch
|
||||
flow attributes agents against).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from ... import log
|
||||
from .util import run_docker
|
||||
from .gateway import DockerGateway
|
||||
from ...paths import (
|
||||
ORCHESTRATOR_TOKEN_ENV,
|
||||
bot_bottle_root,
|
||||
host_orchestrator_token,
|
||||
from .orchestrator import (
|
||||
DockerOrchestrator,
|
||||
ORCHESTRATOR_IMAGE,
|
||||
ORCHESTRATOR_LABEL,
|
||||
ORCHESTRATOR_NAME,
|
||||
ORCHESTRATOR_NETWORK,
|
||||
ORCHESTRATOR_SOURCE_HASH_LABEL,
|
||||
)
|
||||
from ...paths import bot_bottle_root
|
||||
from ...gateway import (
|
||||
GATEWAY_DOCKERFILE,
|
||||
GATEWAY_IMAGE,
|
||||
GATEWAY_NAME,
|
||||
GATEWAY_NETWORK,
|
||||
GatewayError,
|
||||
)
|
||||
from ...orchestrator_auth import ROLE_GATEWAY, mint
|
||||
from ...orchestrator.lifecycle import (
|
||||
DEFAULT_PORT,
|
||||
DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||
OrchestratorStartError,
|
||||
source_hash,
|
||||
)
|
||||
|
||||
# The control plane's own container + the dedicated control network the gateway
|
||||
# reaches it over. The network is created `--internal`: the orchestrator and
|
||||
# gateway join it, agents never do, so agents have no route to the control
|
||||
# plane (PRD 0070 "Separating the planes"). Same name across docker + macOS.
|
||||
ORCHESTRATOR_NAME = "bot-bottle-orchestrator"
|
||||
ORCHESTRATOR_LABEL = "bot-bottle-orchestrator=1"
|
||||
ORCHESTRATOR_NETWORK = "bot-bottle-orchestrator"
|
||||
ORCHESTRATOR_IMAGE = os.environ.get(
|
||||
"BOT_BOTTLE_ORCHESTRATOR_IMAGE", "bot-bottle-orchestrator:latest"
|
||||
)
|
||||
ORCHESTRATOR_DOCKERFILE = "Dockerfile.orchestrator"
|
||||
# Baked as a container label so `ensure_running` can detect whether the running
|
||||
# orchestrator is executing the current bind-mounted source.
|
||||
ORCHESTRATOR_SOURCE_HASH_LABEL = "bot-bottle-orchestrator-source-hash"
|
||||
|
||||
# The bind-mount path for the live control-plane source inside the orchestrator
|
||||
# container. PYTHONPATH points here so a code change takes effect on the next
|
||||
# launch without an image rebuild.
|
||||
_SRC_IN_CONTAINER = "/bot-bottle-src"
|
||||
# Bot-bottle host-root bind-mount (DB + state) inside the orchestrator. The
|
||||
# control plane opens bot-bottle.db under here (via BOT_BOTTLE_ROOT ->
|
||||
# host_db_path()); it is the ONLY container with a handle on it (issue #469).
|
||||
_ROOT_IN_CONTAINER = "/bot-bottle-root"
|
||||
|
||||
# The URL the gateway's data plane resolves policy against — the orchestrator
|
||||
# container by name on the control network (docker DNS, container↔container).
|
||||
ORCHESTRATOR_URL_IN_NETWORK = f"http://{ORCHESTRATOR_NAME}:{DEFAULT_PORT}"
|
||||
|
||||
# Back-compat aliases: the split retired the combined `bot-bottle-infra`
|
||||
# container, but the fixed-name constants some callers/tests import still map to
|
||||
# the pair's public identity.
|
||||
INFRA_NAME = GATEWAY_NAME # the container agents attribute against is the gateway
|
||||
|
||||
_HEALTH_POLL_SECONDS = 0.25
|
||||
_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
class DockerInfraService:
|
||||
"""Brings up the per-host control plane + gateway as two containers.
|
||||
"""Composes the per-host control plane + gateway as two containers.
|
||||
|
||||
`orchestrator_name` / `gateway_name` let callers run independent pairs on
|
||||
one host without name collisions (e.g. isolated integration tests that
|
||||
@@ -130,99 +94,24 @@ class DockerInfraService:
|
||||
@property
|
||||
def url(self) -> str:
|
||||
"""Host-side control-plane URL (the orchestrator's published loopback)."""
|
||||
return f"http://127.0.0.1:{self.port}"
|
||||
return self.orchestrator().url()
|
||||
|
||||
def is_healthy(self, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS) -> bool:
|
||||
try:
|
||||
with urllib.request.urlopen(f"{self.url}/health", timeout=timeout) as resp:
|
||||
return resp.status == 200
|
||||
except (urllib.error.URLError, TimeoutError, OSError):
|
||||
return False
|
||||
def is_healthy(self) -> bool:
|
||||
return self.orchestrator().is_healthy()
|
||||
|
||||
def _container_running(self, name: str) -> bool:
|
||||
proc = run_docker(["docker", "ps", "--filter", f"name=^/{name}$", "--format", "{{.Names}}"])
|
||||
return name in proc.stdout.split()
|
||||
|
||||
def _orchestrator_source_current(self, current_hash: str) -> bool:
|
||||
"""True iff the running orchestrator was started from the current
|
||||
bind-mounted source."""
|
||||
if not self._container_running(self._orchestrator_name):
|
||||
return False
|
||||
proc = run_docker([
|
||||
"docker", "inspect", "--format",
|
||||
"{{ index .Config.Labels \"" + ORCHESTRATOR_SOURCE_HASH_LABEL + "\" }}",
|
||||
self._orchestrator_name,
|
||||
])
|
||||
if proc.returncode != 0:
|
||||
return True # can't compare → don't churn a working container
|
||||
return proc.stdout.strip() == current_hash
|
||||
|
||||
def _ensure_network(self, name: str, *, internal: bool) -> None:
|
||||
if run_docker(["docker", "network", "inspect", name]).returncode == 0:
|
||||
return
|
||||
argv = ["docker", "network", "create"]
|
||||
if internal:
|
||||
argv.append("--internal")
|
||||
argv.append(name)
|
||||
proc = run_docker(argv)
|
||||
if proc.returncode != 0 and "already exists" not in proc.stderr:
|
||||
raise GatewayError(f"network {name} failed to create: {proc.stderr.strip()}")
|
||||
|
||||
def _build_images(self) -> None:
|
||||
"""Build the gateway (data plane) and orchestrator (control plane)
|
||||
images. Cache-aware: a no-op when nothing changed. The combined
|
||||
`Dockerfile.infra` is no longer built — the planes ship as two images."""
|
||||
for tag, dockerfile in (
|
||||
(self.gateway_image, GATEWAY_DOCKERFILE),
|
||||
(self.orchestrator_image, ORCHESTRATOR_DOCKERFILE),
|
||||
):
|
||||
argv = ["docker", "build", "-t", tag,
|
||||
"-f", str(self._repo_root / dockerfile),
|
||||
str(self._repo_root)]
|
||||
if os.environ.get("BOT_BOTTLE_NO_CACHE"):
|
||||
argv.insert(2, "--no-cache")
|
||||
proc = run_docker(argv)
|
||||
if proc.returncode != 0:
|
||||
raise GatewayError(f"{dockerfile} build failed: {proc.stderr.strip()}")
|
||||
|
||||
def _run_orchestrator_container(self, current_hash: str) -> None:
|
||||
"""Start the lean control-plane container on the control network only,
|
||||
published to host loopback for the CLI. Idempotent (clears a stale
|
||||
fixed-name container first)."""
|
||||
self._ensure_network(self.control_network, internal=True)
|
||||
run_docker(["docker", "rm", "--force", self._orchestrator_name])
|
||||
_signing_key = host_orchestrator_token()
|
||||
proc = run_docker([
|
||||
"docker", "run", "--detach",
|
||||
"--name", self._orchestrator_name,
|
||||
"--label", self._orchestrator_label,
|
||||
"--label", f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={current_hash}",
|
||||
# Control network only — agents are never on it, so they have no
|
||||
# route to the control plane (the L3 block, not just the JWT).
|
||||
"--network", self.control_network,
|
||||
# Host CLI reaches the control plane here (loopback only). The
|
||||
# orchestrator listens on the fixed DEFAULT_PORT inside the
|
||||
# container; self.port is the host-side published port.
|
||||
"--publish", f"127.0.0.1:{self.port}:{DEFAULT_PORT}",
|
||||
# Live control-plane source (code changes without an image rebuild).
|
||||
"--volume", f"{self._repo_root}:{_SRC_IN_CONTAINER}:ro",
|
||||
"--env", f"PYTHONPATH={_SRC_IN_CONTAINER}",
|
||||
# Orchestrator registry DB on the host (sole writer: control plane).
|
||||
"--volume", f"{self._host_root}:{_ROOT_IN_CONTAINER}",
|
||||
"--env", f"BOT_BOTTLE_ROOT={_ROOT_IN_CONTAINER}",
|
||||
# The signing key — held ONLY by the orchestrator (it verifies
|
||||
# tokens); the gateway gets the pre-minted `gateway` JWT, never the
|
||||
# key (issue #469). Bare `--env NAME` keeps the value off argv.
|
||||
"--env", ORCHESTRATOR_TOKEN_ENV,
|
||||
def orchestrator(self) -> DockerOrchestrator:
|
||||
"""The control-plane service. Cheap to reconstruct — `ensure_built`
|
||||
builds its image, `ensure_running` brings it up, and the launch flow
|
||||
reads its `url()` / `gateway_url()` / `mint_gateway_token()` off it."""
|
||||
return DockerOrchestrator(
|
||||
self.orchestrator_image,
|
||||
# Dockerfile.orchestrator ENTRYPOINT is `python3 -m bot_bottle.orchestrator`;
|
||||
# these are its args.
|
||||
"--host", "0.0.0.0", "--port", str(DEFAULT_PORT), "--broker", "stub",
|
||||
], env={**os.environ, ORCHESTRATOR_TOKEN_ENV: _signing_key})
|
||||
if proc.returncode != 0:
|
||||
raise OrchestratorStartError(
|
||||
f"orchestrator container failed to start: {proc.stderr.strip()}"
|
||||
)
|
||||
name=self._orchestrator_name,
|
||||
label=self._orchestrator_label,
|
||||
port=self.port,
|
||||
control_network=self.control_network,
|
||||
repo_root=self._repo_root,
|
||||
host_root=self._host_root,
|
||||
)
|
||||
|
||||
def gateway(self) -> DockerGateway:
|
||||
"""The data-plane gateway, dual-homed on the agent network + the control
|
||||
@@ -236,7 +125,6 @@ class DockerInfraService:
|
||||
network=self.network,
|
||||
control_network=self.control_network,
|
||||
build_context=self._repo_root,
|
||||
dockerfile=None, # images are built by _build_images
|
||||
)
|
||||
|
||||
def ensure_running(
|
||||
@@ -246,37 +134,24 @@ class DockerInfraService:
|
||||
control-plane URL. Idempotent — a healthy orchestrator on current source
|
||||
(and a current-image gateway) is left untouched. Raises
|
||||
`OrchestratorStartError` on control-plane startup timeout."""
|
||||
self._build_images()
|
||||
self._ensure_network(self.network, internal=False)
|
||||
orchestrator = self.orchestrator()
|
||||
gateway = self.gateway()
|
||||
# Build both images (cache-aware; a no-op when nothing changed) before
|
||||
# bringing either plane up.
|
||||
orchestrator.ensure_built()
|
||||
gateway.ensure_built()
|
||||
|
||||
current_hash = source_hash(self._repo_root)
|
||||
if not (self.is_healthy() and self._orchestrator_source_current(current_hash)):
|
||||
log.info("starting orchestrator container",
|
||||
context={"name": self._orchestrator_name})
|
||||
self._run_orchestrator_container(current_hash)
|
||||
|
||||
deadline = time.monotonic() + startup_timeout
|
||||
while time.monotonic() < deadline:
|
||||
if self.is_healthy():
|
||||
log.info("orchestrator healthy", context={"url": self.url})
|
||||
break
|
||||
time.sleep(_HEALTH_POLL_SECONDS)
|
||||
else:
|
||||
raise OrchestratorStartError(
|
||||
f"orchestrator at {self.url} did not become healthy "
|
||||
f"within {startup_timeout:g}s"
|
||||
)
|
||||
orchestrator.ensure_running(startup_timeout=startup_timeout)
|
||||
|
||||
# Bring up (or refresh) the gateway once the control plane it resolves
|
||||
# against is healthy. The orchestrator (which holds the signing key)
|
||||
# mints the role-scoped `gateway` JWT here and hands it to the gateway,
|
||||
# which never sees the key (#469). `connect_to_orchestrator` is
|
||||
# idempotent.
|
||||
gateway_token = mint(ROLE_GATEWAY, host_orchestrator_token())
|
||||
self.gateway().connect_to_orchestrator(
|
||||
ORCHESTRATOR_URL_IN_NETWORK, gateway_token,
|
||||
gateway.connect_to_orchestrator(
|
||||
orchestrator.gateway_url(), orchestrator.mint_gateway_token(),
|
||||
)
|
||||
return self.url
|
||||
return orchestrator.url()
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Remove both containers (idempotent)."""
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
"""The docker orchestrator (control plane) as a single, fixed-name container
|
||||
(PRD 0070).
|
||||
|
||||
`DockerOrchestrator` is the docker implementation of the backend-neutral
|
||||
`Orchestrator` service. The lean control-plane container joins the `--internal`
|
||||
control network only (agents are never on it, so they have no L3 route to it)
|
||||
plus a host-loopback publish for the CLI. It is the sole opener of
|
||||
`bot-bottle.db` and the sole holder of the signing key (#469).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from ... import log
|
||||
from .util import run_docker
|
||||
from ...paths import (
|
||||
ORCHESTRATOR_TOKEN_ENV,
|
||||
bot_bottle_root,
|
||||
host_orchestrator_token,
|
||||
)
|
||||
from ...gateway import GatewayError
|
||||
from ...orchestrator.lifecycle import (
|
||||
DEFAULT_PORT,
|
||||
DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||
Orchestrator,
|
||||
OrchestratorStartError,
|
||||
source_hash,
|
||||
)
|
||||
|
||||
# The control plane's own container + the dedicated `--internal` control network
|
||||
# the gateway reaches it over. The orchestrator + gateway join it, agents never
|
||||
# do, so agents have no route to the control plane (PRD 0070 "Separating the
|
||||
# planes"). Same name across docker + macOS.
|
||||
ORCHESTRATOR_NAME = "bot-bottle-orchestrator"
|
||||
ORCHESTRATOR_LABEL = "bot-bottle-orchestrator=1"
|
||||
ORCHESTRATOR_NETWORK = "bot-bottle-orchestrator"
|
||||
ORCHESTRATOR_IMAGE = os.environ.get(
|
||||
"BOT_BOTTLE_ORCHESTRATOR_IMAGE", "bot-bottle-orchestrator:latest"
|
||||
)
|
||||
ORCHESTRATOR_DOCKERFILE = "Dockerfile.orchestrator"
|
||||
# Baked as a container label so `ensure_running` can detect whether the running
|
||||
# orchestrator is executing the current bind-mounted source.
|
||||
ORCHESTRATOR_SOURCE_HASH_LABEL = "bot-bottle-orchestrator-source-hash"
|
||||
|
||||
# The bind-mount path for the live control-plane source inside the container.
|
||||
# PYTHONPATH points here so a code change takes effect on the next launch
|
||||
# without an image rebuild.
|
||||
_SRC_IN_CONTAINER = "/bot-bottle-src"
|
||||
# Bot-bottle host-root bind-mount (DB + state) inside the orchestrator. The
|
||||
# control plane opens bot-bottle.db under here (via BOT_BOTTLE_ROOT ->
|
||||
# host_db_path()); it is the ONLY container with a handle on it (issue #469).
|
||||
_ROOT_IN_CONTAINER = "/bot-bottle-root"
|
||||
|
||||
_HEALTH_POLL_SECONDS = 0.25
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
class DockerOrchestrator(Orchestrator):
|
||||
"""The control plane as a single fixed-name container. `ensure_built` builds
|
||||
it from `Dockerfile.orchestrator`; `ensure_running` starts it on the control
|
||||
network + host loopback and blocks until `/health` answers."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_ref: str = ORCHESTRATOR_IMAGE,
|
||||
*,
|
||||
name: str = ORCHESTRATOR_NAME,
|
||||
label: str = ORCHESTRATOR_LABEL,
|
||||
port: int = DEFAULT_PORT,
|
||||
control_network: str = ORCHESTRATOR_NETWORK,
|
||||
repo_root: Path = _REPO_ROOT,
|
||||
host_root: Path | None = None,
|
||||
dockerfile: str | None = ORCHESTRATOR_DOCKERFILE,
|
||||
) -> None:
|
||||
self.image_ref = image_ref
|
||||
self.name = name
|
||||
self.label = label
|
||||
self.port = port
|
||||
self.control_network = control_network
|
||||
self._repo_root = repo_root
|
||||
self._host_root = host_root or bot_bottle_root()
|
||||
self._dockerfile = dockerfile
|
||||
|
||||
def url(self) -> str:
|
||||
"""Host-side control-plane URL — the orchestrator's published loopback,
|
||||
which the CLI reaches."""
|
||||
return f"http://127.0.0.1:{self.port}"
|
||||
|
||||
def gateway_url(self) -> str:
|
||||
"""The URL the gateway's data plane resolves policy against — the
|
||||
orchestrator container by name on the control network (docker DNS,
|
||||
container↔container, no host firewall)."""
|
||||
return f"http://{self.name}:{DEFAULT_PORT}"
|
||||
|
||||
def ensure_built(self) -> None:
|
||||
"""Build the control-plane image from its Dockerfile, cache-aware (a
|
||||
no-op when nothing changed). No-op when no dockerfile is configured (a
|
||||
pre-pulled image). BOT_BOTTLE_NO_CACHE forces a full rebuild."""
|
||||
if self._dockerfile is None:
|
||||
return
|
||||
argv = ["docker", "build", "-t", self.image_ref,
|
||||
"-f", str(self._repo_root / self._dockerfile),
|
||||
str(self._repo_root)]
|
||||
if os.environ.get("BOT_BOTTLE_NO_CACHE"):
|
||||
argv.insert(2, "--no-cache")
|
||||
proc = run_docker(argv)
|
||||
if proc.returncode != 0:
|
||||
raise GatewayError(
|
||||
f"{self._dockerfile} build failed: {proc.stderr.strip()}")
|
||||
|
||||
def is_running(self) -> bool:
|
||||
proc = run_docker([
|
||||
"docker", "ps", "--filter", f"name=^/{self.name}$", "--format", "{{.Names}}",
|
||||
])
|
||||
return self.name in proc.stdout.split()
|
||||
|
||||
def _source_current(self, current_hash: str) -> bool:
|
||||
"""True iff the running orchestrator was started from the current
|
||||
bind-mounted source."""
|
||||
if not self.is_running():
|
||||
return False
|
||||
proc = run_docker([
|
||||
"docker", "inspect", "--format",
|
||||
"{{ index .Config.Labels \"" + ORCHESTRATOR_SOURCE_HASH_LABEL + "\" }}",
|
||||
self.name,
|
||||
])
|
||||
if proc.returncode != 0:
|
||||
return True # can't compare → don't churn a working container
|
||||
return proc.stdout.strip() == current_hash
|
||||
|
||||
def _ensure_control_network(self) -> None:
|
||||
if run_docker(["docker", "network", "inspect", self.control_network]).returncode == 0:
|
||||
return
|
||||
proc = run_docker(["docker", "network", "create", "--internal", self.control_network])
|
||||
if proc.returncode != 0 and "already exists" not in proc.stderr:
|
||||
raise GatewayError(
|
||||
f"control network {self.control_network} failed to create: "
|
||||
f"{proc.stderr.strip()}"
|
||||
)
|
||||
|
||||
def ensure_running(
|
||||
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||
) -> None:
|
||||
"""Ensure the control-plane container is up on current source; block
|
||||
until healthy. Idempotent — a healthy orchestrator on current source is
|
||||
left untouched (its in-memory egress tokens survive). Raises
|
||||
`OrchestratorStartError` on startup timeout."""
|
||||
current_hash = source_hash(self._repo_root)
|
||||
if self.is_healthy() and self._source_current(current_hash):
|
||||
return
|
||||
log.info("starting orchestrator container", context={"name": self.name})
|
||||
self._run_container(current_hash)
|
||||
deadline = time.monotonic() + startup_timeout
|
||||
while time.monotonic() < deadline:
|
||||
if self.is_healthy():
|
||||
log.info("orchestrator healthy", context={"url": self.url()})
|
||||
return
|
||||
time.sleep(_HEALTH_POLL_SECONDS)
|
||||
raise OrchestratorStartError(
|
||||
f"orchestrator at {self.url()} did not become healthy "
|
||||
f"within {startup_timeout:g}s"
|
||||
)
|
||||
|
||||
def _run_container(self, current_hash: str) -> None:
|
||||
"""Start the lean control-plane container on the control network only,
|
||||
published to host loopback for the CLI. Idempotent (clears a stale
|
||||
fixed-name container first)."""
|
||||
self._ensure_control_network()
|
||||
run_docker(["docker", "rm", "--force", self.name])
|
||||
_signing_key = host_orchestrator_token()
|
||||
proc = run_docker([
|
||||
"docker", "run", "--detach",
|
||||
"--name", self.name,
|
||||
"--label", self.label,
|
||||
"--label", f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={current_hash}",
|
||||
# Control network only — agents are never on it, so they have no
|
||||
# route to the control plane (the L3 block, not just the JWT).
|
||||
"--network", self.control_network,
|
||||
# Host CLI reaches the control plane here (loopback only). The
|
||||
# orchestrator listens on the fixed DEFAULT_PORT inside the
|
||||
# container; self.port is the host-side published port.
|
||||
"--publish", f"127.0.0.1:{self.port}:{DEFAULT_PORT}",
|
||||
# Live control-plane source (code changes without an image rebuild).
|
||||
"--volume", f"{self._repo_root}:{_SRC_IN_CONTAINER}:ro",
|
||||
"--env", f"PYTHONPATH={_SRC_IN_CONTAINER}",
|
||||
# Orchestrator registry DB on the host (sole writer: control plane).
|
||||
"--volume", f"{self._host_root}:{_ROOT_IN_CONTAINER}",
|
||||
"--env", f"BOT_BOTTLE_ROOT={_ROOT_IN_CONTAINER}",
|
||||
# The signing key — held ONLY by the orchestrator (it verifies
|
||||
# tokens); the gateway gets the pre-minted `gateway` JWT, never the
|
||||
# key (issue #469). Bare `--env NAME` keeps the value off argv.
|
||||
"--env", ORCHESTRATOR_TOKEN_ENV,
|
||||
self.image_ref,
|
||||
# Dockerfile.orchestrator ENTRYPOINT is `python3 -m bot_bottle.orchestrator`;
|
||||
# these are its args.
|
||||
"--host", "0.0.0.0", "--port", str(DEFAULT_PORT), "--broker", "stub",
|
||||
], env={**os.environ, ORCHESTRATOR_TOKEN_ENV: _signing_key})
|
||||
if proc.returncode != 0:
|
||||
raise OrchestratorStartError(
|
||||
f"orchestrator container failed to start: {proc.stderr.strip()}"
|
||||
)
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Remove the control-plane container (idempotent)."""
|
||||
run_docker(["docker", "rm", "--force", self.name])
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DockerOrchestrator",
|
||||
"ORCHESTRATOR_NAME",
|
||||
"ORCHESTRATOR_LABEL",
|
||||
"ORCHESTRATOR_NETWORK",
|
||||
"ORCHESTRATOR_IMAGE",
|
||||
"ORCHESTRATOR_DOCKERFILE",
|
||||
"ORCHESTRATOR_SOURCE_HASH_LABEL",
|
||||
]
|
||||
@@ -42,6 +42,7 @@ if TYPE_CHECKING:
|
||||
)
|
||||
from .docker_broker import DockerBroker, DockerBrokerError
|
||||
from ..gateway import Gateway, GatewayError
|
||||
from .lifecycle import Orchestrator
|
||||
from .service import OrchestratorCore
|
||||
from .server import OrchestratorServer, dispatch, make_server
|
||||
|
||||
@@ -64,6 +65,7 @@ _LAZY: dict[str, str] = {
|
||||
"DockerBrokerError": ".docker_broker",
|
||||
"Gateway": "..gateway",
|
||||
"GatewayError": "..gateway",
|
||||
"Orchestrator": ".lifecycle",
|
||||
"OrchestratorCore": ".service",
|
||||
"OrchestratorServer": ".server",
|
||||
"dispatch": ".server",
|
||||
@@ -96,6 +98,7 @@ __all__ = [
|
||||
"DockerBrokerError",
|
||||
"Gateway",
|
||||
"GatewayError",
|
||||
"Orchestrator",
|
||||
"OrchestratorCore",
|
||||
"OrchestratorServer",
|
||||
"dispatch",
|
||||
|
||||
@@ -1,23 +1,38 @@
|
||||
"""Shared orchestrator-process constants + helpers (PRD 0070).
|
||||
"""The host-side orchestrator (control-plane) service + shared lifecycle pieces
|
||||
(PRD 0070).
|
||||
|
||||
Backend-neutral pieces the per-backend infra services build on: the
|
||||
control-plane port, the startup timeout, the start-error, and the source hash
|
||||
used to detect a code change. The per-backend infra lifecycle lives with each
|
||||
backend — docker: `backend.docker.infra.DockerInfraService`; macOS:
|
||||
`backend.macos_container.infra`; firecracker: `backend.firecracker.infra_vm`.
|
||||
`Orchestrator` is the backend-neutral host-side contract for the control plane,
|
||||
mirroring `Gateway` for the data plane: build its image/rootfs, bring it up,
|
||||
report where the CLI and the gateway reach it, mint the gateway's token, tear it
|
||||
down. One concrete impl per backend (`backend/*/orchestrator.py`); the host
|
||||
composes it with the `Gateway` service.
|
||||
|
||||
This module also holds the backend-neutral constants those impls build on: the
|
||||
control-plane port, the startup timeout, the start-error, and the `source_hash`
|
||||
used to detect a code change. (The in-guest control-plane *core* — the registry
|
||||
+ broker object the running process wraps — is `service.OrchestratorCore`.)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
import hashlib
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from ..orchestrator_auth import ROLE_GATEWAY, mint
|
||||
from ..paths import host_orchestrator_token
|
||||
|
||||
DEFAULT_PORT = 8099
|
||||
DEFAULT_STARTUP_TIMEOUT_SECONDS = 45.0
|
||||
# A single /health probe's timeout (the poll loop repeats it up to the startup
|
||||
# timeout). Short so a wedged control plane fails fast per attempt.
|
||||
DEFAULT_HEALTH_TIMEOUT_SECONDS = 1.0
|
||||
|
||||
|
||||
class OrchestratorStartError(RuntimeError):
|
||||
"""The infra container/VM did not become healthy within the timeout."""
|
||||
"""The control plane did not become healthy within the timeout."""
|
||||
|
||||
|
||||
def source_hash(repo_root: Path) -> str:
|
||||
@@ -34,9 +49,72 @@ def source_hash(repo_root: Path) -> str:
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
class Orchestrator(abc.ABC):
|
||||
"""Provision + interact with the per-host orchestrator (control plane).
|
||||
|
||||
One concrete impl per backend (`backend/*/orchestrator.py`); the host
|
||||
composes it with the `Gateway` service. The orchestrator is the **sole**
|
||||
holder of the signing key and the **sole** opener of `bot-bottle.db` (#469),
|
||||
so it — not the gateway — mints the gateway's role-scoped token.
|
||||
Backend-neutral."""
|
||||
|
||||
def ensure_built(self) -> None:
|
||||
"""Ensure the orchestrator's image / rootfs exists, building it if
|
||||
needed. Default: nothing to build (e.g. a pre-pulled image). Call before
|
||||
`ensure_running`."""
|
||||
return
|
||||
|
||||
@abc.abstractmethod
|
||||
def ensure_running(
|
||||
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||
) -> None:
|
||||
"""Start the control plane if it isn't already healthy on the current
|
||||
source, then block until `/health` answers. Idempotent — a healthy,
|
||||
current control plane is left running so its in-memory egress tokens
|
||||
survive (#381). Raises `OrchestratorStartError` on startup timeout."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def is_running(self) -> bool:
|
||||
"""True iff the orchestrator instance (container / VM) is currently up."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def stop(self) -> None:
|
||||
"""Remove the orchestrator. Idempotent — absent is success."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def url(self) -> str:
|
||||
"""The host-facing control-plane URL the CLI reaches the orchestrator
|
||||
at (docker: a host-loopback publish; macOS/firecracker: the guest's
|
||||
control-network / guest IP)."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def gateway_url(self) -> str:
|
||||
"""The control-plane URL the gateway's data plane resolves policy
|
||||
against. May differ from `url()` — docker reaches the orchestrator by
|
||||
its container-DNS name on the control network, not the host loopback."""
|
||||
|
||||
def is_healthy(self, *, timeout: float = DEFAULT_HEALTH_TIMEOUT_SECONDS) -> bool:
|
||||
"""True iff the control plane answers 200 on `/health` at `url()`.
|
||||
Backends may override (e.g. to also check the VM process is alive)."""
|
||||
try:
|
||||
with urllib.request.urlopen(f"{self.url()}/health", timeout=timeout) as resp:
|
||||
return resp.status == 200
|
||||
except (urllib.error.URLError, TimeoutError, OSError):
|
||||
return False
|
||||
|
||||
def mint_gateway_token(self) -> str:
|
||||
"""Mint a role-scoped `gateway` JWT from the host signing key for the
|
||||
gateway to present. The orchestrator holds the key; the gateway never
|
||||
does (#469). Backend-neutral — the same host token file is the single
|
||||
source of truth across backends."""
|
||||
return mint(ROLE_GATEWAY, host_orchestrator_token())
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_PORT",
|
||||
"DEFAULT_STARTUP_TIMEOUT_SECONDS",
|
||||
"DEFAULT_HEALTH_TIMEOUT_SECONDS",
|
||||
"OrchestratorStartError",
|
||||
"source_hash",
|
||||
"Orchestrator",
|
||||
]
|
||||
|
||||
+35
-180
@@ -1,209 +1,64 @@
|
||||
"""Unit: orchestrator + gateway container lifecycle (PRD 0070 plane split).
|
||||
"""Unit: DockerInfraService composes the orchestrator + gateway services.
|
||||
|
||||
`DockerInfraService` brings up two containers — the lean orchestrator on the
|
||||
`--internal` control network + host loopback, and the dual-homed gateway. These
|
||||
tests isolate the orchestrator-container logic by mocking `gateway` (the
|
||||
gateway runs via `DockerGateway`, exercised in its own test)."""
|
||||
`DockerInfraService` no longer owns the container lifecycles — it composes the
|
||||
`DockerOrchestrator` (control plane) and `DockerGateway` (data plane) services,
|
||||
each tested in its own module. These tests isolate the *composition*: both are
|
||||
built, the orchestrator comes up first, then the gateway is connected to it with
|
||||
a freshly minted token."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
import urllib.error
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from bot_bottle.gateway import GatewayError
|
||||
from bot_bottle.backend.docker.infra import (
|
||||
ORCHESTRATOR_NAME,
|
||||
ORCHESTRATOR_NETWORK,
|
||||
ORCHESTRATOR_SOURCE_HASH_LABEL,
|
||||
DockerInfraService,
|
||||
)
|
||||
from bot_bottle.gateway import GATEWAY_NAME
|
||||
from bot_bottle.orchestrator.lifecycle import (
|
||||
OrchestratorStartError,
|
||||
source_hash,
|
||||
)
|
||||
from tests.unit import use_bottle_root
|
||||
|
||||
_URLOPEN = "bot_bottle.backend.docker.infra.urllib.request.urlopen"
|
||||
_RUN = "bot_bottle.backend.docker.infra.run_docker"
|
||||
_SLEEP = "bot_bottle.backend.docker.infra.time.sleep"
|
||||
_MONOTONIC = "bot_bottle.backend.docker.infra.time.monotonic"
|
||||
|
||||
|
||||
def _health(status: int) -> MagicMock:
|
||||
m = MagicMock()
|
||||
m.__enter__.return_value.status = status
|
||||
return m
|
||||
|
||||
|
||||
def _proc(returncode: int = 0, stdout: str = "", stderr: str = "") -> Mock:
|
||||
return Mock(returncode=returncode, stdout=stdout, stderr=stderr)
|
||||
|
||||
|
||||
class TestDockerInfraService(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self._tmp.cleanup)
|
||||
self.addCleanup(use_bottle_root(Path(self._tmp.name)))
|
||||
self.svc = DockerInfraService(port=8099)
|
||||
# Isolate the orchestrator-container logic: the gateway is brought up by
|
||||
# DockerGateway (its own module/run_docker), tested separately.
|
||||
self.orch = MagicMock()
|
||||
self.orch.url.return_value = "http://127.0.0.1:8099"
|
||||
self.orch.gateway_url.return_value = f"http://{ORCHESTRATOR_NAME}:8099"
|
||||
self.orch.mint_gateway_token.return_value = "gw.jwt"
|
||||
self.gw = MagicMock()
|
||||
patcher = patch.object(self.svc, "gateway", return_value=self.gw)
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
def test_url(self) -> None:
|
||||
self.assertEqual("http://127.0.0.1:8099", self.svc.url)
|
||||
for name, mock in (("orchestrator", self.orch), ("gateway", self.gw)):
|
||||
p = patch.object(self.svc, name, return_value=mock)
|
||||
p.start()
|
||||
self.addCleanup(p.stop)
|
||||
|
||||
def test_gateway_name(self) -> None:
|
||||
self.assertEqual(GATEWAY_NAME, self.svc.gateway_name)
|
||||
|
||||
def test_is_healthy(self) -> None:
|
||||
with patch(_URLOPEN, return_value=_health(200)):
|
||||
self.assertTrue(self.svc.is_healthy())
|
||||
with patch(_URLOPEN, side_effect=urllib.error.URLError("refused")):
|
||||
self.assertFalse(self.svc.is_healthy())
|
||||
def test_url_delegates_to_the_orchestrator(self) -> None:
|
||||
self.assertEqual("http://127.0.0.1:8099", self.svc.url)
|
||||
|
||||
def test_noop_when_healthy_and_source_unchanged(self) -> None:
|
||||
# A healthy orchestrator on current source is left alone — recreating it
|
||||
# on every launch drops in-memory egress tokens (#381). The gateway is
|
||||
# still refreshed (idempotent).
|
||||
current = source_hash(self.svc._repo_root)
|
||||
def test_is_healthy_delegates_to_the_orchestrator(self) -> None:
|
||||
self.orch.is_healthy.return_value = True
|
||||
self.assertTrue(self.svc.is_healthy())
|
||||
self.orch.is_healthy.assert_called_once()
|
||||
|
||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
||||
if argv[:2] == ["docker", "ps"]:
|
||||
return _proc(stdout=ORCHESTRATOR_NAME)
|
||||
if argv[:2] == ["docker", "inspect"]:
|
||||
return _proc(stdout=current)
|
||||
return _proc()
|
||||
def test_ensure_running_builds_both_then_starts_orchestrator_first(self) -> None:
|
||||
url = self.svc.ensure_running()
|
||||
self.assertEqual("http://127.0.0.1:8099", url)
|
||||
# Both images built; the control plane comes up before the gateway.
|
||||
self.orch.ensure_built.assert_called_once()
|
||||
self.gw.ensure_built.assert_called_once()
|
||||
self.orch.ensure_running.assert_called_once()
|
||||
|
||||
with patch(_URLOPEN, return_value=_health(200)), \
|
||||
patch(_RUN, side_effect=fake) as run, patch(_SLEEP):
|
||||
self.assertEqual(self.svc.url, self.svc.ensure_running())
|
||||
runs = [c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "run"]]
|
||||
self.assertEqual([], runs)
|
||||
self.gw.connect_to_orchestrator.assert_called_once()
|
||||
|
||||
def test_recreates_orchestrator_when_source_changed(self) -> None:
|
||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
||||
if argv[:2] == ["docker", "ps"]:
|
||||
return _proc(stdout=ORCHESTRATOR_NAME)
|
||||
if argv[:2] == ["docker", "inspect"]:
|
||||
return _proc(stdout="stale-hash")
|
||||
return _proc()
|
||||
|
||||
with patch(_URLOPEN, return_value=_health(200)), \
|
||||
patch(_RUN, side_effect=fake) as run, patch(_SLEEP):
|
||||
self.assertEqual(self.svc.url, self.svc.ensure_running())
|
||||
runs = [c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "run"]]
|
||||
self.assertEqual(1, len(runs))
|
||||
self.assertIn(ORCHESTRATOR_NAME, runs[0])
|
||||
current = source_hash(self.svc._repo_root)
|
||||
self.assertIn(f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={current}", runs[0])
|
||||
|
||||
def test_starts_orchestrator_when_absent(self) -> None:
|
||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
||||
if argv[:2] == ["docker", "ps"]:
|
||||
return _proc(stdout="")
|
||||
return _proc()
|
||||
|
||||
with patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \
|
||||
patch(_RUN, side_effect=fake) as run, patch(_SLEEP):
|
||||
self.assertEqual(self.svc.url, self.svc.ensure_running())
|
||||
runs = [c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "run"]]
|
||||
self.assertEqual(1, len(runs))
|
||||
argv = runs[0]
|
||||
self.assertIn(ORCHESTRATOR_NAME, argv)
|
||||
# Control network only — agents are never on it.
|
||||
self.assertEqual(ORCHESTRATOR_NETWORK, argv[argv.index("--network") + 1])
|
||||
# Published on loopback for the CLI, mapped to the fixed internal 8099.
|
||||
self.assertEqual("127.0.0.1:8099:8099", argv[argv.index("--publish") + 1])
|
||||
# The lean control plane: no mitmproxy CA mount, no gateway daemons.
|
||||
self.assertFalse([a for a in argv if a.endswith(":/home/mitmproxy/.mitmproxy")])
|
||||
self.assertNotIn("BOT_BOTTLE_GATEWAY_DAEMONS", " ".join(argv))
|
||||
# Orchestrator entrypoint args (image ENTRYPOINT is `-m bot_bottle.orchestrator`).
|
||||
self.assertIn("--broker", argv)
|
||||
self.assertIn("stub", argv)
|
||||
# Gateway is brought up after the control plane is healthy.
|
||||
self.gw.connect_to_orchestrator.assert_called_once()
|
||||
|
||||
def test_builds_gateway_and_orchestrator_images(self) -> None:
|
||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
||||
if argv[:2] == ["docker", "ps"]:
|
||||
return _proc(stdout="")
|
||||
return _proc()
|
||||
|
||||
with patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \
|
||||
patch(_RUN, side_effect=fake) as run, patch(_SLEEP):
|
||||
self.svc.ensure_running()
|
||||
builds = [c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "build"]]
|
||||
# Two images now (the combined Dockerfile.infra is retired).
|
||||
self.assertEqual(2, len(builds))
|
||||
dockerfiles = [next(a for a in b if "Dockerfile" in a) for b in builds]
|
||||
self.assertIn("Dockerfile.gateway", dockerfiles[0])
|
||||
self.assertIn("Dockerfile.orchestrator", dockerfiles[1])
|
||||
|
||||
def test_publish_maps_host_port_to_fixed_internal_port(self) -> None:
|
||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
||||
if argv[:2] == ["docker", "ps"]:
|
||||
return _proc(stdout="")
|
||||
return _proc()
|
||||
|
||||
svc = DockerInfraService(port=20001)
|
||||
with patch.object(svc, "gateway", return_value=MagicMock()), \
|
||||
patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \
|
||||
patch(_RUN, side_effect=fake) as run, patch(_SLEEP):
|
||||
svc.ensure_running()
|
||||
argv = next(c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "run"])
|
||||
self.assertEqual("127.0.0.1:20001:8099", argv[argv.index("--publish") + 1])
|
||||
|
||||
def test_raises_on_control_plane_timeout(self) -> None:
|
||||
with patch(_URLOPEN, side_effect=urllib.error.URLError("down")), \
|
||||
patch(_RUN, return_value=_proc()), \
|
||||
patch(_SLEEP), patch(_MONOTONIC, side_effect=[0.0, 0.5, 2.0]):
|
||||
with self.assertRaises(OrchestratorStartError):
|
||||
self.svc.ensure_running(startup_timeout=1.0)
|
||||
|
||||
def test_noop_when_healthy_and_inspect_fails(self) -> None:
|
||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
||||
if argv[:2] == ["docker", "ps"]:
|
||||
return _proc(stdout=ORCHESTRATOR_NAME)
|
||||
if argv[:2] == ["docker", "inspect"]:
|
||||
return _proc(returncode=1, stderr="daemon error")
|
||||
return _proc()
|
||||
|
||||
with patch(_URLOPEN, return_value=_health(200)), \
|
||||
patch(_RUN, side_effect=fake) as run, patch(_SLEEP):
|
||||
self.svc.ensure_running()
|
||||
runs = [c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "run"]]
|
||||
self.assertEqual([], runs)
|
||||
|
||||
def test_build_failure_raises(self) -> None:
|
||||
with patch(_URLOPEN, side_effect=urllib.error.URLError("down")), \
|
||||
patch(_RUN, return_value=_proc(returncode=1, stderr="no space left")):
|
||||
with self.assertRaises(GatewayError):
|
||||
self.svc.ensure_running()
|
||||
|
||||
def test_creates_control_network_internal(self) -> None:
|
||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
||||
if argv[:3] == ["docker", "network", "inspect"]:
|
||||
return _proc(returncode=1, stderr="not found")
|
||||
if argv[:2] == ["docker", "ps"]:
|
||||
return _proc(stdout="")
|
||||
return _proc()
|
||||
|
||||
with patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \
|
||||
patch(_RUN, side_effect=fake) as run, patch(_SLEEP):
|
||||
self.svc.ensure_running()
|
||||
creates = [c.args[0] for c in run.call_args_list if c.args[0][:3] == ["docker", "network", "create"]]
|
||||
# The control network is created --internal; the data network isn't.
|
||||
control = [c for c in creates if ORCHESTRATOR_NETWORK in c]
|
||||
self.assertEqual(1, len(control))
|
||||
self.assertIn("--internal", control[0])
|
||||
def test_ensure_running_connects_gateway_with_minted_token(self) -> None:
|
||||
self.svc.ensure_running()
|
||||
# The gateway is bound to the orchestrator's *gateway_url* (the control
|
||||
# DNS name), carrying the orchestrator-minted `gateway` token.
|
||||
self.gw.connect_to_orchestrator.assert_called_once_with(
|
||||
f"http://{ORCHESTRATOR_NAME}:8099", "gw.jwt")
|
||||
self.orch.mint_gateway_token.assert_called_once()
|
||||
|
||||
def test_stop_removes_both_containers(self) -> None:
|
||||
with patch(_RUN) as run:
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
"""Unit: the docker orchestrator (control plane) container lifecycle (PRD 0070)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
import urllib.error
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
from bot_bottle.backend.docker.orchestrator import (
|
||||
ORCHESTRATOR_NAME,
|
||||
ORCHESTRATOR_NETWORK,
|
||||
ORCHESTRATOR_SOURCE_HASH_LABEL,
|
||||
DockerOrchestrator,
|
||||
)
|
||||
from bot_bottle.gateway import GatewayError
|
||||
from bot_bottle.orchestrator.lifecycle import OrchestratorStartError, source_hash
|
||||
|
||||
_ORCH = "bot_bottle.backend.docker.orchestrator"
|
||||
_RUN = f"{_ORCH}.run_docker"
|
||||
_SLEEP = f"{_ORCH}.time.sleep"
|
||||
_MONOTONIC = f"{_ORCH}.time.monotonic"
|
||||
_TOKEN = f"{_ORCH}.host_orchestrator_token"
|
||||
# The ABC's is_healthy probes /health via urllib in the lifecycle module.
|
||||
_URLOPEN = "bot_bottle.orchestrator.lifecycle.urllib.request.urlopen"
|
||||
|
||||
|
||||
def _health(status: int) -> MagicMock:
|
||||
m = MagicMock()
|
||||
m.__enter__.return_value.status = status
|
||||
return m
|
||||
|
||||
|
||||
def _proc(returncode: int = 0, stdout: str = "", stderr: str = "") -> Mock:
|
||||
return Mock(returncode=returncode, stdout=stdout, stderr=stderr)
|
||||
|
||||
|
||||
class TestDockerOrchestrator(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.orch = DockerOrchestrator(port=8099)
|
||||
# _run_container seeds the signing key; keep it off the real host file.
|
||||
p = patch(_TOKEN, return_value="signing-key")
|
||||
p.start()
|
||||
self.addCleanup(p.stop)
|
||||
|
||||
def test_url_is_host_loopback(self) -> None:
|
||||
self.assertEqual("http://127.0.0.1:8099", self.orch.url())
|
||||
|
||||
def test_gateway_url_is_the_container_dns_name(self) -> None:
|
||||
# The gateway reaches the orchestrator by name on the control network.
|
||||
self.assertEqual(f"http://{ORCHESTRATOR_NAME}:8099", self.orch.gateway_url())
|
||||
|
||||
def test_is_healthy_probes_health_on_the_loopback_url(self) -> None:
|
||||
with patch(_URLOPEN, return_value=_health(200)):
|
||||
self.assertTrue(self.orch.is_healthy())
|
||||
with patch(_URLOPEN, side_effect=urllib.error.URLError("refused")):
|
||||
self.assertFalse(self.orch.is_healthy())
|
||||
|
||||
def test_noop_when_healthy_and_source_unchanged(self) -> None:
|
||||
# A healthy orchestrator on current source is left alone — recreating it
|
||||
# on every launch drops in-memory egress tokens (#381).
|
||||
current = source_hash(self.orch._repo_root)
|
||||
|
||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
||||
if argv[:2] == ["docker", "ps"]:
|
||||
return _proc(stdout=ORCHESTRATOR_NAME)
|
||||
if argv[:2] == ["docker", "inspect"]:
|
||||
return _proc(stdout=current)
|
||||
return _proc()
|
||||
|
||||
with patch(_URLOPEN, return_value=_health(200)), \
|
||||
patch(_RUN, side_effect=fake) as run, patch(_SLEEP):
|
||||
self.orch.ensure_running()
|
||||
self.assertEqual([], [c.args[0] for c in run.call_args_list
|
||||
if c.args[0][:2] == ["docker", "run"]])
|
||||
|
||||
def test_recreates_when_source_changed(self) -> None:
|
||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
||||
if argv[:2] == ["docker", "ps"]:
|
||||
return _proc(stdout=ORCHESTRATOR_NAME)
|
||||
if argv[:2] == ["docker", "inspect"]:
|
||||
return _proc(stdout="stale-hash")
|
||||
return _proc()
|
||||
|
||||
with patch(_URLOPEN, return_value=_health(200)), \
|
||||
patch(_RUN, side_effect=fake) as run, patch(_SLEEP):
|
||||
self.orch.ensure_running()
|
||||
runs = [c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "run"]]
|
||||
self.assertEqual(1, len(runs))
|
||||
self.assertIn(ORCHESTRATOR_NAME, runs[0])
|
||||
current = source_hash(self.orch._repo_root)
|
||||
self.assertIn(f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={current}", runs[0])
|
||||
|
||||
def test_starts_when_absent_on_control_net_and_loopback(self) -> None:
|
||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
||||
if argv[:2] == ["docker", "ps"]:
|
||||
return _proc(stdout="")
|
||||
return _proc()
|
||||
|
||||
with patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \
|
||||
patch(_RUN, side_effect=fake) as run, patch(_SLEEP):
|
||||
self.orch.ensure_running()
|
||||
argv = next(c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "run"])
|
||||
self.assertIn(ORCHESTRATOR_NAME, argv)
|
||||
# Control network only — agents are never on it.
|
||||
self.assertEqual(ORCHESTRATOR_NETWORK, argv[argv.index("--network") + 1])
|
||||
# Published on loopback for the CLI, mapped to the fixed internal 8099.
|
||||
self.assertEqual("127.0.0.1:8099:8099", argv[argv.index("--publish") + 1])
|
||||
# The lean control plane: no mitmproxy CA mount, no gateway daemons.
|
||||
self.assertFalse([a for a in argv if a.endswith(":/home/mitmproxy/.mitmproxy")])
|
||||
self.assertNotIn("BOT_BOTTLE_GATEWAY_DAEMONS", " ".join(argv))
|
||||
# Orchestrator entrypoint args (image ENTRYPOINT is `-m bot_bottle.orchestrator`).
|
||||
self.assertIn("--broker", argv)
|
||||
self.assertIn("stub", argv)
|
||||
|
||||
def test_publish_maps_host_port_to_fixed_internal_port(self) -> None:
|
||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
||||
return _proc(stdout="") if argv[:2] == ["docker", "ps"] else _proc()
|
||||
|
||||
orch = DockerOrchestrator(port=20001)
|
||||
with patch(_TOKEN, return_value="k"), \
|
||||
patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \
|
||||
patch(_RUN, side_effect=fake) as run, patch(_SLEEP):
|
||||
orch.ensure_running()
|
||||
argv = next(c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "run"])
|
||||
self.assertEqual("127.0.0.1:20001:8099", argv[argv.index("--publish") + 1])
|
||||
|
||||
def test_raises_on_startup_timeout(self) -> None:
|
||||
with patch(_URLOPEN, side_effect=urllib.error.URLError("down")), \
|
||||
patch(_RUN, return_value=_proc()), \
|
||||
patch(_SLEEP), patch(_MONOTONIC, side_effect=[0.0, 0.5, 2.0]):
|
||||
with self.assertRaises(OrchestratorStartError):
|
||||
self.orch.ensure_running(startup_timeout=1.0)
|
||||
|
||||
def test_noop_when_healthy_and_inspect_fails(self) -> None:
|
||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
||||
if argv[:2] == ["docker", "ps"]:
|
||||
return _proc(stdout=ORCHESTRATOR_NAME)
|
||||
if argv[:2] == ["docker", "inspect"]:
|
||||
return _proc(returncode=1, stderr="daemon error")
|
||||
return _proc()
|
||||
|
||||
with patch(_URLOPEN, return_value=_health(200)), \
|
||||
patch(_RUN, side_effect=fake) as run, patch(_SLEEP):
|
||||
self.orch.ensure_running()
|
||||
self.assertEqual([], [c.args[0] for c in run.call_args_list
|
||||
if c.args[0][:2] == ["docker", "run"]])
|
||||
|
||||
def test_creates_control_network_internal(self) -> None:
|
||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
||||
if argv[:3] == ["docker", "network", "inspect"]:
|
||||
return _proc(returncode=1, stderr="not found")
|
||||
if argv[:2] == ["docker", "ps"]:
|
||||
return _proc(stdout="")
|
||||
return _proc()
|
||||
|
||||
with patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \
|
||||
patch(_RUN, side_effect=fake) as run, patch(_SLEEP):
|
||||
self.orch.ensure_running()
|
||||
creates = [c.args[0] for c in run.call_args_list if c.args[0][:3] == ["docker", "network", "create"]]
|
||||
control = [c for c in creates if ORCHESTRATOR_NETWORK in c]
|
||||
self.assertEqual(1, len(control))
|
||||
self.assertIn("--internal", control[0])
|
||||
|
||||
def test_ensure_built_builds_the_orchestrator_image(self) -> None:
|
||||
with patch(_RUN, return_value=_proc()) as run:
|
||||
self.orch.ensure_built()
|
||||
builds = [c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "build"]]
|
||||
self.assertEqual(1, len(builds))
|
||||
self.assertTrue(any(a.endswith("Dockerfile.orchestrator") for a in builds[0]))
|
||||
|
||||
def test_ensure_built_raises_on_build_failure(self) -> None:
|
||||
with patch(_RUN, return_value=_proc(returncode=1, stderr="no space left")):
|
||||
with self.assertRaises(GatewayError):
|
||||
self.orch.ensure_built()
|
||||
|
||||
def test_ensure_built_noop_without_dockerfile(self) -> None:
|
||||
orch = DockerOrchestrator(dockerfile=None)
|
||||
with patch(_RUN) as run:
|
||||
orch.ensure_built()
|
||||
run.assert_not_called()
|
||||
|
||||
def test_is_running_reads_docker_ps(self) -> None:
|
||||
with patch(_RUN, return_value=_proc(stdout=ORCHESTRATOR_NAME)):
|
||||
self.assertTrue(self.orch.is_running())
|
||||
with patch(_RUN, return_value=_proc(stdout="")):
|
||||
self.assertFalse(self.orch.is_running())
|
||||
|
||||
def test_stop_removes_the_container(self) -> None:
|
||||
with patch(_RUN) as run:
|
||||
self.orch.stop()
|
||||
argv = run.call_args.args[0]
|
||||
self.assertEqual(["docker", "rm", "--force", ORCHESTRATOR_NAME], argv)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user