Compare commits
2 Commits
0bccd34d4e
...
5f1d3fbd15
| Author | SHA1 | Date | |
|---|---|---|---|
| 5f1d3fbd15 | |||
| 1553a98275 |
@@ -17,9 +17,10 @@ from pathlib import Path
|
||||
|
||||
from .. import log
|
||||
from .store.store_manager import StoreManager
|
||||
from ..paths import LAUNCH_BROKER_KEY_ENV
|
||||
from .broker import StubBroker, SubmitBroker
|
||||
from .broker_client import BrokerClient
|
||||
from .host_server import BROKER_SECRET_ENV, DEFAULT_PORT, broker_secret_from_env
|
||||
from .host_server import DEFAULT_PORT, broker_secret
|
||||
from .server import make_server
|
||||
from .docker_broker import DockerBroker
|
||||
from .store.registry_store import RegistryStore, default_db_path
|
||||
@@ -57,17 +58,19 @@ def main(argv: list[str] | None = None) -> int:
|
||||
# A signing secret ties the orchestrator (signer) to its broker (verifier).
|
||||
# 'stub' records launches instead of starting anything; 'docker' runs real
|
||||
# containers in-process; 'http' relays signed requests to a separate host
|
||||
# control server, which verifies and launches. For 'stub'/'docker' the
|
||||
# secret is ephemeral (signer and verifier share this process); for 'http'
|
||||
# it must be the SAME secret the host controller holds, so it is read from
|
||||
# the shared env var (the chunk-1 stand-in for out-of-band provisioning).
|
||||
# control server, which verifies and launches. For 'stub'/'docker' the secret
|
||||
# is ephemeral (signer and verifier share this process). For 'http' it must be
|
||||
# the SAME key the host controller holds — and this process is the *guest*
|
||||
# (signer), so it must be given that key by injection, NOT mint its own
|
||||
# process-local one (which would diverge from the host's and 401 every launch).
|
||||
broker: SubmitBroker
|
||||
if args.broker == "http":
|
||||
secret = broker_secret_from_env()
|
||||
secret = broker_secret() # env-injected only; no host-file fallback here
|
||||
if secret is None:
|
||||
parser.error(
|
||||
f"--broker http requires a shared signing secret in "
|
||||
f"${BROKER_SECRET_ENV} (hex), matching the host control server"
|
||||
f"--broker http requires the launch-broker key injected as "
|
||||
f"${LAUNCH_BROKER_KEY_ENV} (the host controller owns/mints it); the "
|
||||
"orchestrator must not mint its own or it would diverge from the host's"
|
||||
)
|
||||
broker = BrokerClient(args.host_controller_url)
|
||||
else:
|
||||
|
||||
@@ -38,7 +38,21 @@ _ALLOWED_OPS = ("launch", "teardown")
|
||||
class BrokerAuthError(Exception):
|
||||
"""A broker request failed provenance or schema verification —
|
||||
bad/absent signature, malformed token, or a payload that doesn't match
|
||||
the fixed launch-request shape. Fail-closed: the broker must not act."""
|
||||
the fixed launch-request shape. Fail-closed: the broker must not act.
|
||||
|
||||
A **definite** negative: nothing was launched, so a caller may safely roll
|
||||
back as if the op never happened."""
|
||||
|
||||
|
||||
class BrokerUnavailableError(Exception):
|
||||
"""A brokered request could not be carried to a verdict: the broker (or the
|
||||
wire to it) was unreachable, timed out, or dropped the response.
|
||||
|
||||
Crucially **ambiguous** — unlike `BrokerAuthError`, the op MAY already have
|
||||
taken effect on the backend before the response was lost, so a caller must
|
||||
NOT assume it did nothing (e.g. must not roll a registry row back as if no
|
||||
launch happened, which would orphan a running container). Only the in-process
|
||||
brokers never raise this; the out-of-process `BrokerClient` does."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -179,6 +193,7 @@ class StubBroker(LaunchBroker):
|
||||
|
||||
__all__ = [
|
||||
"BrokerAuthError",
|
||||
"BrokerUnavailableError",
|
||||
"LaunchRequest",
|
||||
"SubmitBroker",
|
||||
"LaunchBroker",
|
||||
|
||||
@@ -20,15 +20,17 @@ import json
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from .broker import BrokerAuthError, LaunchRequest
|
||||
from .broker import BrokerAuthError, BrokerUnavailableError, LaunchRequest
|
||||
|
||||
DEFAULT_TIMEOUT_SECONDS = 5.0
|
||||
|
||||
|
||||
class BrokerClientError(RuntimeError):
|
||||
"""A brokered launch/teardown could not be delivered to the host control
|
||||
server: unreachable, or an unexpected status other than the fail-closed 401
|
||||
(which surfaces as `BrokerAuthError`, matching the in-process broker)."""
|
||||
"""The host control server *responded*, but with an unexpected status other
|
||||
than the fail-closed 401 (which surfaces as `BrokerAuthError`) — e.g. a 502
|
||||
backend failure or a malformed body. A definite negative: the host processed
|
||||
the request and it did not launch. (A *no-response* failure — unreachable /
|
||||
timeout / dropped — is the ambiguous `BrokerUnavailableError` instead.)"""
|
||||
|
||||
|
||||
class BrokerClient:
|
||||
@@ -45,9 +47,11 @@ class BrokerClient:
|
||||
verified and acted on.
|
||||
|
||||
Raises `BrokerAuthError` on a fail-closed 401 (bad provenance/schema —
|
||||
the same exception the in-process broker raises), or `BrokerClientError`
|
||||
if the controller is unreachable, times out, or returns any other
|
||||
non-success status."""
|
||||
the same exception the in-process broker raises); `BrokerClientError` if
|
||||
the host *responds* with any other non-success status or a malformed
|
||||
body (a definite negative); or `BrokerUnavailableError` if no response is
|
||||
obtained (unreachable / timeout / dropped) — the **ambiguous** case, where
|
||||
the host may already have acted, so the caller must not roll back."""
|
||||
data = json.dumps({"token": token}).encode()
|
||||
req = urllib.request.Request(
|
||||
f"{self._base}/broker", data=data, method="POST",
|
||||
@@ -65,8 +69,11 @@ class BrokerClient:
|
||||
raise BrokerClientError(
|
||||
f"POST /broker: HTTP {e.code} {detail}".rstrip()
|
||||
) from e
|
||||
except (urllib.error.URLError, TimeoutError, OSError, ValueError) as e:
|
||||
raise BrokerClientError(f"POST /broker: {e}") from e
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as e:
|
||||
# No usable response — unreachable, timed out, or the connection
|
||||
# dropped mid-exchange. Ambiguous: the request may already have
|
||||
# launched the bottle, so this is NOT a definite failure.
|
||||
raise BrokerUnavailableError(f"POST /broker: {e}") from e
|
||||
|
||||
|
||||
def _json_object(raw: bytes) -> dict[str, object]:
|
||||
|
||||
@@ -23,12 +23,15 @@ controller down.
|
||||
|
||||
The signed launch token *is* the endpoint's authentication (its provenance is the
|
||||
whole point of the JWS), so `/broker` needs no separate caller credential; the
|
||||
host controller's own lifecycle endpoints, which do, arrive with the durable
|
||||
`TrustDomain` key in a later chunk.
|
||||
host controller's own lifecycle endpoints, which do, arrive with the `host`-role
|
||||
tokens of the separate `HOST_CONTROLLER` trust domain in a later chunk.
|
||||
|
||||
The shared signing secret is read from `$BOT_BOTTLE_BROKER_SECRET` (hex). That is
|
||||
a **chunk-1 stopgap**: it must be provisioned to signer and verifier out of band,
|
||||
which is exactly what the durable `TrustDomain` key in chunk 2 (#476) replaces.
|
||||
The shared signing secret is the durable **launch-broker `TrustDomain` key**
|
||||
(#468/#476): a host-canonical key file minted 0600 on first use, provisioned to
|
||||
the orchestrator (signer) and this server (verifier). A backend launcher injects
|
||||
it via `$BOT_BOTTLE_LAUNCH_BROKER_KEY`; a host-side dev-harness process reads the
|
||||
key file directly. Durability is the point — a restarted orchestrator re-verifies
|
||||
against the same key, so re-adoption works.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -43,20 +46,28 @@ import typing
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from .. import log
|
||||
from ..paths import LAUNCH_BROKER_KEY_ENV
|
||||
from ..trust_domain import LAUNCH_BROKER
|
||||
from .broker import BrokerAuthError, LaunchBroker
|
||||
from .docker_broker import DockerBroker
|
||||
|
||||
# JSON body payload type (parsed request / rendered response).
|
||||
Json = dict[str, object]
|
||||
|
||||
# The hex-encoded HS256 secret shared with the request signer (the orchestrator).
|
||||
# Chunk-1 stopgap for the durable, out-of-band `TrustDomain` key of chunk 2.
|
||||
BROKER_SECRET_ENV = "BOT_BOTTLE_BROKER_SECRET"
|
||||
|
||||
# Default host-controller port. Distinct from the orchestrator control plane
|
||||
# (8099) — a separate privileged component listening on its own socket.
|
||||
DEFAULT_PORT = 8091
|
||||
|
||||
# Cap on the request body. A signed broker request is tiny, so rejecting anything
|
||||
# larger *before reading it* keeps a caller that can merely reach the socket (no
|
||||
# signed token needed) from exhausting memory or a handler thread with a huge
|
||||
# Content-Length — the signed token, not mere reachability, is the authority.
|
||||
MAX_BODY_BYTES = 64 * 1024
|
||||
|
||||
# Per-request socket timeout, bounding how long a stalled / slow-loris caller can
|
||||
# hold a handler thread on this privileged listener.
|
||||
REQUEST_TIMEOUT_SECONDS = 15
|
||||
|
||||
|
||||
def _parse_json_object(body: bytes) -> Json:
|
||||
"""Parse a JSON object body. Raises ValueError for non-objects / bad JSON."""
|
||||
@@ -68,19 +79,34 @@ def _parse_json_object(body: bytes) -> Json:
|
||||
return obj
|
||||
|
||||
|
||||
def broker_secret_from_env(environ: typing.Mapping[str, str] | None = None) -> bytes | None:
|
||||
"""The shared HS256 secret from `$BOT_BOTTLE_BROKER_SECRET` (hex), or None
|
||||
when unset or not valid hex. The signer (orchestrator, `--broker http`) and
|
||||
the verifier (this server) read the same env var so both hold the same key —
|
||||
the chunk-1 stand-in for out-of-band provisioning."""
|
||||
env = os.environ if environ is None else environ
|
||||
raw = env.get(BROKER_SECRET_ENV, "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return bytes.fromhex(raw)
|
||||
except ValueError:
|
||||
return None
|
||||
def broker_secret(
|
||||
environ: typing.Mapping[str, str] | None = None, *, allow_host_file: bool = False,
|
||||
) -> bytes | None:
|
||||
"""The shared launch-broker HS256 secret, as this process should use it.
|
||||
|
||||
Always prefers the key injected into this process's env
|
||||
(`$BOT_BOTTLE_LAUNCH_BROKER_KEY`). `allow_host_file` decides the fallback when
|
||||
it is absent, and the distinction is a security boundary:
|
||||
|
||||
- **Host-side** processes — the host controller and the host dev-harness — pass
|
||||
``allow_host_file=True`` to read (minting on first use) the durable host key
|
||||
file (``bot_bottle_root()/launch-broker-key``) they legitimately own.
|
||||
- The **guest orchestrator** (``--broker http``) keeps the default ``False``.
|
||||
It runs inside a container/VM whose ``bot_bottle_root()`` is process-local,
|
||||
so minting a file there would silently create a key UNRELATED to the host
|
||||
controller's — startup would succeed but every launch would be rejected 401.
|
||||
It must instead be *given* the key by its launcher, and fail closed (None)
|
||||
if it wasn't, rather than diverge.
|
||||
|
||||
None when no key is available (a guest with no injection, or an unwritable
|
||||
host root)."""
|
||||
key = LAUNCH_BROKER.key_from_env(environ)
|
||||
if not key and allow_host_file:
|
||||
try:
|
||||
key = LAUNCH_BROKER.signing_key() # host-canonical, minted on first use
|
||||
except OSError:
|
||||
return None
|
||||
return key.encode("utf-8") if key else None
|
||||
|
||||
|
||||
def dispatch( # pylint: disable=too-many-return-statements
|
||||
@@ -130,6 +156,10 @@ def dispatch( # pylint: disable=too-many-return-statements
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
"""Thin stdlib adapter: read the body, call `dispatch`, write JSON."""
|
||||
|
||||
# Socket timeout per request (applied by StreamRequestHandler.setup) so a
|
||||
# stalled caller can't pin a handler thread on this privileged listener.
|
||||
timeout = REQUEST_TIMEOUT_SECONDS
|
||||
|
||||
# Quiet by default; opt back into stdlib access logging with
|
||||
# BOT_BOTTLE_HOST_CONTROLLER_DEBUG (the controller has its own logging).
|
||||
def log_message(self, format: str, *args: typing.Any) -> None: # noqa: A002
|
||||
@@ -137,12 +167,21 @@ class Handler(http.server.BaseHTTPRequestHandler):
|
||||
super().log_message(format, *args)
|
||||
|
||||
def _serve(self, method: str) -> None:
|
||||
"""Read the request body, dispatch it, and write the JSON reply. A
|
||||
dispatch that raises (it shouldn't — dispatch is total) still returns a
|
||||
500 rather than dropping the connection."""
|
||||
"""Read the request body (bounded), dispatch it, and write the JSON
|
||||
reply. A dispatch that raises (it shouldn't — dispatch is total) still
|
||||
returns a 500 rather than dropping the connection."""
|
||||
server = self.server
|
||||
assert isinstance(server, HostControlServer)
|
||||
length = int(self.headers.get("Content-Length") or 0)
|
||||
try:
|
||||
length = int(self.headers.get("Content-Length") or 0)
|
||||
except ValueError:
|
||||
self._reply(400, {"error": "invalid Content-Length"})
|
||||
return
|
||||
if length < 0 or length > MAX_BODY_BYTES:
|
||||
# Reject before reading: nothing legitimate is this big, so an
|
||||
# oversized declared length is a bug or a resource-exhaustion attempt.
|
||||
self._reply(413, {"error": "request body too large"})
|
||||
return
|
||||
body = self.rfile.read(length) if length > 0 else b""
|
||||
try:
|
||||
status, payload = dispatch(server.broker, method, self.path, body)
|
||||
@@ -150,6 +189,10 @@ class Handler(http.server.BaseHTTPRequestHandler):
|
||||
sys.stderr.write(f"host controller: {method} {self.path} failed: {e!r}\n")
|
||||
sys.stderr.flush()
|
||||
status, payload = 500, {"error": f"internal error: {e}"}
|
||||
self._reply(status, payload)
|
||||
|
||||
def _reply(self, status: int, payload: typing.Mapping[str, object]) -> None:
|
||||
"""Write one JSON response with an explicit Content-Length."""
|
||||
data = json.dumps(payload).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
@@ -192,20 +235,22 @@ def main(argv: list[str] | None = None) -> int:
|
||||
|
||||
python -m bot_bottle.orchestrator.host_server [--host H] [--port P]
|
||||
|
||||
Fail-closed: without a shared `$BOT_BOTTLE_BROKER_SECRET` the server can
|
||||
verify no request's provenance, so it refuses to start rather than run a
|
||||
launcher that accepts unsigned input."""
|
||||
Fail-closed: without the launch-broker key the server can verify no request's
|
||||
provenance, so it refuses to start rather than run a launcher that accepts
|
||||
unsigned input. As the host-side owner of the key, it may mint/read the host
|
||||
key file (`allow_host_file=True`)."""
|
||||
parser = argparse.ArgumentParser(prog="bot_bottle.orchestrator.host_server")
|
||||
parser.add_argument("--host", default="127.0.0.1", help="bind address")
|
||||
parser.add_argument("--port", type=int, default=DEFAULT_PORT, help="bind port (0 = ephemeral)")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
secret = broker_secret_from_env()
|
||||
secret = broker_secret(allow_host_file=True)
|
||||
if secret is None:
|
||||
sys.stderr.write(
|
||||
f"host controller: refusing to start without a shared signing secret "
|
||||
f"(${BROKER_SECRET_ENV}, hex) — it could verify no request's "
|
||||
"provenance and would relay unsigned launches to the backend\n"
|
||||
f"host controller: refusing to start without the launch-broker key "
|
||||
f"(${LAUNCH_BROKER_KEY_ENV}, or a writable host root to mint it) — it "
|
||||
"could verify no request's provenance and would relay unsigned "
|
||||
"launches to the backend\n"
|
||||
)
|
||||
sys.stderr.flush()
|
||||
return 2
|
||||
@@ -231,10 +276,9 @@ __all__ = [
|
||||
"Handler",
|
||||
"HostControlServer",
|
||||
"make_host_server",
|
||||
"broker_secret_from_env",
|
||||
"broker_secret",
|
||||
"main",
|
||||
"Json",
|
||||
"BROKER_SECRET_ENV",
|
||||
"DEFAULT_PORT",
|
||||
]
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ import json
|
||||
from collections.abc import Iterable
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from .broker import LaunchRequest, SubmitBroker, sign_request
|
||||
from .broker import BrokerUnavailableError, LaunchRequest, SubmitBroker, sign_request
|
||||
from .store.registry_store import DEFAULT_REAP_GRACE_SECONDS, BottleRecord, RegistryStore
|
||||
from .supervisor import (
|
||||
AuditEntry,
|
||||
@@ -111,14 +111,23 @@ class OrchestratorCore:
|
||||
image_ref=image_ref,
|
||||
slot=slot,
|
||||
)
|
||||
launched = False
|
||||
try:
|
||||
self._broker.submit(sign_request(req, self._secret))
|
||||
launched = True
|
||||
finally:
|
||||
if not launched:
|
||||
self.registry.deregister(rec.bottle_id)
|
||||
self._tokens.pop(rec.bottle_id, None)
|
||||
except BrokerUnavailableError:
|
||||
# Ambiguous delivery failure (timeout / dropped response): the broker
|
||||
# may already have launched the bottle before the response was lost.
|
||||
# Do NOT deregister — that would orphan a running container with no
|
||||
# registry row (reconcile reaps rows, never containers). Keep the row
|
||||
# so reconcile reaps it iff the bottle is not actually live; surface
|
||||
# the error so the caller knows the launch is unconfirmed.
|
||||
raise
|
||||
except Exception:
|
||||
# A definite failure — a fail-closed rejection, a backend launch
|
||||
# error, or the host reporting it did not launch: nothing is running,
|
||||
# so roll the registry entry back to leave no orphan.
|
||||
self.registry.deregister(rec.bottle_id)
|
||||
self._tokens.pop(rec.bottle_id, None)
|
||||
raise
|
||||
return rec
|
||||
|
||||
def teardown_bottle(self, bottle_id: str) -> bool:
|
||||
|
||||
@@ -36,6 +36,13 @@ ROLE_GATEWAY = "gateway"
|
||||
ROLE_CLI = "cli"
|
||||
ROLES: frozenset[str] = frozenset({ROLE_GATEWAY, ROLE_CLI})
|
||||
|
||||
# The host controller's own lifecycle role (#468). Deliberately OUTSIDE `ROLES`:
|
||||
# it belongs to a separate trust domain (`HOST_CONTROLLER`) signed by a key the
|
||||
# orchestrator never holds, so the orchestrator's control-plane key can neither
|
||||
# mint nor accept it — the orchestrator must not be able to forge the credential
|
||||
# used to start and stop it.
|
||||
ROLE_HOST = "host"
|
||||
|
||||
_ALG = "HS256"
|
||||
|
||||
|
||||
@@ -103,4 +110,4 @@ def verify(token: str, secret: str, *, roles: frozenset[str] = ROLES) -> str | N
|
||||
return role if isinstance(role, str) and role in roles else None
|
||||
|
||||
|
||||
__all__ = ["ROLE_GATEWAY", "ROLE_CLI", "ROLES", "mint", "verify"]
|
||||
__all__ = ["ROLE_GATEWAY", "ROLE_CLI", "ROLE_HOST", "ROLES", "mint", "verify"]
|
||||
|
||||
@@ -47,6 +47,22 @@ ORCHESTRATOR_TOKEN_ENV = "BOT_BOTTLE_ORCHESTRATOR_TOKEN"
|
||||
# cannot forge a higher-privilege `cli` token (issue #469 review).
|
||||
ORCHESTRATOR_AUTH_JWT_ENV = "BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT"
|
||||
|
||||
# The durable launch-broker signing key: the HS256 secret the orchestrator
|
||||
# (signer) and the host control server (verifier) share to sign/verify launch
|
||||
# requests (#468). A host-canonical key file (minted 0600 on first use) so it
|
||||
# survives orchestrator restarts — re-adoption re-verifies against the same key —
|
||||
# instead of the ephemeral per-process secret of the in-process broker.
|
||||
LAUNCH_BROKER_KEY_FILENAME = "launch-broker-key"
|
||||
LAUNCH_BROKER_KEY_ENV = "BOT_BOTTLE_LAUNCH_BROKER_KEY"
|
||||
# The host controller's OWN key, for its lifecycle endpoints (the direct
|
||||
# cli -> host controller path that starts/stops the orchestrator). Separate from
|
||||
# the launch-broker key and never held by the orchestrator: the controller starts
|
||||
# and stops the orchestrator, so the orchestrator must not be able to mint the
|
||||
# credentials used to drive it (#468/#476).
|
||||
HOST_CONTROLLER_KEY_FILENAME = "host-controller-key"
|
||||
HOST_CONTROLLER_KEY_ENV = "BOT_BOTTLE_HOST_CONTROLLER_KEY"
|
||||
HOST_CONTROLLER_AUTH_JWT_ENV = "BOT_BOTTLE_HOST_CONTROLLER_AUTH_JWT"
|
||||
|
||||
# The host directory holding the gateway's persistent mitmproxy CA. Bind-mounted
|
||||
# into the infra/gateway container at mitmproxy's confdir so the self-generated
|
||||
# CA survives container recreation — every agent installs this one CA to trust
|
||||
@@ -142,6 +158,11 @@ __all__ = [
|
||||
"ORCHESTRATOR_TOKEN_FILENAME",
|
||||
"ORCHESTRATOR_TOKEN_ENV",
|
||||
"ORCHESTRATOR_AUTH_JWT_ENV",
|
||||
"LAUNCH_BROKER_KEY_FILENAME",
|
||||
"LAUNCH_BROKER_KEY_ENV",
|
||||
"HOST_CONTROLLER_KEY_FILENAME",
|
||||
"HOST_CONTROLLER_KEY_ENV",
|
||||
"HOST_CONTROLLER_AUTH_JWT_ENV",
|
||||
"GATEWAY_CA_DIRNAME",
|
||||
"bot_bottle_root",
|
||||
"host_db_path",
|
||||
|
||||
@@ -29,8 +29,13 @@ from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
|
||||
from . import orchestrator_auth
|
||||
from .orchestrator_auth import ROLE_GATEWAY
|
||||
from .orchestrator_auth import ROLE_GATEWAY, ROLE_HOST
|
||||
from .paths import (
|
||||
HOST_CONTROLLER_AUTH_JWT_ENV,
|
||||
HOST_CONTROLLER_KEY_ENV,
|
||||
HOST_CONTROLLER_KEY_FILENAME,
|
||||
LAUNCH_BROKER_KEY_ENV,
|
||||
LAUNCH_BROKER_KEY_FILENAME,
|
||||
ORCHESTRATOR_AUTH_JWT_ENV,
|
||||
ORCHESTRATOR_TOKEN_ENV,
|
||||
ORCHESTRATOR_TOKEN_FILENAME,
|
||||
@@ -99,6 +104,40 @@ CONTROL_PLANE = TrustDomain(
|
||||
)
|
||||
|
||||
|
||||
# The launch-broker domain (#468): durable key material for the broker's own
|
||||
# signed launch requests (`broker.py`'s HS256 launch JWT), shared by the
|
||||
# orchestrator (signer) and the host control server (verifier). Unlike
|
||||
# `CONTROL_PLANE` it mints no role tokens — the broker's provenance is the launch
|
||||
# JWT, not a role token — so its `roles` set is empty and it is used only as a
|
||||
# provider of durable, host-canonical key material (`signing_key` / `key_from_env`).
|
||||
# The durability is the point: the key survives orchestrator restarts, so a
|
||||
# restarted orchestrator re-verifies against the same key instead of the
|
||||
# ephemeral per-process secret the in-process broker used.
|
||||
LAUNCH_BROKER = TrustDomain(
|
||||
name="launch-broker",
|
||||
key_filename=LAUNCH_BROKER_KEY_FILENAME,
|
||||
roles=frozenset(),
|
||||
key_env=LAUNCH_BROKER_KEY_ENV,
|
||||
token_env="",
|
||||
)
|
||||
|
||||
# The host controller's own domain (#468) — the SECOND domain #476 reserves. Its
|
||||
# key, which the orchestrator never holds, signs the `host`-role tokens the CLI
|
||||
# presents on the host controller's lifecycle endpoints (start / restart / status
|
||||
# of the orchestrator itself). Keeping it separate from `CONTROL_PLANE` is the
|
||||
# whole point: the host controller starts and stops the orchestrator, so the
|
||||
# orchestrator must not be able to mint the credentials used to drive it. (The
|
||||
# lifecycle endpoints themselves arrive in a later chunk; the domain is
|
||||
# established here alongside the durable launch-broker key.)
|
||||
HOST_CONTROLLER = TrustDomain(
|
||||
name="host-controller",
|
||||
key_filename=HOST_CONTROLLER_KEY_FILENAME,
|
||||
roles=frozenset({ROLE_HOST}),
|
||||
key_env=HOST_CONTROLLER_KEY_ENV,
|
||||
token_env=HOST_CONTROLLER_AUTH_JWT_ENV,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ControlPlaneProvisioning:
|
||||
"""The one seam every backend launcher uses to provision control-plane auth,
|
||||
@@ -132,9 +171,56 @@ class ControlPlaneProvisioning:
|
||||
return self.domain.mint(ROLE_GATEWAY)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LaunchBrokerProvisioning:
|
||||
"""The seam that provisions the host-side launch broker's durable keys (#468),
|
||||
the counterpart to `ControlPlaneProvisioning`. Both the orchestrator (signer)
|
||||
and the host control server (verifier) receive the SAME launch-broker key
|
||||
(carry it in `broker_domain.key_env`); the host controller ALSO receives its
|
||||
own lifecycle key (`controller_domain.key_env`) the orchestrator never holds.
|
||||
|
||||
Fail-closed like the control-plane seam: minting returns "" only if the host
|
||||
root is unwritable, and an empty launch-broker key would leave the verifier
|
||||
unable to authenticate any launch — so we raise rather than hand back a key
|
||||
that would make the host controller reject (or, if a caller defaulted it,
|
||||
accept) unsigned input."""
|
||||
|
||||
broker_domain: TrustDomain = LAUNCH_BROKER
|
||||
controller_domain: TrustDomain = HOST_CONTROLLER
|
||||
|
||||
def broker_key(self) -> str:
|
||||
"""The durable launch-broker key both the orchestrator and the host
|
||||
control server must receive (in `broker_domain.key_env`). Raises rather
|
||||
than return ""."""
|
||||
key = self.broker_domain.signing_key()
|
||||
if not key:
|
||||
raise ProvisioningError(
|
||||
f"refusing to provision the {self.broker_domain.name} broker "
|
||||
"without a signing key: the host controller could then verify no "
|
||||
"launch request's provenance"
|
||||
)
|
||||
return key
|
||||
|
||||
def controller_key(self) -> str:
|
||||
"""The host controller's own lifecycle key — provisioned ONLY to the host
|
||||
controller (in `controller_domain.key_env`), never to the orchestrator, so
|
||||
the orchestrator cannot mint the `host`-role tokens that start and stop
|
||||
it. Raises rather than return ""."""
|
||||
key = self.controller_domain.signing_key()
|
||||
if not key:
|
||||
raise ProvisioningError(
|
||||
f"refusing to provision the {self.controller_domain.name} without "
|
||||
"a signing key: its lifecycle endpoints would authenticate no one"
|
||||
)
|
||||
return key
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ProvisioningError",
|
||||
"TrustDomain",
|
||||
"CONTROL_PLANE",
|
||||
"LAUNCH_BROKER",
|
||||
"HOST_CONTROLLER",
|
||||
"ControlPlaneProvisioning",
|
||||
"LaunchBrokerProvisioning",
|
||||
]
|
||||
|
||||
@@ -8,7 +8,11 @@ import unittest
|
||||
import urllib.error
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from bot_bottle.orchestrator.broker import BrokerAuthError, LaunchRequest
|
||||
from bot_bottle.orchestrator.broker import (
|
||||
BrokerAuthError,
|
||||
BrokerUnavailableError,
|
||||
LaunchRequest,
|
||||
)
|
||||
from bot_bottle.orchestrator.broker_client import BrokerClient, BrokerClientError
|
||||
|
||||
_URLOPEN = "bot_bottle.orchestrator.broker_client.urllib.request.urlopen"
|
||||
@@ -58,14 +62,25 @@ class TestSubmit(unittest.TestCase):
|
||||
with self.assertRaises(BrokerAuthError):
|
||||
self.c.submit("forged")
|
||||
|
||||
def test_502_raises_broker_client_error(self) -> None:
|
||||
def test_502_is_a_definite_client_error(self) -> None:
|
||||
# The host responded — it processed the request and did not launch, so a
|
||||
# definite BrokerClientError (the caller may safely roll back).
|
||||
with patch(_URLOPEN, side_effect=_http_error(502, {"error": "docker down"})):
|
||||
with self.assertRaises(BrokerClientError):
|
||||
self.c.submit("tok")
|
||||
|
||||
def test_unreachable_raises_broker_client_error(self) -> None:
|
||||
def test_unreachable_is_ambiguous_unavailable(self) -> None:
|
||||
# No response at all — the request may already have launched, so the
|
||||
# AMBIGUOUS BrokerUnavailableError (the caller must NOT roll back).
|
||||
with patch(_URLOPEN, side_effect=urllib.error.URLError("refused")):
|
||||
with self.assertRaises(BrokerClientError):
|
||||
with self.assertRaises(BrokerUnavailableError):
|
||||
self.c.submit("tok")
|
||||
|
||||
def test_timeout_is_ambiguous_unavailable(self) -> None:
|
||||
# A dropped/late response after the request was sent is the exact orphan
|
||||
# risk: the host may have launched. Must be ambiguous, not a definite fail.
|
||||
with patch(_URLOPEN, side_effect=TimeoutError("read timed out")):
|
||||
with self.assertRaises(BrokerUnavailableError):
|
||||
self.c.submit("tok")
|
||||
|
||||
def test_malformed_success_body_raises(self) -> None:
|
||||
@@ -79,6 +94,24 @@ class TestSubmit(unittest.TestCase):
|
||||
with self.assertRaises(BrokerAuthError):
|
||||
self.c.submit("forged")
|
||||
|
||||
def test_non_json_success_body_raises(self) -> None:
|
||||
# A 200 whose body isn't JSON is tolerated into {} then fails the
|
||||
# missing-field check — a definite client error, not a crash.
|
||||
m = MagicMock()
|
||||
m.__enter__.return_value.read.return_value = b"not json at all"
|
||||
with patch(_URLOPEN, return_value=m):
|
||||
with self.assertRaises(BrokerClientError):
|
||||
self.c.submit("tok")
|
||||
|
||||
def test_unreadable_error_body_is_tolerated(self) -> None:
|
||||
# An HTTPError whose body can't be read (fp=None) still classifies by
|
||||
# status — the error detail is best-effort.
|
||||
err = urllib.error.HTTPError(
|
||||
"http://host/broker", 502, "err", {}, None) # type: ignore[arg-type]
|
||||
with patch(_URLOPEN, side_effect=err):
|
||||
with self.assertRaises(BrokerClientError):
|
||||
self.c.submit("tok")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -7,10 +7,16 @@ the full sign -> POST -> verify -> act seam over HTTP.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import http.client
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import tempfile
|
||||
import threading
|
||||
import typing
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from bot_bottle.orchestrator.broker import (
|
||||
BrokerAuthError,
|
||||
@@ -21,10 +27,15 @@ from bot_bottle.orchestrator.broker import (
|
||||
)
|
||||
from bot_bottle.orchestrator.broker_client import BrokerClient
|
||||
from bot_bottle.orchestrator.host_server import (
|
||||
broker_secret_from_env,
|
||||
MAX_BODY_BYTES,
|
||||
Handler,
|
||||
HostControlServer,
|
||||
broker_secret,
|
||||
dispatch,
|
||||
main,
|
||||
make_host_server,
|
||||
)
|
||||
from bot_bottle.paths import LAUNCH_BROKER_KEY_ENV
|
||||
|
||||
|
||||
def _body(obj: object) -> bytes:
|
||||
@@ -97,6 +108,16 @@ class TestDispatch(unittest.TestCase):
|
||||
status, _ = dispatch(self.broker, "POST", "/broker", b"{not json")
|
||||
self.assertEqual(400, status)
|
||||
|
||||
def test_empty_body_is_missing_token_400(self) -> None:
|
||||
# Empty body parses to {} (no token) → 400, never reaching the broker.
|
||||
status, _ = dispatch(self.broker, "POST", "/broker", b"")
|
||||
self.assertEqual(400, status)
|
||||
self.assertEqual([], self.broker.launched)
|
||||
|
||||
def test_non_object_body_is_400(self) -> None:
|
||||
status, _ = dispatch(self.broker, "POST", "/broker", b"[1, 2]")
|
||||
self.assertEqual(400, status)
|
||||
|
||||
def test_unknown_route_404(self) -> None:
|
||||
status, _ = dispatch(self.broker, "GET", "/nope", b"")
|
||||
self.assertEqual(404, status)
|
||||
@@ -106,16 +127,34 @@ class TestDispatch(unittest.TestCase):
|
||||
self.assertEqual(200, status)
|
||||
|
||||
|
||||
class TestBrokerSecretFromEnv(unittest.TestCase):
|
||||
def test_reads_hex_secret(self) -> None:
|
||||
s = secrets.token_bytes(16)
|
||||
self.assertEqual(s, broker_secret_from_env({"BOT_BOTTLE_BROKER_SECRET": s.hex()}))
|
||||
class TestBrokerSecret(unittest.TestCase):
|
||||
"""The durable launch-broker key (#468/#476): prefer the env-injected key,
|
||||
else the durable host key file, so signer and verifier resolve the same one."""
|
||||
|
||||
def test_unset_is_none(self) -> None:
|
||||
self.assertIsNone(broker_secret_from_env({}))
|
||||
def test_reads_injected_key_from_env(self) -> None:
|
||||
# The injected key is honoured regardless of allow_host_file — both the
|
||||
# host controller and the guest orchestrator take an injected key.
|
||||
self.assertEqual(
|
||||
b"injected-key", broker_secret({LAUNCH_BROKER_KEY_ENV: "injected-key"}))
|
||||
self.assertEqual(
|
||||
b"injected-key",
|
||||
broker_secret({LAUNCH_BROKER_KEY_ENV: "injected-key"}, allow_host_file=True))
|
||||
|
||||
def test_invalid_hex_is_none(self) -> None:
|
||||
self.assertIsNone(broker_secret_from_env({"BOT_BOTTLE_BROKER_SECRET": "not-hex"}))
|
||||
def test_guest_without_injection_fails_closed(self) -> None:
|
||||
# The default (guest orchestrator): no env key and NO host-file fallback,
|
||||
# so it returns None rather than mint a divergent process-local key.
|
||||
self.assertIsNone(broker_secret({}))
|
||||
|
||||
def test_host_side_falls_back_to_the_durable_key_file(self) -> None:
|
||||
# allow_host_file=True (host controller / dev-harness): mint/read the
|
||||
# durable host key file, the same key on every call (restart re-adoption).
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
with patch.dict("os.environ", {"BOT_BOTTLE_ROOT": root}, clear=False):
|
||||
os.environ.pop(LAUNCH_BROKER_KEY_ENV, None)
|
||||
first = broker_secret(allow_host_file=True)
|
||||
second = broker_secret(allow_host_file=True)
|
||||
self.assertTrue(first)
|
||||
self.assertEqual(first, second)
|
||||
|
||||
|
||||
class TestSeamRoundTrip(unittest.TestCase):
|
||||
@@ -152,5 +191,116 @@ class TestSeamRoundTrip(unittest.TestCase):
|
||||
self.assertEqual([], broker.launched) # fail-closed across the wire
|
||||
|
||||
|
||||
class TestRequestLimits(unittest.TestCase):
|
||||
"""The privileged listener must not let a caller that can merely reach the
|
||||
socket (no signed token) exhaust it via an oversized declared body — and it
|
||||
rejects on the Content-Length *header*, before reading the body."""
|
||||
|
||||
def _addr(self) -> tuple[str, int]:
|
||||
self.broker = StubBroker(secrets.token_bytes(16))
|
||||
server = make_host_server(self.broker, "127.0.0.1", 0)
|
||||
self.addCleanup(server.server_close)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
self.addCleanup(server.shutdown)
|
||||
host, port = server.server_address[:2]
|
||||
return typing.cast(str, host), port
|
||||
|
||||
def test_oversized_content_length_is_rejected_before_reading(self) -> None:
|
||||
host, port = self._addr()
|
||||
conn = http.client.HTTPConnection(host, port, timeout=5)
|
||||
self.addCleanup(conn.close)
|
||||
# Declare an oversized body but send only a sliver: the server must reject
|
||||
# on the header before reading, so the caller gets a clean, deterministic
|
||||
# 413 (no large unread body to race a connection reset).
|
||||
conn.putrequest("POST", "/broker", skip_accept_encoding=True)
|
||||
conn.putheader("Content-Type", "application/json")
|
||||
conn.putheader("Content-Length", str(MAX_BODY_BYTES + 1))
|
||||
conn.endheaders()
|
||||
conn.send(b"{}") # far short of the declared length; never read
|
||||
resp = conn.getresponse()
|
||||
self.assertEqual(413, resp.status)
|
||||
self.assertEqual([], self.broker.launched) # never reached the broker
|
||||
|
||||
|
||||
class TestServeUnit(unittest.TestCase):
|
||||
"""Drive `Handler._serve` directly (no socket). The real per-request handler
|
||||
runs in a daemon thread whose coverage/trace data is lost, so the
|
||||
bounded-body and error paths are exercised here in the main thread instead."""
|
||||
|
||||
def _handler(self, broker: LaunchBroker, headers: dict[str, str],
|
||||
body: bytes = b"") -> tuple[Handler, MagicMock]:
|
||||
server = HostControlServer.__new__(HostControlServer)
|
||||
server.broker = broker
|
||||
h = Handler.__new__(Handler)
|
||||
h.server = server
|
||||
h.headers = headers # type: ignore[assignment] — dict is a valid .get() stand-in
|
||||
h.path = "/broker"
|
||||
h.rfile = io.BytesIO(body)
|
||||
h.wfile = io.BytesIO()
|
||||
send_response = MagicMock()
|
||||
h.send_response = send_response # type: ignore[method-assign]
|
||||
h.send_header = MagicMock() # type: ignore[method-assign]
|
||||
h.end_headers = MagicMock() # type: ignore[method-assign]
|
||||
return h, send_response
|
||||
|
||||
def test_oversized_content_length_is_413(self) -> None:
|
||||
broker = StubBroker(secrets.token_bytes(16))
|
||||
h, send_response = self._handler(broker, {"Content-Length": str(MAX_BODY_BYTES + 1)})
|
||||
h.do_POST() # exercises do_POST -> _serve
|
||||
send_response.assert_called_once_with(413)
|
||||
self.assertEqual([], broker.launched) # rejected before the broker
|
||||
|
||||
def test_invalid_content_length_is_400(self) -> None:
|
||||
h, send_response = self._handler(StubBroker(secrets.token_bytes(16)),
|
||||
{"Content-Length": "not-a-number"})
|
||||
h._serve("POST")
|
||||
send_response.assert_called_once_with(400)
|
||||
|
||||
def test_valid_request_dispatches_200(self) -> None:
|
||||
secret = secrets.token_bytes(16)
|
||||
broker = StubBroker(secret)
|
||||
body = _body({"token": sign_request(
|
||||
LaunchRequest(op="teardown", bottle_id="b1"), secret)})
|
||||
h, send_response = self._handler(broker, {"Content-Length": str(len(body))}, body)
|
||||
h._serve("POST")
|
||||
send_response.assert_called_once_with(200)
|
||||
self.assertEqual(["b1"], [r.bottle_id for r in broker.torn_down])
|
||||
|
||||
def test_dispatch_exception_becomes_500(self) -> None:
|
||||
# dispatch is total, but the handler still guards it: a raised dispatch
|
||||
# returns 500 rather than dropping the connection.
|
||||
h, send_response = self._handler(
|
||||
StubBroker(secrets.token_bytes(16)), {"Content-Length": "0"})
|
||||
with patch("bot_bottle.orchestrator.host_server.dispatch",
|
||||
side_effect=RuntimeError("boom")):
|
||||
h._serve("POST")
|
||||
send_response.assert_called_once_with(500)
|
||||
|
||||
def test_health_over_do_get(self) -> None:
|
||||
h, send_response = self._handler(StubBroker(secrets.token_bytes(16)), {})
|
||||
h.path = "/health"
|
||||
h.do_GET()
|
||||
send_response.assert_called_once_with(200)
|
||||
|
||||
|
||||
class TestMain(unittest.TestCase):
|
||||
def test_fail_closed_without_secret(self) -> None:
|
||||
with patch("bot_bottle.orchestrator.host_server.broker_secret",
|
||||
return_value=None):
|
||||
self.assertEqual(2, main(["--port", "0"]))
|
||||
|
||||
def test_serves_then_shuts_down_cleanly(self) -> None:
|
||||
fake = MagicMock()
|
||||
fake.server_address = ("127.0.0.1", 0)
|
||||
fake.serve_forever.side_effect = KeyboardInterrupt
|
||||
with patch("bot_bottle.orchestrator.host_server.broker_secret",
|
||||
return_value=b"k"), \
|
||||
patch("bot_bottle.orchestrator.host_server.make_host_server",
|
||||
return_value=fake):
|
||||
self.assertEqual(0, main(["--port", "0"]))
|
||||
fake.serve_forever.assert_called_once()
|
||||
fake.server_close.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Unit: the orchestrator dev-harness entrypoint (`python -m bot_bottle.orchestrator`).
|
||||
|
||||
Exercises broker selection (stub / docker / http) and the fail-closed http path,
|
||||
patching `make_server` so the serve loop returns instead of blocking.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import secrets
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from bot_bottle.orchestrator.__main__ import main
|
||||
from bot_bottle.paths import LAUNCH_BROKER_KEY_ENV
|
||||
|
||||
|
||||
def _fake_server() -> MagicMock:
|
||||
fake = MagicMock()
|
||||
fake.server_address = ("127.0.0.1", 0)
|
||||
# Break out of serve_forever immediately, exercising the try/finally.
|
||||
fake.serve_forever.side_effect = KeyboardInterrupt
|
||||
return fake
|
||||
|
||||
|
||||
class TestMain(unittest.TestCase):
|
||||
def _run(self, broker: str, env: dict[str, str] | None = None) -> tuple[int, MagicMock]:
|
||||
fake = _fake_server()
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
argv = ["--db", str(Path(d) / "r.db"), "--port", "0", "--broker", broker]
|
||||
with patch("bot_bottle.orchestrator.__main__.make_server", return_value=fake), \
|
||||
patch.dict("os.environ", env or {}, clear=False):
|
||||
if env is None:
|
||||
os.environ.pop(LAUNCH_BROKER_KEY_ENV, None)
|
||||
rc = main(argv)
|
||||
return rc, fake
|
||||
|
||||
def test_stub_broker_serves_and_closes(self) -> None:
|
||||
rc, fake = self._run("stub")
|
||||
self.assertEqual(0, rc)
|
||||
fake.serve_forever.assert_called_once()
|
||||
fake.server_close.assert_called_once()
|
||||
|
||||
def test_docker_broker_serves(self) -> None:
|
||||
rc, _ = self._run("docker")
|
||||
self.assertEqual(0, rc)
|
||||
|
||||
def test_http_broker_with_injected_key_serves(self) -> None:
|
||||
# The guest orchestrator takes the launch-broker key by injection.
|
||||
rc, _ = self._run(
|
||||
"http", env={LAUNCH_BROKER_KEY_ENV: secrets.token_urlsafe(16)})
|
||||
self.assertEqual(0, rc)
|
||||
|
||||
def test_http_broker_without_injected_key_exits(self) -> None:
|
||||
# Fail-closed: no host-file fallback for the guest, so a missing injected
|
||||
# key is a usage error rather than a silently-minted divergent key.
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
with patch.dict("os.environ", {}, clear=False):
|
||||
os.environ.pop(LAUNCH_BROKER_KEY_ENV, None)
|
||||
with self.assertRaises(SystemExit):
|
||||
main(["--db", str(Path(d) / "r.db"), "--broker", "http"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -11,7 +11,12 @@ from contextlib import closing
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from bot_bottle.orchestrator.broker import LaunchBroker, LaunchRequest, StubBroker
|
||||
from bot_bottle.orchestrator.broker import (
|
||||
BrokerUnavailableError,
|
||||
LaunchBroker,
|
||||
LaunchRequest,
|
||||
StubBroker,
|
||||
)
|
||||
from bot_bottle.orchestrator.store.registry_store import RegistryStore
|
||||
from bot_bottle.orchestrator.service import OrchestratorCore
|
||||
from bot_bottle.orchestrator.store.secret_store import new_env_var_secret
|
||||
@@ -25,8 +30,8 @@ from bot_bottle.orchestrator.supervisor import (
|
||||
|
||||
|
||||
class _FailingBroker(LaunchBroker):
|
||||
"""Verifies the token like any broker, then fails the launch — to
|
||||
exercise the orchestrator's registry rollback."""
|
||||
"""Verifies the token like any broker, then fails the launch *definitely* —
|
||||
to exercise the orchestrator's registry rollback."""
|
||||
|
||||
def _launch(self, req: LaunchRequest) -> None:
|
||||
raise RuntimeError("launch failed")
|
||||
@@ -35,6 +40,18 @@ class _FailingBroker(LaunchBroker):
|
||||
pass
|
||||
|
||||
|
||||
class _UnavailableBroker(LaunchBroker):
|
||||
"""Verifies the token, then raises the *ambiguous* BrokerUnavailableError —
|
||||
the host may already have launched — so the orchestrator must KEEP the
|
||||
registry row rather than orphan a running container."""
|
||||
|
||||
def _launch(self, req: LaunchRequest) -> None:
|
||||
raise BrokerUnavailableError("delivery dropped after send")
|
||||
|
||||
def _teardown(self, req: LaunchRequest) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class TestOrchestrator(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
@@ -136,11 +153,20 @@ class TestOrchestrator(unittest.TestCase):
|
||||
self.assertIsNotNone(self.orch.resolve("10.243.0.3", rec.identity_token))
|
||||
self.assertIsNone(self.orch.resolve("10.243.0.3", "wrong-token"))
|
||||
|
||||
def test_launch_rolls_back_registry_on_broker_failure(self) -> None:
|
||||
def test_launch_rolls_back_registry_on_definite_broker_failure(self) -> None:
|
||||
orch = OrchestratorCore(self.store, _FailingBroker(self.secret), self.secret)
|
||||
with self.assertRaises(RuntimeError):
|
||||
orch.launch_bottle("10.243.0.9")
|
||||
self.assertEqual([], self.store.all()) # no orphan
|
||||
self.assertEqual([], self.store.all()) # no orphan row
|
||||
|
||||
def test_launch_keeps_registry_on_ambiguous_broker_failure(self) -> None:
|
||||
# The host may already have launched the bottle before the response was
|
||||
# lost, so deregistering would orphan a running container with no row.
|
||||
# The row is kept for reconcile to reap iff the bottle is not live.
|
||||
orch = OrchestratorCore(self.store, _UnavailableBroker(self.secret), self.secret)
|
||||
with self.assertRaises(BrokerUnavailableError):
|
||||
orch.launch_bottle("10.243.0.9")
|
||||
self.assertEqual(1, len(self.store.all())) # row survives — no orphan container
|
||||
|
||||
def test_gateway_status_reports_unconfigured(self) -> None:
|
||||
# The orchestrator no longer owns a standalone gateway lifecycle; the
|
||||
|
||||
@@ -6,10 +6,13 @@ import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from bot_bottle import orchestrator_auth
|
||||
from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY
|
||||
from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY, ROLE_HOST
|
||||
from bot_bottle.trust_domain import (
|
||||
CONTROL_PLANE,
|
||||
HOST_CONTROLLER,
|
||||
LAUNCH_BROKER,
|
||||
ControlPlaneProvisioning,
|
||||
LaunchBrokerProvisioning,
|
||||
ProvisioningError,
|
||||
TrustDomain,
|
||||
)
|
||||
@@ -101,5 +104,73 @@ class TestControlPlaneProvisioning(unittest.TestCase):
|
||||
self.assertNotEqual(ROLE_CLI, CONTROL_PLANE.verify(tok, "k"))
|
||||
|
||||
|
||||
class TestLaunchBrokerAndHostControllerDomains(unittest.TestCase):
|
||||
"""The real #468 domains: the launch-broker key (shared by orchestrator +
|
||||
host controller) and the host controller's own lifecycle key."""
|
||||
|
||||
def test_launch_broker_mints_no_role_tokens(self) -> None:
|
||||
# Empty role set — it provides durable key material for the broker's own
|
||||
# launch JWT, not orchestrator_auth role tokens.
|
||||
self.assertEqual(frozenset(), LAUNCH_BROKER.roles)
|
||||
with patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
|
||||
with self.assertRaises(ValueError):
|
||||
LAUNCH_BROKER.mint(ROLE_CLI)
|
||||
|
||||
def test_host_controller_signs_host_role_only(self) -> None:
|
||||
with patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
|
||||
tok = HOST_CONTROLLER.mint(ROLE_HOST)
|
||||
self.assertEqual(ROLE_HOST, HOST_CONTROLLER.verify(tok, "k"))
|
||||
# A control-plane `cli` token (the orchestrator's key) never verifies as a
|
||||
# host-controller role — the orchestrator can't forge lifecycle creds.
|
||||
cli_tok = orchestrator_auth.mint(ROLE_CLI, "k")
|
||||
self.assertIsNone(HOST_CONTROLLER.verify(cli_tok, "k"))
|
||||
|
||||
def test_control_plane_cannot_mint_the_host_role(self) -> None:
|
||||
# `host` is outside the control-plane role set on purpose.
|
||||
with patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
|
||||
with self.assertRaises(ValueError):
|
||||
CONTROL_PLANE.mint(ROLE_HOST)
|
||||
|
||||
def test_the_three_domains_use_distinct_keys_and_env_vars(self) -> None:
|
||||
self.assertEqual(3, len({
|
||||
CONTROL_PLANE.key_filename,
|
||||
LAUNCH_BROKER.key_filename,
|
||||
HOST_CONTROLLER.key_filename,
|
||||
}))
|
||||
self.assertEqual(3, len({
|
||||
CONTROL_PLANE.key_env, LAUNCH_BROKER.key_env, HOST_CONTROLLER.key_env,
|
||||
}))
|
||||
|
||||
|
||||
class TestLaunchBrokerProvisioning(unittest.TestCase):
|
||||
def test_broker_key_returns_the_durable_key(self) -> None:
|
||||
prov = LaunchBrokerProvisioning()
|
||||
with patch("bot_bottle.trust_domain.host_signing_key", return_value="bk"):
|
||||
self.assertEqual("bk", prov.broker_key())
|
||||
|
||||
def test_broker_key_fail_closes_when_empty(self) -> None:
|
||||
# An empty key would leave the host controller unable to verify any
|
||||
# launch — fail-closed rather than hand back a useless/dangerous key.
|
||||
prov = LaunchBrokerProvisioning()
|
||||
with patch("bot_bottle.trust_domain.host_signing_key", return_value=""):
|
||||
with self.assertRaises(ProvisioningError):
|
||||
prov.broker_key()
|
||||
|
||||
def test_controller_key_is_distinct_from_the_broker_key(self) -> None:
|
||||
# The orchestrator holds the broker key but NEVER the controller key.
|
||||
prov = LaunchBrokerProvisioning()
|
||||
keys = {"launch-broker-key": "bk", "host-controller-key": "ck"}
|
||||
with patch("bot_bottle.trust_domain.host_signing_key",
|
||||
side_effect=keys.__getitem__):
|
||||
self.assertEqual("bk", prov.broker_key())
|
||||
self.assertEqual("ck", prov.controller_key())
|
||||
|
||||
def test_controller_key_fail_closes_when_empty(self) -> None:
|
||||
prov = LaunchBrokerProvisioning()
|
||||
with patch("bot_bottle.trust_domain.host_signing_key", return_value=""):
|
||||
with self.assertRaises(ProvisioningError):
|
||||
prov.controller_key()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user