7005b22bcf
Chunk 3 of the host-control-server stack: grow the broker op vocabulary
(PRD gap 3), starting with `list_live`, and invert `reconcile` onto it.
- broker: split the closed op vocabulary into mutation (`launch`/`teardown`,
carry a bottle id + static flags) and query (`list_live`, carries nothing
but its op name) kinds. `verify_request` now enforces a **strict schema**
(open question 1, resolved yes): unknown claim keys are rejected, a mutation
must name its bottle, and a query that smuggles any id/flag is refused.
- broker verb: `LaunchBroker.list_live` / `SubmitBroker.list_live` return the
backend's live source IPs; a backend enumeration failure is converted to the
single `BrokerUnavailableError` "live set unknown" signal. `DockerBroker`
enumerates its labelled containers; `StubBroker` derives from launches (or a
test override).
- host controller: `POST /broker/live` verifies a signed `list_live` token and
returns `{source_ips}`; `BrokerClient.list_live` is its drop-in client.
- reconcile: `OrchestratorCore.reconcile()` drops the `live_source_ips`
parameter and pulls the live set from the broker itself — the tell that the
orchestrator couldn't see the backend goes away. **Fail-safe**: if the broker
can't return an authoritative set the sweep is skipped, never run against an
empty/partial set (which would reap healthy rows). The `/reconcile` HTTP
contract + `OrchestratorClient.reconcile` become a bare trigger.
The macOS launcher's Apple-container enumeration stays for now; it becomes the
host controller's `list_live` when launch itself moves behind the broker (the
pulled-forward chunk 5, next in the stack).
Tests + pyright clean; pylint 10.0 on broker.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
169 lines
7.6 KiB
Python
169 lines
7.6 KiB
Python
"""Orchestrator-side broker transport (issue #468, chunk 1).
|
|
|
|
The signer's half of the launch-broker transport gap. `BrokerClient` satisfies
|
|
the exact `submit(token)` contract `OrchestratorCore` already depends on (see
|
|
`broker.SubmitBroker`), but instead of verifying and launching in-process it POSTs
|
|
the signed token to the host control server over HTTP (stdlib `urllib`, like
|
|
`orchestrator/client.py`). Because it is drop-in for that interface, wiring a real
|
|
out-of-process backend does not change the core: it still signs a request and
|
|
calls `submit()`; only the wire is new.
|
|
|
|
A provenance/schema rejection from the host controller (HTTP 401) is re-raised as
|
|
the same `BrokerAuthError` the in-process broker raises, so the launch path's
|
|
rollback-on-failure (`OrchestratorCore.launch_bottle`) behaves identically whether
|
|
the broker is local or remote.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
from .broker import BrokerAuthError, BrokerUnavailableError, LaunchRequest
|
|
|
|
DEFAULT_TIMEOUT_SECONDS = 5.0
|
|
|
|
|
|
class BrokerClientError(RuntimeError):
|
|
"""The host control server *responded*, but with an unexpected status other
|
|
than the fail-closed 401 (which surfaces as `BrokerAuthError`) — e.g. a 502
|
|
backend failure or a malformed body. A definite negative: the host processed
|
|
the request and it did not launch. (A *no-response* failure — unreachable /
|
|
timeout / dropped — is the ambiguous `BrokerUnavailableError` instead.)"""
|
|
|
|
|
|
class BrokerClient:
|
|
"""Drop-in `submit(token)` that relays a signed request to the host control
|
|
server. Holds no secret — provenance rides entirely in the signed token, so a
|
|
caller that can reach this client still cannot forge a launch."""
|
|
|
|
def __init__(self, base_url: str, *, timeout: float = DEFAULT_TIMEOUT_SECONDS) -> None:
|
|
self._base = base_url.rstrip("/")
|
|
self._timeout = timeout
|
|
|
|
def submit(self, token: str) -> LaunchRequest:
|
|
"""POST the signed token to the host controller and return the request it
|
|
verified and acted on.
|
|
|
|
Raises `BrokerAuthError` on a fail-closed 401 (bad provenance/schema —
|
|
the same exception the in-process broker raises); `BrokerClientError` if
|
|
the host *responds* with any other non-success status or a malformed
|
|
body (a definite negative); or `BrokerUnavailableError` if no response is
|
|
obtained (unreachable / timeout / dropped) — the **ambiguous** case, where
|
|
the host may already have acted, so the caller must not roll back."""
|
|
data = json.dumps({"token": token}).encode()
|
|
req = urllib.request.Request(
|
|
f"{self._base}/broker", data=data, method="POST",
|
|
headers={"Content-Type": "application/json"},
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=self._timeout) as resp:
|
|
return _request_from(_json_object(resp.read()))
|
|
except urllib.error.HTTPError as e:
|
|
detail = _error_detail(e)
|
|
if e.code == 401:
|
|
raise BrokerAuthError(
|
|
detail or "host controller rejected the request"
|
|
) from e
|
|
raise BrokerClientError(
|
|
f"POST /broker: HTTP {e.code} {detail}".rstrip()
|
|
) from e
|
|
except (urllib.error.URLError, TimeoutError, OSError) as e:
|
|
# No usable response — unreachable, timed out, or the connection
|
|
# dropped mid-exchange. Ambiguous: the request may already have
|
|
# launched the bottle, so this is NOT a definite failure.
|
|
raise BrokerUnavailableError(f"POST /broker: {e}") from e
|
|
|
|
def list_live(self, token: str) -> list[str]:
|
|
"""POST the signed `list_live` token to the host controller and return
|
|
the source IPs of the bottles it enumerated as running.
|
|
|
|
Raises `BrokerAuthError` on a fail-closed 401 (bad provenance/schema);
|
|
`BrokerClientError` on any other non-success status or a malformed body
|
|
(a backend-enumeration 502 included); or `BrokerUnavailableError` if no
|
|
response is obtained. A query has no backend side effect, so — unlike
|
|
`submit` — every one of these is a *definite* "no live set"; reconcile
|
|
catches all three and skips the sweep rather than reaping against an
|
|
unknown or partial set."""
|
|
data = json.dumps({"token": token}).encode()
|
|
req = urllib.request.Request(
|
|
f"{self._base}/broker/live", data=data, method="POST",
|
|
headers={"Content-Type": "application/json"},
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=self._timeout) as resp:
|
|
return _source_ips_from(_json_object(resp.read()))
|
|
except urllib.error.HTTPError as e:
|
|
detail = _error_detail(e)
|
|
if e.code == 401:
|
|
raise BrokerAuthError(
|
|
detail or "host controller rejected the request"
|
|
) from e
|
|
raise BrokerClientError(
|
|
f"POST /broker/live: HTTP {e.code} {detail}".rstrip()
|
|
) from e
|
|
except (urllib.error.URLError, TimeoutError, OSError) as e:
|
|
raise BrokerUnavailableError(f"POST /broker/live: {e}") from e
|
|
|
|
|
|
def _json_object(raw: bytes) -> dict[str, object]:
|
|
"""Parse a JSON object, tolerating an empty or malformed body (→ {}), like
|
|
the orchestrator client — a bad body becomes a clean 'missing field' error
|
|
downstream rather than an opaque JSON crash."""
|
|
if not raw:
|
|
return {}
|
|
try:
|
|
obj = json.loads(raw)
|
|
except ValueError:
|
|
return {}
|
|
return obj if isinstance(obj, dict) else {}
|
|
|
|
|
|
def _error_detail(e: urllib.error.HTTPError) -> str:
|
|
"""The `error` string from a structured error response, best-effort — an
|
|
error body may be absent or unreadable, in which case there is no detail."""
|
|
try:
|
|
detail = _json_object(e.read()).get("error", "")
|
|
except Exception: # noqa: BLE001 — the error body is advisory only
|
|
return ""
|
|
return detail if isinstance(detail, str) else ""
|
|
|
|
|
|
def _source_ips_from(payload: dict[str, object]) -> list[str]:
|
|
"""The `source_ips` list from a `/broker/live` response — every string
|
|
entry, ignoring any non-string the host controller should never send. A
|
|
missing/!list field is a malformed response (the query is meaningless
|
|
without it), so it fails rather than silently reconciling against []."""
|
|
raw = payload.get("source_ips")
|
|
if not isinstance(raw, list):
|
|
raise BrokerClientError("host controller response missing source_ips")
|
|
return [ip for ip in raw if isinstance(ip, str) and ip]
|
|
|
|
|
|
def _request_from(payload: dict[str, object]) -> LaunchRequest:
|
|
"""Reconstruct the verified `LaunchRequest` the controller echoed, so the
|
|
returned value matches the in-process broker's (which returns the request it
|
|
acted on). A missing op/bottle_id means a malformed response."""
|
|
op = payload.get("op")
|
|
bottle_id = payload.get("bottle_id")
|
|
if not isinstance(op, str) or not isinstance(bottle_id, str) or not bottle_id:
|
|
raise BrokerClientError("host controller response missing op/bottle_id")
|
|
source_ip = payload.get("source_ip")
|
|
image_ref = payload.get("image_ref")
|
|
slot = payload.get("slot")
|
|
return LaunchRequest(
|
|
op=op,
|
|
bottle_id=bottle_id,
|
|
source_ip=source_ip if isinstance(source_ip, str) else "",
|
|
image_ref=image_ref if isinstance(image_ref, str) else "",
|
|
slot=slot if isinstance(slot, int) and not isinstance(slot, bool) else None,
|
|
)
|
|
|
|
|
|
__all__ = [
|
|
"BrokerClient",
|
|
"BrokerClientError",
|
|
"DEFAULT_TIMEOUT_SECONDS",
|
|
]
|