feat(orchestrator): list_live broker op + reconcile via broker (#468)
Chunk 3 of the host-control-server stack: grow the broker op vocabulary
(PRD gap 3), starting with `list_live`, and invert `reconcile` onto it.
- broker: split the closed op vocabulary into mutation (`launch`/`teardown`,
carry a bottle id + static flags) and query (`list_live`, carries nothing
but its op name) kinds. `verify_request` now enforces a **strict schema**
(open question 1, resolved yes): unknown claim keys are rejected, a mutation
must name its bottle, and a query that smuggles any id/flag is refused.
- broker verb: `LaunchBroker.list_live` / `SubmitBroker.list_live` return the
backend's live source IPs; a backend enumeration failure is converted to the
single `BrokerUnavailableError` "live set unknown" signal. `DockerBroker`
enumerates its labelled containers; `StubBroker` derives from launches (or a
test override).
- host controller: `POST /broker/live` verifies a signed `list_live` token and
returns `{source_ips}`; `BrokerClient.list_live` is its drop-in client.
- reconcile: `OrchestratorCore.reconcile()` drops the `live_source_ips`
parameter and pulls the live set from the broker itself — the tell that the
orchestrator couldn't see the backend goes away. **Fail-safe**: if the broker
can't return an authoritative set the sweep is skipped, never run against an
empty/partial set (which would reap healthy rows). The `/reconcile` HTTP
contract + `OrchestratorClient.reconcile` become a bare trigger.
The macOS launcher's Apple-container enumeration stays for now; it becomes the
host controller's `list_live` when launch itself moves behind the broker (the
pulled-forward chunk 5, next in the stack).
Tests + pyright clean; pylint 10.0 on broker.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -43,8 +43,9 @@ def _body(obj: object) -> bytes:
|
||||
|
||||
|
||||
class _RaisingBroker(LaunchBroker):
|
||||
"""A broker whose backend launch always fails — exercises the 502 path (an
|
||||
operational backend failure, distinct from a fail-closed provenance 401)."""
|
||||
"""A broker whose backend launch/enumeration always fails — exercises the
|
||||
502 path (an operational backend failure, distinct from a fail-closed
|
||||
provenance 401)."""
|
||||
|
||||
def _launch(self, req: LaunchRequest) -> None:
|
||||
raise RuntimeError("docker down")
|
||||
@@ -52,6 +53,9 @@ class _RaisingBroker(LaunchBroker):
|
||||
def _teardown(self, req: LaunchRequest) -> None:
|
||||
raise RuntimeError("docker down")
|
||||
|
||||
def _list_live(self) -> list[str]:
|
||||
raise RuntimeError("docker down")
|
||||
|
||||
|
||||
class TestDispatch(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
@@ -100,6 +104,42 @@ class TestDispatch(unittest.TestCase):
|
||||
self.assertEqual(502, status)
|
||||
self.assertIn("backend launch failed", str(payload["error"]))
|
||||
|
||||
def test_broker_live_returns_source_ips(self) -> None:
|
||||
# Two launched bottles -> the stub reports both as live.
|
||||
self.broker.submit(self._token(op="launch", bottle_id="b1", source_ip="10.0.0.1"))
|
||||
self.broker.submit(self._token(op="launch", bottle_id="b2", source_ip="10.0.0.2"))
|
||||
token = self._token(op="list_live")
|
||||
status, payload = dispatch(self.broker, "POST", "/broker/live", _body({"token": token}))
|
||||
self.assertEqual(200, status)
|
||||
self.assertEqual(
|
||||
["10.0.0.1", "10.0.0.2"],
|
||||
sorted(typing.cast("list[str]", payload["source_ips"])),
|
||||
)
|
||||
|
||||
def test_broker_live_forged_token_is_401(self) -> None:
|
||||
forged = sign_request(LaunchRequest(op="list_live"), secrets.token_bytes(16))
|
||||
status, payload = dispatch(self.broker, "POST", "/broker/live", _body({"token": forged}))
|
||||
self.assertEqual(401, status)
|
||||
self.assertIn("broker auth failed", str(payload["error"]))
|
||||
|
||||
def test_broker_live_rejects_a_mutation_token(self) -> None:
|
||||
# A launch token routed to the query endpoint is a fail-closed 401 — the
|
||||
# endpoints don't share a schema even though they share a secret.
|
||||
token = self._token(op="launch", bottle_id="b1", image_ref="img")
|
||||
status, _ = dispatch(self.broker, "POST", "/broker/live", _body({"token": token}))
|
||||
self.assertEqual(401, status)
|
||||
|
||||
def test_broker_live_backend_failure_is_502(self) -> None:
|
||||
broker = _RaisingBroker(self.secret)
|
||||
token = self._token(op="list_live")
|
||||
status, payload = dispatch(broker, "POST", "/broker/live", _body({"token": token}))
|
||||
self.assertEqual(502, status)
|
||||
self.assertIn("backend enumeration failed", str(payload["error"]))
|
||||
|
||||
def test_broker_live_missing_token_is_400(self) -> None:
|
||||
status, _ = dispatch(self.broker, "POST", "/broker/live", _body({}))
|
||||
self.assertEqual(400, status)
|
||||
|
||||
def test_missing_token_is_400(self) -> None:
|
||||
status, _ = dispatch(self.broker, "POST", "/broker", _body({}))
|
||||
self.assertEqual(400, status)
|
||||
@@ -190,6 +230,14 @@ class TestSeamRoundTrip(unittest.TestCase):
|
||||
client.submit(forged)
|
||||
self.assertEqual([], broker.launched) # fail-closed across the wire
|
||||
|
||||
def test_list_live_enumerates_over_http(self) -> None:
|
||||
secret = secrets.token_bytes(16)
|
||||
broker = StubBroker(secret)
|
||||
broker.live_source_ips = ["10.0.0.7", "10.0.0.8"]
|
||||
client = self._serve(broker)
|
||||
got = client.list_live(sign_request(LaunchRequest(op="list_live"), secret))
|
||||
self.assertEqual(["10.0.0.7", "10.0.0.8"], got)
|
||||
|
||||
|
||||
class TestRequestLimits(unittest.TestCase):
|
||||
"""The privileged listener must not let a caller that can merely reach the
|
||||
|
||||
Reference in New Issue
Block a user