feat(orchestrator): list_live broker op + reconcile via broker (#468)
lint / lint (push) Successful in 1m5s
refresh-image-locks / refresh (push) Successful in 34s
test / unit (pull_request) Has started running
test / image-input-builds (pull_request) Has started running
test / coverage (pull_request) Blocked by required conditions
test / integration-docker (pull_request) Has been cancelled
tracker-policy-pr / check-pr (pull_request) Successful in 7s
lint / lint (push) Successful in 1m5s
refresh-image-locks / refresh (push) Successful in 34s
test / unit (pull_request) Has started running
test / image-input-builds (pull_request) Has started running
test / coverage (pull_request) Blocked by required conditions
test / integration-docker (pull_request) Has been cancelled
tracker-policy-pr / check-pr (pull_request) Successful in 7s
Chunk 3 of the host-control-server stack: grow the broker op vocabulary
(PRD gap 3), starting with `list_live`, and invert `reconcile` onto it.
- broker: split the closed op vocabulary into mutation (`launch`/`teardown`,
carry a bottle id + static flags) and query (`list_live`, carries nothing
but its op name) kinds. `verify_request` now enforces a **strict schema**
(open question 1, resolved yes): unknown claim keys are rejected, a mutation
must name its bottle, and a query that smuggles any id/flag is refused.
- broker verb: `LaunchBroker.list_live` / `SubmitBroker.list_live` return the
backend's live source IPs; a backend enumeration failure is converted to the
single `BrokerUnavailableError` "live set unknown" signal. `DockerBroker`
enumerates its labelled containers; `StubBroker` derives from launches (or a
test override).
- host controller: `POST /broker/live` verifies a signed `list_live` token and
returns `{source_ips}`; `BrokerClient.list_live` is its drop-in client.
- reconcile: `OrchestratorCore.reconcile()` drops the `live_source_ips`
parameter and pulls the live set from the broker itself — the tell that the
orchestrator couldn't see the backend goes away. **Fail-safe**: if the broker
can't return an authoritative set the sweep is skipped, never run against an
empty/partial set (which would reap healthy rows). The `/reconcile` HTTP
contract + `OrchestratorClient.reconcile` become a bare trigger.
The macOS launcher's Apple-container enumeration stays for now; it becomes the
host controller's `list_live` when launch itself moves behind the broker (the
pulled-forward chunk 5, next in the stack).
Tests + pyright clean; pylint 10.0 on broker.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -624,13 +624,18 @@ if __name__ == "__main__":
|
||||
|
||||
|
||||
class TestReconcileRoute(unittest.TestCase):
|
||||
"""`POST /reconcile` — the host tells the orchestrator which bottles are
|
||||
actually up, since the orchestrator can't see the backend from inside the
|
||||
infra container."""
|
||||
"""`POST /reconcile` — a bare self-heal trigger. The orchestrator pulls its
|
||||
own live set from the broker (`list_live`); the caller no longer supplies it,
|
||||
so the request body carries at most `grace_seconds`."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.orch = _orchestrator(Path(self._tmp.name) / "r.db")
|
||||
store = RegistryStore(Path(self._tmp.name) / "r.db")
|
||||
store.migrate()
|
||||
secret = secrets.token_bytes(16)
|
||||
# Hold a typed StubBroker so tests can set its live set directly.
|
||||
self.broker = StubBroker(secret)
|
||||
self.orch = OrchestratorCore(store, self.broker, secret)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
@@ -647,44 +652,35 @@ class TestReconcileRoute(unittest.TestCase):
|
||||
def test_reaps_absent_and_reports_ids(self) -> None:
|
||||
dead = self._old("10.0.0.1")
|
||||
alive = self._old("10.0.0.2")
|
||||
status, payload = dispatch(
|
||||
self.orch, "POST", "/reconcile", _body({"live_source_ips": ["10.0.0.2"]}))
|
||||
self.broker.live_source_ips = ["10.0.0.2"] # broker: only .2 is up
|
||||
status, payload = dispatch(self.orch, "POST", "/reconcile", _body({}))
|
||||
self.assertEqual(200, status)
|
||||
self.assertEqual([dead], payload["reaped"])
|
||||
self.assertIsNone(self.orch.registry.get(dead))
|
||||
self.assertIsNotNone(self.orch.registry.get(alive))
|
||||
|
||||
def test_missing_live_source_ips_is_400(self) -> None:
|
||||
status, _ = dispatch(self.orch, "POST", "/reconcile", _body({}))
|
||||
self.assertEqual(400, status)
|
||||
def test_empty_body_triggers_the_sweep(self) -> None:
|
||||
"""No body is fine — the live set comes from the broker, not the body."""
|
||||
dead = self._old("10.0.0.1")
|
||||
self.broker.live_source_ips = [] # broker: nothing running
|
||||
status, payload = dispatch(self.orch, "POST", "/reconcile", b"")
|
||||
self.assertEqual(200, status)
|
||||
self.assertEqual([dead], payload["reaped"])
|
||||
|
||||
def test_grace_seconds_is_honoured(self) -> None:
|
||||
"""A grace window wide enough to cover the row protects it."""
|
||||
self.orch.registry.register("10.0.0.3")
|
||||
self.broker.live_source_ips = []
|
||||
status, payload = dispatch(
|
||||
self.orch, "POST", "/reconcile",
|
||||
_body({"live_source_ips": [], "grace_seconds": 3600}))
|
||||
self.orch, "POST", "/reconcile", _body({"grace_seconds": 3600}))
|
||||
self.assertEqual(200, status)
|
||||
self.assertEqual([], payload["reaped"])
|
||||
|
||||
def test_non_string_entries_are_rejected(self) -> None:
|
||||
status, payload = dispatch(
|
||||
self.orch, "POST", "/reconcile",
|
||||
_body({"live_source_ips": [None, 7, "10.0.0.9"]}))
|
||||
self.assertEqual(400, status)
|
||||
self.assertIn("live_source_ips", str(payload["error"]))
|
||||
|
||||
def test_empty_live_source_ip_is_rejected(self) -> None:
|
||||
status, payload = dispatch(
|
||||
self.orch, "POST", "/reconcile", _body({"live_source_ips": [""]}))
|
||||
self.assertEqual(400, status)
|
||||
self.assertIn("live_source_ips", str(payload["error"]))
|
||||
|
||||
def test_invalid_grace_seconds_is_rejected(self) -> None:
|
||||
for value in (True, "30", -1, float("inf"), float("nan")):
|
||||
with self.subTest(value=value):
|
||||
status, payload = dispatch(
|
||||
self.orch, "POST", "/reconcile",
|
||||
_body({"live_source_ips": [], "grace_seconds": value}))
|
||||
_body({"grace_seconds": value}))
|
||||
self.assertEqual(400, status)
|
||||
self.assertIn("grace_seconds", str(payload["error"]))
|
||||
|
||||
Reference in New Issue
Block a user