76037f36a6
lint / lint (push) Successful in 1m5s
refresh-image-locks / refresh (push) Successful in 34s
tracker-policy-pr / check-pr (pull_request) Successful in 11s
test / unit (pull_request) Has started running
test / image-input-builds (pull_request) Has started running
test / coverage (pull_request) Blocked by required conditions
test / integration-docker (pull_request) Has been cancelled
Chunk 3 of the host-control-server stack: grow the broker op vocabulary
(PRD gap 3), starting with `list_live`, and invert `reconcile` onto it.
- broker: split the closed op vocabulary into mutation (`launch`/`teardown`,
carry a bottle id + static flags) and query (`list_live`, carries nothing
but its op name) kinds. `verify_request` now enforces a **strict schema**
(open question 1, resolved yes): unknown claim keys are rejected, a mutation
must name its bottle, and a query that smuggles any id/flag is refused.
- broker verb: `LaunchBroker.list_live` / `SubmitBroker.list_live` return the
backend's live source IPs; a backend enumeration failure is converted to the
single `BrokerUnavailableError` "live set unknown" signal. `DockerBroker`
enumerates its labelled containers; `StubBroker` derives from launches (or a
test override).
- host controller: `POST /broker/live` verifies a signed `list_live` token and
returns `{source_ips}`; `BrokerClient.list_live` is its drop-in client.
- reconcile: `OrchestratorCore.reconcile()` drops the `live_source_ips`
parameter and pulls the live set from the broker itself — the tell that the
orchestrator couldn't see the backend goes away. **Fail-safe**: if the broker
can't return an authoritative set the sweep is skipped, never run against an
empty/partial set (which would reap healthy rows). The `/reconcile` HTTP
contract + `OrchestratorClient.reconcile` become a bare trigger.
The macOS launcher's Apple-container enumeration stays for now; it becomes the
host controller's `list_live` when launch itself moves behind the broker (the
pulled-forward chunk 5, next in the stack).
Tests + pyright clean; pylint 10.0 on broker.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
161 lines
6.9 KiB
Python
161 lines
6.9 KiB
Python
"""Unit: orchestrator-side broker client (issue #468, chunk 1). HTTP mocked."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import json
|
|
import unittest
|
|
import urllib.error
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from bot_bottle.orchestrator.broker import (
|
|
BrokerAuthError,
|
|
BrokerUnavailableError,
|
|
LaunchRequest,
|
|
)
|
|
from bot_bottle.orchestrator.broker_client import BrokerClient, BrokerClientError
|
|
|
|
_URLOPEN = "bot_bottle.orchestrator.broker_client.urllib.request.urlopen"
|
|
|
|
|
|
def _resp(payload: object) -> MagicMock:
|
|
m = MagicMock()
|
|
m.__enter__.return_value.read.return_value = json.dumps(payload).encode()
|
|
return m
|
|
|
|
|
|
def _http_error(code: int, payload: object = None) -> urllib.error.HTTPError:
|
|
body = json.dumps(payload).encode() if payload is not None else b""
|
|
return urllib.error.HTTPError(
|
|
"http://host/broker", code, "err", {}, io.BytesIO(body)) # type: ignore[arg-type]
|
|
|
|
|
|
class TestSubmit(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.c = BrokerClient("http://host:8091")
|
|
|
|
def test_returns_the_verified_request(self) -> None:
|
|
echo = {
|
|
"op": "launch", "bottle_id": "b1", "source_ip": "10.0.0.1",
|
|
"image_ref": "img", "slot": 3,
|
|
}
|
|
with patch(_URLOPEN, return_value=_resp(echo)):
|
|
got = self.c.submit("tok")
|
|
self.assertEqual(
|
|
LaunchRequest(op="launch", bottle_id="b1", source_ip="10.0.0.1",
|
|
image_ref="img", slot=3),
|
|
got,
|
|
)
|
|
|
|
def test_posts_token_to_broker_endpoint(self) -> None:
|
|
with patch(_URLOPEN, return_value=_resp({"op": "teardown", "bottle_id": "b1"})) as m:
|
|
self.c.submit("signed-token")
|
|
request = m.call_args.args[0]
|
|
self.assertEqual("POST", request.get_method())
|
|
self.assertTrue(request.full_url.endswith("/broker"))
|
|
self.assertEqual({"token": "signed-token"}, json.loads(request.data))
|
|
|
|
def test_401_raises_broker_auth_error(self) -> None:
|
|
# A fail-closed provenance/schema rejection surfaces as the SAME exception
|
|
# the in-process broker raises, so the launch path's rollback is identical.
|
|
with patch(_URLOPEN, side_effect=_http_error(401, {"error": "bad signature"})):
|
|
with self.assertRaises(BrokerAuthError):
|
|
self.c.submit("forged")
|
|
|
|
def test_502_is_a_definite_client_error(self) -> None:
|
|
# The host responded — it processed the request and did not launch, so a
|
|
# definite BrokerClientError (the caller may safely roll back).
|
|
with patch(_URLOPEN, side_effect=_http_error(502, {"error": "docker down"})):
|
|
with self.assertRaises(BrokerClientError):
|
|
self.c.submit("tok")
|
|
|
|
def test_unreachable_is_ambiguous_unavailable(self) -> None:
|
|
# No response at all — the request may already have launched, so the
|
|
# AMBIGUOUS BrokerUnavailableError (the caller must NOT roll back).
|
|
with patch(_URLOPEN, side_effect=urllib.error.URLError("refused")):
|
|
with self.assertRaises(BrokerUnavailableError):
|
|
self.c.submit("tok")
|
|
|
|
def test_timeout_is_ambiguous_unavailable(self) -> None:
|
|
# A dropped/late response after the request was sent is the exact orphan
|
|
# risk: the host may have launched. Must be ambiguous, not a definite fail.
|
|
with patch(_URLOPEN, side_effect=TimeoutError("read timed out")):
|
|
with self.assertRaises(BrokerUnavailableError):
|
|
self.c.submit("tok")
|
|
|
|
def test_malformed_success_body_raises(self) -> None:
|
|
with patch(_URLOPEN, return_value=_resp({"op": "launch"})): # missing bottle_id
|
|
with self.assertRaises(BrokerClientError):
|
|
self.c.submit("tok")
|
|
|
|
def test_empty_error_body_is_tolerated(self) -> None:
|
|
# An error with no readable JSON body still classifies by status code.
|
|
with patch(_URLOPEN, side_effect=_http_error(401)):
|
|
with self.assertRaises(BrokerAuthError):
|
|
self.c.submit("forged")
|
|
|
|
def test_non_json_success_body_raises(self) -> None:
|
|
# A 200 whose body isn't JSON is tolerated into {} then fails the
|
|
# missing-field check — a definite client error, not a crash.
|
|
m = MagicMock()
|
|
m.__enter__.return_value.read.return_value = b"not json at all"
|
|
with patch(_URLOPEN, return_value=m):
|
|
with self.assertRaises(BrokerClientError):
|
|
self.c.submit("tok")
|
|
|
|
def test_unreadable_error_body_is_tolerated(self) -> None:
|
|
# An HTTPError whose body can't be read (fp=None) still classifies by
|
|
# status — the error detail is best-effort.
|
|
err = urllib.error.HTTPError(
|
|
"http://host/broker", 502, "err", {}, None) # type: ignore[arg-type]
|
|
with patch(_URLOPEN, side_effect=err):
|
|
with self.assertRaises(BrokerClientError):
|
|
self.c.submit("tok")
|
|
|
|
|
|
class TestListLive(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.c = BrokerClient("http://host:8091")
|
|
|
|
def test_returns_source_ips(self) -> None:
|
|
with patch(_URLOPEN, return_value=_resp({"source_ips": ["10.0.0.1", "10.0.0.2"]})):
|
|
self.assertEqual(["10.0.0.1", "10.0.0.2"], self.c.list_live("tok"))
|
|
|
|
def test_posts_token_to_live_endpoint(self) -> None:
|
|
with patch(_URLOPEN, return_value=_resp({"source_ips": []})) as m:
|
|
self.c.list_live("signed-token")
|
|
request = m.call_args.args[0]
|
|
self.assertEqual("POST", request.get_method())
|
|
self.assertTrue(request.full_url.endswith("/broker/live"))
|
|
self.assertEqual({"token": "signed-token"}, json.loads(request.data))
|
|
|
|
def test_non_string_entries_are_filtered(self) -> None:
|
|
with patch(_URLOPEN, return_value=_resp({"source_ips": ["10.0.0.1", 5, None, ""]})):
|
|
self.assertEqual(["10.0.0.1"], self.c.list_live("tok"))
|
|
|
|
def test_missing_source_ips_raises(self) -> None:
|
|
# A response without the field is malformed — better to fail (reconcile
|
|
# skips) than to reconcile against a silent empty set.
|
|
with patch(_URLOPEN, return_value=_resp({})):
|
|
with self.assertRaises(BrokerClientError):
|
|
self.c.list_live("tok")
|
|
|
|
def test_401_raises_broker_auth_error(self) -> None:
|
|
with patch(_URLOPEN, side_effect=_http_error(401, {"error": "bad signature"})):
|
|
with self.assertRaises(BrokerAuthError):
|
|
self.c.list_live("forged")
|
|
|
|
def test_502_enumeration_failure_is_client_error(self) -> None:
|
|
with patch(_URLOPEN, side_effect=_http_error(502, {"error": "docker down"})):
|
|
with self.assertRaises(BrokerClientError):
|
|
self.c.list_live("tok")
|
|
|
|
def test_unreachable_is_unavailable(self) -> None:
|
|
with patch(_URLOPEN, side_effect=urllib.error.URLError("refused")):
|
|
with self.assertRaises(BrokerUnavailableError):
|
|
self.c.list_live("tok")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|