Files
bot-bottle/tests/unit/test_orchestrator_control_plane.py
T
didericis-claude 2c496dc3d0 fix(supervise): reach the queue over RPC, get bot-bottle.db off the data plane
PRD 0070's rule — only the orchestrator opens bot-bottle.db; the data
plane reaches state through the control-plane RPC — was not in force.
Three data-plane daemons held a direct read-write handle on the shared
SQLite file: the supervise MCP server, the egress DLP addon (the most
attack-exposed process, TLS-bumping hostile traffic), and the git-gate
pre-receive hook. An RCE in any of them could read every bottle's
plaintext identity_token and forge attribution fleet-wide (issue #469).

Add the agent half of the supervise flow to the control plane:

  POST /supervise/propose  -> queue a proposal, 201 {proposal_id}
  POST /supervise/poll     -> non-blocking decision poll, 200 {status,...}

Both attribute the caller by (source_ip, identity_token) exactly like
/resolve — never a caller-supplied slug — so a bottle can only ever queue
or read its own proposals even if the data plane is compromised. A decided
poll archives server-side, preserving the archive-after-read contract.

Data plane: the supervise server, egress addon, and git-gate hook now
queue/poll through PolicyResolver.propose_supervise / poll_supervise
instead of opening the DB. supervise_server keeps its ~30s grace window
by polling the RPC; egress keeps its safelist keyed by resolved bottle;
the git-gate hook gets (source_ip, identity_token) from the CGI env.

Packaging: drop the DB bind-mount and SUPERVISE_DB_PATH from the
data-plane containers/VMs (docker gateway + infra, macOS infra,
firecracker infra). The orchestrator remains the sole opener of the one
file via BOT_BOTTLE_ROOT / host_db_path().

Update PRD 0070: the rule is now in force; remove the transitional caveat.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-24 02:49:51 +00:00

590 lines
24 KiB
Python

"""Unit tests for the orchestrator HTTP control plane (PRD 0070).
Mostly exercises the pure `dispatch()` (socket-free, like the supervise
server tests), plus one real-socket round-trip to prove the handler wiring.
"""
from __future__ import annotations
import base64
import json
import secrets
import sqlite3
import tempfile
import threading
import unittest
import urllib.error
import urllib.request
from contextlib import closing
from pathlib import Path
from unittest.mock import patch
from bot_bottle.orchestrator.broker import StubBroker
from bot_bottle.orchestrator.control_plane import dispatch, make_server
from bot_bottle.orchestrator.registry import RegistryStore
from bot_bottle.orchestrator.service import Orchestrator
from bot_bottle.store_manager import StoreManager
from bot_bottle.supervise import (
Proposal,
TOOL_EGRESS_ALLOW,
sha256_hex,
write_proposal,
)
def _body(obj: object) -> bytes:
return json.dumps(obj).encode()
def _orchestrator(db_path: Path) -> Orchestrator:
store = RegistryStore(db_path)
store.migrate()
secret = secrets.token_bytes(16)
return Orchestrator(store, StubBroker(secret), secret)
class TestDispatch(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.orch = _orchestrator(Path(self._tmp.name) / "r.db")
def tearDown(self) -> None:
self._tmp.cleanup()
def test_health(self) -> None:
status, payload = dispatch(self.orch, "GET", "/health", b"")
self.assertEqual(200, status)
self.assertEqual("ok", payload["status"])
def test_register_returns_id_and_token(self) -> None:
status, payload = dispatch(
self.orch, "POST", "/bottles", _body({"source_ip": "10.243.0.1"})
)
self.assertEqual(201, status)
self.assertTrue(payload["bottle_id"])
self.assertTrue(payload["identity_token"])
def test_register_and_reprovision_encrypted_tokens(self) -> None:
key = base64.urlsafe_b64encode(b"unit-test-key").rstrip(b"=").decode()
status, payload = dispatch(
self.orch, "POST", "/bottles", _body({
"source_ip": "10.243.0.11",
"tokens": {"EGRESS_TOKEN_0": "upstream-secret"},
"env_var_secret": key,
}),
)
self.assertEqual(201, status)
bottle_id = payload["bottle_id"]
assert isinstance(bottle_id, str)
self.orch._tokens.clear()
status, response = dispatch(
self.orch, "POST", f"/bottles/{bottle_id}/reprovision_gateway",
_body({"env_var_secret": key}),
)
self.assertEqual((200, {"reprovisioned": True}), (status, response))
self.assertEqual(
{"EGRESS_TOKEN_0": "upstream-secret"}, self.orch.tokens_for(bottle_id),
)
def test_reprovision_validates_request_and_missing_rows(self) -> None:
status, _ = dispatch(
self.orch, "POST", "/bottles/b1/reprovision_gateway", b"not-json",
)
self.assertEqual(400, status)
status, _ = dispatch(
self.orch, "POST", "/bottles/b1/reprovision_gateway", _body({}),
)
self.assertEqual(400, status)
status, _ = dispatch(
self.orch, "POST", "/bottles/b1/reprovision_gateway",
_body({"env_var_secret": "key"}),
)
self.assertEqual(404, status)
def test_register_requires_source_ip(self) -> None:
status, _ = dispatch(self.orch, "POST", "/bottles", _body({}))
self.assertEqual(400, status)
def test_register_rejects_bad_json(self) -> None:
status, _ = dispatch(self.orch, "POST", "/bottles", b"{not json")
self.assertEqual(400, status)
def test_list_redacts_identity_token(self) -> None:
dispatch(self.orch, "POST", "/bottles", _body({"source_ip": "10.243.0.1"}))
status, payload = dispatch(self.orch, "GET", "/bottles", b"")
self.assertEqual(200, status)
bottles = payload["bottles"]
assert isinstance(bottles, list)
self.assertEqual(1, len(bottles))
first = bottles[0]
assert isinstance(first, dict)
self.assertNotIn("identity_token", first)
self.assertEqual("10.243.0.1", first["source_ip"])
def test_attribute_ok_and_forbidden(self) -> None:
_, reg = dispatch(
self.orch, "POST", "/bottles", _body({"source_ip": "10.243.0.5"})
)
token = reg["identity_token"]
ok_status, ok = dispatch(
self.orch, "POST", "/attribute",
_body({"source_ip": "10.243.0.5", "identity_token": token}),
)
self.assertEqual(200, ok_status)
self.assertEqual(reg["bottle_id"], ok["bottle_id"])
bad_status, _ = dispatch(
self.orch, "POST", "/attribute",
_body({"source_ip": "10.243.0.5", "identity_token": "nope"}),
)
self.assertEqual(403, bad_status)
def test_attribute_requires_both_fields(self) -> None:
status, _ = dispatch(
self.orch, "POST", "/attribute", _body({"source_ip": "10.243.0.5"})
)
self.assertEqual(400, status)
def test_delete(self) -> None:
_, reg = dispatch(
self.orch, "POST", "/bottles", _body({"source_ip": "10.243.0.1"})
)
bid = reg["bottle_id"]
status, payload = dispatch(self.orch, "DELETE", f"/bottles/{bid}", b"")
self.assertEqual(200, status)
self.assertEqual(True, payload["torn_down"])
self.assertEqual([], self.orch.registry.all())
def test_delete_missing_404(self) -> None:
status, _ = dispatch(self.orch, "DELETE", "/bottles/ghost", b"")
self.assertEqual(404, status)
def test_unknown_route_404(self) -> None:
status, _ = dispatch(self.orch, "GET", "/nope", b"")
self.assertEqual(404, status)
def test_trailing_slash_normalized(self) -> None:
status, _ = dispatch(self.orch, "GET", "/health/", b"")
self.assertEqual(200, status)
def test_gateway_status_unconfigured(self) -> None:
status, payload = dispatch(self.orch, "GET", "/gateway", b"")
self.assertEqual(200, status)
self.assertEqual(False, payload["configured"])
def test_launch_stores_policy_and_resolve_returns_it(self) -> None:
_, reg = dispatch(
self.orch, "POST", "/bottles",
_body({"source_ip": "10.243.0.5", "policy": '{"a":1}'}),
)
status, payload = dispatch(
self.orch, "POST", "/resolve",
_body({"source_ip": "10.243.0.5", "identity_token": reg["identity_token"]}),
)
self.assertEqual(200, status)
self.assertEqual(reg["bottle_id"], payload["bottle_id"])
self.assertEqual('{"a":1}', payload["policy"])
def test_resolve_forbidden_on_bad_token(self) -> None:
dispatch(self.orch, "POST", "/bottles", _body({"source_ip": "10.243.0.5"}))
status, _ = dispatch(
self.orch, "POST", "/resolve",
_body({"source_ip": "10.243.0.5", "identity_token": "nope"}),
)
self.assertEqual(403, status)
def test_put_policy_updates_live(self) -> None:
_, reg = dispatch(self.orch, "POST", "/bottles", _body({"source_ip": "10.243.0.1"}))
bid = reg["bottle_id"]
status, payload = dispatch(
self.orch, "PUT", f"/bottles/{bid}/policy", _body({"policy": '{"v":9}'})
)
self.assertEqual(200, status)
self.assertEqual(True, payload["updated"])
_, resolved = dispatch(
self.orch, "POST", "/resolve",
_body({"source_ip": "10.243.0.1", "identity_token": reg["identity_token"]}),
)
self.assertEqual('{"v":9}', resolved["policy"])
def test_put_policy_missing_bottle_404(self) -> None:
status, _ = dispatch(
self.orch, "PUT", "/bottles/ghost/policy", _body({"policy": "{}"})
)
self.assertEqual(404, status)
def test_put_policy_requires_string(self) -> None:
_, reg = dispatch(self.orch, "POST", "/bottles", _body({"source_ip": "10.243.0.1"}))
status, _ = dispatch(
self.orch, "PUT", f"/bottles/{reg['bottle_id']}/policy", _body({})
)
self.assertEqual(400, status)
def test_resolve_without_token_denies(self) -> None:
# Mandatory token: source-IP alone no longer resolves (fail-closed 403).
dispatch(
self.orch, "POST", "/bottles",
_body({"source_ip": "10.243.0.5", "policy": "P"}),
)
status, _ = dispatch(
self.orch, "POST", "/resolve", _body({"source_ip": "10.243.0.5"})
)
self.assertEqual(403, status)
def test_resolve_with_matching_token(self) -> None:
_, reg = dispatch(
self.orch, "POST", "/bottles",
_body({"source_ip": "10.243.0.6", "policy": "P"}),
)
status, payload = dispatch(
self.orch, "POST", "/resolve",
_body({"source_ip": "10.243.0.6", "identity_token": reg["identity_token"]}),
)
self.assertEqual(200, status)
self.assertEqual(reg["bottle_id"], payload["bottle_id"])
self.assertEqual("P", payload["policy"])
def test_resolve_requires_source_ip(self) -> None:
status, _ = dispatch(self.orch, "POST", "/resolve", _body({}))
self.assertEqual(400, status)
class TestServerRoundTrip(unittest.TestCase):
def test_http_register_health_attribute(self) -> None:
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
orch = _orchestrator(Path(tmp.name) / "r.db")
server = make_server(orch, "127.0.0.1", 0)
self.addCleanup(server.server_close)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
self.addCleanup(server.shutdown)
host, port = server.server_address[0], server.server_address[1]
base = f"http://{host}:{port}"
reg = json.load(urllib.request.urlopen(
urllib.request.Request(
f"{base}/bottles", data=_body({"source_ip": "10.243.0.7"}),
method="POST", headers={"Content-Type": "application/json"},
), timeout=5,
))
self.assertTrue(reg["bottle_id"])
health = json.load(urllib.request.urlopen(f"{base}/health", timeout=5))
self.assertEqual("ok", health["status"])
attr = json.load(urllib.request.urlopen(
urllib.request.Request(
f"{base}/attribute",
data=_body({"source_ip": "10.243.0.7", "identity_token": reg["identity_token"]}),
method="POST", headers={"Content-Type": "application/json"},
), timeout=5,
))
self.assertEqual(reg["bottle_id"], attr["bottle_id"])
class TestControlPlaneAuth(unittest.TestCase):
"""The per-host control-plane secret (issue #400): every route but /health
is a trusted-caller op an agent must not be able to drive just because it
can reach the port."""
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.addCleanup(self._tmp.cleanup)
self.orch = _orchestrator(Path(self._tmp.name) / "r.db")
def test_health_is_public_even_unauthorized(self) -> None:
status, _ = dispatch(self.orch, "GET", "/health", b"", authorized=False)
self.assertEqual(200, status)
def test_unauthorized_denies_every_other_route(self) -> None:
for method, path, body in [
("GET", "/bottles", b""),
("POST", "/bottles", _body({"source_ip": "10.0.0.1"})),
("PUT", "/bottles/x/policy", _body({"policy": "routes: []"})),
("DELETE", "/bottles/x", b""),
("POST", "/reconcile", _body({"live_source_ips": []})),
("POST", "/resolve", _body({"source_ip": "10.0.0.1", "identity_token": "t"})),
("POST", "/attribute", _body({"source_ip": "10.0.0.1", "identity_token": "t"})),
("GET", "/supervise/proposals", b""),
("POST", "/supervise/respond", _body({"proposal_id": "p", "bottle_slug": "s", "decision": "approve"})),
]:
status, _ = dispatch(self.orch, method, path, body, authorized=False)
self.assertEqual(401, status, f"{method} {path} should be 401 unauthorized")
def test_deny_happens_before_the_registry_is_touched(self) -> None:
"""An unauthorized DELETE must not tear a bottle down. 401, and the
bottle is still there."""
rec = self.orch.registry.register("10.0.0.9", policy="", metadata="")
status, _ = dispatch(
self.orch, "DELETE", f"/bottles/{rec.bottle_id}", b"", authorized=False)
self.assertEqual(401, status)
self.assertIsNotNone(self.orch.registry.get(rec.bottle_id))
def _server_with_secret(self, secret: str):
with patch.dict("os.environ", {"BOT_BOTTLE_CONTROL_PLANE_TOKEN": secret}):
server = make_server(self.orch, "127.0.0.1", 0)
self.addCleanup(server.server_close)
threading.Thread(target=server.serve_forever, daemon=True).start()
self.addCleanup(server.shutdown)
host, port = server.server_address[0], server.server_address[1]
return f"http://{host}:{port}"
def _status(self, url: str, *, header: str | None = None) -> int:
req = urllib.request.Request(url)
if header is not None:
req.add_header("x-bot-bottle-control-auth", header)
try:
return urllib.request.urlopen(req, timeout=5).status
except urllib.error.HTTPError as e:
return e.code
def test_configured_server_enforces_the_header_over_http(self) -> None:
base = self._server_with_secret("s3cret-admin")
# /health is public — no header needed.
self.assertEqual(200, self._status(f"{base}/health"))
# /bottles requires the secret.
self.assertEqual(401, self._status(f"{base}/bottles"))
self.assertEqual(401, self._status(f"{base}/bottles", header="wrong"))
self.assertEqual(200, self._status(f"{base}/bottles", header="s3cret-admin"))
def test_unconfigured_server_runs_open(self) -> None:
"""No secret set (tests / nft-protected Firecracker): open mode, so the
existing round-trip and unit behavior are unchanged."""
with patch.dict("os.environ", {}, clear=False):
import os
os.environ.pop("BOT_BOTTLE_CONTROL_PLANE_TOKEN", None)
server = make_server(self.orch, "127.0.0.1", 0)
self.addCleanup(server.server_close)
self.assertTrue(server.is_authorized(""))
self.assertTrue(server.is_authorized("anything"))
class TestDispatchSupervise(unittest.TestCase):
"""The /supervise/* routes over the pure dispatch()."""
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
root = Path(self._tmp.name)
db = root / "db" / "bot-bottle.db"
db.parent.mkdir(parents=True)
self._env = patch.dict("os.environ", {
"BOT_BOTTLE_ROOT": str(root),
"SUPERVISE_DB_PATH": str(db),
})
self._env.start()
self.store = RegistryStore(db)
self.store.migrate()
StoreManager(db).migrate()
secret = secrets.token_bytes(16)
self.orch = Orchestrator(self.store, StubBroker(secret), secret)
def tearDown(self) -> None:
self._env.stop()
self._tmp.cleanup()
def _queue(self, slug: str, proposed: str) -> str:
self.store.register(
"10.243.0.1", metadata=json.dumps({"slug": slug}), policy="routes: []\n")
p = Proposal.new(
bottle_slug=slug, tool=TOOL_EGRESS_ALLOW, proposed_file=proposed,
justification="need it", current_file_hash=sha256_hex(proposed))
write_proposal(p)
return p.id
def test_list_pending(self) -> None:
pid = self._queue("demo", "routes:\n - host: google.com\n")
status, payload = dispatch(self.orch, "GET", "/supervise/proposals", b"")
self.assertEqual(200, status)
proposals = payload["proposals"]
assert isinstance(proposals, list)
self.assertEqual(pid, proposals[0]["id"])
def test_respond_approve_applies_and_clears(self) -> None:
pid = self._queue("demo", "routes:\n - host: google.com\n")
status, payload = dispatch(
self.orch, "POST", "/supervise/respond",
_body({"proposal_id": pid, "bottle_slug": "demo", "decision": "approve"}),
)
self.assertEqual(200, status)
self.assertTrue(payload["responded"])
_, listing = dispatch(self.orch, "GET", "/supervise/proposals", b"")
self.assertEqual([], listing["proposals"])
def test_respond_requires_fields(self) -> None:
status, _ = dispatch(
self.orch, "POST", "/supervise/respond", _body({"decision": "approve"}))
self.assertEqual(400, status)
def test_respond_unknown_proposal_conflicts(self) -> None:
status, payload = dispatch(
self.orch, "POST", "/supervise/respond",
_body({"proposal_id": "ghost", "bottle_slug": "demo", "decision": "approve"}),
)
self.assertEqual(409, status)
self.assertIn("no such proposal", str(payload["error"]))
class TestDispatchSuperviseAgentRpc(unittest.TestCase):
"""The agent half — `/supervise/propose` + `/supervise/poll` — attributed by
(source_ip, identity_token) like /resolve, so a bottle can only ever queue
or read its own proposals (PRD 0070 / issue #469)."""
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
root = Path(self._tmp.name)
db = root / "db" / "bot-bottle.db"
db.parent.mkdir(parents=True)
self._env = patch.dict("os.environ", {
"BOT_BOTTLE_ROOT": str(root),
"SUPERVISE_DB_PATH": str(db),
})
self._env.start()
self.store = RegistryStore(db)
self.store.migrate()
StoreManager(db).migrate()
secret = secrets.token_bytes(16)
self.orch = Orchestrator(self.store, StubBroker(secret), secret)
def tearDown(self) -> None:
self._env.stop()
self._tmp.cleanup()
def _register(self, source_ip: str = "10.243.0.9", slug: str = "demo"):
return self.store.register(
source_ip, metadata=json.dumps({"slug": slug}), policy="routes: []\n")
def _propose(self, rec, proposed: str = "routes:\n - host: g.com\n"):
return dispatch(self.orch, "POST", "/supervise/propose", _body({
"source_ip": rec.source_ip, "identity_token": rec.identity_token,
"tool": TOOL_EGRESS_ALLOW, "proposed_file": proposed, "justification": "need it",
}))
def _poll(self, rec, proposal_id: str):
return dispatch(self.orch, "POST", "/supervise/poll", _body({
"source_ip": rec.source_ip, "identity_token": rec.identity_token,
"proposal_id": proposal_id,
}))
def test_propose_queues_under_the_resolved_bottle(self) -> None:
rec = self._register()
status, payload = self._propose(rec)
self.assertEqual(201, status)
pid = payload["proposal_id"]
assert isinstance(pid, str) and pid
_, listing = dispatch(self.orch, "GET", "/supervise/proposals", b"")
self.assertEqual(pid, listing["proposals"][0]["id"])
# Queued under the orchestrator-resolved bottle id, never a caller slug.
self.assertEqual(rec.bottle_id, listing["proposals"][0]["bottle_slug"])
def test_propose_unattributed_is_403(self) -> None:
status, payload = dispatch(self.orch, "POST", "/supervise/propose", _body({
"source_ip": "10.9.9.9", "identity_token": "wrong",
"tool": TOOL_EGRESS_ALLOW, "proposed_file": "x\n", "justification": "j",
}))
self.assertEqual(403, status)
self.assertIn("unattributed", str(payload["error"]))
def test_propose_rejects_unknown_tool(self) -> None:
rec = self._register()
status, _ = dispatch(self.orch, "POST", "/supervise/propose", _body({
"source_ip": rec.source_ip, "identity_token": rec.identity_token,
"tool": "not-a-tool", "proposed_file": "x\n", "justification": "j",
}))
self.assertEqual(400, status)
def test_poll_pending_then_decided_then_archived(self) -> None:
rec = self._register()
_, proposed = self._propose(rec)
pid = proposed["proposal_id"]
assert isinstance(pid, str)
status, poll = self._poll(rec, pid)
self.assertEqual(200, status)
self.assertEqual("pending", poll["status"])
# Operator decides server-side.
dispatch(self.orch, "POST", "/supervise/respond", _body({
"proposal_id": pid, "bottle_slug": rec.bottle_id,
"decision": "approve", "notes": "ok",
}))
_, decided = self._poll(rec, pid)
self.assertEqual("approved", decided["status"])
self.assertEqual("ok", decided["notes"])
# The decided poll archived it: gone from pending, and a re-poll is
# 'unknown' rather than replaying the decision forever.
_, listing = dispatch(self.orch, "GET", "/supervise/proposals", b"")
self.assertEqual([], listing["proposals"])
_, again = self._poll(rec, pid)
self.assertEqual("unknown", again["status"])
def test_poll_cannot_read_another_bottles_proposal(self) -> None:
rec_a = self._register("10.0.0.1", "a")
rec_b = self._register("10.0.0.2", "b")
_, proposed = self._propose(rec_a)
pid = proposed["proposal_id"]
assert isinstance(pid, str)
# b polls a's proposal id: scoped to b's own queue → never a's response.
status, poll = self._poll(rec_b, pid)
self.assertEqual(200, status)
self.assertEqual("unknown", poll["status"])
if __name__ == "__main__":
unittest.main()
class TestReconcileRoute(unittest.TestCase):
"""`POST /reconcile` — the host tells the orchestrator which bottles are
actually up, since the orchestrator can't see the backend from inside the
infra container."""
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.orch = _orchestrator(Path(self._tmp.name) / "r.db")
def tearDown(self) -> None:
self._tmp.cleanup()
def _old(self, source_ip: str) -> str:
rec = self.orch.registry.register(source_ip)
with closing(sqlite3.connect(self.orch.registry.db_path)) as conn:
conn.execute(
"UPDATE orchestrator_bottles SET created_at = 0.0 WHERE bottle_id = ?",
(rec.bottle_id,))
conn.commit()
return rec.bottle_id
def test_reaps_absent_and_reports_ids(self) -> None:
dead = self._old("10.0.0.1")
alive = self._old("10.0.0.2")
status, payload = dispatch(
self.orch, "POST", "/reconcile", _body({"live_source_ips": ["10.0.0.2"]}))
self.assertEqual(200, status)
self.assertEqual([dead], payload["reaped"])
self.assertIsNone(self.orch.registry.get(dead))
self.assertIsNotNone(self.orch.registry.get(alive))
def test_missing_live_source_ips_is_400(self) -> None:
status, _ = dispatch(self.orch, "POST", "/reconcile", _body({}))
self.assertEqual(400, status)
def test_grace_seconds_is_honoured(self) -> None:
"""A grace window wide enough to cover the row protects it."""
self.orch.registry.register("10.0.0.3")
status, payload = dispatch(
self.orch, "POST", "/reconcile",
_body({"live_source_ips": [], "grace_seconds": 3600}))
self.assertEqual(200, status)
self.assertEqual([], payload["reaped"])
def test_non_string_entries_are_ignored(self) -> None:
dead = self._old("10.0.0.4")
status, payload = dispatch(
self.orch, "POST", "/reconcile",
_body({"live_source_ips": [None, 7, "10.0.0.9"]}))
self.assertEqual(200, status)
self.assertEqual([dead], payload["reaped"])