From 2c496dc3d0c3ba8a8d37f7c411b38200df850a90 Mon Sep 17 00:00:00 2001 From: claude Date: Fri, 24 Jul 2026 02:07:21 +0000 Subject: [PATCH 01/39] fix(supervise): reach the queue over RPC, get bot-bottle.db off the data plane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PRD 0070's rule — only the orchestrator opens bot-bottle.db; the data plane reaches state through the control-plane RPC — was not in force. Three data-plane daemons held a direct read-write handle on the shared SQLite file: the supervise MCP server, the egress DLP addon (the most attack-exposed process, TLS-bumping hostile traffic), and the git-gate pre-receive hook. An RCE in any of them could read every bottle's plaintext identity_token and forge attribution fleet-wide (issue #469). Add the agent half of the supervise flow to the control plane: POST /supervise/propose -> queue a proposal, 201 {proposal_id} POST /supervise/poll -> non-blocking decision poll, 200 {status,...} Both attribute the caller by (source_ip, identity_token) exactly like /resolve — never a caller-supplied slug — so a bottle can only ever queue or read its own proposals even if the data plane is compromised. A decided poll archives server-side, preserving the archive-after-read contract. Data plane: the supervise server, egress addon, and git-gate hook now queue/poll through PolicyResolver.propose_supervise / poll_supervise instead of opening the DB. supervise_server keeps its ~30s grace window by polling the RPC; egress keeps its safelist keyed by resolved bottle; the git-gate hook gets (source_ip, identity_token) from the CGI env. Packaging: drop the DB bind-mount and SUPERVISE_DB_PATH from the data-plane containers/VMs (docker gateway + infra, macOS infra, firecracker infra). The orchestrator remains the sole opener of the one file via BOT_BOTTLE_ROOT / host_db_path(). Update PRD 0070: the rule is now in force; remove the transitional caveat. Co-Authored-By: Claude Opus 4.8 --- Dockerfile.gateway | 3 +- bot_bottle/backend/firecracker/infra_vm.py | 5 +- bot_bottle/backend/macos_container/infra.py | 6 +- bot_bottle/egress_addon.py | 78 ++-- bot_bottle/git_gate_render.py | 81 ++-- bot_bottle/git_http_backend.py | 11 +- bot_bottle/orchestrator/control_plane.py | 63 +++ bot_bottle/orchestrator/gateway.py | 24 +- bot_bottle/orchestrator/lifecycle.py | 13 +- bot_bottle/orchestrator/service.py | 61 +++ bot_bottle/policy_resolver.py | 84 +++- bot_bottle/supervise.py | 4 + bot_bottle/supervise_server.py | 261 +++++++----- bot_bottle/supervise_types.py | 11 + docs/prds/0070-per-host-orchestrator.md | 13 +- tests/unit/test_egress_addon_request_flow.py | 143 ++++--- tests/unit/test_git_gate.py | 23 +- tests/unit/test_git_http_backend.py | 18 +- tests/unit/test_orchestrator_control_plane.py | 106 +++++ tests/unit/test_orchestrator_gateway.py | 12 +- tests/unit/test_orchestrator_service.py | 37 ++ tests/unit/test_policy_resolver.py | 52 +++ tests/unit/test_supervise_server.py | 377 +++++++++--------- 23 files changed, 968 insertions(+), 518 deletions(-) diff --git a/Dockerfile.gateway b/Dockerfile.gateway index a9f6d6d..8849d21 100644 --- a/Dockerfile.gateway +++ b/Dockerfile.gateway @@ -26,8 +26,9 @@ # /git-gate-entrypoint.sh docker-cp'd at start time # /git-gate/creds/* docker-cp'd at start time # /git/* bare repos, populated at runtime -# /run/supervise/bot-bottle.db bind-mounted at run time # /home/mitmproxy/.mitmproxy/ mitmproxy CA dir +# (No bot-bottle.db mount: the data plane reaches the supervise queue over the +# control-plane RPC and never opens the DB — PRD 0070 / issue #469.) # # Exposed ports inside the container: # 9099 egress (mitmproxy, agent-facing HTTPS proxy) diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py index 8ccf6e9..00d962f 100644 --- a/bot_bottle/backend/firecracker/infra_vm.py +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -493,10 +493,11 @@ BOT_BOTTLE_ROOT=/var/lib/bot-bottle python3 -m bot_bottle.orchestrator \\ # Gateway data plane, multi-tenant: each request resolves source-IP -> # policy against the local control plane. The VM backend reaches git over # git-http (9420), so the git:// daemon (git-gate, needs a per-bottle -# entrypoint the consolidated model doesn't use) is left out. +# entrypoint the consolidated model doesn't use) is left out. No +# SUPERVISE_DB_PATH: the data plane reaches the supervise queue over the +# control-plane RPC and never opens bot-bottle.db (PRD 0070 / #469). BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise \\ BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{CONTROL_PLANE_PORT} \\ -SUPERVISE_DB_PATH=/var/lib/bot-bottle/db/bot-bottle.db \\ python3 -m bot_bottle.gateway_init & # Reap as PID 1; children are backgrounded, so `wait` blocks. diff --git a/bot_bottle/backend/macos_container/infra.py b/bot_bottle/backend/macos_container/infra.py index 1574bdb..cb92540 100644 --- a/bot_bottle/backend/macos_container/infra.py +++ b/bot_bottle/backend/macos_container/infra.py @@ -100,10 +100,12 @@ def _init_script(port: int) -> str: f"( cd {_SRC_IN_CONTAINER} && BOT_BOTTLE_ROOT={_DB_ROOT_IN_CONTAINER} " f"python3 -m bot_bottle.orchestrator --host 0.0.0.0 --port {port} " "--broker stub ) &\n" - # Gateway data plane, multi-tenant against the local control plane. + # Gateway data plane, multi-tenant against the local control plane. No + # SUPERVISE_DB_PATH: the data plane reaches the supervise queue over the + # control-plane RPC and never opens bot-bottle.db (PRD 0070 / #469). f"( cd /app && BOT_BOTTLE_GATEWAY_DAEMONS={_GATEWAY_DAEMONS} " f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{port} " - f"SUPERVISE_DB_PATH={_DB_PATH_IN_CONTAINER} python3 -m bot_bottle.gateway_init ) &\n" + f"python3 -m bot_bottle.gateway_init ) &\n" "while : ; do wait ; done\n" ) diff --git a/bot_bottle/egress_addon.py b/bot_bottle/egress_addon.py index 04dd3fa..8766623 100644 --- a/bot_bottle/egress_addon.py +++ b/bot_bottle/egress_addon.py @@ -40,8 +40,13 @@ from bot_bottle.egress_addon_core import ( scan_inbound, scan_outbound, ) -from bot_bottle import supervise as _sv -from bot_bottle.policy_resolver import PolicyResolver +from bot_bottle.policy_resolver import PolicyResolveError, PolicyResolver +from bot_bottle.supervise_types import ( + STATUS_APPROVED, + STATUS_MODIFIED, + STATUSES, + TOOL_EGRESS_TOKEN_ALLOW, +) INTROSPECT_HOST = "_egress.local" @@ -314,8 +319,15 @@ class EgressAddon: flow.request.headers.pop("Proxy-Authorization", None) flow.request.headers.pop(IDENTITY_HEADER, None) conn = flow.client_conn - if not token and conn is not None: - token = self._conn_tokens.get(getattr(conn, "id", ""), "") + conn_id = getattr(conn, "id", "") if conn is not None else "" + if not token and conn_id: + token = self._conn_tokens.get(conn_id, "") + # Remember the token per connection so a later token-block on this flow + # can attribute its supervise proposal by (source_ip, identity_token) + # over RPC — plain-HTTP requests carry it here, HTTPS tunnels captured + # it at CONNECT (http_connect); both land in `_conn_tokens`. + if token and conn_id: + self._conn_tokens[conn_id] = token return token def http_connect(self, flow: http.HTTPFlow) -> None: @@ -592,42 +604,48 @@ class EgressAddon: redact_tokens(request_path, env=env), result, ) - proposal = _sv.Proposal.new( - bottle_slug=slug, - tool=_sv.TOOL_EGRESS_TOKEN_ALLOW, - proposed_file=payload, - justification=_TOKEN_ALLOW_JUSTIFICATION, - current_file_hash=_sv.sha256_hex(payload), - ) + # Attribute the proposal by (source_ip, identity_token) over the control + # plane — the data plane no longer opens bot-bottle.db (PRD 0070 / #469). + # source_ip + token come from this bottle's connection; `slug` is kept + # only for its own DLP safelist. + conn = flow.client_conn + source_ip = conn.peername[0] if conn is not None and conn.peername else "" + token = self._conn_tokens.get(getattr(conn, "id", ""), "") if conn is not None else "" try: - _sv.write_proposal(proposal) - except OSError as e: + proposal_id = self._resolver.propose_supervise( + source_ip, token, + tool=TOOL_EGRESS_TOKEN_ALLOW, + proposed_file=payload, + justification=_TOKEN_ALLOW_JUSTIFICATION, + ) + except PolicyResolveError as e: sys.stderr.write( f"egress: could not queue token-allow proposal: {e}; " "blocking request\n" ) + proposal_id = None + if proposal_id is None: self._block(flow, f"egress DLP: {result.reason}", ctx=self._req_ctx(flow)) return False sys.stderr.write(json.dumps({ "event": "egress_token_supervise", "reason": f"egress DLP: {result.reason}", - "proposal": proposal.id, + "proposal": proposal_id, **self._req_ctx(flow), }) + "\n") - response = await self._await_token_response(proposal.id, slug) - _sv.archive_proposal(slug, proposal.id) + response = await self._await_token_response(proposal_id, source_ip, token) - if response is not None and response.status in ( - _sv.STATUS_APPROVED, _sv.STATUS_MODIFIED, + if response is not None and response.get("status") in ( + STATUS_APPROVED, STATUS_MODIFIED, ): self._safe_tokens_for(slug).add(result.matched) if self._flow_log(flow) >= LOG_BLOCKS: sys.stderr.write(json.dumps({ "event": "egress_token_allowed", "reason": f"egress DLP: {result.reason}", - "proposal": proposal.id, + "proposal": proposal_id, **self._req_ctx(flow), }) + "\n") return True @@ -645,19 +663,23 @@ class EgressAddon: async def _await_token_response( self, proposal_id: str, - slug: str, - ) -> "_sv.Response | None": - """Poll the DB for the operator's response without blocking the - proxy event loop. Returns the Response, or None on timeout.""" + source_ip: str, + token: str, + ) -> "dict[str, object] | None": + """Poll the control plane for the operator's decision without blocking + the proxy event loop. Returns the terminal `{status, ...}` payload once + decided, or None on timeout. A transient orchestrator error (or a + `pending`/`unknown` status) is retried until the deadline, then fails + closed — the caller blocks the request.""" loop = asyncio.get_running_loop() deadline = loop.time() + self._token_allow_timeout while True: try: - return _sv.read_response(slug, proposal_id) - except (OSError, ValueError, KeyError): - # Not written yet, or a partial/malformed write — retry until - # the deadline, then fail closed. - pass + result = self._resolver.poll_supervise(source_ip, token, proposal_id) + except PolicyResolveError: + result = None + if result is not None and result.get("status") in STATUSES: + return result if loop.time() >= deadline: return None await asyncio.sleep(TOKEN_ALLOW_POLL_INTERVAL_SECONDS) diff --git a/bot_bottle/git_gate_render.py b/bot_bottle/git_gate_render.py index 2a2378c..04c038c 100644 --- a/bot_bottle/git_gate_render.py +++ b/bot_bottle/git_gate_render.py @@ -272,21 +272,27 @@ supervise_gitleaks_allow() { proposal_id=$( PYTHONPATH="/app${PYTHONPATH:+:$PYTHONPATH}" GITLEAKS_ALLOW_REF="$ref" python3 - "$report_file" <<'PY' -import datetime -import hashlib import json import os import sys from pathlib import Path +# Queue over the control-plane RPC — the git-gate data plane no longer opens +# bot-bottle.db (PRD 0070 / issue #469). Attribution is by (source_ip, +# identity_token), resolved server-side, so the proposal lands under the +# calling bottle exactly as a direct write once did. try: - import supervise as _sv + from bot_bottle.policy_resolver import PolicyResolver, PolicyResolveError + from bot_bottle.supervise_types import TOOL_GITLEAKS_ALLOW except ImportError: - from bot_bottle import supervise as _sv + from policy_resolver import PolicyResolver, PolicyResolveError + from supervise_types import TOOL_GITLEAKS_ALLOW report_path = Path(sys.argv[1]) -slug = os.environ.get("SUPERVISE_BOTTLE_SLUG", "") -if not slug: +source_ip = os.environ.get("SUPERVISE_SOURCE_IP", "") +token = os.environ.get("SUPERVISE_IDENTITY_TOKEN", "") +orch_url = os.environ.get("BOT_BOTTLE_ORCHESTRATOR_URL", "") +if not source_ip or not orch_url: sys.exit(2) try: @@ -323,19 +329,21 @@ for i, finding in enumerate(raw, 1): ]) payload = "\n".join(lines).rstrip() + "\n" -proposal = _sv.Proposal.new( - bottle_slug=slug, - tool=_sv.TOOL_GITLEAKS_ALLOW, - proposed_file=payload, - justification=( - "git-gate found gitleaks findings hidden by # gitleaks:allow; " - "approve only for dummy test fixtures or confirmed false positives" - ), - current_file_hash=hashlib.sha256(payload.encode("utf-8")).hexdigest(), - now=datetime.datetime.now(datetime.timezone.utc), -) -_sv.write_proposal(proposal) -print(proposal.id) +try: + proposal_id = PolicyResolver(orch_url).propose_supervise( + source_ip, token, + tool=TOOL_GITLEAKS_ALLOW, + proposed_file=payload, + justification=( + "git-gate found gitleaks findings hidden by # gitleaks:allow; " + "approve only for dummy test fixtures or confirmed false positives" + ), + ) +except PolicyResolveError: + sys.exit(4) +if not proposal_id: + sys.exit(2) +print(proposal_id) PY ) rc=$? @@ -348,7 +356,6 @@ PY return 1 fi - slug=${SUPERVISE_BOTTLE_SLUG:-} timeout=${SUPERVISE_GITLEAKS_ALLOW_TIMEOUT_SECONDS:-300} case "$timeout" in ''|*[!0-9]*) @@ -360,20 +367,30 @@ PY echo "git-gate: approve with './cli.py supervise' to continue this push" >&2 waited=0 while [ "$waited" -lt "$timeout" ]; do - status=$(PYTHONPATH="/app${PYTHONPATH:+:$PYTHONPATH}" python3 - "$slug" "$proposal_id" <<'PY' + status=$(PYTHONPATH="/app${PYTHONPATH:+:$PYTHONPATH}" python3 - "$proposal_id" <<'PY' +import os import sys +# Non-blocking poll over the control plane. A decided proposal is archived +# server-side on read, so no separate archive step is needed here. try: - import supervise as _sv + from bot_bottle.policy_resolver import PolicyResolver, PolicyResolveError except ImportError: - from bot_bottle import supervise as _sv + from policy_resolver import PolicyResolver, PolicyResolveError -slug = sys.argv[1] +source_ip = os.environ.get("SUPERVISE_SOURCE_IP", "") +token = os.environ.get("SUPERVISE_IDENTITY_TOKEN", "") +orch_url = os.environ.get("BOT_BOTTLE_ORCHESTRATOR_URL", "") try: - response = _sv.read_response(slug, sys.argv[2]) -except FileNotFoundError: + result = PolicyResolver(orch_url).poll_supervise(source_ip, token, sys.argv[1]) +except PolicyResolveError: sys.exit(2) -print(response.status) +if result is None: + sys.exit(2) +status = result.get("status", "") +if status in ("pending", "unknown"): + sys.exit(2) +print(status) PY ) rc=$? @@ -385,16 +402,6 @@ PY if [ -n "$status" ]; then case "$status" in approved|modified) - PYTHONPATH="/app${PYTHONPATH:+:$PYTHONPATH}" python3 - "$slug" "$proposal_id" <<'PY' || true -import sys - -try: - import supervise as _sv -except ImportError: - from bot_bottle import supervise as _sv - -_sv.archive_proposal(sys.argv[1], sys.argv[2]) -PY echo "git-gate: supervisor approved # gitleaks:allow for $ref" >&2 return 0 ;; diff --git a/bot_bottle/git_http_backend.py b/bot_bottle/git_http_backend.py index 22e538e..21f924f 100644 --- a/bot_bottle/git_http_backend.py +++ b/bot_bottle/git_http_backend.py @@ -167,11 +167,14 @@ class GitHttpHandler(BaseHTTPRequestHandler): "SERVER_PORT": str(self.server.server_port), # type: ignore "SERVER_PROTOCOL": self.request_version, }) - # Attribute the gitleaks-allow supervise proposal (written by + # Attribute the gitleaks-allow supervise proposal (queued by # receive-pack's pre-receive hook, a child of the CGI we spawn below) to - # the calling bottle. The namespaced root is `/`, so its - # final component is the bottle id — the same per-bottle key egress uses. - env["SUPERVISE_BOTTLE_SLUG"] = sandbox_root.name + # the calling bottle. The hook queues over the control-plane RPC (PRD + # 0070 / issue #469), which re-resolves the bottle from these — the same + # (source_ip, identity_token) pair that selected the namespaced root + # above — so it can only ever queue its own proposals. + env["SUPERVISE_SOURCE_IP"] = self.client_address[0] + env["SUPERVISE_IDENTITY_TOKEN"] = self.headers.get(IDENTITY_HEADER, "") for header, variable in ( ("accept", "HTTP_ACCEPT"), ("content-encoding", "HTTP_CONTENT_ENCODING"), diff --git a/bot_bottle/orchestrator/control_plane.py b/bot_bottle/orchestrator/control_plane.py index 1625cec..87d7714 100644 --- a/bot_bottle/orchestrator/control_plane.py +++ b/bot_bottle/orchestrator/control_plane.py @@ -27,6 +27,18 @@ vsock / unix-socket portability caveats): POST /supervise/respond -> 200 {"responded": true} | 409 (operator) body: {"proposal_id","bottle_slug", "decision", ["notes"],["final_file"]} + POST /supervise/propose -> 201 {"proposal_id"} | 403 (agent) + body: {"source_ip","identity_token", + "tool","proposed_file","justification"} + POST /supervise/poll -> 200 {"status", ["notes"],["final_file"]} | 403 + body: {"source_ip","identity_token", + "proposal_id"} + +The `/supervise/propose` + `/supervise/poll` pair is the **agent** half of the +supervise flow: the data plane (supervise / egress / git-gate) queues a +proposal and polls for its response over RPC instead of opening `bot-bottle.db` +directly. Both attribute the caller by `(source_ip, identity_token)` exactly +like `/resolve`, so a bottle can only ever queue or read its own proposals. `POST /bottles` / `DELETE` drive the full launch lifecycle: they mint (or tear down) the bottle in the registry AND broker the backend-native launch @@ -51,6 +63,7 @@ import typing from urllib.parse import urlsplit from ..paths import CONTROL_PLANE_TOKEN_ENV +from ..supervise_types import TOOLS from .service import Orchestrator # JSON body payload type (parsed request / rendered response). @@ -235,6 +248,56 @@ def dispatch( # pylint: disable=too-many-return-statements,too-many-branches return 200, {"responded": True} return 409, {"error": err} + if method == "POST" and route == "/supervise/propose": + # Agent half: queue a proposal, attributed to the caller resolved from + # (source_ip, identity_token) — never a caller-supplied slug — so the + # data plane can't forge attribution. Fail-closed 403 when unattributed. + try: + data = _parse_json_object(body) + except ValueError as e: + return 400, {"error": f"invalid JSON: {e}"} + source_ip = data.get("source_ip") + token = data.get("identity_token") + tool = data.get("tool") + proposed_file = data.get("proposed_file") + justification = data.get("justification") + if not isinstance(source_ip, str) or not source_ip: + return 400, {"error": "source_ip (string) is required"} + if not isinstance(tool, str) or tool not in TOOLS: + return 400, {"error": f"tool (string) must be one of {TOOLS}"} + if not isinstance(proposed_file, str) or not proposed_file: + return 400, {"error": "proposed_file (string) is required"} + if not isinstance(justification, str) or not justification: + return 400, {"error": "justification (string) is required"} + rec = orch.resolve(source_ip, token if isinstance(token, str) else "") + if rec is None: + return 403, {"error": "unattributed"} + proposal_id = orch.supervise_queue_proposal( + rec.bottle_id, tool=tool, proposed_file=proposed_file, + justification=justification, + ) + return 201, {"proposal_id": proposal_id} + + if method == "POST" and route == "/supervise/poll": + # Agent half: non-blocking read of the caller's own proposal decision. + # Attributed like /propose, and scoped to the resolved bottle id, so a + # guessed proposal_id can never read another bottle's response. + try: + data = _parse_json_object(body) + except ValueError as e: + return 400, {"error": f"invalid JSON: {e}"} + source_ip = data.get("source_ip") + token = data.get("identity_token") + proposal_id = data.get("proposal_id") + if not isinstance(source_ip, str) or not source_ip: + return 400, {"error": "source_ip (string) is required"} + if not isinstance(proposal_id, str) or not proposal_id: + return 400, {"error": "proposal_id (string) is required"} + rec = orch.resolve(source_ip, token if isinstance(token, str) else "") + if rec is None: + return 403, {"error": "unattributed"} + return 200, orch.supervise_poll_response(rec.bottle_id, proposal_id) + if method == "POST" and route == "/resolve": # The per-request lookup the multi-tenant gateway makes: returns the # bottle's policy. Requires a matching (source_ip, identity_token) diff --git a/bot_bottle/orchestrator/gateway.py b/bot_bottle/orchestrator/gateway.py index 741e20e..16bf1e6 100644 --- a/bot_bottle/orchestrator/gateway.py +++ b/bot_bottle/orchestrator/gateway.py @@ -26,15 +26,8 @@ from ..docker_cmd import run_docker from ..paths import ( CONTROL_PLANE_TOKEN_ENV, host_control_plane_token, - host_db_path, host_gateway_ca_dir, ) -from ..supervise import DB_PATH_IN_CONTAINER - -# The host DB dir is bind-mounted here so the gateway's supervise daemon -# writes its queued proposals into the ONE host DB (the same file the -# orchestrator container opens and the operator reaches over HTTP). -_SUPERVISE_DB_DIR_IN_CONTAINER = os.path.dirname(DB_PATH_IN_CONTAINER) # The gateway's mitmproxy writes its CA a beat after the container starts, so # reads poll for it rather than assuming it's there on a fresh launch. @@ -75,14 +68,6 @@ GATEWAY_DOCKERFILE = "Dockerfile.gateway" _REPO_ROOT = Path(__file__).resolve().parents[2] -def _host_db_dir() -> str: - """The host DB directory (created if missing), for the gateway's - supervise-DB bind-mount.""" - db_dir = host_db_path().parent - db_dir.mkdir(parents=True, exist_ok=True) - return str(db_dir) - - def rotate_gateway_ca(ca_dir: Path | None = None) -> list[Path]: """Delete the persisted mitmproxy CA so the next gateway start mints a fresh one — the explicit, deliberate CA-rollover path (issue #450). @@ -255,11 +240,10 @@ class DockerGateway(Gateway): # container recreation AND docker volume pruning (agents trust it) # — see host_gateway_ca_dir / issue #450. "--volume", f"{host_gateway_ca_dir()}:{MITMPROXY_HOME}", - # Share the one host DB: the supervise daemon queues proposals - # into the same file the orchestrator (and the operator, over - # HTTP) reads — no second, disconnected DB in the container. - "--volume", f"{_host_db_dir()}:{_SUPERVISE_DB_DIR_IN_CONTAINER}", - "--env", f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}", + # No DB mount: the data plane (egress / supervise / git-gate) reaches + # the supervise queue over the control-plane RPC and never opens + # bot-bottle.db, so the gateway container gets no file handle on it + # (PRD 0070 / issue #469). ] for port in self._host_port_bindings: argv += ["--publish", f"0.0.0.0:{port}:{port}"] diff --git a/bot_bottle/orchestrator/lifecycle.py b/bot_bottle/orchestrator/lifecycle.py index cc3d82a..9560c07 100644 --- a/bot_bottle/orchestrator/lifecycle.py +++ b/bot_bottle/orchestrator/lifecycle.py @@ -29,14 +29,12 @@ from ..paths import ( host_control_plane_token, host_gateway_ca_dir, ) -from ..supervise import DB_PATH_IN_CONTAINER from .gateway import ( GATEWAY_DOCKERFILE, GATEWAY_IMAGE, GATEWAY_NETWORK, GatewayError, MITMPROXY_HOME, - _host_db_dir, ) DEFAULT_PORT = 8099 @@ -69,12 +67,12 @@ _INFRA_DAEMONS = "egress,git-http,supervise,orchestrator" # container. Separate from /app so the gateway's baked scripts # (egress_addon.py, egress-entrypoint.sh) are not overlaid. _SRC_IN_CONTAINER = "/bot-bottle-src" -# Bot-bottle host-root bind-mount inside the container (DB + state). +# Bot-bottle host-root bind-mount inside the container (DB + state). The +# orchestrator control plane opens bot-bottle.db under here (via BOT_BOTTLE_ROOT +# -> host_db_path()); it is the ONLY process in the container with a file +# handle on it (PRD 0070 / issue #469). _ROOT_IN_CONTAINER = "/bot-bottle-root" -# The supervise daemon writes proposals into the host DB directory. -_SUPERVISE_DB_DIR_IN_CONTAINER = os.path.dirname(DB_PATH_IN_CONTAINER) - _HEALTH_POLL_SECONDS = 0.25 _HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0 @@ -202,9 +200,6 @@ class OrchestratorService: # recreation AND docker volume pruning (issue #450): every agent # trusts this one CA, so a fresh one would break all running bottles. "--volume", f"{host_gateway_ca_dir()}:{MITMPROXY_HOME}", - # Shared supervise DB (same file the operator reads over HTTP). - "--volume", f"{_host_db_dir()}:{_SUPERVISE_DB_DIR_IN_CONTAINER}", - "--env", f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}", # Live control-plane source, mounted to a path that does not # overlay the gateway's baked /app scripts. "--volume", f"{self._repo_root}:{_SRC_IN_CONTAINER}:ro", diff --git a/bot_bottle/orchestrator/service.py b/bot_bottle/orchestrator/service.py index f1d2963..1f042f3 100644 --- a/bot_bottle/orchestrator/service.py +++ b/bot_bottle/orchestrator/service.py @@ -31,16 +31,23 @@ from .gateway import Gateway from ..supervise import ( AuditEntry, COMPONENT_FOR_TOOL, + POLL_STATUS_PENDING, + POLL_STATUS_UNKNOWN, + Proposal, Response, STATUS_APPROVED, STATUS_MODIFIED, STATUS_REJECTED, TOOL_EGRESS_ALLOW, TOOL_EGRESS_BLOCK, + archive_proposal, list_all_pending_proposals, read_proposal, + read_response, render_diff, + sha256_hex, write_audit_entry, + write_proposal, write_response, ) @@ -188,6 +195,60 @@ class Orchestrator: # The orchestrator owns the single DB *and* the live policy, so operator # decisions are applied here, server-side, and reached over HTTP by the # host TUI (no direct-DB access, one path for every backend). + # + # The *agent* half of the queue (queue a proposal, poll for its response) + # is served here too (PRD 0070): the data plane no longer opens the DB, so + # supervise_server / egress / git-gate reach the queue over the control + # plane. Both are keyed on the caller's orchestrator-resolved bottle id — + # never a caller-supplied slug — so a bottle can only queue or read its own + # proposals even if the data plane is compromised. + + def supervise_queue_proposal( + self, bottle_id: str, *, tool: str, proposed_file: str, justification: str, + ) -> str: + """Queue an agent-side proposal under `bottle_id` and return its id. + + `bottle_id` is the caller resolved from `(source_ip, identity_token)` by + the control plane, so the proposal is attributed to the calling bottle, + not to anything the (possibly hostile) data plane asserts. The + proposed-file self-hash is computed here so the agent never supplies it.""" + proposal = Proposal.new( + bottle_slug=bottle_id, + tool=tool, + proposed_file=proposed_file, + justification=justification, + current_file_hash=sha256_hex(proposed_file), + ) + write_proposal(proposal) + return proposal.id + + def supervise_poll_response(self, bottle_id: str, proposal_id: str) -> dict[str, object]: + """Non-blocking poll of one of `bottle_id`'s queued proposals. + + Returns `{"status": ...}`: a terminal decision (`approved`/`modified`/ + `rejected`, with `notes` + `final_file`) once the operator has responded, + `pending` while it's still queued undecided, or `unknown` when no such + queued proposal exists for this bottle (wrong id, or already resolved and + read). A decided proposal is archived here — the same terminal step the + data plane used to run after reading the response — so `pending` + proposals stay visible to the operator until decided *and* polled. + + Reads are scoped to `bottle_id` (the queue key), so a caller can never + read another bottle's proposal even with a guessed id.""" + try: + response = read_response(bottle_id, proposal_id) + except FileNotFoundError: + try: + read_proposal(bottle_id, proposal_id) + except FileNotFoundError: + return {"status": POLL_STATUS_UNKNOWN} + return {"status": POLL_STATUS_PENDING} + archive_proposal(bottle_id, proposal_id) + return { + "status": response.status, + "notes": response.notes, + "final_file": response.final_file, + } def supervise_pending(self) -> list[dict[str, object]]: """All pending proposals across bottles, FIFO, as JSON dicts diff --git a/bot_bottle/policy_resolver.py b/bot_bottle/policy_resolver.py index 06ed563..5a5dd15 100644 --- a/bot_bottle/policy_resolver.py +++ b/bot_bottle/policy_resolver.py @@ -64,28 +64,80 @@ class PolicyResolver: self._base = base_url.rstrip("/") self._timeout = timeout + def _post_json(self, path: str, payload: dict[str, object]) -> dict[str, object] | None: + """POST `payload` to control-plane `path`, returning the JSON object body + — or None when the orchestrator answers `403` (unattributed / fail + closed). Raises `PolicyResolveError` on an unreachable / unexpected-status + / malformed response so every caller can fail closed. Shared by + `_post_resolve` and the supervise propose/poll RPCs.""" + body = json.dumps(payload).encode() + req = urllib.request.Request( + f"{self._base}{path}", data=body, method="POST", + headers={"Content-Type": "application/json", **_control_auth_headers()}, + ) + try: + with urllib.request.urlopen(req, timeout=self._timeout) as resp: + data = json.loads(resp.read()) + except urllib.error.HTTPError as e: + if e.code == 403: + return None # unattributed → fail closed (caller denies) + raise PolicyResolveError(f"{path} returned HTTP {e.code}") from e + except (urllib.error.URLError, TimeoutError, OSError, ValueError) as e: + raise PolicyResolveError(f"{path} unreachable or malformed: {e}") from e + return data if isinstance(data, dict) else {} + def _post_resolve(self, source_ip: str, identity_token: str) -> dict[str, object] | None: """The orchestrator's `/resolve` payload for this client, or None if unattributed (a clean `403`). Raises `PolicyResolveError` on an unreachable / unexpected-status / malformed response so every caller can fail closed. Shared by `resolve` and `resolve_bottle_id`.""" - body = json.dumps( - {"source_ip": source_ip, "identity_token": identity_token} - ).encode() - req = urllib.request.Request( - f"{self._base}/resolve", data=body, method="POST", - headers={"Content-Type": "application/json", **_control_auth_headers()}, + return self._post_json( + "/resolve", {"source_ip": source_ip, "identity_token": identity_token}, ) - try: - with urllib.request.urlopen(req, timeout=self._timeout) as resp: - payload = json.loads(resp.read()) - except urllib.error.HTTPError as e: - if e.code == 403: - return None # unattributed → fail closed (caller denies) - raise PolicyResolveError(f"/resolve returned HTTP {e.code}") from e - except (urllib.error.URLError, TimeoutError, OSError, ValueError) as e: - raise PolicyResolveError(f"/resolve unreachable or malformed: {e}") from e - return payload if isinstance(payload, dict) else {} + + def propose_supervise( + self, + source_ip: str, + identity_token: str, + *, + tool: str, + proposed_file: str, + justification: str, + ) -> str | None: + """Queue a supervise proposal on the control plane and return its + `proposal_id`. The orchestrator attributes the proposal to the calling + bottle by `(source_ip, identity_token)` — exactly like `/resolve` — so a + bottle can only ever queue its *own* proposals. Returns None when the + pair is unattributed (a clean `403`); raises `PolicyResolveError` if the + orchestrator can't be reached, so the data-plane caller fails closed + (blocks / refuses) rather than silently dropping the proposal.""" + payload = self._post_json("/supervise/propose", { + "source_ip": source_ip, + "identity_token": identity_token, + "tool": tool, + "proposed_file": proposed_file, + "justification": justification, + }) + if payload is None: + return None + proposal_id = payload.get("proposal_id") + return proposal_id if isinstance(proposal_id, str) and proposal_id else None + + def poll_supervise( + self, source_ip: str, identity_token: str, proposal_id: str, + ) -> dict[str, object] | None: + """Poll a queued proposal for the operator's decision, **non-blocking**. + Attributed by `(source_ip, identity_token)` so a bottle can only read its + *own* proposal's response. Returns `{"status": ...}` where status is one + of the terminal decisions (`approved`/`modified`/`rejected`, carrying + `notes` + `final_file`), `pending` (queued, no decision yet), or + `unknown` (no such queued proposal for this bottle). Returns None when + unattributed; raises `PolicyResolveError` if unreachable.""" + return self._post_json("/supervise/poll", { + "source_ip": source_ip, + "identity_token": identity_token, + "proposal_id": proposal_id, + }) def resolve(self, source_ip: str, identity_token: str = "") -> str | None: """The calling bottle's policy blob, or None if unattributed. Always diff --git a/bot_bottle/supervise.py b/bot_bottle/supervise.py index a1bc0aa..827bd30 100644 --- a/bot_bottle/supervise.py +++ b/bot_bottle/supervise.py @@ -40,6 +40,8 @@ from pathlib import Path from .supervise_types import ( ACTION_OPERATOR_EDIT, AuditEntry, + POLL_STATUS_PENDING, + POLL_STATUS_UNKNOWN, Proposal, Response, STATUSES, @@ -257,6 +259,8 @@ __all__ = [ "STATUS_APPROVED", "STATUS_MODIFIED", "STATUS_REJECTED", + "POLL_STATUS_PENDING", + "POLL_STATUS_UNKNOWN", "SUPERVISE_HOSTNAME", "SUPERVISE_PORT", "Supervise", diff --git a/bot_bottle/supervise_server.py b/bot_bottle/supervise_server.py index 64c4069..7a90d21 100644 --- a/bot_bottle/supervise_server.py +++ b/bot_bottle/supervise_server.py @@ -7,9 +7,13 @@ config changes when stuck. The tools are `egress-allow`, Each queued proposal tool call: 1. Validates the proposed file syntactically. - 2. Writes a Proposal to the host SQLite database. - 3. Blocks polling for a matching Response row, up to a short grace - window (`SUPERVISE_RESPONSE_TIMEOUT_SECONDS`, default 30s). + 2. Queues a Proposal over the control-plane RPC (`POST /supervise/propose`), + which attributes it to the calling bottle by (source_ip, identity_token) + and writes it to the one orchestrator-owned SQLite database. This daemon + never opens `bot-bottle.db` itself (PRD 0070 / issue #469). + 3. Polls the control plane (`POST /supervise/poll`) for a matching Response, + up to a short grace window (`SUPERVISE_RESPONSE_TIMEOUT_SECONDS`, + default 30s). 4. On a decision within the window, returns the operator's `{status, notes}`. On timeout, returns `status: pending` **with the proposal id** and leaves the proposal queued — the flow is @@ -24,8 +28,8 @@ without holding an HTTP request open. One shared server fronts every bottle (PRD 0070) and attributes each proposal to the calling bottle by source IP, resolved from the orchestrator — an unattributed or unreachable source fails closed. BOT_BOTTLE_ORCHESTRATOR_URL -is mandatory: there is no fixed-slug single-tenant fallback. SUPERVISE_DB_PATH -points at the bind-mounted host database. +is mandatory: there is no fixed-slug single-tenant fallback, and the queue is +reached only over that control-plane RPC (no direct database handle). Speaks MCP over HTTP+JSON-RPC. Methods handled: @@ -51,7 +55,7 @@ import socketserver import sys import time import typing -from dataclasses import dataclass, replace +from dataclasses import dataclass from bot_bottle.constants import IDENTITY_HEADER from bot_bottle.egress_addon_core import ( @@ -312,7 +316,9 @@ def validate_proposed_file(tool: str, content: str) -> None: @dataclass(frozen=True) class ServerConfig: - bottle_slug: str + # No bottle_slug: the calling bottle is attributed per request by the + # control plane from (source_ip, identity_token), so nothing on this daemon + # is keyed by a single slug (PRD 0070 / issue #469). response_timeout_seconds: float = DEFAULT_RESPONSE_TIMEOUT_SECONDS @@ -331,14 +337,22 @@ def handle_tools_list(_params: dict[str, object]) -> dict[str, object]: def handle_tools_call( params: dict[str, object], config: ServerConfig, + *, + resolver: "PolicyResolver", + source_ip: str, + identity_token: str, ) -> dict[str, object]: - """Validates the proposal, writes it to the queue, blocks waiting - for a Response, returns the result wrapped in MCP `content`. + """Validates the proposal, queues it over the control-plane RPC, polls for + a Response through the grace window, and returns the result wrapped in MCP + `content`. - `list-egress-routes` never reaches here — the handler answers it from - the calling bottle's resolved policy before dispatching (see - `MCPHandler._dispatch`); this path is the queued, operator-approved - `egress-allow` / `egress-block` tools.""" + The queue lives behind the orchestrator now — `propose_supervise` attributes + the proposal to the calling bottle by `(source_ip, identity_token)` and + `poll_supervise` reads only that bottle's own responses, so this daemon + never opens `bot-bottle.db`. `list-egress-routes` never reaches here — the + handler answers it from the calling bottle's resolved policy before + dispatching (see `MCPHandler._dispatch`); this path is the queued, + operator-approved `egress-allow` / `egress-block` tools.""" name = params.get("name") if not isinstance(name, str): raise _RpcClientError(ERR_INVALID_PARAMS, "tools/call missing 'name'") @@ -366,59 +380,100 @@ def handle_tools_call( else: raise _RpcClientError(ERR_INVALID_PARAMS, f"unknown tool {name!r}") - proposal = _sv.Proposal.new( - bottle_slug=config.bottle_slug, - tool=name, - proposed_file=proposed_file, - justification=justification, - current_file_hash=_sv.sha256_hex(proposed_file), + proposal_id = _queue_proposal( + resolver, source_ip, identity_token, + tool=name, proposed_file=proposed_file, justification=justification, ) - try: - _sv.write_proposal(proposal) - except OSError as e: - raise _RpcInternalError(f"failed to write proposal to queue: {e}") from e sys.stderr.write( - f"supervise: queued proposal {proposal.id} ({name}) " - f"for bottle {config.bottle_slug}; waiting for operator...\n" + f"supervise: queued proposal {proposal_id} ({name}); waiting for operator...\n" ) sys.stderr.flush() - deadline = time.monotonic() + config.response_timeout_seconds - try: - response = _sv.wait_for_response( - config.bottle_slug, - proposal.id, - poll_interval=MIN_RESPONSE_POLL_INTERVAL_SECONDS, - deadline=deadline, - ) - except TimeoutError: - text = format_pending_response_text(proposal.id, config.response_timeout_seconds) - return { - "content": [{"type": "text", "text": text}], - "isError": False, - } - try: - _sv.archive_proposal(config.bottle_slug, proposal.id) - except OSError as e: - raise _RpcInternalError(f"failed to archive proposal: {e}") from e - text = format_response_text(response) + deadline = time.monotonic() + config.response_timeout_seconds + while time.monotonic() < deadline: + response = _poll_response(resolver, source_ip, identity_token, proposal_id) + if response is not None: + return { + "content": [{"type": "text", "text": format_response_text(response)}], + "isError": response.status == _sv.STATUS_REJECTED, + } + time.sleep(MIN_RESPONSE_POLL_INTERVAL_SECONDS) + + text = format_pending_response_text(proposal_id, config.response_timeout_seconds) return { "content": [{"type": "text", "text": text}], - "isError": response.status == _sv.STATUS_REJECTED, + "isError": False, } +def _queue_proposal( + resolver: "PolicyResolver", + source_ip: str, + identity_token: str, + *, + tool: str, + proposed_file: str, + justification: str, +) -> str: + """Queue a proposal over the control plane and return its id, mapping the + resolver's fail-closed outcomes to internal errors so `do_POST` returns a + clean -32603 rather than leaking the cause to the agent.""" + try: + proposal_id = resolver.propose_supervise( + source_ip, identity_token, + tool=tool, proposed_file=proposed_file, justification=justification, + ) + except PolicyResolveError as e: + raise _RpcInternalError(f"orchestrator unreachable, cannot queue proposal: {e}") from e + if proposal_id is None: + raise _RpcInternalError("request source is not attributed to a bottle") + return proposal_id + + +def _poll_response( + resolver: "PolicyResolver", source_ip: str, identity_token: str, proposal_id: str, +) -> "_sv.Response | None": + """One non-blocking control-plane poll for `proposal_id`'s decision. Returns + the operator's Response once decided, or None while it is still pending + (or `unknown` — treated as still-waiting so a transient blip degrades to the + pending-timeout path rather than a spurious error mid grace window).""" + try: + result = resolver.poll_supervise(source_ip, identity_token, proposal_id) + except PolicyResolveError as e: + raise _RpcInternalError(f"orchestrator unreachable, cannot poll proposal: {e}") from e + if result is None: + raise _RpcInternalError("request source is not attributed to a bottle") + if result.get("status") in _sv.STATUSES: + return _response_from_poll(proposal_id, result) + return None + + +def _response_from_poll(proposal_id: str, result: "dict[str, object]") -> "_sv.Response": + """Build a Response from a decided `/supervise/poll` payload.""" + final_file = result.get("final_file") + return _sv.Response( + proposal_id=proposal_id, + status=str(result.get("status")), + notes=str(result.get("notes") or ""), + final_file=final_file if isinstance(final_file, str) else None, + ) + + def handle_check_proposal( params: dict[str, object], - config: ServerConfig, + *, + resolver: "PolicyResolver", + source_ip: str, + identity_token: str, ) -> dict[str, object]: """Non-blocking poll of a queued proposal's decision, by id. Never creates a Proposal (so `check-proposal` isn't in `TOOLS`); it only - reads the queue. Resolution order mirrors the synchronous path's terminal - step — a decided proposal is archived here exactly as `handle_tools_call` - archives it after `wait_for_response`, so `pending` proposals stay visible - to the operator until they're both decided *and* polled.""" + reads the queue over the control-plane RPC, attributed by + `(source_ip, identity_token)` so it can only read *this* bottle's own + proposals. The orchestrator archives a decided proposal on read — the same + terminal step the synchronous path triggers — so `pending` proposals stay + visible to the operator until they're both decided *and* polled.""" args_raw = params.get("arguments", {}) if not isinstance(args_raw, dict): raise _RpcClientError(ERR_INVALID_PARAMS, "tools/call 'arguments' must be an object") @@ -431,25 +486,24 @@ def handle_check_proposal( proposal_id = proposal_id.strip() try: - response = _sv.read_response(config.bottle_slug, proposal_id) - except FileNotFoundError: - # No decision yet — distinguish "still queued" from "unknown id". - try: - _sv.read_proposal(config.bottle_slug, proposal_id) - except FileNotFoundError: - return { - "content": [{"type": "text", "text": format_unknown_proposal_text(proposal_id)}], - "isError": True, - } + result = resolver.poll_supervise(source_ip, identity_token, proposal_id) + except PolicyResolveError as e: + raise _RpcInternalError(f"orchestrator unreachable, cannot poll proposal: {e}") from e + if result is None: + raise _RpcInternalError("request source is not attributed to a bottle") + status = result.get("status") + + if status == _sv.POLL_STATUS_UNKNOWN: + return { + "content": [{"type": "text", "text": format_unknown_proposal_text(proposal_id)}], + "isError": True, + } + if status == _sv.POLL_STATUS_PENDING: return { "content": [{"type": "text", "text": format_still_pending_text(proposal_id)}], "isError": False, } - - try: - _sv.archive_proposal(config.bottle_slug, proposal_id) - except OSError as e: - raise _RpcInternalError(f"failed to archive proposal: {e}") from e + response = _response_from_poll(proposal_id, result) return { "content": [{"type": "text", "text": format_response_text(response)}], "isError": response.status == _sv.STATUS_REJECTED, @@ -591,16 +645,34 @@ class MCPHandler(http.server.BaseHTTPRequestHandler): # — silently dropping base routes like api.anthropic.com on approval. if req.params.get("name") == _sv.TOOL_LIST_EGRESS_ROUTES: return self._resolved_routes_payload() + resolver = self._resolver_or_fail() + source_ip = self.client_address[0] + token = self._identity_token() # `check-proposal` is a non-blocking read of the calling bottle's - # own queue — attributed by source IP like a proposal, but it - # never queues or blocks. + # own queue — attributed by (source_ip, identity_token) like a + # proposal, but it never queues or blocks. if req.params.get("name") == _sv.TOOL_CHECK_PROPOSAL: - return handle_check_proposal(req.params, self._attributed_config(config)) - # Attribute the proposal to the source-IP-resolved bottle, so the one - # shared server queues each bottle's proposal under its own slug. - return handle_tools_call(req.params, self._attributed_config(config)) + return handle_check_proposal( + req.params, resolver=resolver, + source_ip=source_ip, identity_token=token, + ) + # The control plane attributes the proposal to the source-IP + token + # resolved bottle, so the one shared queue holds each bottle's + # proposal under its own id — no slug is asserted by this daemon. + return handle_tools_call( + req.params, config, resolver=resolver, + source_ip=source_ip, identity_token=token, + ) raise _RpcClientError(ERR_METHOD_NOT_FOUND, f"method not found: {method}") + def _identity_token(self) -> str: + """The agent's per-bottle identity token from the request header (the + MCP client sends it via `mcp add --header`), or empty. The control plane + requires the (source_ip, token) pair, so a missing/wrong token + fail-closes there.""" + headers = getattr(self, "headers", None) + return headers.get(IDENTITY_HEADER, "") if headers is not None else "" + def _resolver_or_fail(self) -> "PolicyResolver": """This server's policy resolver. A server started without one is a misconfiguration, not a tenancy mode — fail closed rather than @@ -612,40 +684,18 @@ class MCPHandler(http.server.BaseHTTPRequestHandler): def _resolved_routes_payload(self) -> dict[str, object]: """The calling bottle's live egress routes as the `list-egress-routes` - JSON payload, resolved by (source_ip, identity token). Fail-closed like - `_attributed_config`: an unattributed source or an unreachable - orchestrator yields an empty route list (never another bottle's), - courtesy of `resolve_client_context`.""" + JSON payload, resolved by (source_ip, identity token). Fail-closed: an + unattributed source or an unreachable orchestrator yields an empty route + list (never another bottle's), courtesy of `resolve_client_context`.""" resolver = self._resolver_or_fail() - headers = getattr(self, "headers", None) - token = headers.get(IDENTITY_HEADER, "") if headers is not None else "" conf, _slug, _tokens = resolve_client_context( - resolver, self.client_address[0], token, + resolver, self.client_address[0], self._identity_token(), ) body = json.dumps( {"routes": [route_to_yaml_dict(r) for r in conf.routes]}, indent=2, ) return {"content": [{"type": "text", "text": body}], "isError": False} - def _attributed_config(self, config: ServerConfig) -> ServerConfig: - """The ServerConfig with `bottle_slug` bound to *this request's* bottle: - the bottle id attributed from the source IP — **fail-closed**, an - unattributed or unreachable source raises so no proposal is queued under - the wrong (or empty) slug.""" - resolver = self._resolver_or_fail() - # The agent's MCP client sends the identity token as a request header - # (provisioned via `mcp add --header`); the orchestrator requires the - # (source_ip, token) pair, so a missing/wrong token fail-closes below. - headers = getattr(self, "headers", None) - token = headers.get(IDENTITY_HEADER, "") if headers is not None else "" - try: - bottle_id = resolver.resolve_bottle_id(self.client_address[0], token) - except PolicyResolveError as e: - raise _RpcInternalError(f"orchestrator unreachable, cannot attribute: {e}") from e - if not bottle_id: - raise _RpcInternalError("request source is not attributed to a bottle") - return replace(config, bottle_slug=bottle_id) - def _write_jsonrpc(self, body: bytes) -> None: self.send_response(200) self.send_header("Content-Type", "application/json") @@ -668,10 +718,10 @@ class MCPHandler(http.server.BaseHTTPRequestHandler): class MCPServer(socketserver.ThreadingMixIn, http.server.HTTPServer): allow_reuse_address = True daemon_threads = True - config: ServerConfig = ServerConfig(bottle_slug="") - # Set by `serve`; every proposal is attributed to the source-IP-resolved - # bottle. The class default is a placeholder — a server without a resolver - # fails closed per request (see `_resolver_or_fail`). + config: ServerConfig = ServerConfig() + # Set by `serve`; every proposal is attributed to the source-IP + token + # resolved bottle by the control plane. A server without a resolver fails + # closed per request (see `_resolver_or_fail`). policy_resolver: "PolicyResolver | None" = None @@ -686,12 +736,9 @@ def serve( response_timeout_seconds: float = DEFAULT_RESPONSE_TIMEOUT_SECONDS, ) -> typing.NoReturn: server = MCPServer((bind, port), MCPHandler) - # bottle_slug is a placeholder: every request's proposal is attributed to - # the source-IP-resolved bottle (see MCPHandler._attributed_config). - server.config = ServerConfig( - bottle_slug="", - response_timeout_seconds=response_timeout_seconds, - ) + # Every request's proposal is attributed to the source-IP + token resolved + # bottle by the control plane (see MCPHandler._dispatch). + server.config = ServerConfig(response_timeout_seconds=response_timeout_seconds) server.policy_resolver = resolver sys.stderr.write( f"supervise listening on {bind}:{port}; multi-tenant; " diff --git a/bot_bottle/supervise_types.py b/bot_bottle/supervise_types.py index f88203a..6bb65e5 100644 --- a/bot_bottle/supervise_types.py +++ b/bot_bottle/supervise_types.py @@ -37,6 +37,15 @@ STATUS_MODIFIED = "modified" STATUS_REJECTED = "rejected" STATUSES: tuple[str, ...] = (STATUS_APPROVED, STATUS_MODIFIED, STATUS_REJECTED) +# Non-terminal markers the control-plane supervise-poll RPC returns when a +# proposal has no operator decision yet (`PENDING`) or no queued proposal with +# that id exists for the calling bottle (`UNKNOWN`). They are never a +# `Response.status` — only the poll wire contract (see +# `Orchestrator.supervise_poll_response`) — so they are deliberately outside +# `STATUSES`. +POLL_STATUS_PENDING = "pending" +POLL_STATUS_UNKNOWN = "unknown" + ACTION_OPERATOR_EDIT = "operator-edit" @@ -157,6 +166,8 @@ __all__ = [ "STATUS_APPROVED", "STATUS_MODIFIED", "STATUS_REJECTED", + "POLL_STATUS_PENDING", + "POLL_STATUS_UNKNOWN", "TOOLS", "TOOL_EGRESS_ALLOW", "TOOL_EGRESS_BLOCK", diff --git a/docs/prds/0070-per-host-orchestrator.md b/docs/prds/0070-per-host-orchestrator.md index 40e7fb3..0b7c579 100644 --- a/docs/prds/0070-per-host-orchestrator.md +++ b/docs/prds/0070-per-host-orchestrator.md @@ -299,11 +299,14 @@ and integrate a console against. the data plane and the console reach state through the control-plane RPC, never a direct file handle.** No agent-facing component gets the file, so none can forge attribution. (This supersedes the earlier `ro`-mount idea.) - - *Transitional caveat:* today the per-bottle **supervise gateway - rw-bind-mounts `bot-bottle.db`** to write proposals — exactly the - pattern the orchestrator removes (supervise consolidates into the - orchestrator; gateway writes become RPC calls). Until that lands, don't - put the attribution registry behind a data-plane-writable mount. + This rule is now **in force** across the data plane (issue #469): the + supervise daemon, the egress addon, and the git-gate pre-receive hook queue + proposals and poll for their responses over the agent-side supervise RPC + (`POST /supervise/propose` + `POST /supervise/poll`, attributed by + `(source_ip, identity_token)` exactly like `/resolve`), so a bottle can only + ever queue or read its own proposals. The gateway/data-plane containers no + longer bind-mount the DB directory or carry `SUPERVISE_DB_PATH`; the + orchestrator is the sole opener of the one file. Implementation note for the VM slices: SQLite **WAL** over a guest share (virtiofs/9p) is finicky (the `-shm`/`-wal` files need real mmap/locking), diff --git a/tests/unit/test_egress_addon_request_flow.py b/tests/unit/test_egress_addon_request_flow.py index fd90c76..3de75b9 100644 --- a/tests/unit/test_egress_addon_request_flow.py +++ b/tests/unit/test_egress_addon_request_flow.py @@ -209,6 +209,7 @@ from bot_bottle.egress_addon_core import ( # noqa: E402 Route, route_to_yaml_dict, ) +from bot_bottle.policy_resolver import PolicyResolveError # noqa: E402 # --------------------------------------------------------------------------- @@ -266,7 +267,42 @@ def _config_to_policy(config: Config) -> str: }) + "\n" -class _StaticResolver: +class _SuperviseRpcFake: + """Mixin adding the control-plane supervise RPCs to a fake resolver, now + that the egress data plane queues/polls proposals over RPC instead of + opening the DB (issue #469). `supervise_status` is the operator's eventual + decision (None models a timeout — poll stays `pending`); `propose_error` + models an unreachable orchestrator. `propose_calls` records what was queued + so a test can assert the proposal was attributed by the caller's source IP.""" + + supervise_status: "str | None" = None + propose_error: bool = False + + @property + def propose_calls(self) -> list: + if not hasattr(self, "_propose_calls"): + self._propose_calls: list = [] + return self._propose_calls + + def propose_supervise( + self, source_ip, identity_token, *, tool, proposed_file, justification, + ): + del proposed_file, justification + self.propose_calls.append( + {"source_ip": source_ip, "identity_token": identity_token, "tool": tool} + ) + if self.propose_error: + raise PolicyResolveError("orchestrator down") + return "prop-1" + + def poll_supervise(self, source_ip, identity_token, proposal_id): + del source_ip, identity_token, proposal_id + if self.supervise_status is None: + return {"status": "pending"} + return {"status": self.supervise_status, "notes": "", "final_file": None} + + +class _StaticResolver(_SuperviseRpcFake): """Fake orchestrator resolver that serves one Config (+ optional bottle id and per-bottle tokens) for every client — the host-test stand-in for a bottle's policy now that egress is resolver-only.""" @@ -323,7 +359,7 @@ def _with_client_ip(flow: _Flow, ip: str) -> _Flow: return flow -class _CtxResolver: +class _CtxResolver(_SuperviseRpcFake): """Fake orchestrator resolver: maps source IP -> bottle id, and grants the same allow-list to any attributed bottle (unattributed -> deny).""" @@ -516,66 +552,33 @@ class TestOutboundDlpPolicy(unittest.TestCase): # --------------------------------------------------------------------------- -def _fake_sv(response_status: str | None) -> types.SimpleNamespace: - """Stand-in for the `supervise` module the adapter queues proposals to. - - `response_status` of None models a timeout (read_response never returns a - decision); a status string models the operator's eventual answer.""" - def _new_proposal(**_kw: Any) -> Any: - return types.SimpleNamespace(id="prop-1") - - def _sha256_hex(_payload: Any) -> str: - return "hash" - - def _noop(*_args: Any) -> None: - return None - - def _read_response(_slug: Any, _pid: Any) -> Any: - if response_status is None: - raise OSError("not written yet") # forces poll -> timeout - return types.SimpleNamespace(status=response_status) - - ns = types.SimpleNamespace() - ns.STATUS_APPROVED = "approved" - ns.STATUS_MODIFIED = "modified" - ns.TOOL_EGRESS_TOKEN_ALLOW = "egress_token_allow" - ns.Proposal = types.SimpleNamespace(new=_new_proposal) - ns.sha256_hex = _sha256_hex - ns.write_proposal = _noop - ns.archive_proposal = _noop - ns.read_response = _read_response - return ns - - class TestSuperviseBranch(unittest.TestCase): - def _supervised_addon(self) -> EgressAddon: + def _supervised_addon(self, status: str | None) -> EgressAddon: addon = _addon(Config(routes=(Route(host="api.example.com"),)), slug="test-bottle") addon._token_allow_timeout = 0.05 + cast(Any, addon._resolver).supervise_status = status return addon def test_operator_approval_allows_token_and_forwards(self) -> None: - addon = self._supervised_addon() + addon = self._supervised_addon("approved") flow = _Flow(_Request(host="api.example.com", method="POST", body=f"k={_OPENAI_KEY}")) - with patch.object(_ea_mod, "_sv", _fake_sv("approved")): - _run_request(addon, flow) + _run_request(addon, flow) self.assertIsNone(flow.response) # forwarded after approval # Approval lands in the calling bottle's safelist (keyed by slug). self.assertIn(_OPENAI_KEY, addon._safe_tokens_for("test-bottle")) def test_operator_rejection_blocks(self) -> None: - addon = self._supervised_addon() + addon = self._supervised_addon("rejected") flow = _Flow(_Request(host="api.example.com", method="POST", body=f"k={_OPENAI_KEY}")) - with patch.object(_ea_mod, "_sv", _fake_sv("rejected")): - _run_request(addon, flow) + _run_request(addon, flow) assert flow.response is not None self.assertEqual(403, flow.response.status_code) self.assertIn("rejected", flow.response.get_text()) def test_supervise_timeout_blocks(self) -> None: - addon = self._supervised_addon() + addon = self._supervised_addon(None) # poll stays pending -> timeout flow = _Flow(_Request(host="api.example.com", method="POST", body=f"k={_OPENAI_KEY}")) - with patch.object(_ea_mod, "_sv", _fake_sv(None)): - _run_request(addon, flow) + _run_request(addon, flow) assert flow.response is not None self.assertEqual(403, flow.response.status_code) self.assertIn("timed out", flow.response.get_text()) @@ -772,19 +775,14 @@ class TestRedactSurfaces(unittest.TestCase): class TestSuperviseWriteFailure(unittest.TestCase): - def test_write_proposal_oserror_blocks(self) -> None: + def test_propose_rpc_error_blocks(self) -> None: + # An unreachable orchestrator (propose RPC raises) fails closed: the + # request is blocked rather than forwarded unsupervised. addon = _addon(Config(routes=(Route(host="api.example.com"),)), slug="test-bottle") addon._token_allow_timeout = 0.05 + cast(Any, addon._resolver).propose_error = True flow = _Flow(_Request(host="api.example.com", method="POST", body=f"k={_OPENAI_KEY}")) - - fake = _fake_sv("approved") - - def _raise(_p: Any) -> None: - raise OSError("disk full") - - fake.write_proposal = _raise - with patch.object(_ea_mod, "_sv", fake): - _run_request(addon, flow) + _run_request(addon, flow) assert flow.response is not None self.assertEqual(403, flow.response.status_code) @@ -844,22 +842,23 @@ class TestSuperviseMultiTenant(unittest.TestCase): """Consolidated gateway: supervise proposals + the DLP safelist are keyed per bottle, resolved by source IP (PRD 0070).""" - def _consolidated_addon(self) -> EgressAddon: + def _consolidated_addon(self, status: str | None = "approved") -> EgressAddon: # Static config is empty; the resolver supplies each bottle's config. addon = _addon(Config(routes=())) - addon._resolver = cast(Any, _CtxResolver({"10.0.0.1": "bottle-a", "10.0.0.2": "bottle-b"})) + resolver = _CtxResolver({"10.0.0.1": "bottle-a", "10.0.0.2": "bottle-b"}) + resolver.supervise_status = status + addon._resolver = cast(Any, resolver) addon._token_allow_timeout = 0.05 return addon def test_approval_is_scoped_to_the_calling_bottle(self) -> None: - addon = self._consolidated_addon() + addon = self._consolidated_addon("approved") # bottle-a (10.0.0.1) sends the token; the operator approves. flow = _with_client_ip( _Flow(_Request(host="api.example.com", method="POST", body=f"k={_OPENAI_KEY}")), "10.0.0.1", ) - with patch.object(_ea_mod, "_sv", _fake_sv("approved")): - _run_request(addon, flow) + _run_request(addon, flow) self.assertIsNone(flow.response) # forwarded after approval # The approval lands ONLY in bottle-a's safelist — never bottle-b's. # A global set here would be the cross-tenant leak this slice closes. @@ -867,22 +866,19 @@ class TestSuperviseMultiTenant(unittest.TestCase): self.assertNotIn(_OPENAI_KEY, addon._safe_tokens_for("bottle-b")) def test_proposal_is_attributed_to_the_source_ip_bottle(self) -> None: - addon = self._consolidated_addon() - seen: list[str] = [] - fake = _fake_sv("approved") - - def _capture(**kw: Any) -> Any: - seen.append(kw["bottle_slug"]) - return types.SimpleNamespace(id="p") - - fake.Proposal = types.SimpleNamespace(new=_capture) + addon = self._consolidated_addon("approved") flow = _with_client_ip( _Flow(_Request(host="api.example.com", method="POST", body=f"k={_OPENAI_KEY}")), "10.0.0.2", ) - with patch.object(_ea_mod, "_sv", fake): - _run_request(addon, flow) - self.assertEqual(["bottle-b"], seen) # proposal keyed by the resolved bottle + _run_request(addon, flow) + # The addon forwards the caller's source IP to the control plane, which + # attributes the proposal server-side by (source_ip, identity_token) — + # the addon never asserts a slug. The resolved bottle keys the safelist. + calls = cast(Any, addon._resolver).propose_calls + self.assertEqual(["10.0.0.2"], [c["source_ip"] for c in calls]) + self.assertIn(_OPENAI_KEY, addon._safe_tokens_for("bottle-b")) + self.assertNotIn(_OPENAI_KEY, addon._safe_tokens_for("bottle-a")) def test_auth_token_injected_from_resolved_tokens(self) -> None: # The bottle's upstream token comes from /resolve (in-memory on the @@ -925,16 +921,17 @@ class TestSuperviseMultiTenant(unittest.TestCase): self.assertIsNotNone(flow.response) # blocked — token unset def test_unattributed_source_ip_cannot_supervise(self) -> None: - addon = self._consolidated_addon() + addon = self._consolidated_addon("approved") # 10.9.9.9 is not in the resolver map -> deny-all config, empty slug. flow = _with_client_ip( _Flow(_Request(host="api.example.com", method="POST", body=f"k={_OPENAI_KEY}")), "10.9.9.9", ) - with patch.object(_ea_mod, "_sv", _fake_sv("approved")): - _run_request(addon, flow) + _run_request(addon, flow) self.assertIsNotNone(flow.response) # blocked (no route, no supervise) self.assertNotIn(_OPENAI_KEY, addon._safe_tokens_for("")) + # Never even reached the queue — no route means no supervise proposal. + self.assertEqual([], cast(Any, addon._resolver).propose_calls) class TestMultiTenantInboundDlp(unittest.TestCase): diff --git a/tests/unit/test_git_gate.py b/tests/unit/test_git_gate.py index cca39c5..fd899f0 100644 --- a/tests/unit/test_git_gate.py +++ b/tests/unit/test_git_gate.py @@ -213,22 +213,25 @@ class TestHookRender(unittest.TestCase): # the suppressed findings for human approval. self.assertIn("--ignore-gitleaks-allow", hook) self.assertIn("--report-format=json", hook) - self.assertIn("tool=_sv.TOOL_GITLEAKS_ALLOW", hook) - self.assertIn("_sv.write_proposal", hook) - self.assertIn("_sv.read_response", hook) - self.assertIn("SUPERVISE_BOTTLE_SLUG", hook) + # The hook queues + polls over the control-plane RPC — it no longer + # opens the DB directly (PRD 0070 / issue #469). + self.assertIn("tool=TOOL_GITLEAKS_ALLOW", hook) + self.assertIn("propose_supervise", hook) + self.assertIn("poll_supervise", hook) + self.assertIn("SUPERVISE_SOURCE_IP", hook) + self.assertIn("SUPERVISE_IDENTITY_TOKEN", hook) self.assertIn("supervisor approved # gitleaks:allow", hook) self.assertIn("supervisor rejected # gitleaks:allow", hook) def test_inline_gitleaks_allow_python_imports_work_in_gateway_layout(self): hook = git_gate_render_hook() - # The gateway image copies supervise.py flat under /app, while - # host-side tests import it through the bot_bottle package. - # Hooks execute from the bare repo directory, so the embedded - # Python must include /app and support both import layouts. + # The gateway image copies the package modules flat under /app, while + # host-side tests import them through the bot_bottle package. Hooks + # execute from the bare repo directory, so the embedded Python must + # include /app and support both import layouts. self.assertIn('PYTHONPATH="/app${PYTHONPATH:+:$PYTHONPATH}"', hook) - self.assertIn("import supervise as _sv", hook) - self.assertIn("from bot_bottle import supervise as _sv", hook) + self.assertIn("from bot_bottle.policy_resolver import PolicyResolver", hook) + self.assertIn("from policy_resolver import PolicyResolver", hook) def test_inline_gitleaks_allow_fails_closed_without_supervisor(self): hook = git_gate_render_hook() diff --git a/tests/unit/test_git_http_backend.py b/tests/unit/test_git_http_backend.py index afe264c..405f0b0 100644 --- a/tests/unit/test_git_http_backend.py +++ b/tests/unit/test_git_http_backend.py @@ -112,11 +112,13 @@ class TestGitHttpBackend(unittest.TestCase): ).strip() self.assertEqual(head, cloned) - def test_consolidated_push_stamps_bottle_slug_for_the_hook(self): - # In consolidated mode the backend attributes the push by source IP and - # stamps SUPERVISE_BOTTLE_SLUG= into the CGI env, so the - # gitleaks-allow pre-receive hook queues its proposal under the right - # bottle. The hook here just records what it received. + def test_consolidated_push_stamps_supervise_attribution_for_the_hook(self): + # In consolidated mode the backend attributes the push by (source IP, + # identity token) and stamps SUPERVISE_SOURCE_IP + SUPERVISE_IDENTITY_TOKEN + # into the CGI env, so the gitleaks-allow pre-receive hook can queue its + # proposal over the control-plane RPC (which re-resolves the bottle from + # exactly that pair — PRD 0070 / issue #469). The hook here just records + # the source IP it received. from http.server import ThreadingHTTPServer bottle_id = "bottleab12" @@ -130,10 +132,10 @@ class TestGitHttpBackend(unittest.TestCase): ["git", "-C", str(bare), "config", "http.receivepack", "true"], check=True, ) - capture = root / "slug-capture" + capture = root / "source-ip-capture" hook = bare / "hooks" / "pre-receive" hook.write_text( - f"#!/bin/sh\nprintf '%s' \"${{SUPERVISE_BOTTLE_SLUG:-UNSET}}\" > " + f"#!/bin/sh\nprintf '%s' \"${{SUPERVISE_SOURCE_IP:-UNSET}}\" > " f"{capture}\ncat >/dev/null\nexit 0\n" ) hook.chmod(0o755) @@ -166,7 +168,7 @@ class TestGitHttpBackend(unittest.TestCase): ["git", "push", url, "HEAD:refs/heads/main"], cwd=work, check=True, capture_output=True, text=True, timeout=5, ) - self.assertEqual(bottle_id, capture.read_text()) + self.assertEqual("127.0.0.1", capture.read_text()) def test_post_forwards_git_cgi_headers(self): from http.server import ThreadingHTTPServer diff --git a/tests/unit/test_orchestrator_control_plane.py b/tests/unit/test_orchestrator_control_plane.py index 9ebd8b6..fe445b2 100644 --- a/tests/unit/test_orchestrator_control_plane.py +++ b/tests/unit/test_orchestrator_control_plane.py @@ -426,6 +426,112 @@ class TestDispatchSupervise(unittest.TestCase): self.assertIn("no such proposal", str(payload["error"])) +class TestDispatchSuperviseAgentRpc(unittest.TestCase): + """The agent half — `/supervise/propose` + `/supervise/poll` — attributed by + (source_ip, identity_token) like /resolve, so a bottle can only ever queue + or read its own proposals (PRD 0070 / issue #469).""" + + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + root = Path(self._tmp.name) + db = root / "db" / "bot-bottle.db" + db.parent.mkdir(parents=True) + self._env = patch.dict("os.environ", { + "BOT_BOTTLE_ROOT": str(root), + "SUPERVISE_DB_PATH": str(db), + }) + self._env.start() + self.store = RegistryStore(db) + self.store.migrate() + StoreManager(db).migrate() + secret = secrets.token_bytes(16) + self.orch = Orchestrator(self.store, StubBroker(secret), secret) + + def tearDown(self) -> None: + self._env.stop() + self._tmp.cleanup() + + def _register(self, source_ip: str = "10.243.0.9", slug: str = "demo"): + return self.store.register( + source_ip, metadata=json.dumps({"slug": slug}), policy="routes: []\n") + + def _propose(self, rec, proposed: str = "routes:\n - host: g.com\n"): + return dispatch(self.orch, "POST", "/supervise/propose", _body({ + "source_ip": rec.source_ip, "identity_token": rec.identity_token, + "tool": TOOL_EGRESS_ALLOW, "proposed_file": proposed, "justification": "need it", + })) + + def _poll(self, rec, proposal_id: str): + return dispatch(self.orch, "POST", "/supervise/poll", _body({ + "source_ip": rec.source_ip, "identity_token": rec.identity_token, + "proposal_id": proposal_id, + })) + + def test_propose_queues_under_the_resolved_bottle(self) -> None: + rec = self._register() + status, payload = self._propose(rec) + self.assertEqual(201, status) + pid = payload["proposal_id"] + assert isinstance(pid, str) and pid + _, listing = dispatch(self.orch, "GET", "/supervise/proposals", b"") + self.assertEqual(pid, listing["proposals"][0]["id"]) + # Queued under the orchestrator-resolved bottle id, never a caller slug. + self.assertEqual(rec.bottle_id, listing["proposals"][0]["bottle_slug"]) + + def test_propose_unattributed_is_403(self) -> None: + status, payload = dispatch(self.orch, "POST", "/supervise/propose", _body({ + "source_ip": "10.9.9.9", "identity_token": "wrong", + "tool": TOOL_EGRESS_ALLOW, "proposed_file": "x\n", "justification": "j", + })) + self.assertEqual(403, status) + self.assertIn("unattributed", str(payload["error"])) + + def test_propose_rejects_unknown_tool(self) -> None: + rec = self._register() + status, _ = dispatch(self.orch, "POST", "/supervise/propose", _body({ + "source_ip": rec.source_ip, "identity_token": rec.identity_token, + "tool": "not-a-tool", "proposed_file": "x\n", "justification": "j", + })) + self.assertEqual(400, status) + + def test_poll_pending_then_decided_then_archived(self) -> None: + rec = self._register() + _, proposed = self._propose(rec) + pid = proposed["proposal_id"] + assert isinstance(pid, str) + + status, poll = self._poll(rec, pid) + self.assertEqual(200, status) + self.assertEqual("pending", poll["status"]) + + # Operator decides server-side. + dispatch(self.orch, "POST", "/supervise/respond", _body({ + "proposal_id": pid, "bottle_slug": rec.bottle_id, + "decision": "approve", "notes": "ok", + })) + _, decided = self._poll(rec, pid) + self.assertEqual("approved", decided["status"]) + self.assertEqual("ok", decided["notes"]) + + # The decided poll archived it: gone from pending, and a re-poll is + # 'unknown' rather than replaying the decision forever. + _, listing = dispatch(self.orch, "GET", "/supervise/proposals", b"") + self.assertEqual([], listing["proposals"]) + _, again = self._poll(rec, pid) + self.assertEqual("unknown", again["status"]) + + def test_poll_cannot_read_another_bottles_proposal(self) -> None: + rec_a = self._register("10.0.0.1", "a") + rec_b = self._register("10.0.0.2", "b") + _, proposed = self._propose(rec_a) + pid = proposed["proposal_id"] + assert isinstance(pid, str) + # b polls a's proposal id: scoped to b's own queue → never a's response. + status, poll = self._poll(rec_b, pid) + self.assertEqual(200, status) + self.assertEqual("unknown", poll["status"]) + + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_orchestrator_gateway.py b/tests/unit/test_orchestrator_gateway.py index a413e66..40a14c8 100644 --- a/tests/unit/test_orchestrator_gateway.py +++ b/tests/unit/test_orchestrator_gateway.py @@ -124,13 +124,11 @@ class TestDockerGateway(unittest.TestCase): src = ca_mounts[0].rsplit(":", 1)[0] self.assertTrue(src.endswith("/" + GATEWAY_CA_DIRNAME), src) self.assertTrue(Path(src).is_absolute(), src) - # Shares the ONE host DB: the supervise daemon queues into the same - # file the orchestrator + operator (over HTTP) use. - self.assertTrue(any( - a.startswith("SUPERVISE_DB_PATH=") and a.endswith("/run/supervise/bot-bottle.db") - for a in runs[0])) - self.assertTrue(any( - a.endswith(":/run/supervise") for a in runs[0])) + # No DB handle on the data plane: the supervise queue is reached over + # the control-plane RPC, so the gateway container carries neither the + # DB bind-mount nor SUPERVISE_DB_PATH (PRD 0070 / issue #469). + self.assertFalse(any(a.startswith("SUPERVISE_DB_PATH=") for a in runs[0])) + self.assertFalse(any(a.endswith(":/run/supervise") for a in runs[0])) # Data plane resolves policy against the orchestrator control plane. self.assertIn(f"BOT_BOTTLE_ORCHESTRATOR_URL={_ORCH_URL}", runs[0]) diff --git a/tests/unit/test_orchestrator_service.py b/tests/unit/test_orchestrator_service.py index 61ed48d..cd50bc6 100644 --- a/tests/unit/test_orchestrator_service.py +++ b/tests/unit/test_orchestrator_service.py @@ -323,6 +323,43 @@ class TestOrchestratorSupervise(unittest.TestCase): self.assertFalse(ok) self.assertIn("no longer registered", err) + # --- agent half: queue + poll (issue #469) ----------------------------- + + def test_queue_proposal_then_poll_pending(self) -> None: + bottle_id = self._register("demo", "routes: []\n") + pid = self.orch.supervise_queue_proposal( + bottle_id, tool=TOOL_EGRESS_ALLOW, + proposed_file="routes:\n - host: google.com\n", justification="need it") + # Visible to the operator, keyed by the bottle id. + pending = self.orch.supervise_pending() + self.assertEqual([pid], [p["id"] for p in pending]) + self.assertEqual( + {"status": "pending"}, self.orch.supervise_poll_response(bottle_id, pid)) + + def test_poll_returns_decision_and_archives(self) -> None: + bottle_id = self._register("demo", "routes: []\n") + pid = self.orch.supervise_queue_proposal( + bottle_id, tool=TOOL_EGRESS_ALLOW, + proposed_file="routes:\n - host: google.com\n", justification="need it") + self.orch.supervise_respond( + pid, bottle_slug=bottle_id, decision="approve", notes="ok") + decided = self.orch.supervise_poll_response(bottle_id, pid) + self.assertEqual("approved", decided["status"]) + self.assertEqual("ok", decided["notes"]) + # Archived on read → gone from pending, and a re-poll is 'unknown'. + self.assertEqual([], self.orch.supervise_pending()) + self.assertEqual( + {"status": "unknown"}, self.orch.supervise_poll_response(bottle_id, pid)) + + def test_poll_unknown_for_other_bottle(self) -> None: + bottle_id = self._register("demo", "routes: []\n") + pid = self.orch.supervise_queue_proposal( + bottle_id, tool=TOOL_EGRESS_ALLOW, + proposed_file="routes:\n - host: google.com\n", justification="j") + # A different bottle id can't read demo's proposal (scoped by queue key). + self.assertEqual( + {"status": "unknown"}, self.orch.supervise_poll_response("other-bottle", pid)) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_policy_resolver.py b/tests/unit/test_policy_resolver.py index c06b7a4..6ab5d1c 100644 --- a/tests/unit/test_policy_resolver.py +++ b/tests/unit/test_policy_resolver.py @@ -106,6 +106,58 @@ class TestPolicyResolver(unittest.TestCase): with self.assertRaises(PolicyResolveError): self.r.resolve_policy_and_bottle_id("10.243.0.1") + # --- supervise agent RPCs (issue #469) --------------------------------- + + def test_propose_supervise_returns_id_and_posts_payload(self) -> None: + with patch(_URLOPEN, return_value=_resp({"proposal_id": "p-7"})) as m: + pid = self.r.propose_supervise( + "10.243.0.7", "the-token", + tool="egress-allow", proposed_file="routes:\n", justification="j", + ) + self.assertEqual("p-7", pid) + req = m.call_args.args[0] + self.assertTrue(req.full_url.endswith("/supervise/propose")) + sent = json.loads(req.data) + self.assertEqual("10.243.0.7", sent["source_ip"]) + self.assertEqual("the-token", sent["identity_token"]) + self.assertEqual("egress-allow", sent["tool"]) + self.assertEqual("routes:\n", sent["proposed_file"]) + + def test_propose_supervise_unattributed_is_none(self) -> None: + with patch(_URLOPEN, side_effect=_http_error(403)): + self.assertIsNone(self.r.propose_supervise( + "10.9.9.9", "t", tool="egress-allow", proposed_file="x", justification="j")) + + def test_propose_supervise_missing_id_is_none(self) -> None: + with patch(_URLOPEN, return_value=_resp({})): + self.assertIsNone(self.r.propose_supervise( + "10.243.0.1", "t", tool="egress-allow", proposed_file="x", justification="j")) + + def test_propose_supervise_unreachable_raises(self) -> None: + with patch(_URLOPEN, side_effect=urllib.error.URLError("refused")): + with self.assertRaises(PolicyResolveError): + self.r.propose_supervise( + "10.243.0.1", "t", tool="egress-allow", proposed_file="x", justification="j") + + def test_poll_supervise_returns_status(self) -> None: + with patch(_URLOPEN, return_value=_resp( + {"status": "approved", "notes": "ok", "final_file": None}) + ) as m: + result = self.r.poll_supervise("10.243.0.7", "tok", "p-7") + self.assertEqual("approved", result["status"]) + req = m.call_args.args[0] + self.assertTrue(req.full_url.endswith("/supervise/poll")) + self.assertEqual("p-7", json.loads(req.data)["proposal_id"]) + + def test_poll_supervise_unattributed_is_none(self) -> None: + with patch(_URLOPEN, side_effect=_http_error(403)): + self.assertIsNone(self.r.poll_supervise("10.9.9.9", "t", "p-7")) + + def test_poll_supervise_unreachable_raises(self) -> None: + with patch(_URLOPEN, side_effect=urllib.error.URLError("refused")): + with self.assertRaises(PolicyResolveError): + self.r.poll_supervise("10.243.0.1", "t", "p-7") + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_supervise_server.py b/tests/unit/test_supervise_server.py index 115b7e0..725943f 100644 --- a/tests/unit/test_supervise_server.py +++ b/tests/unit/test_supervise_server.py @@ -1,4 +1,11 @@ -"""Unit: supervise daemon MCP server (PRD 0013).""" +"""Unit: supervise daemon MCP server (PRD 0013, PRD 0070). + +The daemon no longer opens bot-bottle.db: it queues proposals and polls for +their responses over the control-plane RPC (issue #469). These tests drive the +handlers with `_FakeSuperviseResolver`, an in-process stand-in for +`PolicyResolver.propose_supervise` / `poll_supervise` backed by the real queue +store — so the operator-response and archive contracts are still exercised +end-to-end, just through the RPC seam instead of a direct file handle.""" import http.client import json @@ -44,6 +51,81 @@ from bot_bottle.supervise_server import ( validate_proposed_file, ) +# Fixed caller identity for the handler tests. The control plane attributes by +# (source_ip, identity_token); the fake resolver ignores them and answers for a +# fixed bottle, since attribution itself is covered by the orchestrator tests. +_SRC = "10.0.0.7" +_TOK = "tok" + + +class _FakeSuperviseResolver: + """Stand-in for `PolicyResolver`'s supervise RPCs, backed by the real queue + store (as the orchestrator is). `bottle_id=None` models an unattributed + caller (a clean 403 → None); `raises=True` models an unreachable + orchestrator (`PolicyResolveError`).""" + + def __init__( + self, bottle_id: str | None = "dev", raises: bool = False, policy: str = "", + ) -> None: + self.bottle_id = bottle_id + self.raises = raises + self._policy = policy + + def propose_supervise( + self, source_ip, identity_token, *, tool, proposed_file, justification, + ): + del source_ip, identity_token + if self.raises: + raise supervise_server.PolicyResolveError("orchestrator down") + if self.bottle_id is None: + return None + proposal = _sv.Proposal.new( + bottle_slug=self.bottle_id, tool=tool, proposed_file=proposed_file, + justification=justification, current_file_hash=_sv.sha256_hex(proposed_file), + ) + _sv.write_proposal(proposal) + return proposal.id + + def poll_supervise(self, source_ip, identity_token, proposal_id): + del source_ip, identity_token + if self.raises: + raise supervise_server.PolicyResolveError("orchestrator down") + if self.bottle_id is None: + return None + try: + response = _sv.read_response(self.bottle_id, proposal_id) + except FileNotFoundError: + try: + _sv.read_proposal(self.bottle_id, proposal_id) + except FileNotFoundError: + return {"status": _sv.POLL_STATUS_UNKNOWN} + return {"status": _sv.POLL_STATUS_PENDING} + _sv.archive_proposal(self.bottle_id, proposal_id) + return { + "status": response.status, "notes": response.notes, + "final_file": response.final_file, + } + + # Used by list-egress-routes (`_resolved_routes_payload`), unchanged path. + def resolve_policy_and_bottle_id(self, source_ip, identity_token=""): + del source_ip, identity_token + if self.raises: + raise supervise_server.PolicyResolveError("orchestrator down") + return self._policy, self.bottle_id, {} + + +def _tools_call(resolver, params, config=None): + return handle_tools_call( + params, config or ServerConfig(), + resolver=resolver, source_ip=_SRC, identity_token=_TOK, + ) + + +def _check(resolver, params): + return handle_check_proposal( + params, resolver=resolver, source_ip=_SRC, identity_token=_TOK, + ) + # --- Validation ------------------------------------------------------------ @@ -111,32 +193,35 @@ class TestRpcErrorTaxonomy(unittest.TestCase): validate_proposed_file(_sv.TOOL_EGRESS_ALLOW, "routes: nope\n") def test_unknown_tool_in_tools_call_is_client_error(self): - config = ServerConfig(bottle_slug="dev") with self.assertRaises(_RpcClientError) as cm: - handle_tools_call({"name": "no-such-tool", "arguments": {}}, config) + _tools_call(_FakeSuperviseResolver(), {"name": "no-such-tool", "arguments": {}}) self.assertEqual(ERR_INVALID_PARAMS, cm.exception.code) -class TestRpcInternalErrorOnIoFailure(unittest.TestCase): - def test_write_proposal_os_error_raises_internal(self): - config = ServerConfig( - bottle_slug="dev", - ) - with patch.object(_sv, "write_proposal", side_effect=OSError("disk full")), \ - self.assertRaises(_RpcInternalError) as cm: - handle_tools_call( - { - "name": _sv.TOOL_EGRESS_ALLOW, - "arguments": { - "routes_yaml": "routes:\n - host: example.com\n", - "justification": "x", - }, - }, - config, - ) +class TestRpcInternalErrorOnRpcFailure(unittest.TestCase): + """A queue RPC that can't reach the orchestrator (or returns unattributed) + surfaces as ERR_INTERNAL — the daemon fails closed rather than leaking the + cause to the agent.""" + + _ARGS = { + "name": _sv.TOOL_EGRESS_ALLOW, + "arguments": { + "routes_yaml": "routes:\n - host: example.com\n", + "justification": "x", + }, + } + + def test_unreachable_orchestrator_raises_internal(self): + with self.assertRaises(_RpcInternalError) as cm: + _tools_call(_FakeSuperviseResolver(raises=True), self._ARGS) self.assertEqual(ERR_INTERNAL, cm.exception.code) self.assertIsNotNone(cm.exception.__cause__) + def test_unattributed_source_raises_internal(self): + with self.assertRaises(_RpcInternalError) as cm: + _tools_call(_FakeSuperviseResolver(bottle_id=None), self._ARGS) + self.assertEqual(ERR_INTERNAL, cm.exception.code) + # --- JSON-RPC parsing ------------------------------------------------------ @@ -265,8 +350,8 @@ class TestHandleToolsList(unittest.TestCase): class TestHandleToolsCall(unittest.TestCase): def setUp(self): self._tmp = tempfile.TemporaryDirectory(prefix="supervise-server-test.") - self._home_patch = self._patch_home(Path(self._tmp.name)) - self.config = ServerConfig(bottle_slug="dev") + self._home_patch = use_bottle_root(Path(self._tmp.name) / ".bot-bottle") + self.resolver = _FakeSuperviseResolver("dev") _qs.QueueStore("dev").migrate() _as.AuditStore().migrate() @@ -274,12 +359,9 @@ class TestHandleToolsCall(unittest.TestCase): self._home_patch() self._tmp.cleanup() - def _patch_home(self, fake_home: Path): - return use_bottle_root(fake_home / ".bot-bottle") - def _respond_when_proposal_appears(self, status: str, notes: str = "") -> threading.Thread: """Background thread: poll the queue for a fresh proposal, write a - matching response. Returns the thread so the test can join it.""" + matching response — the operator half, out of band.""" def runner(): for _ in range(200): pending = _sv.list_pending_proposals("dev") @@ -298,16 +380,13 @@ class TestHandleToolsCall(unittest.TestCase): def test_call_round_trips_through_queue(self): responder = self._respond_when_proposal_appears(_sv.STATUS_APPROVED, notes="lgtm") try: - result = handle_tools_call( - { - "name": _sv.TOOL_EGRESS_BLOCK, - "arguments": { - "routes_yaml": "routes:\n - host: example.com\n", - "justification": "need example.com", - }, + result = _tools_call(self.resolver, { + "name": _sv.TOOL_EGRESS_BLOCK, + "arguments": { + "routes_yaml": "routes:\n - host: example.com\n", + "justification": "need example.com", }, - self.config, - ) + }) finally: responder.join() self.assertFalse(result["isError"]) # type: ignore[index] @@ -318,16 +397,13 @@ class TestHandleToolsCall(unittest.TestCase): def test_allow_round_trips_through_queue(self): responder = self._respond_when_proposal_appears(_sv.STATUS_APPROVED, notes="ok") try: - result = handle_tools_call( - { - "name": _sv.TOOL_EGRESS_ALLOW, - "arguments": { - "routes_yaml": "routes:\n - host: example.com\n", - "justification": "need example.com", - }, + result = _tools_call(self.resolver, { + "name": _sv.TOOL_EGRESS_ALLOW, + "arguments": { + "routes_yaml": "routes:\n - host: example.com\n", + "justification": "need example.com", }, - self.config, - ) + }) finally: responder.join() self.assertFalse(result["isError"]) # type: ignore[index] @@ -338,94 +414,70 @@ class TestHandleToolsCall(unittest.TestCase): def test_rejected_response_sets_isError(self): responder = self._respond_when_proposal_appears(_sv.STATUS_REJECTED, notes="nope") try: - result = handle_tools_call( - { - "name": _sv.TOOL_EGRESS_ALLOW, - "arguments": { - "routes_yaml": "routes:\n - host: example.com\n", - "justification": "needed for tests", - }, + result = _tools_call(self.resolver, { + "name": _sv.TOOL_EGRESS_ALLOW, + "arguments": { + "routes_yaml": "routes:\n - host: example.com\n", + "justification": "needed for tests", }, - self.config, - ) + }) finally: responder.join() self.assertTrue(result["isError"]) # type: ignore[index] def test_invalid_tool_name_raises(self): with self.assertRaises(_RpcError) as cm: - handle_tools_call( - {"name": "not-a-tool", "arguments": {}}, - self.config, - ) + _tools_call(self.resolver, {"name": "not-a-tool", "arguments": {}}) self.assertEqual(ERR_INVALID_PARAMS, cm.exception.code) def test_missing_justification_raises(self): with self.assertRaises(_RpcError): - handle_tools_call( - { - "name": _sv.TOOL_EGRESS_ALLOW, - "arguments": {"routes_yaml": "routes:\n - host: example.com\n"}, - }, - self.config, - ) + _tools_call(self.resolver, { + "name": _sv.TOOL_EGRESS_ALLOW, + "arguments": {"routes_yaml": "routes:\n - host: example.com\n"}, + }) def test_missing_name_raises(self): with self.assertRaises(_RpcError) as cm: - handle_tools_call({"arguments": {}}, self.config) + _tools_call(self.resolver, {"arguments": {}}) self.assertEqual(ERR_INVALID_PARAMS, cm.exception.code) def test_arguments_must_be_object(self): with self.assertRaises(_RpcError) as cm: - handle_tools_call( - { - "name": _sv.TOOL_EGRESS_ALLOW, - "arguments": [], - }, - self.config, - ) + _tools_call(self.resolver, {"name": _sv.TOOL_EGRESS_ALLOW, "arguments": []}) self.assertEqual(ERR_INVALID_PARAMS, cm.exception.code) self.assertIn("must be an object", cm.exception.message) def test_capability_block_call_raises_unknown_tool(self): with self.assertRaises(_RpcError) as cm: - handle_tools_call( - { - "name": "capability-block", - "arguments": { - "dockerfile": "FROM python:3.13\n", - "justification": "need git", - }, + _tools_call(self.resolver, { + "name": "capability-block", + "arguments": { + "dockerfile": "FROM python:3.13\n", + "justification": "need git", }, - self.config, - ) + }) self.assertEqual(ERR_INVALID_PARAMS, cm.exception.code) self.assertIn("unknown tool", cm.exception.message) def test_archives_proposal_after_response(self): responder = self._respond_when_proposal_appears(_sv.STATUS_APPROVED) try: - handle_tools_call( - { - "name": _sv.TOOL_EGRESS_ALLOW, - "arguments": { - "routes_yaml": "routes:\n - host: example.com\n", - "justification": "x", - }, + _tools_call(self.resolver, { + "name": _sv.TOOL_EGRESS_ALLOW, + "arguments": { + "routes_yaml": "routes:\n - host: example.com\n", + "justification": "x", }, - self.config, - ) + }) finally: responder.join() - # No pending proposals left after archive. + # No pending proposals left after the decided poll archives it. self.assertEqual([], _sv.list_pending_proposals("dev")) def test_pending_response_times_out_without_archive(self): - config = ServerConfig( - bottle_slug="dev", - response_timeout_seconds=0.05, - ) - result = handle_tools_call( + result = _tools_call( + self.resolver, { "name": _sv.TOOL_EGRESS_ALLOW, "arguments": { @@ -433,9 +485,8 @@ class TestHandleToolsCall(unittest.TestCase): "justification": "need egress", }, }, - config, + ServerConfig(response_timeout_seconds=0.05), ) - self.assertFalse(result["isError"]) # type: ignore[index] text = result["content"][0]["text"] # type: ignore[index] self.assertIn("status: pending", text) @@ -510,7 +561,7 @@ class TestHttpEndToEnd(unittest.TestCase): self.port = s.getsockname()[1] s.close() self.server = MCPServer(("127.0.0.1", self.port), MCPHandler) - self.server.config = ServerConfig(bottle_slug="dev") + self.server.config = ServerConfig() self.thread = threading.Thread( target=self.server.serve_forever, daemon=True, ) @@ -552,23 +603,21 @@ class TestHttpEndToEnd(unittest.TestCase): ) self.assertEqual(ERR_METHOD_NOT_FOUND, result["error"]["code"]) # type: ignore[index] - def test_internal_error_returns_err_internal_over_http(self): - with patch.object( - supervise_server._sv, "write_proposal", - side_effect=OSError("disk full"), - ): - result = self._post_jsonrpc({ - "jsonrpc": "2.0", - "id": 99, - "method": "tools/call", - "params": { - "name": _sv.TOOL_EGRESS_ALLOW, - "arguments": { - "routes_yaml": "routes:\n - host: example.com\n", - "justification": "x", - }, + def test_no_resolver_fails_closed_over_http(self): + # The test server has no policy_resolver wired, so a proposal tools/call + # fails closed with ERR_INTERNAL rather than queuing anything. + result = self._post_jsonrpc({ + "jsonrpc": "2.0", + "id": 99, + "method": "tools/call", + "params": { + "name": _sv.TOOL_EGRESS_ALLOW, + "arguments": { + "routes_yaml": "routes:\n - host: example.com\n", + "justification": "x", }, - }) + }, + }) self.assertIn("error", result) self.assertEqual(ERR_INTERNAL, result["error"]["code"]) # type: ignore[index] @@ -583,72 +632,23 @@ class TestHttpEndToEnd(unittest.TestCase): conn.close() -class _FakeResolver: - def __init__( - self, - bottle_id: str | None = None, - raises: bool = False, - policy: str = "", - ) -> None: - self._bottle_id = bottle_id - self._raises = raises - self._policy = policy - self.calls: list[str] = [] - - def resolve_bottle_id(self, source_ip: str, identity_token: str = "") -> str | None: - del identity_token - self.calls.append(source_ip) - if self._raises: - # Raise the exact class supervise_server catches (it imports - # policy_resolver flat inside the bundle, package-side in tests). - raise supervise_server.PolicyResolveError("orchestrator down") - return self._bottle_id - - def resolve_policy_and_bottle_id( - self, source_ip: str, identity_token: str = "", - ) -> "tuple[str, str | None, dict[str, str]]": - del identity_token - self.calls.append(source_ip) - if self._raises: - raise supervise_server.PolicyResolveError("orchestrator down") - return self._policy, self._bottle_id, {} - - def _handler(resolver: object) -> MCPHandler: """A bare MCPHandler wired with a server (carrying the resolver) and a - client address, enough to exercise `_attributed_config` off-socket.""" + client address, enough to exercise the resolver-backed paths off-socket.""" h: MCPHandler = MCPHandler.__new__(MCPHandler) h.server = types.SimpleNamespace(policy_resolver=resolver) # type: ignore[assignment] h.client_address = ("10.0.0.7", 4321) + h.headers = {} # type: ignore[assignment] return h -class TestAttributedConfig(unittest.TestCase): - """Each proposal is attributed to the calling bottle by source IP (PRD - 0070); a server without a resolver fails closed rather than queuing under an - unattributed slug.""" +class TestResolverFailClosed(unittest.TestCase): + """A dispatch without a resolver is a misconfiguration, not a tenancy mode + — fail closed rather than queue (or list) anything (PRD 0070).""" - def test_missing_resolver_fails_closed(self) -> None: + def test_missing_resolver_raises(self) -> None: with self.assertRaises(_RpcInternalError): - _handler(None)._attributed_config(ServerConfig(bottle_slug="dev")) - - def test_consolidated_binds_source_ip_bottle(self) -> None: - r = _FakeResolver(bottle_id="bottle-x") - cfg = _handler(r)._attributed_config(ServerConfig(bottle_slug="ignored")) - self.assertEqual("bottle-x", cfg.bottle_slug) # resolved slug wins - self.assertEqual(["10.0.0.7"], r.calls) - - def test_unattributed_source_fails_closed(self) -> None: - with self.assertRaises(_RpcInternalError): - _handler(_FakeResolver(bottle_id=None))._attributed_config( - ServerConfig(bottle_slug="x") - ) - - def test_resolver_error_fails_closed(self) -> None: - with self.assertRaises(_RpcInternalError): - _handler(_FakeResolver(raises=True))._attributed_config( - ServerConfig(bottle_slug="x") - ) + _handler(None)._resolver_or_fail() class TestResolvedRoutesPayload(unittest.TestCase): @@ -664,7 +664,7 @@ class TestResolvedRoutesPayload(unittest.TestCase): " - host: www.google.com\n" ) payload = _handler( - _FakeResolver(bottle_id="b1", policy=policy) + _FakeSuperviseResolver(bottle_id="b1", policy=policy) )._resolved_routes_payload() assert payload is not None self.assertFalse(payload["isError"]) # type: ignore[index] @@ -676,7 +676,7 @@ class TestResolvedRoutesPayload(unittest.TestCase): # resolve_client_context swallows resolver errors → deny-all (empty), # never another bottle's routes. payload = _handler( - _FakeResolver(raises=True) + _FakeSuperviseResolver(raises=True) )._resolved_routes_payload() assert payload is not None data = json.loads(payload["content"][0]["text"]) # type: ignore[index] @@ -698,7 +698,7 @@ class TestNonBlockingSupervise(unittest.TestCase): def setUp(self): self._tmp = tempfile.TemporaryDirectory(prefix="supervise-nonblock-test.") self._home_patch = use_bottle_root(Path(self._tmp.name) / ".bot-bottle") - self.config = ServerConfig(bottle_slug="dev") + self.resolver = _FakeSuperviseResolver("dev") _qs.QueueStore("dev").migrate() _as.AuditStore().migrate() @@ -717,9 +717,6 @@ class TestNonBlockingSupervise(unittest.TestCase): _sv.write_proposal(p) return p - def _check(self, proposal_id: str) -> dict[str, object]: - return handle_check_proposal({"arguments": {"proposal_id": proposal_id}}, self.config) - # --- pending response carries the id --- def test_pending_text_includes_id_and_pointer(self): @@ -730,12 +727,13 @@ class TestNonBlockingSupervise(unittest.TestCase): def test_tools_call_timeout_returns_pending_with_id_and_stays_queued(self): # No responder → the grace window expires → pending, not blocked forever. - result = handle_tools_call( + result = _tools_call( + self.resolver, { "name": _sv.TOOL_EGRESS_ALLOW, "arguments": {"routes_yaml": self._ROUTES, "justification": "x"}, }, - ServerConfig(bottle_slug="dev", response_timeout_seconds=0.05), + ServerConfig(response_timeout_seconds=0.05), ) self.assertFalse(result["isError"]) # type: ignore[index] text = result["content"][0]["text"] # type: ignore[index] @@ -749,7 +747,7 @@ class TestNonBlockingSupervise(unittest.TestCase): def test_check_returns_approved_and_archives(self): p = self._seed_proposal() _sv.write_response("dev", _sv.Response(proposal_id=p.id, status=_sv.STATUS_APPROVED, notes="ok")) - result = self._check(p.id) + result = _check(self.resolver, {"arguments": {"proposal_id": p.id}}) self.assertFalse(result["isError"]) text = result["content"][0]["text"] # type: ignore[index] self.assertIn("status: approved", text) @@ -760,13 +758,13 @@ class TestNonBlockingSupervise(unittest.TestCase): def test_check_rejected_sets_isError(self): p = self._seed_proposal() _sv.write_response("dev", _sv.Response(proposal_id=p.id, status=_sv.STATUS_REJECTED, notes="no")) - result = self._check(p.id) + result = _check(self.resolver, {"arguments": {"proposal_id": p.id}}) self.assertTrue(result["isError"]) self.assertIn("status: rejected", result["content"][0]["text"]) # type: ignore[index] def test_check_pending_when_no_decision_yet(self): p = self._seed_proposal() - result = self._check(p.id) + result = _check(self.resolver, {"arguments": {"proposal_id": p.id}}) self.assertFalse(result["isError"]) text = result["content"][0]["text"] # type: ignore[index] self.assertIn("status: pending", text) @@ -774,40 +772,41 @@ class TestNonBlockingSupervise(unittest.TestCase): self.assertEqual(1, len(_sv.list_pending_proposals("dev"))) # not archived def test_check_unknown_id_is_error(self): - result = self._check("no-such-proposal") + result = _check(self.resolver, {"arguments": {"proposal_id": "no-such-proposal"}}) self.assertTrue(result["isError"]) self.assertIn("status: unknown", result["content"][0]["text"]) # type: ignore[index] def test_check_missing_id_raises(self): with self.assertRaises(_RpcClientError) as cm: - handle_check_proposal({"arguments": {}}, self.config) + _check(self.resolver, {"arguments": {}}) self.assertEqual(ERR_INVALID_PARAMS, cm.exception.code) def test_check_empty_id_raises(self): with self.assertRaises(_RpcClientError) as cm: - handle_check_proposal({"arguments": {"proposal_id": " "}}, self.config) + _check(self.resolver, {"arguments": {"proposal_id": " "}}) self.assertEqual(ERR_INVALID_PARAMS, cm.exception.code) def test_check_arguments_must_be_object(self): with self.assertRaises(_RpcClientError) as cm: - handle_check_proposal({"arguments": []}, self.config) + _check(self.resolver, {"arguments": []}) self.assertEqual(ERR_INVALID_PARAMS, cm.exception.code) def test_full_nonblocking_round_trip(self): # 1. tools/call times out → pending with id - result = handle_tools_call( + result = _tools_call( + self.resolver, { "name": _sv.TOOL_EGRESS_ALLOW, "arguments": {"routes_yaml": self._ROUTES, "justification": "x"}, }, - ServerConfig(bottle_slug="dev", response_timeout_seconds=0.05), + ServerConfig(response_timeout_seconds=0.05), ) pid = _sv.list_pending_proposals("dev")[0].id self.assertIn(pid, result["content"][0]["text"]) # type: ignore[index] # 2. operator decides out-of-band _sv.write_response("dev", _sv.Response(proposal_id=pid, status=_sv.STATUS_APPROVED, notes="ok")) # 3. agent resumes by polling — no re-proposing - poll = self._check(pid) + poll = _check(self.resolver, {"arguments": {"proposal_id": pid}}) self.assertFalse(poll["isError"]) self.assertIn("status: approved", poll["content"][0]["text"]) # type: ignore[index] self.assertEqual([], _sv.list_pending_proposals("dev")) # resolved + archived -- 2.52.0 From 72fdb1d14bc7524f0e61e310afbe5a8241526240 Mon Sep 17 00:00:00 2001 From: claude Date: Fri, 24 Jul 2026 03:24:30 +0000 Subject: [PATCH 02/39] test: annotations + coverage for the supervise RPC seam Add pyright-strict parameter/return annotations to the fake resolvers and test helpers, and cover the new control-plane validation/403 branches (/supervise/propose + /supervise/poll) plus the supervise-server and egress poll-error fail-closed paths, so the diff-coverage gate stays above 90%. No production behavior change. Co-Authored-By: Claude Opus 4.8 --- tests/unit/test_egress_addon_request_flow.py | 26 +++++-- tests/unit/test_orchestrator_control_plane.py | 47 +++++++++++-- tests/unit/test_policy_resolver.py | 1 + tests/unit/test_supervise_server.py | 68 +++++++++++++++---- 4 files changed, 120 insertions(+), 22 deletions(-) diff --git a/tests/unit/test_egress_addon_request_flow.py b/tests/unit/test_egress_addon_request_flow.py index 3de75b9..6896e9d 100644 --- a/tests/unit/test_egress_addon_request_flow.py +++ b/tests/unit/test_egress_addon_request_flow.py @@ -277,16 +277,18 @@ class _SuperviseRpcFake: supervise_status: "str | None" = None propose_error: bool = False + poll_error: bool = False @property - def propose_calls(self) -> list: + def propose_calls(self) -> list[dict[str, str]]: if not hasattr(self, "_propose_calls"): - self._propose_calls: list = [] + self._propose_calls: list[dict[str, str]] = [] return self._propose_calls def propose_supervise( - self, source_ip, identity_token, *, tool, proposed_file, justification, - ): + self, source_ip: str, identity_token: str, *, + tool: str, proposed_file: str, justification: str, + ) -> str: del proposed_file, justification self.propose_calls.append( {"source_ip": source_ip, "identity_token": identity_token, "tool": tool} @@ -295,8 +297,12 @@ class _SuperviseRpcFake: raise PolicyResolveError("orchestrator down") return "prop-1" - def poll_supervise(self, source_ip, identity_token, proposal_id): + def poll_supervise( + self, source_ip: str, identity_token: str, proposal_id: str, + ) -> dict[str, object]: del source_ip, identity_token, proposal_id + if self.poll_error: + raise PolicyResolveError("orchestrator down") if self.supervise_status is None: return {"status": "pending"} return {"status": self.supervise_status, "notes": "", "final_file": None} @@ -583,6 +589,16 @@ class TestSuperviseBranch(unittest.TestCase): self.assertEqual(403, flow.response.status_code) self.assertIn("timed out", flow.response.get_text()) + def test_poll_error_during_wait_times_out_and_blocks(self) -> None: + # A transient orchestrator error on each poll is retried until the + # deadline, then fails closed (blocked) — never forwarded unsupervised. + addon = self._supervised_addon("approved") + cast(Any, addon._resolver).poll_error = True + flow = _Flow(_Request(host="api.example.com", method="POST", body=f"k={_OPENAI_KEY}")) + _run_request(addon, flow) + assert flow.response is not None + self.assertEqual(403, flow.response.status_code) + # --------------------------------------------------------------------------- # Inbound DLP on responses diff --git a/tests/unit/test_orchestrator_control_plane.py b/tests/unit/test_orchestrator_control_plane.py index fe445b2..c30222f 100644 --- a/tests/unit/test_orchestrator_control_plane.py +++ b/tests/unit/test_orchestrator_control_plane.py @@ -21,7 +21,7 @@ from unittest.mock import patch from bot_bottle.orchestrator.broker import StubBroker from bot_bottle.orchestrator.control_plane import dispatch, make_server -from bot_bottle.orchestrator.registry import RegistryStore +from bot_bottle.orchestrator.registry import BottleRecord, RegistryStore from bot_bottle.orchestrator.service import Orchestrator from bot_bottle.store_manager import StoreManager from bot_bottle.supervise import ( @@ -455,13 +455,13 @@ class TestDispatchSuperviseAgentRpc(unittest.TestCase): return self.store.register( source_ip, metadata=json.dumps({"slug": slug}), policy="routes: []\n") - def _propose(self, rec, proposed: str = "routes:\n - host: g.com\n"): + def _propose(self, rec: BottleRecord, proposed: str = "routes:\n - host: g.com\n"): return dispatch(self.orch, "POST", "/supervise/propose", _body({ "source_ip": rec.source_ip, "identity_token": rec.identity_token, "tool": TOOL_EGRESS_ALLOW, "proposed_file": proposed, "justification": "need it", })) - def _poll(self, rec, proposal_id: str): + def _poll(self, rec: BottleRecord, proposal_id: str): return dispatch(self.orch, "POST", "/supervise/poll", _body({ "source_ip": rec.source_ip, "identity_token": rec.identity_token, "proposal_id": proposal_id, @@ -474,9 +474,11 @@ class TestDispatchSuperviseAgentRpc(unittest.TestCase): pid = payload["proposal_id"] assert isinstance(pid, str) and pid _, listing = dispatch(self.orch, "GET", "/supervise/proposals", b"") - self.assertEqual(pid, listing["proposals"][0]["id"]) + proposals = listing["proposals"] + assert isinstance(proposals, list) + self.assertEqual(pid, proposals[0]["id"]) # Queued under the orchestrator-resolved bottle id, never a caller slug. - self.assertEqual(rec.bottle_id, listing["proposals"][0]["bottle_slug"]) + self.assertEqual(rec.bottle_id, proposals[0]["bottle_slug"]) def test_propose_unattributed_is_403(self) -> None: status, payload = dispatch(self.orch, "POST", "/supervise/propose", _body({ @@ -531,6 +533,41 @@ class TestDispatchSuperviseAgentRpc(unittest.TestCase): self.assertEqual(200, status) self.assertEqual("unknown", poll["status"]) + # --- request validation (400) + fail-closed (403) ---------------------- + + def test_propose_invalid_json_is_400(self) -> None: + status, _ = dispatch(self.orch, "POST", "/supervise/propose", b"{not json") + self.assertEqual(400, status) + + def test_propose_missing_fields_are_400(self) -> None: + rec = self._register() + base = { + "source_ip": rec.source_ip, "identity_token": rec.identity_token, + "tool": TOOL_EGRESS_ALLOW, "proposed_file": "routes:\n", "justification": "j", + } + for drop in ("source_ip", "proposed_file", "justification"): + body = {k: v for k, v in base.items() if k != drop} + status, _ = dispatch(self.orch, "POST", "/supervise/propose", _body(body)) + self.assertEqual(400, status, drop) + + def test_poll_invalid_json_is_400(self) -> None: + status, _ = dispatch(self.orch, "POST", "/supervise/poll", b"{not json") + self.assertEqual(400, status) + + def test_poll_missing_fields_are_400(self) -> None: + rec = self._register() + for body in ({"identity_token": rec.identity_token, "proposal_id": "p"}, + {"source_ip": rec.source_ip, "identity_token": rec.identity_token}): + status, _ = dispatch(self.orch, "POST", "/supervise/poll", _body(body)) + self.assertEqual(400, status) + + def test_poll_unattributed_is_403(self) -> None: + status, payload = dispatch(self.orch, "POST", "/supervise/poll", _body({ + "source_ip": "10.9.9.9", "identity_token": "wrong", "proposal_id": "p", + })) + self.assertEqual(403, status) + self.assertIn("unattributed", str(payload["error"])) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_policy_resolver.py b/tests/unit/test_policy_resolver.py index 6ab5d1c..cc5bde0 100644 --- a/tests/unit/test_policy_resolver.py +++ b/tests/unit/test_policy_resolver.py @@ -144,6 +144,7 @@ class TestPolicyResolver(unittest.TestCase): {"status": "approved", "notes": "ok", "final_file": None}) ) as m: result = self.r.poll_supervise("10.243.0.7", "tok", "p-7") + assert result is not None self.assertEqual("approved", result["status"]) req = m.call_args.args[0] self.assertTrue(req.full_url.endswith("/supervise/poll")) diff --git a/tests/unit/test_supervise_server.py b/tests/unit/test_supervise_server.py index 725943f..a3f956f 100644 --- a/tests/unit/test_supervise_server.py +++ b/tests/unit/test_supervise_server.py @@ -15,7 +15,6 @@ import time import types import unittest from pathlib import Path -from unittest.mock import patch from tests.unit import use_bottle_root @@ -66,14 +65,21 @@ class _FakeSuperviseResolver: def __init__( self, bottle_id: str | None = "dev", raises: bool = False, policy: str = "", + poll_raises: bool = False, poll_none: bool = False, ) -> None: self.bottle_id = bottle_id self.raises = raises self._policy = policy + # propose succeeds, but the later poll fails — models an orchestrator + # that becomes unreachable (poll_raises) or unattributes mid-flight + # (poll_none) after the proposal was queued. + self.poll_raises = poll_raises + self.poll_none = poll_none def propose_supervise( - self, source_ip, identity_token, *, tool, proposed_file, justification, - ): + self, source_ip: str, identity_token: str, *, + tool: str, proposed_file: str, justification: str, + ) -> str | None: del source_ip, identity_token if self.raises: raise supervise_server.PolicyResolveError("orchestrator down") @@ -86,11 +92,13 @@ class _FakeSuperviseResolver: _sv.write_proposal(proposal) return proposal.id - def poll_supervise(self, source_ip, identity_token, proposal_id): + def poll_supervise( + self, source_ip: str, identity_token: str, proposal_id: str, + ) -> dict[str, object] | None: del source_ip, identity_token - if self.raises: + if self.raises or self.poll_raises: raise supervise_server.PolicyResolveError("orchestrator down") - if self.bottle_id is None: + if self.bottle_id is None or self.poll_none: return None try: response = _sv.read_response(self.bottle_id, proposal_id) @@ -107,23 +115,28 @@ class _FakeSuperviseResolver: } # Used by list-egress-routes (`_resolved_routes_payload`), unchanged path. - def resolve_policy_and_bottle_id(self, source_ip, identity_token=""): + def resolve_policy_and_bottle_id( + self, source_ip: str, identity_token: str = "", + ) -> tuple[str | None, str | None, dict[str, str]]: del source_ip, identity_token if self.raises: raise supervise_server.PolicyResolveError("orchestrator down") return self._policy, self.bottle_id, {} -def _tools_call(resolver, params, config=None): +def _tools_call( + resolver: object, params: dict[str, object], + config: "ServerConfig | None" = None, +) -> dict[str, object]: return handle_tools_call( params, config or ServerConfig(), - resolver=resolver, source_ip=_SRC, identity_token=_TOK, + resolver=resolver, source_ip=_SRC, identity_token=_TOK, # type: ignore[arg-type] ) -def _check(resolver, params): +def _check(resolver: object, params: dict[str, object]) -> dict[str, object]: return handle_check_proposal( - params, resolver=resolver, source_ip=_SRC, identity_token=_TOK, + params, resolver=resolver, source_ip=_SRC, identity_token=_TOK, # type: ignore[arg-type] ) @@ -203,7 +216,7 @@ class TestRpcInternalErrorOnRpcFailure(unittest.TestCase): surfaces as ERR_INTERNAL — the daemon fails closed rather than leaking the cause to the agent.""" - _ARGS = { + _ARGS: dict[str, object] = { "name": _sv.TOOL_EGRESS_ALLOW, "arguments": { "routes_yaml": "routes:\n - host: example.com\n", @@ -493,6 +506,25 @@ class TestHandleToolsCall(unittest.TestCase): self.assertIn("proposal remains queued", text) self.assertEqual(1, len(_sv.list_pending_proposals("dev"))) + _ALLOW: dict[str, object] = { + "name": _sv.TOOL_EGRESS_ALLOW, + "arguments": {"routes_yaml": "routes:\n - host: example.com\n", "justification": "x"}, + } + + def test_poll_unreachable_after_queue_raises_internal(self): + # Proposal queues, then the orchestrator becomes unreachable on poll. + self.resolver.poll_raises = True + with self.assertRaises(_RpcInternalError) as cm: + _tools_call(self.resolver, self._ALLOW, ServerConfig(response_timeout_seconds=5)) + self.assertEqual(ERR_INTERNAL, cm.exception.code) + + def test_poll_unattributed_after_queue_raises_internal(self): + # Proposal queues, then poll comes back unattributed (fail-closed). + self.resolver.poll_none = True + with self.assertRaises(_RpcInternalError) as cm: + _tools_call(self.resolver, self._ALLOW, ServerConfig(response_timeout_seconds=5)) + self.assertEqual(ERR_INTERNAL, cm.exception.code) + class TestResponseTimeoutEnv(unittest.TestCase): def test_unset_uses_default(self): @@ -776,6 +808,18 @@ class TestNonBlockingSupervise(unittest.TestCase): self.assertTrue(result["isError"]) self.assertIn("status: unknown", result["content"][0]["text"]) # type: ignore[index] + def test_check_poll_unreachable_raises_internal(self): + self.resolver.poll_raises = True + with self.assertRaises(_RpcInternalError) as cm: + _check(self.resolver, {"arguments": {"proposal_id": "p"}}) + self.assertEqual(ERR_INTERNAL, cm.exception.code) + + def test_check_poll_unattributed_raises_internal(self): + self.resolver.poll_none = True + with self.assertRaises(_RpcInternalError) as cm: + _check(self.resolver, {"arguments": {"proposal_id": "p"}}) + self.assertEqual(ERR_INTERNAL, cm.exception.code) + def test_check_missing_id_raises(self): with self.assertRaises(_RpcClientError) as cm: _check(self.resolver, {"arguments": {}}) -- 2.52.0 From 1db2a9eb671a60adb1d6773e5437de7396bfdba0 Mon Sep 17 00:00:00 2001 From: claude Date: Fri, 24 Jul 2026 05:18:39 +0000 Subject: [PATCH 03/39] feat(control-plane): role-scoped signed tokens so the gateway can't drive operator routes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up on #469: the data plane held the same control-plane secret that authorizes every route, so a compromised egress/git-gate could queue a supervise proposal AND approve it (or rewrite policy, read injected tokens) — the (source_ip, identity_token) checks attribute the *bottle*, not the caller. Replace the single shared bearer secret with role-scoped, HMAC-signed tokens (compact HS256 JWTs, stdlib-only — no new dependency): * new `control_auth` mints/verifies `{role}` tokens; roles are `gateway` (data plane) and `cli` (host operator/launcher). * the orchestrator holds only the signing *key* and verifies; `dispatch` gates each route by role — `gateway` reaches /resolve + /supervise/ {propose,poll}, everything else is `cli`-only (401 unauthenticated, 403 wrong role). * the gateway is handed a pre-minted `gateway` token it cannot rewrite into `cli`; the host CLI mints its own `cli` token from the host key. * `gateway_init` scopes the signing key to the orchestrator process and the gateway token to the data-plane daemons, so even in the combined infra container a compromised data-plane daemon never sees the key. Launchers (docker gateway + infra, macOS infra) inject the minted token(s); Firecracker stays open behind its nft boundary. Open mode (no key) still grants full `cli` access — the fail-visible fallback for tests. Co-Authored-By: Claude Opus 4.8 --- bot_bottle/backend/macos_container/infra.py | 24 +++-- bot_bottle/control_auth.py | 100 +++++++++++++++++ bot_bottle/gateway_init.py | 37 +++++-- bot_bottle/orchestrator/client.py | 13 ++- bot_bottle/orchestrator/control_plane.py | 101 +++++++++++------- bot_bottle/orchestrator/gateway.py | 16 +-- bot_bottle/orchestrator/lifecycle.py | 16 ++- bot_bottle/paths.py | 10 ++ bot_bottle/policy_resolver.py | 19 ++-- ..._orchestrator_docker_control_plane_auth.py | 24 ++++- tests/unit/test_control_auth.py | 86 +++++++++++++++ tests/unit/test_gateway_init.py | 24 +++++ tests/unit/test_orchestrator_client.py | 15 +++ tests/unit/test_orchestrator_control_plane.py | 91 ++++++++++------ tests/unit/test_policy_resolver.py | 20 +++- 15 files changed, 485 insertions(+), 111 deletions(-) create mode 100644 bot_bottle/control_auth.py create mode 100644 tests/unit/test_control_auth.py diff --git a/bot_bottle/backend/macos_container/infra.py b/bot_bottle/backend/macos_container/infra.py index cb92540..731a62a 100644 --- a/bot_bottle/backend/macos_container/infra.py +++ b/bot_bottle/backend/macos_container/infra.py @@ -48,7 +48,9 @@ from ...orchestrator.lifecycle import ( OrchestratorStartError, source_hash, ) +from ...control_auth import ROLE_GATEWAY, mint from ...paths import ( + CONTROL_AUTH_JWT_ENV, CONTROL_PLANE_TOKEN_ENV, HOST_DB_FILENAME, host_control_plane_token, @@ -237,18 +239,26 @@ class MacosInfraService: # Baked onto the container so `_source_current` can detect a real # control-plane code change and recreate. "--env", f"BOT_BOTTLE_SOURCE_HASH={current_hash}", - # The control-plane secret, for BOTH the control plane (to require - # it) and the gateway's PolicyResolver (to present it) — they share - # this one container. Bare `--env NAME` inherits the value from the - # run process below, so the secret never lands on argv or in - # `container inspect`'s command line. The agent runs in a SEPARATE - # container that is never given this var, which is the whole point. + # The control-plane signing key (control plane: verifies tokens) and + # the pre-minted `gateway` JWT (the gateway's PolicyResolver: presents + # it) — they share this one container, and gateway_init scopes each to + # its process so a compromised data-plane daemon never sees the key + # (issue #469 review). Bare `--env NAME` inherits the value from the + # run process below, so neither lands on argv or in `container + # inspect`'s command line. The agent runs in a SEPARATE container that + # is never given these vars, which is the whole point. "--env", CONTROL_PLANE_TOKEN_ENV, + "--env", CONTROL_AUTH_JWT_ENV, "--entrypoint", "sh", self.image, "-c", _init_script(self.port), ] - run_env = {**os.environ, CONTROL_PLANE_TOKEN_ENV: host_control_plane_token()} + _signing_key = host_control_plane_token() + run_env = { + **os.environ, + CONTROL_PLANE_TOKEN_ENV: _signing_key, + CONTROL_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key), + } result = container_mod.run_container_argv(argv, env=run_env) if result.returncode != 0: raise OrchestratorStartError( diff --git a/bot_bottle/control_auth.py b/bot_bottle/control_auth.py new file mode 100644 index 0000000..23cf351 --- /dev/null +++ b/bot_bottle/control_auth.py @@ -0,0 +1,100 @@ +"""Role-scoped control-plane credentials (issue #469 review follow-up). + +The control plane no longer trusts a single shared bearer secret for every +route. Instead the orchestrator holds a *signing key* and issues short, +HMAC-signed tokens (compact HS256 JWTs) that embed a **role** naming the kind +of caller: + + * ``gateway`` — the data plane (egress / git-gate / supervise). Restricted to + the agent-facing routes it actually needs (``/resolve``, + ``/supervise/propose``, ``/supervise/poll``). + * ``cli`` — the host operator / launcher. Full access to the mutating and + operator routes (launch/teardown, policy, ``/supervise/respond``, …). + +Only the orchestrator (and the host CLI, which shares the host trust domain) +holds the signing key; the gateway is handed a pre-minted ``gateway`` token it +cannot rewrite into a ``cli`` token. So a compromised data-plane process can no +longer approve its own supervise proposals or drive operator routes — it can +only present the ``gateway`` role it was issued (control_plane rejects it on +operator routes with 403). + +Stdlib-only (HMAC-SHA256 over a JSON payload); no JWT dependency — the project +carries no runtime pip deps. Tokens are **signed, not encrypted** (the role is +not a secret) and **long-lived** (parity with the static token they replace; +the security win is the unforgeable role claim, not rotation). +""" + +from __future__ import annotations + +import base64 +import binascii +import hashlib +import hmac +import json + +ROLE_GATEWAY = "gateway" +ROLE_CLI = "cli" +ROLES: frozenset[str] = frozenset({ROLE_GATEWAY, ROLE_CLI}) + +_ALG = "HS256" + + +def _b64url_encode(raw: bytes) -> str: + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii") + + +def _b64url_decode(text: str) -> bytes: + padded = text + "=" * (-len(text) % 4) + return base64.urlsafe_b64decode(padded.encode("ascii")) + + +def _sign(secret: str, signing_input: str) -> str: + mac = hmac.new(secret.encode("utf-8"), signing_input.encode("ascii"), hashlib.sha256) + return _b64url_encode(mac.digest()) + + +# The fixed, canonical JOSE header — the same for every token we mint. +_HEADER_SEGMENT = _b64url_encode( + json.dumps({"alg": _ALG, "typ": "JWT"}, separators=(",", ":")).encode("utf-8") +) + + +def mint(role: str, secret: str) -> str: + """A compact HS256 token asserting `role`, signed with `secret`. + + Raises ValueError for an unknown role (mint only what the control plane will + accept) or an empty signing key (an unsigned credential is never valid).""" + if role not in ROLES: + raise ValueError(f"unknown control-plane role {role!r}") + if not secret: + raise ValueError("cannot mint a control-plane token without a signing key") + payload = _b64url_encode(json.dumps({"role": role}, separators=(",", ":")).encode("utf-8")) + signing_input = f"{_HEADER_SEGMENT}.{payload}" + return f"{signing_input}.{_sign(secret, signing_input)}" + + +def verify(token: str, secret: str) -> str | None: + """The role a valid `token` carries, or None if it is malformed, wrongly + signed, or names an unknown role. Constant-time signature check; rejects any + header whose alg isn't HS256 (no alg-confusion / `none`).""" + if not token or not secret: + return None + parts = token.split(".") + if len(parts) != 3: + return None + header_b64, payload_b64, sig = parts + expected = _sign(secret, f"{header_b64}.{payload_b64}") + if not hmac.compare_digest(sig, expected): + return None + try: + header = json.loads(_b64url_decode(header_b64)) + payload = json.loads(_b64url_decode(payload_b64)) + except (ValueError, binascii.Error): + return None + if not isinstance(header, dict) or header.get("alg") != _ALG: + return None + role = payload.get("role") if isinstance(payload, dict) else None + return role if isinstance(role, str) and role in ROLES else None + + +__all__ = ["ROLE_GATEWAY", "ROLE_CLI", "ROLES", "mint", "verify"] diff --git a/bot_bottle/gateway_init.py b/bot_bottle/gateway_init.py index 0f24b26..7e9df7e 100644 --- a/bot_bottle/gateway_init.py +++ b/bot_bottle/gateway_init.py @@ -54,6 +54,15 @@ class _DaemonSpec: _EGRESS_ONLY_ENV_PREFIXES: tuple[str, ...] = ("EGRESS_TOKEN_",) _READY_GATED_DAEMONS: tuple[str, ...] = ("git-gate", "git-http") +# The control-plane signing key is the orchestrator's alone — verifying tokens. +# The data-plane daemons instead hold the pre-minted `gateway` JWT they present. +# Scoping each to its process (even in the combined infra container) keeps a +# compromised data-plane daemon from reading the key and minting a `cli` token +# (issue #469 review). Values match paths.CONTROL_PLANE_TOKEN_ENV / +# CONTROL_AUTH_JWT_ENV; hardcoded here so this supervisor stays import-light. +_SIGNING_KEY_ENV = "BOT_BOTTLE_CONTROL_PLANE_TOKEN" +_GATEWAY_JWT_ENV = "BOT_BOTTLE_CONTROL_AUTH_JWT" + # Daemons that must be requested explicitly via BOT_BOTTLE_GATEWAY_DAEMONS # and are NOT started in the default (env-var-unset) case. The orchestrator # only runs in the combined infra container, never in a standalone gateway. @@ -61,16 +70,24 @@ _OPT_IN_DAEMONS: frozenset[str] = frozenset({"orchestrator"}) def _env_for_daemon(name: str, base_env: dict[str, str]) -> dict[str, str]: - """Egress sees the full bundle env. Everyone else gets a copy - with `EGRESS_TOKEN_*` (and any other future egress-only - credential slots) stripped. Returns a fresh dict — callers - can mutate without affecting `base_env`.""" - if name == "egress": - return dict(base_env) - return { - k: v for k, v in base_env.items() - if not any(k.startswith(p) for p in _EGRESS_ONLY_ENV_PREFIXES) - } + """Per-daemon env, scoped to what each process legitimately needs. + Returns a fresh dict — callers can mutate without affecting `base_env`. + + * `EGRESS_TOKEN_*` upstream-auth slots go to egress only. + * the control-plane signing key goes to the orchestrator only. + * the pre-minted `gateway` JWT goes to the data-plane daemons only (the + orchestrator verifies tokens, it never presents one).""" + env = dict(base_env) + if name != "egress": + env = { + k: v for k, v in env.items() + if not any(k.startswith(p) for p in _EGRESS_ONLY_ENV_PREFIXES) + } + if name == "orchestrator": + env.pop(_GATEWAY_JWT_ENV, None) + else: + env.pop(_SIGNING_KEY_ENV, None) + return env # The orchestrator is listed first so it starts before the gateway daemons, diff --git a/bot_bottle/orchestrator/client.py b/bot_bottle/orchestrator/client.py index b3128bf..a349c64 100644 --- a/bot_bottle/orchestrator/client.py +++ b/bot_bottle/orchestrator/client.py @@ -18,6 +18,7 @@ import urllib.request from collections.abc import Iterable from dataclasses import dataclass +from ..control_auth import ROLE_CLI, mint from ..paths import host_control_plane_token from .control_plane import CONTROL_AUTH_HEADER @@ -25,12 +26,14 @@ DEFAULT_TIMEOUT_SECONDS = 5.0 def _host_auth_token() -> str: - """The per-host control-plane secret, or "" if it can't be read. "" means - 'send no auth header' — correct against an open (unconfigured) control - plane, and harmlessly rejected by a secured one.""" + """A freshly minted `cli`-role token, signed with the host control-plane + key, or "" if the key can't be read. The host CLI shares the orchestrator's + trust domain, so it holds the signing key and mints its own operator token; + "" means 'send no auth header' — correct against an open (unconfigured) + control plane, and harmlessly rejected by a secured one.""" try: - return host_control_plane_token() - except OSError: + return mint(ROLE_CLI, host_control_plane_token()) + except (OSError, ValueError): return "" diff --git a/bot_bottle/orchestrator/control_plane.py b/bot_bottle/orchestrator/control_plane.py index 87d7714..add9420 100644 --- a/bot_bottle/orchestrator/control_plane.py +++ b/bot_bottle/orchestrator/control_plane.py @@ -53,7 +53,6 @@ returned only once, to the caller that launches the bottle. from __future__ import annotations -import hmac import http.server import json import os @@ -62,6 +61,7 @@ import sys import typing from urllib.parse import urlsplit +from ..control_auth import ROLE_CLI, ROLES, verify from ..paths import CONTROL_PLANE_TOKEN_ENV from ..supervise_types import TOOLS from .service import Orchestrator @@ -69,14 +69,32 @@ from .service import Orchestrator # JSON body payload type (parsed request / rendered response). Json = dict[str, object] -# The request header carrying the per-host control-plane secret. Every route -# except `GET /health` requires it (see `dispatch`). The trusted callers hold -# the secret (the gateway's PolicyResolver, the host CLI's OrchestratorClient); -# an agent that can merely *reach* the port cannot present it, so it can't -# enumerate bottles, rewrite policy, read injected upstream tokens, or approve -# its own supervise proposals. +# The request header carrying the caller's role-scoped control-plane token (a +# signed JWT naming the caller's role — see control_auth). The role gates which +# routes the caller may reach: the data plane holds a `gateway` token good only +# for the agent-facing lookups; the host CLI holds a `cli` token for the +# operator/mutating routes. An agent that can merely *reach* the port holds no +# token at all, and a compromised gateway holds only `gateway` — neither can +# drive the operator routes (approve proposals, rewrite policy, read tokens). CONTROL_AUTH_HEADER = "x-bot-bottle-control-auth" +# The routes the data plane (role `gateway`) is allowed to reach — exactly the +# per-request lookups PolicyResolver makes. Every other authenticated route is +# operator-only. `cli` is a superset role: it may reach any route. +_GATEWAY_ROUTES: frozenset[tuple[str, str]] = frozenset({ + ("POST", "/resolve"), + ("POST", "/supervise/propose"), + ("POST", "/supervise/poll"), +}) + + +def _allowed_roles(method: str, route: str) -> frozenset[str]: + """The roles permitted on `(method, route)`: `gateway` or `cli` on the + data-plane routes, `cli`-only everywhere else.""" + if (method, route) in _GATEWAY_ROUTES: + return ROLES + return frozenset({ROLE_CLI}) + def _parse_json_object(body: bytes) -> Json: """Parse a JSON object body. Raises ValueError for non-objects / bad JSON.""" @@ -89,28 +107,33 @@ def _parse_json_object(body: bytes) -> Json: def dispatch( # pylint: disable=too-many-return-statements,too-many-branches - orch: Orchestrator, method: str, path: str, body: bytes, *, authorized: bool = True, + orch: Orchestrator, method: str, path: str, body: bytes, *, role: str | None = ROLE_CLI, ) -> tuple[int, Json]: """Route one control-plane request to a (status, payload) pair. Pure — no I/O beyond the orchestrator — so it is fully testable without a socket. - `authorized` is whether the request presented the control-plane secret (or - no secret is configured — see `ControlPlaneServer`). Every route except - `GET /health` requires it: the source-IP + identity-token checks inside - `/resolve` and `/attribute` authenticate the *bottle* a request is about, - not the *caller*, so without this gate any agent that can reach the port - could rewrite another bottle's policy, read the injected upstream tokens, - or approve its own supervise proposals. Defaults True so unit tests of the - routing logic don't have to thread it through.""" + `role` is the caller's verified control-plane role (`gateway` or `cli`), or + None for an unauthenticated request; an open-mode server (no signing key + configured — see `ControlPlaneServer`) passes `cli`. Every route except + `GET /health` requires a role: a missing role is 401, and a role that + doesn't cover the route is 403 — so a `gateway` data-plane token can reach + `/resolve` + `/supervise/{propose,poll}` but not the operator routes + (rewrite policy, read injected tokens, approve its own supervise proposals). + The source-IP + identity-token checks inside `/resolve` and `/attribute` + authenticate the *bottle* a request is about, not the *caller*, so this role + gate is what protects the caller-privileged routes. Defaults `cli` so unit + tests of the routing logic don't have to thread it through.""" route = urlsplit(path).path.rstrip("/") or "/" if method == "GET" and route == "/health": return 200, {"status": "ok"} - if not authorized: - # Everything below is a trusted-caller operation. Deny before touching - # the registry / broker / supervise store. + # Role gate — every route below is a trusted-caller operation. Deny before + # touching the registry / broker / supervise store. + if role is None: return 401, {"error": "control-plane authentication required"} + if role not in _allowed_roles(method, route): + return 403, {"error": "insufficient role for this route"} if method == "GET" and route == "/gateway": return 200, orch.gateway_status() @@ -343,10 +366,10 @@ class Handler(http.server.BaseHTTPRequestHandler): assert isinstance(server, ControlPlaneServer) length = int(self.headers.get("Content-Length") or 0) body = self.rfile.read(length) if length > 0 else b"" - authorized = server.is_authorized(self.headers.get(CONTROL_AUTH_HEADER, "")) + role = server.role_for(self.headers.get(CONTROL_AUTH_HEADER, "")) try: status, payload = dispatch( - server.orchestrator, method, self.path, body, authorized=authorized) + server.orchestrator, method, self.path, body, role=role) except Exception as e: # noqa: BLE001 — the control plane must stay up sys.stderr.write(f"orchestrator: {method} {self.path} failed: {e!r}\n") sys.stderr.flush() @@ -374,22 +397,24 @@ class Handler(http.server.BaseHTTPRequestHandler): class ControlPlaneServer(socketserver.ThreadingMixIn, http.server.HTTPServer): """Threading HTTP server that carries the orchestrator for its handlers. - Holds the per-host control-plane secret (from `$BOT_BOTTLE_CONTROL_PLANE_TOKEN`, - injected by the launcher into this container only). When a secret is set, - every route but `/health` requires it; when it is unset the server runs - **open** and says so loudly at startup — a fail-visible fallback for tests - and any backend that hasn't wired the secret yet (e.g. Firecracker, whose - nft boundary already blocks agents from the control-plane port).""" + Holds the per-host control-plane *signing key* (from + `$BOT_BOTTLE_CONTROL_PLANE_TOKEN`, injected by the launcher into the + orchestrator process only) and verifies each request's role-scoped token + against it. When a key is set, every route but `/health` requires a valid + token whose role covers the route; when it is unset the server runs **open** + (full `cli` access) and says so loudly at startup — a fail-visible fallback + for tests and any backend that hasn't wired the key yet (e.g. Firecracker, + whose nft boundary already blocks agents from the control-plane port).""" daemon_threads = True allow_reuse_address = True def __init__(self, address: tuple[str, int], orchestrator: Orchestrator) -> None: self.orchestrator = orchestrator - self._auth_token = os.environ.get(CONTROL_PLANE_TOKEN_ENV, "").strip() - if not self._auth_token: + self._signing_key = os.environ.get(CONTROL_PLANE_TOKEN_ENV, "").strip() + if not self._signing_key: sys.stderr.write( - "orchestrator: WARNING — no control-plane secret " + "orchestrator: WARNING — no control-plane signing key " f"(${CONTROL_PLANE_TOKEN_ENV}); running WITHOUT caller " "authentication. Any client that can reach this port can drive " "it. Backends that put the control plane on an agent-reachable " @@ -398,13 +423,15 @@ class ControlPlaneServer(socketserver.ThreadingMixIn, http.server.HTTPServer): sys.stderr.flush() super().__init__(address, Handler) - def is_authorized(self, presented: str) -> bool: - """True iff the request may proceed past `/health`: either no secret is - configured (open mode) or the presented header matches it. Constant-time - compare so a wrong token leaks nothing timing-wise.""" - if not self._auth_token: - return True - return hmac.compare_digest(presented, self._auth_token) + def role_for(self, presented: str) -> str | None: + """The role the request is authorized as, or None if unauthenticated. + Open mode (no signing key) grants full `cli` access — the fail-visible + fallback. Otherwise verify the presented signed token; a missing/invalid + token yields None (→ 401), a valid one yields its `gateway`/`cli` + role (→ per-route 401/403 in `dispatch`).""" + if not self._signing_key: + return ROLE_CLI + return verify(presented, self._signing_key) def make_server( diff --git a/bot_bottle/orchestrator/gateway.py b/bot_bottle/orchestrator/gateway.py index 16bf1e6..cc12330 100644 --- a/bot_bottle/orchestrator/gateway.py +++ b/bot_bottle/orchestrator/gateway.py @@ -22,9 +22,10 @@ import os import time from pathlib import Path +from ..control_auth import ROLE_GATEWAY, mint from ..docker_cmd import run_docker from ..paths import ( - CONTROL_PLANE_TOKEN_ENV, + CONTROL_AUTH_JWT_ENV, host_control_plane_token, host_gateway_ca_dir, ) @@ -252,11 +253,14 @@ class DockerGateway(Gateway): # policy against the control plane per request (guaranteed non-empty by # the check above). argv += ["--env", f"BOT_BOTTLE_ORCHESTRATOR_URL={self._orchestrator_url}"] - # ...and present the control-plane secret on those /resolve calls (the - # control plane requires it). Bare `--env NAME` keeps the value off argv - # / `docker inspect`; only the gateway (not the agent) is given it. - argv += ["--env", CONTROL_PLANE_TOKEN_ENV] - run_env[CONTROL_PLANE_TOKEN_ENV] = host_control_plane_token() + # ...presenting a role-scoped `gateway` token (a signed JWT minted from + # the host signing key) on those calls. The gateway never receives the + # signing key — only this pre-minted token, which it cannot rewrite into + # a `cli` token, so a compromised data-plane process can't drive the + # operator routes (issue #469 review). Bare `--env NAME` keeps the value + # off argv / `docker inspect`; only the gateway (not the agent) is given it. + argv += ["--env", CONTROL_AUTH_JWT_ENV] + run_env[CONTROL_AUTH_JWT_ENV] = mint(ROLE_GATEWAY, host_control_plane_token()) argv.append(self.image_ref) proc = run_docker(argv, env=run_env) if proc.returncode != 0: diff --git a/bot_bottle/orchestrator/lifecycle.py b/bot_bottle/orchestrator/lifecycle.py index 9560c07..b75e87d 100644 --- a/bot_bottle/orchestrator/lifecycle.py +++ b/bot_bottle/orchestrator/lifecycle.py @@ -22,8 +22,10 @@ import urllib.request from pathlib import Path from .. import log +from ..control_auth import ROLE_GATEWAY, mint from ..docker_cmd import run_docker from ..paths import ( + CONTROL_AUTH_JWT_ENV, CONTROL_PLANE_TOKEN_ENV, bot_bottle_root, host_control_plane_token, @@ -186,6 +188,7 @@ class OrchestratorService: so a later `ensure_running` can detect a real code change.""" self._ensure_network() run_docker(["docker", "rm", "--force", self._infra_name]) + _signing_key = host_control_plane_token() proc = run_docker([ "docker", "run", "--detach", "--name", self._infra_name, @@ -209,16 +212,23 @@ class OrchestratorService: # Orchestrator registry DB on the host (sole writer: control plane). "--volume", f"{self._host_root}:{_ROOT_IN_CONTAINER}", "--env", f"BOT_BOTTLE_ROOT={_ROOT_IN_CONTAINER}", - # Control-plane secret: required by the orchestrator (to enforce) - # and by the gateway daemons (to present on /resolve calls). + # Control-plane signing key (orchestrator: verifies tokens) + the + # pre-minted `gateway` JWT (data-plane daemons: present it). gateway_init + # scopes each to its process, so a compromised data-plane daemon never + # sees the key and can't mint a `cli` token (issue #469 review). "--env", CONTROL_PLANE_TOKEN_ENV, + "--env", CONTROL_AUTH_JWT_ENV, # Gateway daemons reach the orchestrator over loopback at its # fixed internal port (DEFAULT_PORT), independent of self.port. "--env", f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{DEFAULT_PORT}", # Opt the orchestrator into gateway_init's supervise tree. "--env", f"BOT_BOTTLE_GATEWAY_DAEMONS={_INFRA_DAEMONS}", self.image, - ], env={**os.environ, CONTROL_PLANE_TOKEN_ENV: host_control_plane_token()}) + ], env={ + **os.environ, + CONTROL_PLANE_TOKEN_ENV: _signing_key, + CONTROL_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key), + }) if proc.returncode != 0: raise OrchestratorStartError( f"infra container failed to start: {proc.stderr.strip()}" diff --git a/bot_bottle/paths.py b/bot_bottle/paths.py index a64179c..629edb5 100644 --- a/bot_bottle/paths.py +++ b/bot_bottle/paths.py @@ -37,7 +37,16 @@ HOST_DB_FILENAME = "bot-bottle.db" # trusted callers (control plane, gateway, host CLI) and never handed to an # agent, so an agent that can reach the control-plane port still can't drive it. CONTROL_PLANE_TOKEN_FILENAME = "control-plane-token" +# The env var carrying the control-plane *signing key* — held only by the +# orchestrator (to verify tokens) and the host CLI (to mint its own), never by +# the data plane. Same value as the host token file; the name is unchanged for +# backward compatibility with existing launchers. CONTROL_PLANE_TOKEN_ENV = "BOT_BOTTLE_CONTROL_PLANE_TOKEN" +# The env var carrying the data plane's pre-minted `gateway`-role token (a +# signed JWT the launcher mints from the signing key). The gateway presents this +# on /resolve + /supervise/{propose,poll}; it never holds the signing key, so it +# cannot forge a higher-privilege `cli` token (issue #469 review). +CONTROL_AUTH_JWT_ENV = "BOT_BOTTLE_CONTROL_AUTH_JWT" # The host directory holding the gateway's persistent mitmproxy CA. Bind-mounted # into the infra/gateway container at mitmproxy's confdir so the self-generated @@ -124,6 +133,7 @@ __all__ = [ "HOST_DB_FILENAME", "CONTROL_PLANE_TOKEN_FILENAME", "CONTROL_PLANE_TOKEN_ENV", + "CONTROL_AUTH_JWT_ENV", "GATEWAY_CA_DIRNAME", "bot_bottle_root", "host_db_path", diff --git a/bot_bottle/policy_resolver.py b/bot_bottle/policy_resolver.py index 5a5dd15..9298a12 100644 --- a/bot_bottle/policy_resolver.py +++ b/bot_bottle/policy_resolver.py @@ -34,21 +34,22 @@ import urllib.request DEFAULT_TIMEOUT_SECONDS = 2.0 -# The control-plane secret this gateway presents on every /resolve call, read -# from the env the launcher injects into the gateway container. The control -# plane requires it (orchestrator/control_plane.py). Constant + env-var name are -# duplicated here rather than imported because this module is COPYed flat into -# the gateway image, free of bot-bottle imports — same rationale as -# IDENTITY_HEADER in egress_addon / git_http_backend. +# The role-scoped `gateway` token this data plane presents on every control-plane +# call, read from the env the launcher injects into the gateway (a pre-minted +# signed JWT — the gateway never holds the signing key, so it can't forge a +# higher-privilege `cli` token). Constant + env-var name are duplicated here +# rather than imported because this module is COPYed flat into the gateway image, +# free of bot-bottle imports — same rationale as IDENTITY_HEADER in egress_addon +# / git_http_backend. CONTROL_AUTH_HEADER = "x-bot-bottle-control-auth" -CONTROL_PLANE_TOKEN_ENV = "BOT_BOTTLE_CONTROL_PLANE_TOKEN" +CONTROL_AUTH_JWT_ENV = "BOT_BOTTLE_CONTROL_AUTH_JWT" def _control_auth_headers() -> dict[str, str]: - """The auth header to send, or {} when no secret is configured (an open + """The auth header to send, or {} when no token is configured (an open control plane, e.g. Firecracker behind its nft boundary — sending nothing is correct there and harmlessly ignored).""" - token = os.environ.get(CONTROL_PLANE_TOKEN_ENV, "").strip() + token = os.environ.get(CONTROL_AUTH_JWT_ENV, "").strip() return {CONTROL_AUTH_HEADER: token} if token else {} diff --git a/tests/integration/test_orchestrator_docker_control_plane_auth.py b/tests/integration/test_orchestrator_docker_control_plane_auth.py index 2c66887..aedb8ca 100644 --- a/tests/integration/test_orchestrator_docker_control_plane_auth.py +++ b/tests/integration/test_orchestrator_docker_control_plane_auth.py @@ -25,6 +25,7 @@ import tempfile import unittest from pathlib import Path +from bot_bottle.control_auth import ROLE_CLI, ROLE_GATEWAY, mint from bot_bottle.orchestrator.client import OrchestratorClient from bot_bottle.orchestrator.lifecycle import OrchestratorService from bot_bottle.paths import host_control_plane_token @@ -85,7 +86,11 @@ class TestDockerControlPlaneAuthIntegration(unittest.TestCase): host_root=host_root, ) cls.svc.ensure_running() - cls.token = host_control_plane_token() + # The control plane now verifies role-scoped signed tokens, not the raw + # key. Mint one of each role from the host signing key (issue #469 review). + signing_key = host_control_plane_token() + cls.cli_token = mint(ROLE_CLI, signing_key) + cls.gateway_token = mint(ROLE_GATEWAY, signing_key) @staticmethod def _teardown_docker( @@ -136,11 +141,17 @@ class TestDockerControlPlaneAuthIntegration(unittest.TestCase): status, _ = self._request("GET", "/bottles", token="not-the-real-secret") self.assertEqual(401, status) - def test_bottles_accepts_the_real_token(self) -> None: - status, payload = self._request("GET", "/bottles", token=self.token) + def test_bottles_accepts_the_cli_token(self) -> None: + status, payload = self._request("GET", "/bottles", token=self.cli_token) self.assertEqual(200, status) self.assertEqual([], payload["bottles"]) + def test_bottles_rejects_the_gateway_token(self) -> None: + """A compromised gateway holds only a `gateway` token — it must not be + able to enumerate bottles (an operator route) with it (issue #469).""" + status, _ = self._request("GET", "/bottles", token=self.gateway_token) + self.assertEqual(403, status) + def test_resolve_rejects_a_caller_with_no_token(self) -> None: """The credential-lift attack from issue #400: an agent could POST /resolve directly and read back the upstream tokens it's never meant @@ -148,6 +159,13 @@ class TestDockerControlPlaneAuthIntegration(unittest.TestCase): status, _ = self._request("POST", "/resolve") self.assertEqual(401, status) + def test_resolve_accepts_the_gateway_token(self) -> None: + """The gateway's own token DOES reach /resolve: with an empty body it + gets 400 (missing source_ip) rather than the 401 a rejected caller sees + — proving the role gate let the gateway token through.""" + status, _ = self._request("POST", "/resolve", token=self.gateway_token) + self.assertEqual(400, status) # past the role gate; body validation fails + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_control_auth.py b/tests/unit/test_control_auth.py new file mode 100644 index 0000000..bfcb7cc --- /dev/null +++ b/tests/unit/test_control_auth.py @@ -0,0 +1,86 @@ +"""Unit: role-scoped control-plane tokens (issue #469 review).""" + +from __future__ import annotations + +import base64 +import json +import unittest + +from bot_bottle.control_auth import ROLE_CLI, ROLE_GATEWAY, ROLES, mint, verify + +_KEY = "test-key" + + +def _b64(obj: object) -> str: + return base64.urlsafe_b64encode( + json.dumps(obj).encode()).rstrip(b"=").decode("ascii") + + +class TestMintVerify(unittest.TestCase): + def test_round_trip_each_role(self) -> None: + for role in ROLES: + self.assertEqual(role, verify(mint(role, _KEY), _KEY)) + + def test_gateway_and_cli_are_distinct(self) -> None: + self.assertEqual(ROLE_GATEWAY, verify(mint(ROLE_GATEWAY, _KEY), _KEY)) + self.assertEqual(ROLE_CLI, verify(mint(ROLE_CLI, _KEY), _KEY)) + + def test_wrong_key_rejected(self) -> None: + self.assertIsNone(verify(mint(ROLE_GATEWAY, _KEY), "other-key")) + + def test_tampered_signature_rejected(self) -> None: + tok = mint(ROLE_CLI, _KEY) + self.assertIsNone(verify(tok[:-1] + ("A" if tok[-1] != "A" else "B"), _KEY)) + + def test_tampered_payload_rejected(self) -> None: + header, _payload, sig = mint(ROLE_GATEWAY, _KEY).split(".") + forged = f"{header}.{_b64({'role': 'cli'})}.{sig}" + self.assertIsNone(verify(forged, _KEY)) + + def test_alg_none_rejected(self) -> None: + # An unsigned "alg: none" token must never verify. + tok = f"{_b64({'alg': 'none', 'typ': 'JWT'})}.{_b64({'role': 'cli'})}." + self.assertIsNone(verify(tok, _KEY)) + + def test_validly_signed_but_wrong_alg_rejected(self) -> None: + # Alg-confusion: even a *correctly signed* token whose header claims a + # non-HS256 alg must be rejected. + from bot_bottle.control_auth import _sign # noqa: PLC0415 + signing_input = f"{_b64({'alg': 'none', 'typ': 'JWT'})}.{_b64({'role': 'cli'})}" + self.assertIsNone(verify(f"{signing_input}.{_sign(_KEY, signing_input)}", _KEY)) + + def test_validly_signed_but_undecodable_rejected(self) -> None: + # A correct signature over a header that isn't valid base64/JSON still + # fails closed rather than raising. + from bot_bottle.control_auth import _sign # noqa: PLC0415 + signing_input = f"!!!not-base64!!!.{_b64({'role': 'cli'})}" + self.assertIsNone(verify(f"{signing_input}.{_sign(_KEY, signing_input)}", _KEY)) + + def test_malformed_tokens_rejected(self) -> None: + for bad in ("", "a", "a.b", "a.b.c.d", "not-base64.$$.$$"): + self.assertIsNone(verify(bad, _KEY)) + + def test_empty_key_never_verifies(self) -> None: + self.assertIsNone(verify(mint(ROLE_CLI, _KEY), "")) + + def test_unknown_role_in_payload_rejected(self) -> None: + header, _p, _s = mint(ROLE_CLI, _KEY).split(".") + # Re-sign a token carrying an unknown role — a valid signature but a + # role the control plane doesn't recognise must still be rejected. + from bot_bottle.control_auth import _sign # noqa: PLC0415 + payload = _b64({"role": "root"}) + signing_input = f"{header}.{payload}" + forged = f"{signing_input}.{_sign(_KEY, signing_input)}" + self.assertIsNone(verify(forged, _KEY)) + + def test_mint_rejects_unknown_role(self) -> None: + with self.assertRaises(ValueError): + mint("root", _KEY) + + def test_mint_rejects_empty_key(self) -> None: + with self.assertRaises(ValueError): + mint(ROLE_GATEWAY, "") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_gateway_init.py b/tests/unit/test_gateway_init.py index 3a3406d..17b5213 100644 --- a/tests/unit/test_gateway_init.py +++ b/tests/unit/test_gateway_init.py @@ -64,6 +64,30 @@ class TestEnvForDaemon(unittest.TestCase): self.assertNotIn("X", self._BASE) +class TestControlPlaneEnvScoping(unittest.TestCase): + """The control-plane signing key stays with the orchestrator; the pre-minted + `gateway` JWT goes to the data-plane daemons (issue #469 review). Scoping + them per-process keeps a compromised data-plane daemon from reading the key + and minting a higher-privilege token, even in the combined infra container.""" + + _BASE = { + "PATH": "/usr/bin", + "BOT_BOTTLE_CONTROL_PLANE_TOKEN": "sk-x", + "BOT_BOTTLE_CONTROL_AUTH_JWT": "gw-jwt", + } + + def test_orchestrator_gets_key_not_jwt(self): + env = _env_for_daemon("orchestrator", self._BASE) + self.assertEqual("sk-x", env["BOT_BOTTLE_CONTROL_PLANE_TOKEN"]) + self.assertNotIn("BOT_BOTTLE_CONTROL_AUTH_JWT", env) + + def test_data_plane_daemons_get_jwt_not_key(self): + for name in ("egress", "git-gate", "git-http", "supervise"): + env = _env_for_daemon(name, self._BASE) + self.assertNotIn("BOT_BOTTLE_CONTROL_PLANE_TOKEN", env, name) + self.assertEqual("gw-jwt", env["BOT_BOTTLE_CONTROL_AUTH_JWT"], name) + + class TestSelectedDaemons(unittest.TestCase): """Env-var subset filtering. The compose renderer is the source of truth for which daemons are wired; the supervisor just diff --git a/tests/unit/test_orchestrator_client.py b/tests/unit/test_orchestrator_client.py index 88a060b..4086ade 100644 --- a/tests/unit/test_orchestrator_client.py +++ b/tests/unit/test_orchestrator_client.py @@ -7,15 +7,30 @@ import unittest import urllib.error from unittest.mock import MagicMock, patch +from bot_bottle.control_auth import ROLE_CLI, verify from bot_bottle.orchestrator.client import ( OrchestratorClient, OrchestratorClientError, RegisteredBottle, + _host_auth_token, ) _URLOPEN = "bot_bottle.orchestrator.client.urllib.request.urlopen" +class TestHostAuthToken(unittest.TestCase): + def test_mints_a_cli_token_from_the_host_key(self) -> None: + with patch("bot_bottle.orchestrator.client.host_control_plane_token", + return_value="signing-key"): + tok = _host_auth_token() + self.assertEqual(ROLE_CLI, verify(tok, "signing-key")) + + def test_returns_empty_when_key_unreadable(self) -> None: + with patch("bot_bottle.orchestrator.client.host_control_plane_token", + side_effect=OSError("no host root")): + self.assertEqual("", _host_auth_token()) + + def _resp(status: int, payload: object) -> MagicMock: m = MagicMock() inner = m.__enter__.return_value diff --git a/tests/unit/test_orchestrator_control_plane.py b/tests/unit/test_orchestrator_control_plane.py index c30222f..a47b376 100644 --- a/tests/unit/test_orchestrator_control_plane.py +++ b/tests/unit/test_orchestrator_control_plane.py @@ -19,6 +19,7 @@ from contextlib import closing from pathlib import Path from unittest.mock import patch +from bot_bottle.control_auth import ROLE_CLI, ROLE_GATEWAY, mint from bot_bottle.orchestrator.broker import StubBroker from bot_bottle.orchestrator.control_plane import dispatch, make_server from bot_bottle.orchestrator.registry import BottleRecord, RegistryStore @@ -285,45 +286,71 @@ class TestServerRoundTrip(unittest.TestCase): class TestControlPlaneAuth(unittest.TestCase): - """The per-host control-plane secret (issue #400): every route but /health - is a trusted-caller op an agent must not be able to drive just because it - can reach the port.""" + """Role-scoped control-plane tokens (issue #400 / #469 review): every route + but /health needs a valid token, and the token's role gates which routes it + reaches — a `gateway` data-plane token can't drive the operator routes.""" + + _OPERATOR_ROUTES = [ + ("GET", "/bottles", b""), + ("POST", "/bottles", _body({"source_ip": "10.0.0.1"})), + ("PUT", "/bottles/x/policy", _body({"policy": "routes: []"})), + ("DELETE", "/bottles/x", b""), + ("POST", "/reconcile", _body({"live_source_ips": []})), + ("POST", "/attribute", _body({"source_ip": "10.0.0.1", "identity_token": "t"})), + ("GET", "/supervise/proposals", b""), + ("POST", "/supervise/respond", + _body({"proposal_id": "p", "bottle_slug": "s", "decision": "a"})), + ] def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() self.addCleanup(self._tmp.cleanup) self.orch = _orchestrator(Path(self._tmp.name) / "r.db") - def test_health_is_public_even_unauthorized(self) -> None: - status, _ = dispatch(self.orch, "GET", "/health", b"", authorized=False) + def test_health_is_public_even_unauthenticated(self) -> None: + status, _ = dispatch(self.orch, "GET", "/health", b"", role=None) self.assertEqual(200, status) - def test_unauthorized_denies_every_other_route(self) -> None: - for method, path, body in [ - ("GET", "/bottles", b""), - ("POST", "/bottles", _body({"source_ip": "10.0.0.1"})), - ("PUT", "/bottles/x/policy", _body({"policy": "routes: []"})), - ("DELETE", "/bottles/x", b""), - ("POST", "/reconcile", _body({"live_source_ips": []})), + def test_unauthenticated_denies_every_other_route(self) -> None: + routes = self._OPERATOR_ROUTES + [ ("POST", "/resolve", _body({"source_ip": "10.0.0.1", "identity_token": "t"})), - ("POST", "/attribute", _body({"source_ip": "10.0.0.1", "identity_token": "t"})), - ("GET", "/supervise/proposals", b""), - ("POST", "/supervise/respond", _body({"proposal_id": "p", "bottle_slug": "s", "decision": "approve"})), - ]: - status, _ = dispatch(self.orch, method, path, body, authorized=False) - self.assertEqual(401, status, f"{method} {path} should be 401 unauthorized") + ("POST", "/supervise/propose", + _body({"source_ip": "1", "tool": "egress-allow", "proposed_file": "x", "justification": "j"})), + ("POST", "/supervise/poll", _body({"source_ip": "1", "proposal_id": "p"})), + ] + for method, path, body in routes: + status, _ = dispatch(self.orch, method, path, body, role=None) + self.assertEqual(401, status, f"{method} {path} should be 401 unauthenticated") + + def test_gateway_role_denied_on_operator_routes(self) -> None: + # A compromised data-plane process holds only a `gateway` token — it must + # not reach the operator routes (approve proposals, rewrite policy, …). + for method, path, body in self._OPERATOR_ROUTES: + status, payload = dispatch(self.orch, method, path, body, role=ROLE_GATEWAY) + self.assertEqual(403, status, f"{method} {path} should be 403 for gateway") + self.assertIn("insufficient role", str(payload.get("error"))) + + def test_gateway_role_allowed_on_data_routes(self) -> None: + # The gateway CAN reach its own lookups. Register a bottle so /resolve + # returns 200 rather than a 403 — proving the role gate let it through. + rec = self.orch.registry.register("10.0.0.9", policy="routes: []", metadata="") + status, _ = dispatch( + self.orch, "POST", "/resolve", + _body({"source_ip": "10.0.0.9", "identity_token": rec.identity_token}), + role=ROLE_GATEWAY) + self.assertEqual(200, status) def test_deny_happens_before_the_registry_is_touched(self) -> None: - """An unauthorized DELETE must not tear a bottle down. 401, and the + """An unauthenticated DELETE must not tear a bottle down. 401, and the bottle is still there.""" rec = self.orch.registry.register("10.0.0.9", policy="", metadata="") status, _ = dispatch( - self.orch, "DELETE", f"/bottles/{rec.bottle_id}", b"", authorized=False) + self.orch, "DELETE", f"/bottles/{rec.bottle_id}", b"", role=None) self.assertEqual(401, status) self.assertIsNotNone(self.orch.registry.get(rec.bottle_id)) - def _server_with_secret(self, secret: str): - with patch.dict("os.environ", {"BOT_BOTTLE_CONTROL_PLANE_TOKEN": secret}): + def _server_with_key(self, signing_key: str): + with patch.dict("os.environ", {"BOT_BOTTLE_CONTROL_PLANE_TOKEN": signing_key}): server = make_server(self.orch, "127.0.0.1", 0) self.addCleanup(server.server_close) threading.Thread(target=server.serve_forever, daemon=True).start() @@ -340,25 +367,29 @@ class TestControlPlaneAuth(unittest.TestCase): except urllib.error.HTTPError as e: return e.code - def test_configured_server_enforces_the_header_over_http(self) -> None: - base = self._server_with_secret("s3cret-admin") + def test_configured_server_enforces_roles_over_http(self) -> None: + key = "test-key" + base = self._server_with_key(key) + gateway_tok = mint(ROLE_GATEWAY, key) + cli_tok = mint(ROLE_CLI, key) # /health is public — no header needed. self.assertEqual(200, self._status(f"{base}/health")) - # /bottles requires the secret. + # /bottles (operator) needs a valid cli token. self.assertEqual(401, self._status(f"{base}/bottles")) self.assertEqual(401, self._status(f"{base}/bottles", header="wrong")) - self.assertEqual(200, self._status(f"{base}/bottles", header="s3cret-admin")) + self.assertEqual(403, self._status(f"{base}/bottles", header=gateway_tok)) + self.assertEqual(200, self._status(f"{base}/bottles", header=cli_tok)) def test_unconfigured_server_runs_open(self) -> None: - """No secret set (tests / nft-protected Firecracker): open mode, so the - existing round-trip and unit behavior are unchanged.""" + """No signing key set (tests / nft-protected Firecracker): open mode + grants full cli access, so existing round-trip behavior is unchanged.""" with patch.dict("os.environ", {}, clear=False): import os os.environ.pop("BOT_BOTTLE_CONTROL_PLANE_TOKEN", None) server = make_server(self.orch, "127.0.0.1", 0) self.addCleanup(server.server_close) - self.assertTrue(server.is_authorized("")) - self.assertTrue(server.is_authorized("anything")) + self.assertEqual(ROLE_CLI, server.role_for("")) + self.assertEqual(ROLE_CLI, server.role_for("anything")) class TestDispatchSupervise(unittest.TestCase): diff --git a/tests/unit/test_policy_resolver.py b/tests/unit/test_policy_resolver.py index cc5bde0..83544ff 100644 --- a/tests/unit/test_policy_resolver.py +++ b/tests/unit/test_policy_resolver.py @@ -7,7 +7,13 @@ import unittest import urllib.error from unittest.mock import MagicMock, patch -from bot_bottle.policy_resolver import PolicyResolveError, PolicyResolver +from bot_bottle.policy_resolver import ( + CONTROL_AUTH_HEADER, + CONTROL_AUTH_JWT_ENV, + PolicyResolveError, + PolicyResolver, + _control_auth_headers, +) _URLOPEN = "bot_bottle.policy_resolver.urllib.request.urlopen" @@ -23,6 +29,18 @@ def _http_error(code: int) -> urllib.error.HTTPError: return urllib.error.HTTPError("http://x/resolve", code, "err", {}, None) # type: ignore[arg-type] +class TestControlAuthHeaders(unittest.TestCase): + def test_sends_the_gateway_jwt_when_configured(self) -> None: + with patch.dict("os.environ", {CONTROL_AUTH_JWT_ENV: "gateway.jwt.tok"}): + self.assertEqual({CONTROL_AUTH_HEADER: "gateway.jwt.tok"}, _control_auth_headers()) + + def test_sends_nothing_when_unset(self) -> None: + import os + with patch.dict("os.environ", {}, clear=False): + os.environ.pop(CONTROL_AUTH_JWT_ENV, None) + self.assertEqual({}, _control_auth_headers()) + + class TestPolicyResolver(unittest.TestCase): def setUp(self) -> None: self.r = PolicyResolver("http://orch:8080") -- 2.52.0 From d8b61b36583e69f2898add3cf75875f427eb0534 Mon Sep 17 00:00:00 2001 From: claude Date: Fri, 24 Jul 2026 06:26:06 +0000 Subject: [PATCH 04/39] fix(supervise): make the response poll idempotent and slow it to 250ms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The control-plane supervise poll archived the proposal on the same call that returned the decision, so a dropped connection lost the operator's decision (the retry saw `unknown`). Poll no longer archives — a re-poll returns the same decision. Decided proposals still drop off the operator's pending list (a response row exists); their rows are reaped when the bottle is torn down or reconciled. Also raise the grace-window poll interval from 50ms to 250ms now that each poll is an HTTP round trip rather than a local file read. Co-Authored-By: Claude Opus 4.8 --- bot_bottle/orchestrator/control_plane.py | 10 ++++--- bot_bottle/orchestrator/service.py | 21 ++++++++----- bot_bottle/queue_store.py | 17 +++++++++++ bot_bottle/supervise.py | 7 +++++ bot_bottle/supervise_server.py | 6 +++- tests/unit/test_orchestrator_control_plane.py | 8 ++--- tests/unit/test_orchestrator_service.py | 30 +++++++++++++++++-- tests/unit/test_supervise_server.py | 17 ++++++----- 8 files changed, 90 insertions(+), 26 deletions(-) diff --git a/bot_bottle/orchestrator/control_plane.py b/bot_bottle/orchestrator/control_plane.py index add9420..5760923 100644 --- a/bot_bottle/orchestrator/control_plane.py +++ b/bot_bottle/orchestrator/control_plane.py @@ -35,10 +35,12 @@ vsock / unix-socket portability caveats): "proposal_id"} The `/supervise/propose` + `/supervise/poll` pair is the **agent** half of the -supervise flow: the data plane (supervise / egress / git-gate) queues a -proposal and polls for its response over RPC instead of opening `bot-bottle.db` -directly. Both attribute the caller by `(source_ip, identity_token)` exactly -like `/resolve`, so a bottle can only ever queue or read its own proposals. +supervise flow: the data plane (supervise / egress / git-gate) queues a proposal +and polls for its response over RPC instead of opening `bot-bottle.db` directly. +`poll` is idempotent — it never archives, so a dropped connection can't lose an +operator decision (the row is reaped when the bottle is torn down / reconciled). +Both attribute the caller by `(source_ip, identity_token)` exactly like +`/resolve`, so a bottle can only ever queue or read its own proposals. `POST /bottles` / `DELETE` drive the full launch lifecycle: they mint (or tear down) the bottle in the registry AND broker the backend-native launch diff --git a/bot_bottle/orchestrator/service.py b/bot_bottle/orchestrator/service.py index 1f042f3..629ab16 100644 --- a/bot_bottle/orchestrator/service.py +++ b/bot_bottle/orchestrator/service.py @@ -40,7 +40,7 @@ from ..supervise import ( STATUS_REJECTED, TOOL_EGRESS_ALLOW, TOOL_EGRESS_BLOCK, - archive_proposal, + archive_all_proposals, list_all_pending_proposals, read_proposal, read_response, @@ -136,6 +136,8 @@ class Orchestrator: self._broker.submit(sign_request(req, self._secret)) self.registry.deregister(bottle_id) self._tokens.pop(bottle_id, None) + # Reap any supervise proposals the gone bottle never acknowledged. + archive_all_proposals(bottle_id) return True def reconcile( @@ -160,6 +162,8 @@ class Orchestrator: live_source_ips, grace_seconds=grace_seconds) for rec in reaped: self._tokens.pop(rec.bottle_id, None) + # Reap any supervise proposals the gone bottle never acknowledged. + archive_all_proposals(rec.bottle_id) return [rec.bottle_id for rec in reaped] def tokens_for(self, bottle_id: str) -> dict[str, str]: @@ -223,15 +227,19 @@ class Orchestrator: return proposal.id def supervise_poll_response(self, bottle_id: str, proposal_id: str) -> dict[str, object]: - """Non-blocking poll of one of `bottle_id`'s queued proposals. + """Non-blocking, **idempotent** poll of one of `bottle_id`'s proposals. Returns `{"status": ...}`: a terminal decision (`approved`/`modified`/ `rejected`, with `notes` + `final_file`) once the operator has responded, `pending` while it's still queued undecided, or `unknown` when no such - queued proposal exists for this bottle (wrong id, or already resolved and - read). A decided proposal is archived here — the same terminal step the - data plane used to run after reading the response — so `pending` - proposals stay visible to the operator until decided *and* polled. + queued proposal exists for this bottle (wrong id, or already archived). + + Poll does **not** archive — a decided proposal keeps returning the same + decision so a client whose connection drops after the decision but before + it consumes the response still gets it on retry (issue #469 review). A + decided proposal is already excluded from the operator's pending list (a + response row exists), so it doesn't linger there; the row itself is reaped + when the bottle is torn down / reconciled. Reads are scoped to `bottle_id` (the queue key), so a caller can never read another bottle's proposal even with a guessed id.""" @@ -243,7 +251,6 @@ class Orchestrator: except FileNotFoundError: return {"status": POLL_STATUS_UNKNOWN} return {"status": POLL_STATUS_PENDING} - archive_proposal(bottle_id, proposal_id) return { "status": response.status, "notes": response.notes, diff --git a/bot_bottle/queue_store.py b/bot_bottle/queue_store.py index b547d90..2dae5c9 100644 --- a/bot_bottle/queue_store.py +++ b/bot_bottle/queue_store.py @@ -192,6 +192,23 @@ class QueueStore(DbStore): (self.queue_key, proposal_id), ) + def archive_all(self) -> None: + """Archive every proposal + response for this queue key. Used to reap a + bottle's supervise records when it is torn down or reconciled away — + the retention path for a client that received a decision but died before + acknowledging it. Idempotent.""" + if not self.db_path.is_file(): + return + with self._connection() as conn: + conn.execute( + "UPDATE supervise_proposals SET archived = 1 WHERE queue_key = ?", + (self.queue_key,), + ) + conn.execute( + "UPDATE supervise_responses SET archived = 1 WHERE queue_key = ?", + (self.queue_key,), + ) + @staticmethod def _row_to_proposal(row: sqlite3.Row) -> Proposal: return Proposal( diff --git a/bot_bottle/supervise.py b/bot_bottle/supervise.py index 827bd30..184500f 100644 --- a/bot_bottle/supervise.py +++ b/bot_bottle/supervise.py @@ -171,6 +171,12 @@ def archive_proposal(bottle_slug: str, proposal_id: str) -> None: QueueStore(bottle_slug).archive_proposal(proposal_id) +def archive_all_proposals(bottle_slug: str) -> None: + """Archive every proposal + response for `bottle_slug` (bottle teardown / + reconcile cleanup). Idempotent.""" + QueueStore(bottle_slug).archive_all() + + # --- Audit log ------------------------------------------------------------- @@ -275,6 +281,7 @@ __all__ = [ "TOOL_EGRESS_TOKEN_ALLOW", "TOOL_LIST_EGRESS_ROUTES", "archive_proposal", + "archive_all_proposals", "audit_dir", "audit_log_path", "list_pending_proposals", diff --git a/bot_bottle/supervise_server.py b/bot_bottle/supervise_server.py index 7a90d21..aad06d1 100644 --- a/bot_bottle/supervise_server.py +++ b/bot_bottle/supervise_server.py @@ -82,7 +82,11 @@ ERR_INVALID_PARAMS = -32602 ERR_INTERNAL = -32603 DEFAULT_RESPONSE_TIMEOUT_SECONDS = 30.0 -MIN_RESPONSE_POLL_INTERVAL_SECONDS = 0.05 +# Grace-window poll cadence. 250 ms keeps the operator-visible latency low while +# staying gentle on the shared control plane — each poll is now an HTTP round +# trip, not a local file read, so the old 50 ms would be ~20 req/s per pending +# proposal across every waiting agent (issue #469 review). +MIN_RESPONSE_POLL_INTERVAL_SECONDS = 0.25 # The per-host orchestrator control plane the shared supervise server attributes # each proposal to, by source IP. Mandatory — there is no single-tenant diff --git a/tests/unit/test_orchestrator_control_plane.py b/tests/unit/test_orchestrator_control_plane.py index a47b376..b400bc5 100644 --- a/tests/unit/test_orchestrator_control_plane.py +++ b/tests/unit/test_orchestrator_control_plane.py @@ -527,7 +527,7 @@ class TestDispatchSuperviseAgentRpc(unittest.TestCase): })) self.assertEqual(400, status) - def test_poll_pending_then_decided_then_archived(self) -> None: + def test_poll_pending_then_decided_then_idempotent(self) -> None: rec = self._register() _, proposed = self._propose(rec) pid = proposed["proposal_id"] @@ -546,12 +546,12 @@ class TestDispatchSuperviseAgentRpc(unittest.TestCase): self.assertEqual("approved", decided["status"]) self.assertEqual("ok", decided["notes"]) - # The decided poll archived it: gone from pending, and a re-poll is - # 'unknown' rather than replaying the decision forever. + # Poll doesn't archive: it's gone from the operator's pending list, but a + # re-poll returns the same decision so a dropped connection can't lose it. _, listing = dispatch(self.orch, "GET", "/supervise/proposals", b"") self.assertEqual([], listing["proposals"]) _, again = self._poll(rec, pid) - self.assertEqual("unknown", again["status"]) + self.assertEqual(decided, again) def test_poll_cannot_read_another_bottles_proposal(self) -> None: rec_a = self._register("10.0.0.1", "a") diff --git a/tests/unit/test_orchestrator_service.py b/tests/unit/test_orchestrator_service.py index cd50bc6..06cf611 100644 --- a/tests/unit/test_orchestrator_service.py +++ b/tests/unit/test_orchestrator_service.py @@ -336,7 +336,7 @@ class TestOrchestratorSupervise(unittest.TestCase): self.assertEqual( {"status": "pending"}, self.orch.supervise_poll_response(bottle_id, pid)) - def test_poll_returns_decision_and_archives(self) -> None: + def test_poll_is_idempotent_and_leaves_no_pending(self) -> None: bottle_id = self._register("demo", "routes: []\n") pid = self.orch.supervise_queue_proposal( bottle_id, tool=TOOL_EGRESS_ALLOW, @@ -346,10 +346,34 @@ class TestOrchestratorSupervise(unittest.TestCase): decided = self.orch.supervise_poll_response(bottle_id, pid) self.assertEqual("approved", decided["status"]) self.assertEqual("ok", decided["notes"]) - # Archived on read → gone from pending, and a re-poll is 'unknown'. + # Decided proposal drops off the operator's pending list (a response row + # exists), but poll does NOT archive — a re-poll returns the same + # decision so a dropped connection can't lose it (issue #469 review). self.assertEqual([], self.orch.supervise_pending()) + self.assertEqual(decided, self.orch.supervise_poll_response(bottle_id, pid)) + + def test_teardown_reaps_the_bottles_proposals(self) -> None: + rec = self.store.register("10.243.0.20", policy="routes: []\n") + pid = self.orch.supervise_queue_proposal( + rec.bottle_id, tool=TOOL_EGRESS_ALLOW, + proposed_file="routes:\n - host: google.com\n", justification="j") + self.orch.supervise_respond( + pid, bottle_slug=rec.bottle_id, decision="approve", notes="ok") + self.assertTrue(self.orch.teardown_bottle(rec.bottle_id)) + # The gone bottle's decided-but-unconsumed proposal is archived, so a + # late poll returns 'unknown' rather than lingering forever. self.assertEqual( - {"status": "unknown"}, self.orch.supervise_poll_response(bottle_id, pid)) + {"status": "unknown"}, self.orch.supervise_poll_response(rec.bottle_id, pid)) + + def test_reconcile_reaps_the_bottles_proposals(self) -> None: + rec = self.store.register("10.243.0.21", policy="routes: []\n") + pid = self.orch.supervise_queue_proposal( + rec.bottle_id, tool=TOOL_EGRESS_ALLOW, + proposed_file="routes:\n - host: google.com\n", justification="j") + # No live source IPs -> the bottle is reaped (grace 0 so it's immediate). + self.assertEqual([rec.bottle_id], self.orch.reconcile([], grace_seconds=0)) + self.assertEqual( + {"status": "unknown"}, self.orch.supervise_poll_response(rec.bottle_id, pid)) def test_poll_unknown_for_other_bottle(self) -> None: bottle_id = self._register("demo", "routes: []\n") diff --git a/tests/unit/test_supervise_server.py b/tests/unit/test_supervise_server.py index a3f956f..c666c55 100644 --- a/tests/unit/test_supervise_server.py +++ b/tests/unit/test_supervise_server.py @@ -108,7 +108,7 @@ class _FakeSuperviseResolver: except FileNotFoundError: return {"status": _sv.POLL_STATUS_UNKNOWN} return {"status": _sv.POLL_STATUS_PENDING} - _sv.archive_proposal(self.bottle_id, proposal_id) + # Idempotent: poll never archives (issue #469 review). return { "status": response.status, "notes": response.notes, "final_file": response.final_file, @@ -473,7 +473,7 @@ class TestHandleToolsCall(unittest.TestCase): self.assertEqual(ERR_INVALID_PARAMS, cm.exception.code) self.assertIn("unknown tool", cm.exception.message) - def test_archives_proposal_after_response(self): + def test_decided_proposal_drops_off_pending(self): responder = self._respond_when_proposal_appears(_sv.STATUS_APPROVED) try: _tools_call(self.resolver, { @@ -485,7 +485,8 @@ class TestHandleToolsCall(unittest.TestCase): }) finally: responder.join() - # No pending proposals left after the decided poll archives it. + # A decided proposal drops off the operator's pending list (a response + # row exists) — poll itself no longer archives (issue #469 review). self.assertEqual([], _sv.list_pending_proposals("dev")) def test_pending_response_times_out_without_archive(self): @@ -776,7 +777,7 @@ class TestNonBlockingSupervise(unittest.TestCase): # --- check-proposal poll --- - def test_check_returns_approved_and_archives(self): + def test_check_returns_approved_idempotently(self): p = self._seed_proposal() _sv.write_response("dev", _sv.Response(proposal_id=p.id, status=_sv.STATUS_APPROVED, notes="ok")) result = _check(self.resolver, {"arguments": {"proposal_id": p.id}}) @@ -784,8 +785,10 @@ class TestNonBlockingSupervise(unittest.TestCase): text = result["content"][0]["text"] # type: ignore[index] self.assertIn("status: approved", text) self.assertIn("notes: ok", text) - with self.assertRaises(FileNotFoundError): # archived on read - _sv.read_proposal("dev", p.id) + # Poll doesn't archive, so a re-check returns the same decision + # (issue #469 review) rather than 'unknown'. + again = _check(self.resolver, {"arguments": {"proposal_id": p.id}}) + self.assertEqual(result, again) def test_check_rejected_sets_isError(self): p = self._seed_proposal() @@ -853,7 +856,7 @@ class TestNonBlockingSupervise(unittest.TestCase): poll = _check(self.resolver, {"arguments": {"proposal_id": pid}}) self.assertFalse(poll["isError"]) self.assertIn("status: approved", poll["content"][0]["text"]) # type: ignore[index] - self.assertEqual([], _sv.list_pending_proposals("dev")) # resolved + archived + self.assertEqual([], _sv.list_pending_proposals("dev")) # resolved (response exists) if __name__ == "__main__": -- 2.52.0 From aac27d8a407634aabb718cbf3360d681449e8e4c Mon Sep 17 00:00:00 2001 From: claude Date: Fri, 24 Jul 2026 07:03:05 +0000 Subject: [PATCH 05/39] fix(firecracker): provision role-scoped control-plane auth in the infra VM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Firecracker infra VM started the control plane with no signing key, so it ran open and granted every unauthenticated caller the `cli` role. The nft boundary only fences off the separate agent VM — the egress / supervise / git-http daemons run in this same VM and reach the orchestrator over 127.0.0.1, so a compromised data-plane daemon could still drive the operator routes (approve its own supervise proposals, rewrite policy, read injected tokens). Generate the signing key on the persistent volume, hand it only to the orchestrator process, mint a `gateway` JWT for the data-plane daemons, and mirror the key back to the host token file so the CLI signs `cli` tokens the VM verifies — the same per-process scoping the docker / macOS launchers use. Co-Authored-By: Claude Opus 4.8 --- bot_bottle/backend/firecracker/infra_vm.py | 52 ++++++++++++++++++++-- tests/unit/test_firecracker_infra_vm.py | 40 +++++++++++++++++ 2 files changed, 88 insertions(+), 4 deletions(-) diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py index 00d962f..cb64dd2 100644 --- a/bot_bottle/backend/firecracker/infra_vm.py +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -33,11 +33,17 @@ from pathlib import Path from typing import Generator from ...log import die, info +from ...paths import CONTROL_PLANE_TOKEN_FILENAME, bot_bottle_root from .. import util as backend_util from ..docker import util as docker_mod from ..docker.gateway_provision import GatewayProvisionError from . import firecracker_vm, infra_artifact, netpool, util +# Where the infra VM keeps its control-plane signing key (generated on the +# persistent /dev/vdb volume mounted at BOT_BOTTLE_ROOT). The host mirrors it +# back so the CLI signs `cli` tokens the VM verifies (issue #469 review). +_GUEST_SIGNING_KEY_PATH = "/var/lib/bot-bottle/control-plane-token" + # The single infra-VM image: gateway data plane + baked control-plane source # (Dockerfile.infra FROM the gateway image). Built from source by default; # a pull-from-registry mode lands later. @@ -167,21 +173,45 @@ def ensure_running() -> InfraVm: want = _expected_version() if _adoptable(key, url, want): info(f"adopting running infra VM at {url}") - return InfraVm(guest_ip=slot.guest_ip, private_key=key) + return _with_signing_key(InfraVm(guest_ip=slot.guest_ip, private_key=key)) with _singleton_lock(): # Re-check under the lock: another launcher may have booted it while # we waited for the lock (double-checked, so we adopt not re-boot). if _adoptable(key, url, want): info(f"adopting running infra VM at {url}") - return InfraVm(guest_ip=slot.guest_ip, private_key=key) + return _with_signing_key(InfraVm(guest_ip=slot.guest_ip, private_key=key)) # Clear a stale/hung/OUTDATED VM holding the link before booting fresh. stop() ensure_built() infra = boot() wait_for_health(infra) _record_booted_version(want) + return _with_signing_key(infra) + + +def _with_signing_key(infra: InfraVm) -> InfraVm: + """Mirror the infra VM's control-plane signing key (generated on its + persistent volume) into the host's control-plane-token file, so the host CLI + signs `cli` tokens the VM verifies (issue #469 review). Best-effort: an + unreadable key is logged, not fatal — the VM still enforces auth, but the CLI + may then be rejected until the key is readable. Returns `infra` for chaining.""" + proc = subprocess.run( + util.ssh_base_argv(infra.private_key, infra.guest_ip) + + [f"cat {_GUEST_SIGNING_KEY_PATH}"], + capture_output=True, text=True, check=False, + ) + signing_key = proc.stdout.strip() + if proc.returncode != 0 or not signing_key: + info("infra signing key not yet readable; control-plane auth may fail") return infra + path = bot_bottle_root() / CONTROL_PLANE_TOKEN_FILENAME + path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as f: + f.write(signing_key) + os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) + return infra @contextmanager @@ -487,7 +517,18 @@ mount -t ext4 /dev/vdb /var/lib/bot-bottle 2>/dev/null || true # Control plane. Source is baked at /app; the package is stdlib-only. cd /app -BOT_BOTTLE_ROOT=/var/lib/bot-bottle python3 -m bot_bottle.orchestrator \\ +# Control-plane signing key + a pre-minted `gateway` JWT, scoped per-process +# (issue #469 review). The key is generated once on the persistent volume and +# handed ONLY to the orchestrator (to verify tokens); the data-plane daemons get +# the `gateway` JWT they present, never the key. Without this the control plane +# would run OPEN and a compromised egress / supervise / git-http daemon in this +# same VM — reaching the orchestrator over 127.0.0.1, past the nft boundary that +# only fences off the separate agent VM — could drive the operator routes +# (approve its own supervise proposals, rewrite policy, read injected tokens). +CP_KEY=$(BOT_BOTTLE_ROOT=/var/lib/bot-bottle python3 -c 'from bot_bottle.paths import host_control_plane_token as t; print(t())') +GW_JWT=$(BB_SIGNING_KEY="$CP_KEY" python3 -c 'import os; from bot_bottle.control_auth import mint, ROLE_GATEWAY; print(mint(ROLE_GATEWAY, os.environ["BB_SIGNING_KEY"]))') + +BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_CONTROL_PLANE_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator \\ --host 0.0.0.0 --port {CONTROL_PLANE_PORT} --broker stub & # Gateway data plane, multi-tenant: each request resolves source-IP -> @@ -495,9 +536,12 @@ BOT_BOTTLE_ROOT=/var/lib/bot-bottle python3 -m bot_bottle.orchestrator \\ # git-http (9420), so the git:// daemon (git-gate, needs a per-bottle # entrypoint the consolidated model doesn't use) is left out. No # SUPERVISE_DB_PATH: the data plane reaches the supervise queue over the -# control-plane RPC and never opens bot-bottle.db (PRD 0070 / #469). +# control-plane RPC and never opens bot-bottle.db (PRD 0070 / #469). It presents +# the pre-minted `gateway` JWT; gateway_init keeps the signing key out of the +# data-plane daemons' env. BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise \\ BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{CONTROL_PLANE_PORT} \\ +BOT_BOTTLE_CONTROL_AUTH_JWT="$GW_JWT" \\ python3 -m bot_bottle.gateway_init & # Reap as PID 1; children are backgrounded, so `wait` blocks. diff --git a/tests/unit/test_firecracker_infra_vm.py b/tests/unit/test_firecracker_infra_vm.py index 8cafd8c..a809344 100644 --- a/tests/unit/test_firecracker_infra_vm.py +++ b/tests/unit/test_firecracker_infra_vm.py @@ -8,6 +8,7 @@ decisions that must hold without a VM. from __future__ import annotations import os +import tempfile import unittest from pathlib import Path from unittest.mock import MagicMock, patch @@ -15,6 +16,37 @@ from unittest.mock import MagicMock, patch from bot_bottle.backend.firecracker import infra_vm +class TestSigningKeySync(unittest.TestCase): + """`_with_signing_key` mirrors the VM's control-plane signing key into the + host token file so the CLI signs `cli` tokens the VM verifies (issue #469).""" + + def _infra(self) -> infra_vm.InfraVm: + return infra_vm.InfraVm(guest_ip="10.0.0.1", private_key=Path("/k")) + + def test_mirrors_guest_key_to_host_file(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d) + proc = MagicMock(returncode=0, stdout="the-signing-key\n") + with patch.object(infra_vm.subprocess, "run", return_value=proc) as run, \ + patch.object(infra_vm, "bot_bottle_root", return_value=root): + out = infra_vm._with_signing_key(self._infra()) + self.assertIsInstance(out, infra_vm.InfraVm) # returned for chaining + token = root / infra_vm.CONTROL_PLANE_TOKEN_FILENAME + self.assertEqual("the-signing-key", token.read_text()) + self.assertEqual(0o600, token.stat().st_mode & 0o777) + # It cat'd the guest volume path over SSH. + self.assertIn(infra_vm._GUEST_SIGNING_KEY_PATH, run.call_args.args[0][-1]) + + def test_unreadable_key_is_not_fatal(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d) + proc = MagicMock(returncode=255, stdout="") + with patch.object(infra_vm.subprocess, "run", return_value=proc), \ + patch.object(infra_vm, "bot_bottle_root", return_value=root): + infra_vm._with_signing_key(self._infra()) # no raise + self.assertFalse((root / infra_vm.CONTROL_PLANE_TOKEN_FILENAME).exists()) + + class TestControlPlaneUrl(unittest.TestCase): def test_url_uses_guest_ip_and_port(self): infra = infra_vm.InfraVm( @@ -46,6 +78,14 @@ class TestBuildInfraRootfs(unittest.TestCase): self.assertIn("/dev/vdb", init) # VM backend uses git-http (9420); the git:// daemon is left out. self.assertIn("BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise", init) + # Role-scoped control-plane auth (issue #469 review): the orchestrator + # gets the signing key, the gateway daemons get a pre-minted `gateway` + # JWT — never open mode in the infra VM. + self.assertIn("host_control_plane_token", init) # key generated on the volume + self.assertIn("mint, ROLE_GATEWAY", init) # gateway JWT minted from it + self.assertIn('BOT_BOTTLE_CONTROL_PLANE_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator', + init) # key -> orchestrator only + self.assertIn('BOT_BOTTLE_CONTROL_AUTH_JWT="$GW_JWT"', init) # JWT -> gateway daemons class TestSshGatewayTransport(unittest.TestCase): -- 2.52.0 From dfce3d9505bbb438ceca8d6512759f86b129e8d0 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 12:39:58 -0400 Subject: [PATCH 06/39] refactor(gateway): move DockerGateway to the backend layer, drop the dead standalone-gateway path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `DockerGateway` container-lifecycle impl now lives in `backend/docker/gateway.py` (the shape backend gateway classes will share); `orchestrator/gateway.py` keeps only the backend-neutral pieces (the `Gateway` ABC, constants, `rotate_gateway_ca`). Delete the standalone-gateway path it was the only consumer of. `--gateway` on `python -m bot_bottle.orchestrator` was invoked nowhere — the production docker flow runs the gateway data plane inside the combined `bot-bottle-infra` container via `OrchestratorService`, never this class. Removing it takes with it `Orchestrator.ensure_gateway()` and the `gateway` ctor arg; `gateway_status()` becomes a stub reporting `configured: false` so the documented `GET /gateway` control-plane route keeps its contract. Fix a latent bug the move surfaced: `launch.py` read the shared gateway CA via `docker exec bot-bottle-orch-gateway`, a container the consolidated flow never creates — so the agent CA install was reaching a nonexistent name. Read it from `bot-bottle-infra` (INFRA_NAME) instead. Repoint the affected tests to the new module; drop the unit test + fake that covered the deleted standalone wiring. Co-Authored-By: Claude Opus 4.8 --- bot_bottle/backend/docker/gateway.py | 189 +++++++++++++++++ bot_bottle/backend/docker/launch.py | 10 +- bot_bottle/orchestrator/__init__.py | 11 +- bot_bottle/orchestrator/__main__.py | 20 +- bot_bottle/orchestrator/gateway.py | 192 +----------------- bot_bottle/orchestrator/service.py | 26 +-- .../test_orchestrator_docker_gateway.py | 2 +- .../test_orchestrator_docker_gateway_build.py | 2 +- tests/unit/test_orchestrator_gateway.py | 6 +- tests/unit/test_orchestrator_service.py | 45 +--- 10 files changed, 226 insertions(+), 277 deletions(-) create mode 100644 bot_bottle/backend/docker/gateway.py diff --git a/bot_bottle/backend/docker/gateway.py b/bot_bottle/backend/docker/gateway.py new file mode 100644 index 0000000..9bc9f93 --- /dev/null +++ b/bot_bottle/backend/docker/gateway.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +import os +import time +from pathlib import Path + +from ...control_auth import ROLE_GATEWAY, mint +from ...docker_cmd import run_docker +from ...paths import ( + CONTROL_AUTH_JWT_ENV, + host_control_plane_token, + host_gateway_ca_dir, +) +from ...orchestrator.gateway import ( + Gateway, GATEWAY_IMAGE, GATEWAY_NAME, GATEWAY_NETWORK, GATEWAY_DOCKERFILE, + REPO_ROOT, GATEWAY_LABEL, MITMPROXY_HOME, DEFAULT_CA_TIMEOUT_SECONDS, + CA_POLL_SECONDS, GATEWAY_CA_CERT, GatewayError +) + +class DockerGateway(Gateway): + """The consolidated gateway as a single, fixed-name Docker container. + + `image_ref` defaults to the gateway data-plane image; `ensure_built` + builds it from `Dockerfile.gateway` when it's missing. (Note: slice 5 + builds + launches the bundle container; wiring its per-bottle, + source-IP-keyed config is a later slice — see PRD 0070.)""" + + def __init__( + self, + image_ref: str = GATEWAY_IMAGE, + *, + name: str = GATEWAY_NAME, + network: str = GATEWAY_NETWORK, + orchestrator_url: str = "", + build_context: Path | None = None, + dockerfile: str | None = GATEWAY_DOCKERFILE, + host_port_bindings: tuple[int, ...] = (), + ) -> None: + self.image_ref = image_ref + self.name = name + self.network = network + # The control-plane URL the gateway's data plane resolves per bottle + # against — reached by container name over docker DNS on the shared + # network (container↔container, no host firewall). Mandatory to *run* + # the gateway (see `ensure_running`); empty is tolerated only for the + # construct-then-read-CA path (`ca_cert_pem` on an already-running + # container), which never launches a container. + self._orchestrator_url = orchestrator_url + self._build_context = build_context or REPO_ROOT + self._dockerfile = dockerfile + # Ports published on the host (0.0.0.0). Used by the Firecracker + # backend's dev-harness gateway so VMs can reach it via their TAP link; + # Docker's DNAT + the nft `ct status dnat accept` rule handle the rest. + self._host_port_bindings = host_port_bindings + + def image_exists(self) -> bool: + return run_docker(["docker", "image", "inspect", self.image_ref]).returncode == 0 + + def ensure_built(self) -> None: + """Build the bundle image from its Dockerfile, **cache-aware** — cheap + (a cache check) when nothing changed, a real rebuild when the flat + sources (egress addon / git-http / policy_resolver / supervise) moved. + + This deliberately builds every time rather than build-if-missing: the + per-bottle model kept the image fresh via compose's `build:` on up, and + a stale image silently runs the OLD single-tenant daemons. No-op only + when no dockerfile is configured (a pre-pulled image). BOT_BOTTLE_NO_CACHE + forces a full rebuild (parity with `start --no-cache`).""" + if self._dockerfile is None: + return + argv = ["docker", "build", "-t", self.image_ref, + "-f", str(self._build_context / self._dockerfile), + str(self._build_context)] + if os.environ.get("BOT_BOTTLE_NO_CACHE"): + argv.insert(2, "--no-cache") + proc = run_docker(argv) + if proc.returncode != 0: + raise GatewayError(f"gateway image build failed: {proc.stderr.strip()}") + + def is_running(self) -> bool: + proc = run_docker([ + "docker", "ps", + "--filter", f"name=^{self.name}$", + "--filter", "status=running", + "--format", "{{.Names}}", + ]) + return self.name in proc.stdout.split() + + def _running_image_is_current(self) -> bool: + """True iff the running gateway was created from the *current* + `image_ref`. When `ensure_built` rebuilds the image (a source change), + the running container is still the OLD image running the OLD flat + daemons — so this is how a rebuild actually takes effect: a mismatch + means recreate.""" + running = run_docker(["docker", "inspect", "--format", "{{.Image}}", self.name]) + current = run_docker(["docker", "image", "inspect", "--format", "{{.Id}}", self.image_ref]) + if running.returncode != 0 or current.returncode != 0: + return True # can't compare → don't churn a working container + return running.stdout.strip() == current.stdout.strip() + + def _ensure_network(self) -> None: + """Create the shared gateway network if it doesn't exist. Idempotent — + a concurrent create loses harmlessly (the loser sees 'already exists'). + Docker picks the subnet; the launcher reads it back to allocate IPs.""" + if run_docker(["docker", "network", "inspect", self.network]).returncode == 0: + return + proc = run_docker(["docker", "network", "create", self.network]) + if proc.returncode != 0 and "already exists" not in proc.stderr: + raise GatewayError( + f"gateway network {self.network} failed to create: {proc.stderr.strip()}" + ) + + def ensure_running(self) -> None: + # Fail closed on a missing policy source. The data-plane daemons are + # resolver-only now (PRD 0070) — without an orchestrator URL egress + # raises, git-http exits 1, and supervise exits 2 — so launching a + # gateway without one would only crash-loop its daemons. Refuse here so + # the misconfiguration surfaces as a clear error, not a broken container. + if not self._orchestrator_url: + raise GatewayError( + "gateway requires an orchestrator URL to run " + "(resolver-only data plane; no single-tenant fallback)" + ) + # Recreate when the running container's image is stale (a rebuild), + # so source changes to the gateway's flat daemons take effect — not + # just when the container is absent. + if self.is_running() and self._running_image_is_current(): + return + self._ensure_network() + # Clear any stale (stopped OR outdated-image) container holding the + # fixed name, then start fresh. `rm --force` on an absent name is a + # tolerated no-op. + run_docker(["docker", "rm", "--force", self.name]) + argv = [ + "docker", "run", "--detach", + "--name", self.name, + "--label", GATEWAY_LABEL, + "--network", self.network, + # Persist the self-generated CA on the host so it survives both + # container recreation AND docker volume pruning (agents trust it) + # — see host_gateway_ca_dir / issue #450. + "--volume", f"{host_gateway_ca_dir()}:{MITMPROXY_HOME}", + # No DB mount: the data plane (egress / supervise / git-gate) reaches + # the supervise queue over the control-plane RPC and never opens + # bot-bottle.db, so the gateway container gets no file handle on it + # (PRD 0070 / issue #469). + ] + for port in self._host_port_bindings: + argv += ["--publish", f"0.0.0.0:{port}:{port}"] + run_env = dict(os.environ) + # The gateway's egress / git / supervise daemons resolve source-IP -> + # policy against the control plane per request (guaranteed non-empty by + # the check above). + argv += ["--env", f"BOT_BOTTLE_ORCHESTRATOR_URL={self._orchestrator_url}"] + # ...presenting a role-scoped `gateway` token (a signed JWT minted from + # the host signing key) on those calls. The gateway never receives the + # signing key — only this pre-minted token, which it cannot rewrite into + # a `cli` token, so a compromised data-plane process can't drive the + # operator routes (issue #469 review). Bare `--env NAME` keeps the value + # off argv / `docker inspect`; only the gateway (not the agent) is given it. + argv += ["--env", CONTROL_AUTH_JWT_ENV] + run_env[CONTROL_AUTH_JWT_ENV] = mint(ROLE_GATEWAY, host_control_plane_token()) + argv.append(self.image_ref) + proc = run_docker(argv, env=run_env) + if proc.returncode != 0: + raise GatewayError(f"gateway failed to start: {proc.stderr.strip()}") + + def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str: + """The gateway's CA certificate (PEM) that agents install to trust its + TLS interception. mitmproxy generates it a moment after the container + starts, so this **polls** for it (up to `timeout`) rather than assuming + it's already there on a fresh gateway — raising only if it never + appears.""" + deadline = time.monotonic() + timeout + while True: + proc = run_docker(["docker", "exec", self.name, "cat", GATEWAY_CA_CERT]) + if proc.returncode == 0 and proc.stdout.strip(): + return proc.stdout + if time.monotonic() >= deadline: + raise GatewayError( + f"gateway CA cert not available after {timeout:g}s: " + f"{proc.stderr.strip() or 'empty'}" + ) + time.sleep(CA_POLL_SECONDS) + + def stop(self) -> None: + proc = run_docker(["docker", "rm", "--force", self.name]) + if proc.returncode != 0 and "No such container" not in proc.stderr: + raise GatewayError(f"gateway failed to stop: {proc.stderr.strip()}") \ No newline at end of file diff --git a/bot_bottle/backend/docker/launch.py b/bot_bottle/backend/docker/launch.py index e767d92..47e2f64 100644 --- a/bot_bottle/backend/docker/launch.py +++ b/bot_bottle/backend/docker/launch.py @@ -66,7 +66,8 @@ from .compose import ( from .consolidated_compose import consolidated_agent_compose from ...orchestrator.config_store import resolve_teardown_timeout from .consolidated_launch import launch_consolidated, teardown_consolidated -from ...orchestrator.gateway import DockerGateway +from ...orchestrator.lifecycle import INFRA_NAME +from .gateway import DockerGateway # Where the repo root lives, for `docker build` context. Computed once. @@ -159,11 +160,14 @@ def launch( ) # Step 4: install the SHARED gateway CA into the agent (replaces the - # per-bottle CA) — read it out of the running gateway. + # per-bottle CA) — read it out of the running gateway. The consolidated + # flow runs the gateway data plane inside the per-host infra container + # (INFRA_NAME), so read the CA from there, not the legacy standalone + # gateway container name. ca_dir = egress_state_dir(plan.slug) / "gateway-ca" ca_dir.mkdir(parents=True, exist_ok=True) ca_file = ca_dir / "gateway-ca.pem" - ca_file.write_text(DockerGateway(network=ctx.network).ca_cert_pem()) + ca_file.write_text(DockerGateway(name=INFRA_NAME).ca_cert_pem()) egress_plan = dataclasses.replace( plan.egress_plan, mitmproxy_ca_host_path=ca_file, diff --git a/bot_bottle/orchestrator/__init__.py b/bot_bottle/orchestrator/__init__.py index 47ff282..3f03a7e 100644 --- a/bot_bottle/orchestrator/__init__.py +++ b/bot_bottle/orchestrator/__init__.py @@ -10,10 +10,10 @@ backend-neutral "consolidation core" that needs no VM packaging: * `broker` — the signed, structured launch-request contract + a `LaunchBroker` (stub for the harness) that verifies provenance before acting. - * `gateway` — the consolidated per-host gateway: a `Gateway` - lifecycle contract (idempotent singleton) + a - `DockerGateway` impl. One gateway shared by all - bottles instead of one per bottle. + * `gateway` — the consolidated per-host gateway: the `Gateway` + lifecycle contract (idempotent singleton). One + gateway shared by all bottles instead of one per + bottle; backend impls live under `backend/*/gateway`. * `service` — the `Orchestrator`: owns the registry, brokers the launch lifecycle (launch/teardown), manages the shared gateway, attributes. @@ -38,7 +38,7 @@ from .broker import ( verify_request, ) from .docker_broker import DockerBroker, DockerBrokerError -from .gateway import DockerGateway, Gateway, GatewayError +from .gateway import Gateway, GatewayError from .service import Orchestrator from .control_plane import ControlPlaneServer, dispatch, make_server @@ -53,7 +53,6 @@ __all__ = [ "DockerBroker", "DockerBrokerError", "Gateway", - "DockerGateway", "GatewayError", "sign_request", "verify_request", diff --git a/bot_bottle/orchestrator/__main__.py b/bot_bottle/orchestrator/__main__.py index 1606e12..0869c90 100644 --- a/bot_bottle/orchestrator/__main__.py +++ b/bot_bottle/orchestrator/__main__.py @@ -22,7 +22,6 @@ from .control_plane import make_server from .docker_broker import DockerBroker from .registry import RegistryStore, default_db_path from .service import Orchestrator -from .gateway import DockerGateway, Gateway def main(argv: list[str] | None = None) -> int: @@ -38,10 +37,6 @@ def main(argv: list[str] | None = None) -> int: "--broker", choices=("stub", "docker"), default="stub", help="launch broker: 'stub' records requests; 'docker' runs containers", ) - parser.add_argument( - "--gateway", action="store_true", - help="run one consolidated per-host gateway (build-if-missing)", - ) args = parser.parse_args(argv) registry = RegistryStore(args.db) @@ -57,20 +52,7 @@ def main(argv: list[str] | None = None) -> int: # anything; 'docker' runs real containers (firecracker drops in later). secret = secrets.token_bytes(32) broker: LaunchBroker = DockerBroker(secret) if args.broker == "docker" else StubBroker(secret) - # Standalone `--gateway` (not the consolidated flow, where the host - # lifecycle runs the gateway). The gateway resolves against this same - # process; the URL is only reachable when they share a docker network. - gateway: Gateway | None = ( - DockerGateway(orchestrator_url=f"http://{args.host}:{args.port}") - if args.gateway else None - ) - orchestrator = Orchestrator(registry, broker, secret, gateway) - - # One persistent per-host gateway, shared by every bottle: build the - # bundle image if missing, then bring the singleton up (idempotent). - if gateway is not None: - orchestrator.ensure_gateway() - log.info("consolidated gateway ensured", context={"name": gateway.name}) + orchestrator = Orchestrator(registry, broker, secret) server = make_server(orchestrator, host=args.host, port=args.port) bound_host, bound_port = server.server_address[0], server.server_address[1] diff --git a/bot_bottle/orchestrator/gateway.py b/bot_bottle/orchestrator/gateway.py index cc12330..a1ad85a 100644 --- a/bot_bottle/orchestrator/gateway.py +++ b/bot_bottle/orchestrator/gateway.py @@ -7,8 +7,9 @@ because the attribution invariant (source IP + identity token, see so per-bottle policy lives in one long-lived process keyed on who's calling. `Gateway` is the backend-neutral lifecycle contract (mirrors `LaunchBroker`): -ensure the single instance is up, report it, tear it down. `DockerGateway` -is the docker implementation; a firecracker gateway VM slots in later. +ensure the single instance is up, report it, tear it down. The docker +implementation (`DockerGateway`) lives in `backend/docker/gateway.py`; a +firecracker gateway VM slots in later. The defining behaviour is **idempotent singleton**: `ensure_running` starts the instance if absent and is a no-op if it's already up, so N bottle @@ -19,20 +20,13 @@ from __future__ import annotations import abc import os -import time from pathlib import Path -from ..control_auth import ROLE_GATEWAY, mint -from ..docker_cmd import run_docker -from ..paths import ( - CONTROL_AUTH_JWT_ENV, - host_control_plane_token, - host_gateway_ca_dir, -) +from ..paths import host_gateway_ca_dir # The gateway's mitmproxy writes its CA a beat after the container starts, so # reads poll for it rather than assuming it's there on a fresh launch. -_CA_POLL_SECONDS = 0.5 +CA_POLL_SECONDS = 0.5 DEFAULT_CA_TIMEOUT_SECONDS = 30.0 GATEWAY_NAME = "bot-bottle-orch-gateway" @@ -66,7 +60,7 @@ GATEWAY_CA_GLOB = "mitmproxy-ca*" # that lands. Env override matches the backend's BOT_BOTTLE_GATEWAY_IMAGE. GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest") GATEWAY_DOCKERFILE = "Dockerfile.gateway" -_REPO_ROOT = Path(__file__).resolve().parents[2] +REPO_ROOT = Path(__file__).resolve().parents[2] def rotate_gateway_ca(ca_dir: Path | None = None) -> list[Path]: @@ -118,180 +112,8 @@ class Gateway(abc.ABC): """Remove the gateway. Idempotent — absent is success.""" -class DockerGateway(Gateway): - """The consolidated gateway as a single, fixed-name Docker container. - - `image_ref` defaults to the gateway data-plane image; `ensure_built` - builds it from `Dockerfile.gateway` when it's missing. (Note: slice 5 - builds + launches the bundle container; wiring its per-bottle, - source-IP-keyed config is a later slice — see PRD 0070.)""" - - def __init__( - self, - image_ref: str = GATEWAY_IMAGE, - *, - name: str = GATEWAY_NAME, - network: str = GATEWAY_NETWORK, - orchestrator_url: str = "", - build_context: Path | None = None, - dockerfile: str | None = GATEWAY_DOCKERFILE, - host_port_bindings: tuple[int, ...] = (), - ) -> None: - self.image_ref = image_ref - self.name = name - self.network = network - # The control-plane URL the gateway's data plane resolves per bottle - # against — reached by container name over docker DNS on the shared - # network (container↔container, no host firewall). Mandatory to *run* - # the gateway (see `ensure_running`); empty is tolerated only for the - # construct-then-read-CA path (`ca_cert_pem` on an already-running - # container), which never launches a container. - self._orchestrator_url = orchestrator_url - self._build_context = build_context or _REPO_ROOT - self._dockerfile = dockerfile - # Ports published on the host (0.0.0.0). Used by the Firecracker - # backend's dev-harness gateway so VMs can reach it via their TAP link; - # Docker's DNAT + the nft `ct status dnat accept` rule handle the rest. - self._host_port_bindings = host_port_bindings - - def image_exists(self) -> bool: - return run_docker(["docker", "image", "inspect", self.image_ref]).returncode == 0 - - def ensure_built(self) -> None: - """Build the bundle image from its Dockerfile, **cache-aware** — cheap - (a cache check) when nothing changed, a real rebuild when the flat - sources (egress addon / git-http / policy_resolver / supervise) moved. - - This deliberately builds every time rather than build-if-missing: the - per-bottle model kept the image fresh via compose's `build:` on up, and - a stale image silently runs the OLD single-tenant daemons. No-op only - when no dockerfile is configured (a pre-pulled image). BOT_BOTTLE_NO_CACHE - forces a full rebuild (parity with `start --no-cache`).""" - if self._dockerfile is None: - return - argv = ["docker", "build", "-t", self.image_ref, - "-f", str(self._build_context / self._dockerfile), - str(self._build_context)] - if os.environ.get("BOT_BOTTLE_NO_CACHE"): - argv.insert(2, "--no-cache") - proc = run_docker(argv) - if proc.returncode != 0: - raise GatewayError(f"gateway image build failed: {proc.stderr.strip()}") - - def is_running(self) -> bool: - proc = run_docker([ - "docker", "ps", - "--filter", f"name=^{self.name}$", - "--filter", "status=running", - "--format", "{{.Names}}", - ]) - return self.name in proc.stdout.split() - - def _running_image_is_current(self) -> bool: - """True iff the running gateway was created from the *current* - `image_ref`. When `ensure_built` rebuilds the image (a source change), - the running container is still the OLD image running the OLD flat - daemons — so this is how a rebuild actually takes effect: a mismatch - means recreate.""" - running = run_docker(["docker", "inspect", "--format", "{{.Image}}", self.name]) - current = run_docker(["docker", "image", "inspect", "--format", "{{.Id}}", self.image_ref]) - if running.returncode != 0 or current.returncode != 0: - return True # can't compare → don't churn a working container - return running.stdout.strip() == current.stdout.strip() - - def _ensure_network(self) -> None: - """Create the shared gateway network if it doesn't exist. Idempotent — - a concurrent create loses harmlessly (the loser sees 'already exists'). - Docker picks the subnet; the launcher reads it back to allocate IPs.""" - if run_docker(["docker", "network", "inspect", self.network]).returncode == 0: - return - proc = run_docker(["docker", "network", "create", self.network]) - if proc.returncode != 0 and "already exists" not in proc.stderr: - raise GatewayError( - f"gateway network {self.network} failed to create: {proc.stderr.strip()}" - ) - - def ensure_running(self) -> None: - # Fail closed on a missing policy source. The data-plane daemons are - # resolver-only now (PRD 0070) — without an orchestrator URL egress - # raises, git-http exits 1, and supervise exits 2 — so launching a - # gateway without one would only crash-loop its daemons. Refuse here so - # the misconfiguration surfaces as a clear error, not a broken container. - if not self._orchestrator_url: - raise GatewayError( - "gateway requires an orchestrator URL to run " - "(resolver-only data plane; no single-tenant fallback)" - ) - # Recreate when the running container's image is stale (a rebuild), - # so source changes to the gateway's flat daemons take effect — not - # just when the container is absent. - if self.is_running() and self._running_image_is_current(): - return - self._ensure_network() - # Clear any stale (stopped OR outdated-image) container holding the - # fixed name, then start fresh. `rm --force` on an absent name is a - # tolerated no-op. - run_docker(["docker", "rm", "--force", self.name]) - argv = [ - "docker", "run", "--detach", - "--name", self.name, - "--label", GATEWAY_LABEL, - "--network", self.network, - # Persist the self-generated CA on the host so it survives both - # container recreation AND docker volume pruning (agents trust it) - # — see host_gateway_ca_dir / issue #450. - "--volume", f"{host_gateway_ca_dir()}:{MITMPROXY_HOME}", - # No DB mount: the data plane (egress / supervise / git-gate) reaches - # the supervise queue over the control-plane RPC and never opens - # bot-bottle.db, so the gateway container gets no file handle on it - # (PRD 0070 / issue #469). - ] - for port in self._host_port_bindings: - argv += ["--publish", f"0.0.0.0:{port}:{port}"] - run_env = dict(os.environ) - # The gateway's egress / git / supervise daemons resolve source-IP -> - # policy against the control plane per request (guaranteed non-empty by - # the check above). - argv += ["--env", f"BOT_BOTTLE_ORCHESTRATOR_URL={self._orchestrator_url}"] - # ...presenting a role-scoped `gateway` token (a signed JWT minted from - # the host signing key) on those calls. The gateway never receives the - # signing key — only this pre-minted token, which it cannot rewrite into - # a `cli` token, so a compromised data-plane process can't drive the - # operator routes (issue #469 review). Bare `--env NAME` keeps the value - # off argv / `docker inspect`; only the gateway (not the agent) is given it. - argv += ["--env", CONTROL_AUTH_JWT_ENV] - run_env[CONTROL_AUTH_JWT_ENV] = mint(ROLE_GATEWAY, host_control_plane_token()) - argv.append(self.image_ref) - proc = run_docker(argv, env=run_env) - if proc.returncode != 0: - raise GatewayError(f"gateway failed to start: {proc.stderr.strip()}") - - def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str: - """The gateway's CA certificate (PEM) that agents install to trust its - TLS interception. mitmproxy generates it a moment after the container - starts, so this **polls** for it (up to `timeout`) rather than assuming - it's already there on a fresh gateway — raising only if it never - appears.""" - deadline = time.monotonic() + timeout - while True: - proc = run_docker(["docker", "exec", self.name, "cat", GATEWAY_CA_CERT]) - if proc.returncode == 0 and proc.stdout.strip(): - return proc.stdout - if time.monotonic() >= deadline: - raise GatewayError( - f"gateway CA cert not available after {timeout:g}s: " - f"{proc.stderr.strip() or 'empty'}" - ) - time.sleep(_CA_POLL_SECONDS) - - def stop(self) -> None: - proc = run_docker(["docker", "rm", "--force", self.name]) - if proc.returncode != 0 and "No such container" not in proc.stderr: - raise GatewayError(f"gateway failed to stop: {proc.stderr.strip()}") - - __all__ = [ - "Gateway", "DockerGateway", "GatewayError", "rotate_gateway_ca", + "Gateway", "GatewayError", "rotate_gateway_ca", "GATEWAY_NAME", "GATEWAY_LABEL", "GATEWAY_IMAGE", "GATEWAY_NETWORK", "GATEWAY_CA_CERT", "GATEWAY_CA_GLOB", ] diff --git a/bot_bottle/orchestrator/service.py b/bot_bottle/orchestrator/service.py index 629ab16..d521937 100644 --- a/bot_bottle/orchestrator/service.py +++ b/bot_bottle/orchestrator/service.py @@ -27,7 +27,6 @@ from datetime import datetime, timezone from .broker import LaunchBroker, LaunchRequest, sign_request from .registry import DEFAULT_REAP_GRACE_SECONDS, BottleRecord, RegistryStore -from .gateway import Gateway from ..supervise import ( AuditEntry, COMPONENT_FOR_TOOL, @@ -72,12 +71,10 @@ class Orchestrator: registry: RegistryStore, broker: LaunchBroker, sign_secret: bytes, - gateway: Gateway | None = None, ) -> None: self.registry = registry self._broker = broker self._secret = sign_secret - self._gateway = gateway # Per-bottle egress auth tokens (env_name -> value), keyed by bottle_id. # Held **in memory only** — never written to the registry DB — so the # gateway can inject each bottle's upstream credential without secrets @@ -383,22 +380,15 @@ class Orchestrator: # --- consolidated gateway ---------------------------------------------- - def ensure_gateway(self) -> None: - """Ensure the single per-host gateway is built and up (idempotent). - No-op when no gateway is configured.""" - if self._gateway is not None: - self._gateway.ensure_built() - self._gateway.ensure_running() - def gateway_status(self) -> dict[str, object]: - """Report the shared gateway for the control plane / console.""" - if self._gateway is None: - return {"configured": False} - return { - "configured": True, - "name": self._gateway.name, - "running": self._gateway.is_running(), - } + """Report the shared gateway for the control plane / console. + + The orchestrator no longer owns a standalone gateway lifecycle — the + consolidated flow runs the gateway data plane inside the per-host infra + container/VM (see `backend/*/gateway`), so this reports `configured: + false`. Retained for the documented `GET /gateway` control-plane + contract.""" + return {"configured": False} __all__ = ["Orchestrator"] diff --git a/tests/integration/test_orchestrator_docker_gateway.py b/tests/integration/test_orchestrator_docker_gateway.py index 38e4b76..453dccd 100644 --- a/tests/integration/test_orchestrator_docker_gateway.py +++ b/tests/integration/test_orchestrator_docker_gateway.py @@ -10,7 +10,7 @@ import secrets import subprocess import unittest -from bot_bottle.orchestrator.gateway import DockerGateway +from bot_bottle.backend.docker.gateway import DockerGateway from tests._docker import skip_unless_docker IMAGE = "busybox" diff --git a/tests/integration/test_orchestrator_docker_gateway_build.py b/tests/integration/test_orchestrator_docker_gateway_build.py index 5de444f..4eb59cb 100644 --- a/tests/integration/test_orchestrator_docker_gateway_build.py +++ b/tests/integration/test_orchestrator_docker_gateway_build.py @@ -11,7 +11,7 @@ from __future__ import annotations import subprocess import unittest -from bot_bottle.orchestrator.gateway import DockerGateway +from bot_bottle.backend.docker.gateway import DockerGateway from tests._docker import skip_unless_docker IMAGE = "busybox" diff --git a/tests/unit/test_orchestrator_gateway.py b/tests/unit/test_orchestrator_gateway.py index 40a14c8..f712fae 100644 --- a/tests/unit/test_orchestrator_gateway.py +++ b/tests/unit/test_orchestrator_gateway.py @@ -7,10 +7,10 @@ import unittest from pathlib import Path from unittest.mock import Mock, patch +from bot_bottle.backend.docker.gateway import DockerGateway from bot_bottle.orchestrator.gateway import ( GATEWAY_CA_CERT, GATEWAY_NAME, - DockerGateway, GatewayError, rotate_gateway_ca, ) @@ -20,7 +20,7 @@ from tests.unit import use_bottle_root _CA_PEM = "-----BEGIN CERTIFICATE-----\nMII...\n-----END CERTIFICATE-----\n" -_RUN_DOCKER = "bot_bottle.orchestrator.gateway.run_docker" +_RUN_DOCKER = "bot_bottle.backend.docker.gateway.run_docker" def _proc(returncode: int = 0, stdout: str = "", stderr: str = "") -> Mock: @@ -162,7 +162,7 @@ class TestDockerGateway(unittest.TestCase): # First read: CA not there yet; second read: present. seq = [_proc(returncode=1, stderr="No such file"), _proc(stdout=_CA_PEM)] with patch(_RUN_DOCKER, side_effect=seq), \ - patch("bot_bottle.orchestrator.gateway.time.sleep"): + patch("bot_bottle.backend.docker.gateway.time.sleep"): self.assertEqual(_CA_PEM, self.sc.ca_cert_pem(timeout=5)) def test_ensure_running_reuses_existing_network(self) -> None: diff --git a/tests/unit/test_orchestrator_service.py b/tests/unit/test_orchestrator_service.py index 06cf611..e76c1c8 100644 --- a/tests/unit/test_orchestrator_service.py +++ b/tests/unit/test_orchestrator_service.py @@ -14,7 +14,6 @@ from unittest.mock import patch from bot_bottle.orchestrator.broker import LaunchBroker, LaunchRequest, StubBroker from bot_bottle.orchestrator.registry import RegistryStore from bot_bottle.orchestrator.service import Orchestrator -from bot_bottle.orchestrator.gateway import Gateway from bot_bottle.orchestrator.secret_store import new_env_var_secret from bot_bottle.store_manager import StoreManager from bot_bottle.supervise import ( @@ -38,29 +37,6 @@ class _FailingBroker(LaunchBroker): pass -class _FakeGateway(Gateway): - """In-memory gateway for wiring tests.""" - - def __init__(self) -> None: - self.name = "fake-gateway" - self.ensured = 0 - self.built = 0 - self._running = False - - def ensure_built(self) -> None: - self.built += 1 - - def ensure_running(self) -> None: - self.ensured += 1 - self._running = True - - def is_running(self) -> bool: - return self._running - - def stop(self) -> None: - self._running = False - - class TestOrchestrator(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() @@ -168,24 +144,11 @@ class TestOrchestrator(unittest.TestCase): orch.launch_bottle("10.243.0.9") self.assertEqual([], self.store.all()) # no orphan - def test_gateway_unconfigured_by_default(self) -> None: + def test_gateway_status_reports_unconfigured(self) -> None: + # The orchestrator no longer owns a standalone gateway lifecycle; the + # consolidated flow runs the gateway data plane in the per-host infra + # container/VM. `GET /gateway` therefore always reports unconfigured. self.assertEqual({"configured": False}, self.orch.gateway_status()) - self.orch.ensure_gateway() # no-op, must not raise - - def test_gateway_wired_and_ensured(self) -> None: - sc = _FakeGateway() - orch = Orchestrator(self.store, self.broker, self.secret, gateway=sc) - self.assertEqual( - {"configured": True, "name": "fake-gateway", "running": False}, - orch.gateway_status(), - ) - orch.ensure_gateway() - self.assertEqual(1, sc.built) # ensure_gateway builds first, - self.assertEqual(1, sc.ensured) # then runs - self.assertEqual( - {"configured": True, "name": "fake-gateway", "running": True}, - orch.gateway_status(), - ) class TestOrchestratorSupervise(unittest.TestCase): -- 2.52.0 From d7a58e52fd2ba5a150b4c1b417e988b1d2fd5b66 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 12:53:52 -0400 Subject: [PATCH 07/39] refactor(store): consolidate the SQLite store family into bot_bottle.store Move the DbStore base (db_store), the shared schema-migration base (migrations / TableMigrations), the concrete stores (audit_store, config_store, queue_store), and StoreManager out of the package root into a bot_bottle/store/ package, so persistence lives in one place rather than scattered across the root. Callers use direct submodule imports (from bot_bottle.store.store_manager import StoreManager). Inside the moved modules the relative imports point back up to their non-store siblings (..paths, ..supervise_types) while co-moved siblings stay package-local; the flat-import fallbacks are untouched. The orchestrator's own stores (config_store, secret_store, registry) stay in orchestrator/ and repoint to the moved DbStore / TableMigrations. No behavior change; full unit suite green. Co-Authored-By: Claude Opus 4.8 --- bot_bottle/cli/__init__.py | 2 +- bot_bottle/image_cache.py | 2 +- bot_bottle/orchestrator/__main__.py | 2 +- bot_bottle/orchestrator/config_store.py | 4 ++-- bot_bottle/orchestrator/registry.py | 4 ++-- bot_bottle/store/__init__.py | 9 +++++++++ bot_bottle/{ => store}/audit_store.py | 4 ++-- bot_bottle/{ => store}/config_store.py | 2 +- bot_bottle/{ => store}/db_store.py | 0 bot_bottle/{ => store}/migrations.py | 0 bot_bottle/{ => store}/queue_store.py | 4 ++-- bot_bottle/{ => store}/store_manager.py | 2 +- bot_bottle/supervise.py | 6 +++--- tests/unit/test_cli_dispatch.py | 4 ++-- tests/unit/test_config_store.py | 4 ++-- tests/unit/test_db_store.py | 4 ++-- tests/unit/test_orchestrator_control_plane.py | 2 +- tests/unit/test_orchestrator_service.py | 2 +- tests/unit/test_supervise.py | 4 ++-- tests/unit/test_supervise_edge.py | 4 ++-- tests/unit/test_supervise_server.py | 4 ++-- 21 files changed, 39 insertions(+), 30 deletions(-) create mode 100644 bot_bottle/store/__init__.py rename bot_bottle/{ => store}/audit_store.py (97%) rename bot_bottle/{ => store}/config_store.py (98%) rename bot_bottle/{ => store}/db_store.py (100%) rename bot_bottle/{ => store}/migrations.py (100%) rename bot_bottle/{ => store}/queue_store.py (98%) rename bot_bottle/{ => store}/store_manager.py (97%) diff --git a/bot_bottle/cli/__init__.py b/bot_bottle/cli/__init__.py index cf62f68..c1618b6 100644 --- a/bot_bottle/cli/__init__.py +++ b/bot_bottle/cli/__init__.py @@ -10,7 +10,7 @@ import sys from ..errors import MissingEnvVarError from ..log import Die, die, error from ..manifest import ManifestError -from ..store_manager import StoreManager +from ..store.store_manager import StoreManager from ._common import PROG from . import list as _list_mod from .backend import cmd_backend diff --git a/bot_bottle/image_cache.py b/bot_bottle/image_cache.py index b2f802a..ff16544 100644 --- a/bot_bottle/image_cache.py +++ b/bot_bottle/image_cache.py @@ -6,7 +6,7 @@ from datetime import datetime, timezone from pathlib import Path try: - from .config_store import ConfigStore + from .store.config_store import ConfigStore except ImportError: from config_store import ConfigStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module diff --git a/bot_bottle/orchestrator/__main__.py b/bot_bottle/orchestrator/__main__.py index 0869c90..38d4e0c 100644 --- a/bot_bottle/orchestrator/__main__.py +++ b/bot_bottle/orchestrator/__main__.py @@ -16,7 +16,7 @@ import secrets from pathlib import Path from .. import log -from ..store_manager import StoreManager +from ..store.store_manager import StoreManager from .broker import LaunchBroker, StubBroker from .control_plane import make_server from .docker_broker import DockerBroker diff --git a/bot_bottle/orchestrator/config_store.py b/bot_bottle/orchestrator/config_store.py index 2cd6f85..7779b68 100644 --- a/bot_bottle/orchestrator/config_store.py +++ b/bot_bottle/orchestrator/config_store.py @@ -11,8 +11,8 @@ import os import sqlite3 from pathlib import Path -from ..db_store import DbStore -from ..migrations import TableMigrations +from ..store.db_store import DbStore +from ..store.migrations import TableMigrations from ..paths import host_db_path TEARDOWN_TIMEOUT_ENV = "BOT_BOTTLE_ORCHESTRATOR_TEARDOWN_TIMEOUT_SECONDS" diff --git a/bot_bottle/orchestrator/registry.py b/bot_bottle/orchestrator/registry.py index a7f34b1..a5afe4e 100644 --- a/bot_bottle/orchestrator/registry.py +++ b/bot_bottle/orchestrator/registry.py @@ -36,8 +36,8 @@ from collections.abc import Iterable from dataclasses import dataclass from pathlib import Path -from ..db_store import DbStore -from ..migrations import TableMigrations +from ..store.db_store import DbStore +from ..store.migrations import TableMigrations from ..paths import host_db_path # 256 bits of urandom, URL-safe — unguessable per-bottle identity token. diff --git a/bot_bottle/store/__init__.py b/bot_bottle/store/__init__.py new file mode 100644 index 0000000..ec5e39b --- /dev/null +++ b/bot_bottle/store/__init__.py @@ -0,0 +1,9 @@ +"""SQLite-backed persistence for bot-bottle. + +The store family: a shared `DbStore` base (versioned `migrations`) and the +concrete stores built on it — `ConfigStore`, `AuditStore`, `QueueStore` — plus +`StoreManager`, which migrates them together against the one per-host DB. + +Callers import the concrete module they need directly, e.g. +`from bot_bottle.store.store_manager import StoreManager`. +""" diff --git a/bot_bottle/audit_store.py b/bot_bottle/store/audit_store.py similarity index 97% rename from bot_bottle/audit_store.py rename to bot_bottle/store/audit_store.py index 2f48f7c..13de470 100644 --- a/bot_bottle/audit_store.py +++ b/bot_bottle/store/audit_store.py @@ -6,8 +6,8 @@ import sqlite3 from pathlib import Path try: - from .supervise_types import AuditEntry - from .paths import host_db_path + from ..supervise_types import AuditEntry + from ..paths import host_db_path from .db_store import DbStore from .migrations import TableMigrations except ImportError: diff --git a/bot_bottle/config_store.py b/bot_bottle/store/config_store.py similarity index 98% rename from bot_bottle/config_store.py rename to bot_bottle/store/config_store.py index bc2f607..d282d71 100644 --- a/bot_bottle/config_store.py +++ b/bot_bottle/store/config_store.py @@ -7,7 +7,7 @@ from pathlib import Path try: from .db_store import DbStore from .migrations import TableMigrations - from .paths import host_db_path + from ..paths import host_db_path except ImportError: from db_store import DbStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module diff --git a/bot_bottle/db_store.py b/bot_bottle/store/db_store.py similarity index 100% rename from bot_bottle/db_store.py rename to bot_bottle/store/db_store.py diff --git a/bot_bottle/migrations.py b/bot_bottle/store/migrations.py similarity index 100% rename from bot_bottle/migrations.py rename to bot_bottle/store/migrations.py diff --git a/bot_bottle/queue_store.py b/bot_bottle/store/queue_store.py similarity index 98% rename from bot_bottle/queue_store.py rename to bot_bottle/store/queue_store.py index 2dae5c9..09ef1cf 100644 --- a/bot_bottle/queue_store.py +++ b/bot_bottle/store/queue_store.py @@ -7,8 +7,8 @@ import sqlite3 from pathlib import Path try: - from .supervise_types import Proposal, Response - from .paths import host_db_path + from ..supervise_types import Proposal, Response + from ..paths import host_db_path from .db_store import DbStore from .migrations import TableMigrations except ImportError: diff --git a/bot_bottle/store_manager.py b/bot_bottle/store/store_manager.py similarity index 97% rename from bot_bottle/store_manager.py rename to bot_bottle/store/store_manager.py index f8807a4..e7729cc 100644 --- a/bot_bottle/store_manager.py +++ b/bot_bottle/store/store_manager.py @@ -26,7 +26,7 @@ class StoreManager: def __init__(self, db_path: Path | None = None) -> None: if db_path is None: try: - from .paths import host_db_path + from ..paths import host_db_path except ImportError: from paths import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module db_path = host_db_path() diff --git a/bot_bottle/supervise.py b/bot_bottle/supervise.py index 184500f..96d25a3 100644 --- a/bot_bottle/supervise.py +++ b/bot_bottle/supervise.py @@ -98,9 +98,9 @@ def audit_log_path(component: str, slug: str) -> Path: try: - from .queue_store import QueueStore - from .audit_store import AuditStore - from .store_manager import StoreManager + from .store.queue_store import QueueStore + from .store.audit_store import AuditStore + from .store.store_manager import StoreManager except ImportError: # Gateway: files are flat-copied under /app, not a package. from queue_store import QueueStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module diff --git a/tests/unit/test_cli_dispatch.py b/tests/unit/test_cli_dispatch.py index 2981f73..e5e10d5 100644 --- a/tests/unit/test_cli_dispatch.py +++ b/tests/unit/test_cli_dispatch.py @@ -12,10 +12,10 @@ from unittest.mock import patch import bot_bottle.cli as climod from bot_bottle.cli import main -from bot_bottle.db_store import DbStore +from bot_bottle.store.db_store import DbStore from bot_bottle.log import Die from bot_bottle.manifest import ManifestError -from bot_bottle.store_manager import StoreManager +from bot_bottle.store.store_manager import StoreManager class TestMainDispatch(unittest.TestCase): diff --git a/tests/unit/test_config_store.py b/tests/unit/test_config_store.py index 5987ae0..0f40576 100644 --- a/tests/unit/test_config_store.py +++ b/tests/unit/test_config_store.py @@ -7,11 +7,11 @@ import tempfile import unittest from pathlib import Path -from bot_bottle.config_store import ( +from bot_bottle.store.config_store import ( DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS, ConfigStore, ) -from bot_bottle.store_manager import StoreManager +from bot_bottle.store.store_manager import StoreManager class TestConfigStore(unittest.TestCase): diff --git a/tests/unit/test_db_store.py b/tests/unit/test_db_store.py index c447d1d..04046ee 100644 --- a/tests/unit/test_db_store.py +++ b/tests/unit/test_db_store.py @@ -7,8 +7,8 @@ import tempfile import unittest from pathlib import Path -from bot_bottle.db_store import DbStore -from bot_bottle.migrations import TableMigrations +from bot_bottle.store.db_store import DbStore +from bot_bottle.store.migrations import TableMigrations def _store(tmp: Path) -> DbStore: diff --git a/tests/unit/test_orchestrator_control_plane.py b/tests/unit/test_orchestrator_control_plane.py index b400bc5..1cd06b8 100644 --- a/tests/unit/test_orchestrator_control_plane.py +++ b/tests/unit/test_orchestrator_control_plane.py @@ -24,7 +24,7 @@ from bot_bottle.orchestrator.broker import StubBroker from bot_bottle.orchestrator.control_plane import dispatch, make_server from bot_bottle.orchestrator.registry import BottleRecord, RegistryStore from bot_bottle.orchestrator.service import Orchestrator -from bot_bottle.store_manager import StoreManager +from bot_bottle.store.store_manager import StoreManager from bot_bottle.supervise import ( Proposal, TOOL_EGRESS_ALLOW, diff --git a/tests/unit/test_orchestrator_service.py b/tests/unit/test_orchestrator_service.py index e76c1c8..9441654 100644 --- a/tests/unit/test_orchestrator_service.py +++ b/tests/unit/test_orchestrator_service.py @@ -15,7 +15,7 @@ from bot_bottle.orchestrator.broker import LaunchBroker, LaunchRequest, StubBrok from bot_bottle.orchestrator.registry import RegistryStore from bot_bottle.orchestrator.service import Orchestrator from bot_bottle.orchestrator.secret_store import new_env_var_secret -from bot_bottle.store_manager import StoreManager +from bot_bottle.store.store_manager import StoreManager from bot_bottle.supervise import ( Proposal, STATUS_APPROVED, diff --git a/tests/unit/test_supervise.py b/tests/unit/test_supervise.py index 6381af0..e3360a7 100644 --- a/tests/unit/test_supervise.py +++ b/tests/unit/test_supervise.py @@ -10,8 +10,8 @@ from pathlib import Path from bot_bottle import supervise from bot_bottle.paths import host_db_path from tests.unit import use_bottle_root -from bot_bottle.audit_store import AuditStore -from bot_bottle.queue_store import QueueStore +from bot_bottle.store.audit_store import AuditStore +from bot_bottle.store.queue_store import QueueStore from bot_bottle.supervise import ( AuditEntry, Proposal, diff --git a/tests/unit/test_supervise_edge.py b/tests/unit/test_supervise_edge.py index 4cedbfa..acbd420 100644 --- a/tests/unit/test_supervise_edge.py +++ b/tests/unit/test_supervise_edge.py @@ -11,9 +11,9 @@ from pathlib import Path from unittest.mock import patch from bot_bottle import supervise -from bot_bottle.audit_store import AuditStore +from bot_bottle.store.audit_store import AuditStore from bot_bottle.paths import bot_bottle_root -from bot_bottle.queue_store import QueueStore +from bot_bottle.store.queue_store import QueueStore from bot_bottle.supervise import ( AuditEntry, Proposal, diff --git a/tests/unit/test_supervise_server.py b/tests/unit/test_supervise_server.py index c666c55..3f596b0 100644 --- a/tests/unit/test_supervise_server.py +++ b/tests/unit/test_supervise_server.py @@ -19,8 +19,8 @@ from pathlib import Path from tests.unit import use_bottle_root from bot_bottle import supervise as _sv -from bot_bottle import queue_store as _qs -from bot_bottle import audit_store as _as +from bot_bottle.store import queue_store as _qs +from bot_bottle.store import audit_store as _as from bot_bottle import supervise_server # noqa: E402 from bot_bottle.supervise_server import ( -- 2.52.0 From a845cba925f055582ae40de686b1cb023fc20bc3 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 13:04:39 -0400 Subject: [PATCH 08/39] refactor(manifest): consolidate the manifest modules into a bot_bottle.manifest package Move the nine root-level manifest modules into a bot_bottle/manifest/ package, dropping the redundant manifest_ prefix: the facade (manifest.py) becomes the package __init__ and keeps re-exporting Manifest / ManifestIndex and the piece types, while manifest_.py become manifest/.py (agent, bottle, egress, extends, git, loader, schema, util). Because the facade stays the package __init__, the ~13 callers that import `from bot_bottle.manifest import ...` are unchanged; only direct-piece imports move to `bot_bottle.manifest.`. Inside the package, intra-manifest imports drop the prefix and the non-manifest siblings (log, yaml_subset, agent_provider) move to `..`. No behavior change; full unit suite green. Co-Authored-By: Claude Opus 4.8 --- bot_bottle/cli/start.py | 2 +- bot_bottle/deploy_key_provisioner.py | 2 +- .../{manifest.py => manifest/__init__.py} | 20 ++++++++-------- .../{manifest_agent.py => manifest/agent.py} | 8 +++---- .../bottle.py} | 24 +++++++++---------- .../egress.py} | 2 +- .../extends.py} | 8 +++---- .../{manifest_git.py => manifest/git.py} | 2 +- .../loader.py} | 12 +++++----- .../schema.py} | 2 +- .../{manifest_util.py => manifest/util.py} | 0 tests/unit/test_cli_start_selector.py | 2 +- tests/unit/test_git_gate_render_provision.py | 2 +- tests/unit/test_manifest_bottle_merge.py | 4 ++-- tests/unit/test_manifest_validation.py | 4 ++-- 15 files changed, 47 insertions(+), 47 deletions(-) rename bot_bottle/{manifest.py => manifest/__init__.py} (97%) rename bot_bottle/{manifest_agent.py => manifest/agent.py} (98%) rename bot_bottle/{manifest_bottle.py => manifest/bottle.py} (87%) rename bot_bottle/{manifest_egress.py => manifest/egress.py} (99%) rename bot_bottle/{manifest_extends.py => manifest/extends.py} (98%) rename bot_bottle/{manifest_git.py => manifest/git.py} (99%) rename bot_bottle/{manifest_loader.py => manifest/loader.py} (93%) rename bot_bottle/{manifest_schema.py => manifest/schema.py} (98%) rename bot_bottle/{manifest_util.py => manifest/util.py} (100%) diff --git a/bot_bottle/cli/start.py b/bot_bottle/cli/start.py index 27bcf3a..b587216 100644 --- a/bot_bottle/cli/start.py +++ b/bot_bottle/cli/start.py @@ -380,7 +380,7 @@ def _peek_agent_bottle(manifest: ManifestIndex, agent_name: str) -> str: return manifest.agents[agent_name].bottle return "" - from ..manifest_loader import scan_agent_names + from ..manifest.loader import scan_agent_names from ..yaml_subset import YamlSubsetError, parse_frontmatter home_agents = scan_agent_names(manifest.home_md / "agents") diff --git a/bot_bottle/deploy_key_provisioner.py b/bot_bottle/deploy_key_provisioner.py index e733bc1..a6d90d5 100644 --- a/bot_bottle/deploy_key_provisioner.py +++ b/bot_bottle/deploy_key_provisioner.py @@ -49,7 +49,7 @@ def get_provisioner( GiteaDeployKeyProvisioner, ) return GiteaDeployKeyProvisioner(token=token, api_url=api_url) - from .manifest_util import ManifestError + from .manifest.util import ManifestError raise ManifestError( f"unknown provisioned_key provider: {provider!r}; " f"available: gitea" diff --git a/bot_bottle/manifest.py b/bot_bottle/manifest/__init__.py similarity index 97% rename from bot_bottle/manifest.py rename to bot_bottle/manifest/__init__.py index f8b6307..8289298 100644 --- a/bot_bottle/manifest.py +++ b/bot_bottle/manifest/__init__.py @@ -63,25 +63,25 @@ from dataclasses import dataclass, field, replace from pathlib import Path from typing import Mapping -from .log import warn -from .manifest_util import ManifestError, as_json_object -from .manifest_agent import ManifestAgent, ManifestAgentProvider -from .manifest_bottle import ManifestBottle -from .manifest_egress import ( +from ..log import warn +from .util import ManifestError, as_json_object +from .agent import ManifestAgent, ManifestAgentProvider +from .bottle import ManifestBottle +from .egress import ( EGRESS_AUTH_SCHEMES, ManifestEgressConfig, ManifestEgressRoute, ) -from .manifest_extends import merge_bottles_runtime, resolve_bottles -from .manifest_git import ManifestGitEntry, ManifestGitUser, ManifestKeyConfig -from .manifest_loader import ( +from .extends import merge_bottles_runtime, resolve_bottles +from .git import ManifestGitEntry, ManifestGitUser, ManifestKeyConfig +from .loader import ( check_stale_json, load_bottle_chain_from_dir, scan_agent_names, scan_bottle_names, ) -from .manifest_schema import validate_agent_frontmatter_keys -from .yaml_subset import YamlSubsetError, parse_frontmatter +from .schema import validate_agent_frontmatter_keys +from ..yaml_subset import YamlSubsetError, parse_frontmatter # Re-export everything that callers currently import from this module. __all__ = [ diff --git a/bot_bottle/manifest_agent.py b/bot_bottle/manifest/agent.py similarity index 98% rename from bot_bottle/manifest_agent.py rename to bot_bottle/manifest/agent.py index c6071ad..f5d76ce 100644 --- a/bot_bottle/manifest_agent.py +++ b/bot_bottle/manifest/agent.py @@ -5,10 +5,10 @@ from __future__ import annotations from dataclasses import dataclass, field from typing import cast -from .agent_provider import PROVIDER_TEMPLATES -from .manifest_util import ManifestError, as_json_object -from .manifest_git import ManifestGitUser -from .manifest_schema import AGENT_MODEL_KEYS, is_valid_entity_name +from ..agent_provider import PROVIDER_TEMPLATES +from .util import ManifestError, as_json_object +from .git import ManifestGitUser +from .schema import AGENT_MODEL_KEYS, is_valid_entity_name @dataclass(frozen=True) diff --git a/bot_bottle/manifest_bottle.py b/bot_bottle/manifest/bottle.py similarity index 87% rename from bot_bottle/manifest_bottle.py rename to bot_bottle/manifest/bottle.py index b4b2015..ae76d7e 100644 --- a/bot_bottle/manifest_bottle.py +++ b/bot_bottle/manifest/bottle.py @@ -1,13 +1,13 @@ """The `ManifestBottle` value type. -Split out of `manifest.py` so the `extends:`/loader resolvers can import it -without a circular dependency: `manifest.py` imports those resolvers, while -they only need this value type. Everything here depends on leaf modules -(`manifest_util`, `manifest_agent`, `manifest_egress`, `manifest_git`, -`manifest_schema`), so this module sits at the bottom of the manifest layer. +Split out of the package facade (`manifest/__init__.py`) so the `extends:`/ +loader resolvers can import it without a circular dependency: the facade +imports those resolvers, while they only need this value type. Everything here +depends on leaf modules (`util`, `agent`, `egress`, `git`, `schema`), so this +module sits at the bottom of the manifest layer. -`manifest.py` re-exports `ManifestBottle`, so existing -`from .manifest import ManifestBottle` callers are unaffected. +The facade re-exports `ManifestBottle`, so existing +`from bot_bottle.manifest import ManifestBottle` callers are unaffected. """ from __future__ import annotations @@ -15,11 +15,11 @@ from __future__ import annotations from dataclasses import dataclass, field from typing import Mapping -from .manifest_util import ManifestError, as_json_object -from .manifest_agent import ManifestAgentProvider -from .manifest_egress import ManifestEgressConfig -from .manifest_git import ManifestGitEntry, ManifestGitUser, parse_git_gate_config -from .manifest_schema import BOTTLE_KEYS +from .util import ManifestError, as_json_object +from .agent import ManifestAgentProvider +from .egress import ManifestEgressConfig +from .git import ManifestGitEntry, ManifestGitUser, parse_git_gate_config +from .schema import BOTTLE_KEYS __all__ = ["ManifestBottle"] diff --git a/bot_bottle/manifest_egress.py b/bot_bottle/manifest/egress.py similarity index 99% rename from bot_bottle/manifest_egress.py rename to bot_bottle/manifest/egress.py index c373f32..a9f22cd 100644 --- a/bot_bottle/manifest_egress.py +++ b/bot_bottle/manifest/egress.py @@ -6,7 +6,7 @@ import re from dataclasses import dataclass from typing import cast -from .manifest_util import ManifestError, as_json_object +from .util import ManifestError, as_json_object EGRESS_AUTH_SCHEMES = ("Bearer", "token") diff --git a/bot_bottle/manifest_extends.py b/bot_bottle/manifest/extends.py similarity index 98% rename from bot_bottle/manifest_extends.py rename to bot_bottle/manifest/extends.py index 698d23f..c96b032 100644 --- a/bot_bottle/manifest_extends.py +++ b/bot_bottle/manifest/extends.py @@ -2,10 +2,10 @@ from __future__ import annotations -from .manifest_bottle import ManifestBottle -from .manifest_egress import ManifestEgressConfig, validate_egress_routes -from .manifest_git import ManifestGitUser, parse_git_gate_config -from .manifest_util import ManifestError, as_json_object +from .bottle import ManifestBottle +from .egress import ManifestEgressConfig, validate_egress_routes +from .git import ManifestGitUser, parse_git_gate_config +from .util import ManifestError, as_json_object def _overlay_declared_bool( diff --git a/bot_bottle/manifest_git.py b/bot_bottle/manifest/git.py similarity index 99% rename from bot_bottle/manifest_git.py rename to bot_bottle/manifest/git.py index f620c15..44c87b6 100644 --- a/bot_bottle/manifest_git.py +++ b/bot_bottle/manifest/git.py @@ -5,7 +5,7 @@ from __future__ import annotations import re from dataclasses import dataclass -from .manifest_util import ManifestError, as_json_object +from .util import ManifestError, as_json_object # Shell-safe characters for git-gate repo names. Names are embedded in # the generated entrypoint shell script (shlex.quote is the primary diff --git a/bot_bottle/manifest_loader.py b/bot_bottle/manifest/loader.py similarity index 93% rename from bot_bottle/manifest_loader.py rename to bot_bottle/manifest/loader.py index c88af32..f19130a 100644 --- a/bot_bottle/manifest_loader.py +++ b/bot_bottle/manifest/loader.py @@ -4,15 +4,15 @@ from __future__ import annotations from pathlib import Path -from .log import warn -from .manifest_bottle import ManifestBottle -from .manifest_extends import resolve_bottles -from .manifest_schema import ( +from ..log import warn +from .bottle import ManifestBottle +from .extends import resolve_bottles +from .schema import ( entity_name_from_path, validate_bottle_frontmatter_keys, ) -from .manifest_util import ManifestError -from .yaml_subset import YamlSubsetError, parse_frontmatter +from .util import ManifestError +from ..yaml_subset import YamlSubsetError, parse_frontmatter def check_stale_json(dir_path: Path, md_dir: Path, label: str) -> None: diff --git a/bot_bottle/manifest_schema.py b/bot_bottle/manifest/schema.py similarity index 98% rename from bot_bottle/manifest_schema.py rename to bot_bottle/manifest/schema.py index 165718a..512231a 100644 --- a/bot_bottle/manifest_schema.py +++ b/bot_bottle/manifest/schema.py @@ -68,7 +68,7 @@ def _validate_frontmatter_keys( keys: object, allowed_keys: frozenset[str], ) -> None: - from .manifest_util import ManifestError + from .util import ManifestError key_set = set(keys) # type: ignore unknown = key_set - allowed_keys # type: ignore diff --git a/bot_bottle/manifest_util.py b/bot_bottle/manifest/util.py similarity index 100% rename from bot_bottle/manifest_util.py rename to bot_bottle/manifest/util.py diff --git a/tests/unit/test_cli_start_selector.py b/tests/unit/test_cli_start_selector.py index 6af13b4..4f41f07 100644 --- a/tests/unit/test_cli_start_selector.py +++ b/tests/unit/test_cli_start_selector.py @@ -336,7 +336,7 @@ class TestManifestToYaml(unittest.TestCase): agent_provider_template: str = "claude", ): from bot_bottle.manifest import Manifest, ManifestBottle - from bot_bottle.manifest_agent import ManifestAgent, ManifestAgentProvider + from bot_bottle.manifest.agent import ManifestAgent, ManifestAgentProvider agent = ManifestAgent(skills=tuple(skills)) bottle = ManifestBottle( diff --git a/tests/unit/test_git_gate_render_provision.py b/tests/unit/test_git_gate_render_provision.py index 481a63c..81263e4 100644 --- a/tests/unit/test_git_gate_render_provision.py +++ b/tests/unit/test_git_gate_render_provision.py @@ -21,7 +21,7 @@ from bot_bottle.git_gate import ( git_gate_render_gitconfig, revoke_git_gate_provisioned_keys, ) -from bot_bottle.manifest_git import ManifestGitEntry, ManifestKeyConfig +from bot_bottle.manifest.git import ManifestGitEntry, ManifestKeyConfig def _entry(**kw: Any) -> ManifestGitEntry: diff --git a/tests/unit/test_manifest_bottle_merge.py b/tests/unit/test_manifest_bottle_merge.py index 03fc7de..256680d 100644 --- a/tests/unit/test_manifest_bottle_merge.py +++ b/tests/unit/test_manifest_bottle_merge.py @@ -14,7 +14,7 @@ import unittest from pathlib import Path from bot_bottle.manifest import ManifestBottle, ManifestError, ManifestIndex -from bot_bottle.manifest_extends import merge_bottles_runtime +from bot_bottle.manifest.extends import merge_bottles_runtime def _index(bottles: dict[str, object], agents: dict[str, object]) -> ManifestIndex: @@ -44,7 +44,7 @@ class TestMergeBottlesRuntime(unittest.TestCase): self.assertEqual("y", result.env["ONLY_OVERRIDE"]) def test_egress_routes_concatenated(self): - from bot_bottle.manifest_egress import ManifestEgressConfig, ManifestEgressRoute + from bot_bottle.manifest.egress import ManifestEgressConfig, ManifestEgressRoute r1 = ManifestEgressRoute(Host="api.a.com") r2 = ManifestEgressRoute(Host="api.b.com") base = ManifestBottle(egress=ManifestEgressConfig(routes=(r1,))) diff --git a/tests/unit/test_manifest_validation.py b/tests/unit/test_manifest_validation.py index 618717d..b762c2a 100644 --- a/tests/unit/test_manifest_validation.py +++ b/tests/unit/test_manifest_validation.py @@ -14,12 +14,12 @@ from unittest.mock import patch from bot_bottle.env import resolve_env from bot_bottle.errors import MissingEnvVarError from bot_bottle.manifest import Manifest, ManifestBottle, ManifestIndex -from bot_bottle.manifest_agent import ( +from bot_bottle.manifest.agent import ( ManifestAgent, ManifestAgentProvider, _parse_provider_settings, ) -from bot_bottle.manifest_util import ManifestError +from bot_bottle.manifest.util import ManifestError def _idx(obj: dict[str, object]) -> ManifestIndex: -- 2.52.0 From f77023db1dd872fe2019ddcaef7c8259cfcc8029 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 13:25:10 -0400 Subject: [PATCH 09/39] refactor(gateway): move the data-plane daemons into a bot_bottle.gateway package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Separate the gateway (data plane) from the orchestrator (control plane) at the module level. The gateway runtime files move out of the package root — and the backend-neutral Gateway lifecycle ABC + GATEWAY_* constants move out of orchestrator/ — into a new bot_bottle/gateway/ package: gateway/__init__.py (was orchestrator/gateway.py: Gateway ABC + consts + rotate_gateway_ca) gateway/gateway_init.py (the PID-1 daemon supervisor) gateway/egress_addon.py, egress_addon_core.py, egress_dlp_config.py, dlp_detectors.py (the egress mitmproxy daemon) gateway/git_http_backend.py (the git-http daemon) gateway/git_gate_render.py (the git-gate pre-receive rendering) gateway/supervise_server.py (the supervise MCP daemon) gateway/policy_resolver.py (the data-plane control-plane RPC client) orchestrator/ now holds only control-plane files. The shared plan/types/auth layer (egress.py=EgressPlan, git_gate.py=GitGatePlan, supervise.py, supervise_types.py, control_auth.py) and the launch-time git-gate provisioning helpers stay at root, so orchestrator/ and backend/ still own them. Because these daemons are invoked as `python3 -m bot_bottle.`, loaded flat by mitmproxy, and referenced in Dockerfile.gateway, the move updates more than Python imports: the `-m` invocations (firecracker/macOS infra scripts), the Dockerfile.gateway addon shim + ENTRYPOINT, gateway_init's _DAEMONS module paths, and the git-gate CGI heredocs all now point at bot_bottle.gateway.*. No behavior change; full unit suite green (2251). Co-Authored-By: Claude Opus 4.8 --- Dockerfile.gateway | 4 ++-- .../backend/docker/consolidated_launch.py | 2 +- bot_bottle/backend/docker/gateway.py | 2 +- bot_bottle/backend/egress_apply.py | 2 +- bot_bottle/backend/firecracker/infra_vm.py | 2 +- bot_bottle/backend/macos_container/gateway.py | 2 +- bot_bottle/backend/macos_container/infra.py | 4 ++-- bot_bottle/backend/macos_container/launch.py | 2 +- bot_bottle/egress.py | 2 +- .../gateway.py => gateway/__init__.py} | 0 bot_bottle/{ => gateway}/dlp_detectors.py | 0 bot_bottle/{ => gateway}/egress_addon.py | 6 ++--- bot_bottle/{ => gateway}/egress_addon_core.py | 2 +- bot_bottle/{ => gateway}/egress_dlp_config.py | 0 bot_bottle/{ => gateway}/gateway_init.py | 4 ++-- bot_bottle/{ => gateway}/git_gate_render.py | 8 +++---- bot_bottle/{ => gateway}/git_http_backend.py | 2 +- bot_bottle/{ => gateway}/policy_resolver.py | 0 bot_bottle/{ => gateway}/supervise_server.py | 4 ++-- bot_bottle/git_gate.py | 2 +- bot_bottle/git_gate_provision.py | 2 +- bot_bottle/orchestrator/__init__.py | 2 +- bot_bottle/orchestrator/lifecycle.py | 2 +- bot_bottle/orchestrator/rotate_ca.py | 2 +- bot_bottle/supervise.py | 4 ++-- tests/integration/test_gateway_image.py | 2 +- .../integration/test_multitenant_isolation.py | 2 +- tests/unit/test_dlp_detectors.py | 8 +++---- tests/unit/test_egress.py | 22 +++++++++---------- tests/unit/test_egress_addon_core.py | 8 +++---- tests/unit/test_egress_addon_log_redaction.py | 2 +- tests/unit/test_egress_addon_request_flow.py | 10 ++++----- tests/unit/test_egress_core_parsing.py | 2 +- tests/unit/test_egress_multitenant.py | 4 ++-- tests/unit/test_firecracker_infra_vm.py | 2 +- tests/unit/test_gateway_init.py | 8 +++---- tests/unit/test_git_gate.py | 2 +- tests/unit/test_git_gate_provision_render.py | 2 +- tests/unit/test_git_http_backend.py | 16 +++++++------- tests/unit/test_git_http_multitenant.py | 4 ++-- tests/unit/test_macos_infra.py | 2 +- tests/unit/test_orchestrator_gateway.py | 2 +- tests/unit/test_orchestrator_lifecycle.py | 2 +- tests/unit/test_orchestrator_registration.py | 2 +- tests/unit/test_orchestrator_rotate_ca.py | 2 +- tests/unit/test_policy_resolver.py | 4 ++-- tests/unit/test_supervise_server.py | 4 ++-- 47 files changed, 87 insertions(+), 87 deletions(-) rename bot_bottle/{orchestrator/gateway.py => gateway/__init__.py} (100%) rename bot_bottle/{ => gateway}/dlp_detectors.py (100%) rename bot_bottle/{ => gateway}/egress_addon.py (99%) rename bot_bottle/{ => gateway}/egress_addon_core.py (99%) rename bot_bottle/{ => gateway}/egress_dlp_config.py (100%) rename bot_bottle/{ => gateway}/gateway_init.py (98%) rename bot_bottle/{ => gateway}/git_gate_render.py (98%) rename bot_bottle/{ => gateway}/git_http_backend.py (99%) rename bot_bottle/{ => gateway}/policy_resolver.py (100%) rename bot_bottle/{ => gateway}/supervise_server.py (99%) diff --git a/Dockerfile.gateway b/Dockerfile.gateway index 8849d21..91d0c52 100644 --- a/Dockerfile.gateway +++ b/Dockerfile.gateway @@ -102,7 +102,7 @@ RUN pip install --no-cache-dir /src/ # WORKDIR here also creates /app so the shim + COPYs below can write into it # (nothing created /app before this point). WORKDIR /app -RUN printf 'from bot_bottle.egress_addon import addons\n' > /app/egress_addon.py +RUN printf 'from bot_bottle.gateway.egress_addon import addons\n' > /app/egress_addon.py COPY bot_bottle/egress_entrypoint.sh /app/egress-entrypoint.sh RUN chmod +x /app/egress-entrypoint.sh @@ -123,4 +123,4 @@ EXPOSE 8888 9099 9418 9420 9100 # PID 1 is the supervisor. It owns signal handling and exit-code # propagation; no `exec` chain in the entrypoint itself. -ENTRYPOINT ["python3", "-m", "bot_bottle.gateway_init"] +ENTRYPOINT ["python3", "-m", "bot_bottle.gateway.gateway_init"] diff --git a/bot_bottle/backend/docker/consolidated_launch.py b/bot_bottle/backend/docker/consolidated_launch.py index 9f9768a..482a30f 100644 --- a/bot_bottle/backend/docker/consolidated_launch.py +++ b/bot_bottle/backend/docker/consolidated_launch.py @@ -20,7 +20,7 @@ from ...docker_cmd import run_docker from ...egress import EgressPlan from ...git_gate import GitGatePlan from ...orchestrator.client import OrchestratorClient -from ...orchestrator.gateway import GATEWAY_NETWORK +from ...gateway import GATEWAY_NETWORK from ...orchestrator.lifecycle import INFRA_NAME, OrchestratorService from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME from ...orchestrator.reprovision import reprovision_bottles diff --git a/bot_bottle/backend/docker/gateway.py b/bot_bottle/backend/docker/gateway.py index 9bc9f93..edea8e7 100644 --- a/bot_bottle/backend/docker/gateway.py +++ b/bot_bottle/backend/docker/gateway.py @@ -11,7 +11,7 @@ from ...paths import ( host_control_plane_token, host_gateway_ca_dir, ) -from ...orchestrator.gateway import ( +from ...gateway import ( Gateway, GATEWAY_IMAGE, GATEWAY_NAME, GATEWAY_NETWORK, GATEWAY_DOCKERFILE, REPO_ROOT, GATEWAY_LABEL, MITMPROXY_HOME, DEFAULT_CA_TIMEOUT_SECONDS, CA_POLL_SECONDS, GATEWAY_CA_CERT, GatewayError diff --git a/bot_bottle/backend/egress_apply.py b/bot_bottle/backend/egress_apply.py index ed54cdf..a198107 100644 --- a/bot_bottle/backend/egress_apply.py +++ b/bot_bottle/backend/egress_apply.py @@ -11,7 +11,7 @@ from pathlib import Path from ..bottle_state import egress_state_dir from ..egress import EGRESS_ROUTES_FILENAME -from ..egress_addon_core import LOG_OFF, load_config +from ..gateway.egress_addon_core import LOG_OFF, load_config class EgressApplyError(RuntimeError): diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py index cb64dd2..a7ab12b 100644 --- a/bot_bottle/backend/firecracker/infra_vm.py +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -542,7 +542,7 @@ BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_CONTROL_PLANE_TOKEN="$CP_KEY" pyt BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise \\ BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{CONTROL_PLANE_PORT} \\ BOT_BOTTLE_CONTROL_AUTH_JWT="$GW_JWT" \\ - python3 -m bot_bottle.gateway_init & + python3 -m bot_bottle.gateway.gateway_init & # Reap as PID 1; children are backgrounded, so `wait` blocks. while : ; do wait ; done diff --git a/bot_bottle/backend/macos_container/gateway.py b/bot_bottle/backend/macos_container/gateway.py index 3d296ff..fcac855 100644 --- a/bot_bottle/backend/macos_container/gateway.py +++ b/bot_bottle/backend/macos_container/gateway.py @@ -12,7 +12,7 @@ from __future__ import annotations import os -from ...orchestrator.gateway import GatewayError +from ...gateway import GatewayError from . import util as container_mod # The shared host-only network the infra container and every agent bottle sit diff --git a/bot_bottle/backend/macos_container/infra.py b/bot_bottle/backend/macos_container/infra.py index 731a62a..85bc3f0 100644 --- a/bot_bottle/backend/macos_container/infra.py +++ b/bot_bottle/backend/macos_container/infra.py @@ -41,7 +41,7 @@ from dataclasses import dataclass from pathlib import Path from ... import log -from ...orchestrator.gateway import GATEWAY_CA_CERT, MITMPROXY_HOME +from ...gateway import GATEWAY_CA_CERT, MITMPROXY_HOME from ...orchestrator.lifecycle import ( DEFAULT_PORT, DEFAULT_STARTUP_TIMEOUT_SECONDS, @@ -107,7 +107,7 @@ def _init_script(port: int) -> str: # control-plane RPC and never opens bot-bottle.db (PRD 0070 / #469). f"( cd /app && BOT_BOTTLE_GATEWAY_DAEMONS={_GATEWAY_DAEMONS} " f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{port} " - f"python3 -m bot_bottle.gateway_init ) &\n" + f"python3 -m bot_bottle.gateway.gateway_init ) &\n" "while : ; do wait ; done\n" ) diff --git a/bot_bottle/backend/macos_container/launch.py b/bot_bottle/backend/macos_container/launch.py index 8902266..d54a20a 100644 --- a/bot_bottle/backend/macos_container/launch.py +++ b/bot_bottle/backend/macos_container/launch.py @@ -52,7 +52,7 @@ from ...git_gate import ( provision_git_gate_dynamic_keys, revoke_git_gate_provisioned_keys, ) -from ...git_http_backend import DEFAULT_PORT as _GIT_HTTP_PORT +from ...gateway.git_http_backend import DEFAULT_PORT as _GIT_HTTP_PORT from ...image_cache import check_stale from ...log import die, info, warn from .. import BottleImages diff --git a/bot_bottle/egress.py b/bot_bottle/egress.py index 1abaf6a..b933bad 100644 --- a/bot_bottle/egress.py +++ b/bot_bottle/egress.py @@ -16,7 +16,7 @@ from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING -from .egress_addon_core import ( +from .gateway.egress_addon_core import ( ON_MATCH_REDACT, HeaderMatch as CoreHeaderMatch, MatchEntry as CoreMatchEntry, diff --git a/bot_bottle/orchestrator/gateway.py b/bot_bottle/gateway/__init__.py similarity index 100% rename from bot_bottle/orchestrator/gateway.py rename to bot_bottle/gateway/__init__.py diff --git a/bot_bottle/dlp_detectors.py b/bot_bottle/gateway/dlp_detectors.py similarity index 100% rename from bot_bottle/dlp_detectors.py rename to bot_bottle/gateway/dlp_detectors.py diff --git a/bot_bottle/egress_addon.py b/bot_bottle/gateway/egress_addon.py similarity index 99% rename from bot_bottle/egress_addon.py rename to bot_bottle/gateway/egress_addon.py index 8766623..112de8b 100644 --- a/bot_bottle/egress_addon.py +++ b/bot_bottle/gateway/egress_addon.py @@ -16,8 +16,8 @@ import typing from mitmproxy import http # type: ignore[import-not-found] # pylint: disable=import-error from bot_bottle.constants import IDENTITY_HEADER -from bot_bottle.dlp_detectors import redact_tokens, strip_crlf -from bot_bottle.egress_addon_core import ( +from bot_bottle.gateway.dlp_detectors import redact_tokens, strip_crlf +from bot_bottle.gateway.egress_addon_core import ( LOG_BLOCKS, LOG_FULL, DEFAULT_OUTBOUND_ON_MATCH, @@ -40,7 +40,7 @@ from bot_bottle.egress_addon_core import ( scan_inbound, scan_outbound, ) -from bot_bottle.policy_resolver import PolicyResolveError, PolicyResolver +from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver from bot_bottle.supervise_types import ( STATUS_APPROVED, STATUS_MODIFIED, diff --git a/bot_bottle/egress_addon_core.py b/bot_bottle/gateway/egress_addon_core.py similarity index 99% rename from bot_bottle/egress_addon_core.py rename to bot_bottle/gateway/egress_addon_core.py index bf14fbb..e4800d5 100644 --- a/bot_bottle/egress_addon_core.py +++ b/bot_bottle/gateway/egress_addon_core.py @@ -16,7 +16,7 @@ import re import typing from dataclasses import dataclass -from .yaml_subset import YamlSubsetError, parse_yaml_subset +from ..yaml_subset import YamlSubsetError, parse_yaml_subset # DLP detector-config parsing lives in a sibling module. Re-exported below # so existing `from egress_addon_core import ON_MATCH_*` callers keep working. diff --git a/bot_bottle/egress_dlp_config.py b/bot_bottle/gateway/egress_dlp_config.py similarity index 100% rename from bot_bottle/egress_dlp_config.py rename to bot_bottle/gateway/egress_dlp_config.py diff --git a/bot_bottle/gateway_init.py b/bot_bottle/gateway/gateway_init.py similarity index 98% rename from bot_bottle/gateway_init.py rename to bot_bottle/gateway/gateway_init.py index 7e9df7e..1e23be7 100644 --- a/bot_bottle/gateway_init.py +++ b/bot_bottle/gateway/gateway_init.py @@ -100,8 +100,8 @@ _DAEMONS: tuple[_DaemonSpec, ...] = ( )), _DaemonSpec("egress", ("/bin/sh", "/app/egress-entrypoint.sh")), _DaemonSpec("git-gate", ("/bin/sh", "/git-gate-entrypoint.sh")), - _DaemonSpec("git-http", ("python3", "-m", "bot_bottle.git_http_backend")), - _DaemonSpec("supervise", ("python3", "-m", "bot_bottle.supervise_server")), + _DaemonSpec("git-http", ("python3", "-m", "bot_bottle.gateway.git_http_backend")), + _DaemonSpec("supervise", ("python3", "-m", "bot_bottle.gateway.supervise_server")), ) diff --git a/bot_bottle/git_gate_render.py b/bot_bottle/gateway/git_gate_render.py similarity index 98% rename from bot_bottle/git_gate_render.py rename to bot_bottle/gateway/git_gate_render.py index 04c038c..294d675 100644 --- a/bot_bottle/git_gate_render.py +++ b/bot_bottle/gateway/git_gate_render.py @@ -14,8 +14,8 @@ import shlex from dataclasses import dataclass from pathlib import Path -from .constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER -from .manifest import ManifestBottle, ManifestGitEntry +from ..constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER +from ..manifest import ManifestBottle, ManifestGitEntry # Short network alias for git-gate inside the gateway. The # agent's `.gitconfig` insteadOf rewrites resolve through this name. @@ -282,7 +282,7 @@ from pathlib import Path # identity_token), resolved server-side, so the proposal lands under the # calling bottle exactly as a direct write once did. try: - from bot_bottle.policy_resolver import PolicyResolver, PolicyResolveError + from bot_bottle.gateway.policy_resolver import PolicyResolver, PolicyResolveError from bot_bottle.supervise_types import TOOL_GITLEAKS_ALLOW except ImportError: from policy_resolver import PolicyResolver, PolicyResolveError @@ -374,7 +374,7 @@ import sys # Non-blocking poll over the control plane. A decided proposal is archived # server-side on read, so no separate archive step is needed here. try: - from bot_bottle.policy_resolver import PolicyResolver, PolicyResolveError + from bot_bottle.gateway.policy_resolver import PolicyResolver, PolicyResolveError except ImportError: from policy_resolver import PolicyResolver, PolicyResolveError diff --git a/bot_bottle/git_http_backend.py b/bot_bottle/gateway/git_http_backend.py similarity index 99% rename from bot_bottle/git_http_backend.py rename to bot_bottle/gateway/git_http_backend.py index 21f924f..c4860e7 100644 --- a/bot_bottle/git_http_backend.py +++ b/bot_bottle/gateway/git_http_backend.py @@ -27,7 +27,7 @@ from pathlib import Path from urllib.parse import urlsplit from bot_bottle.constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER -from bot_bottle.policy_resolver import PolicyResolveError, PolicyResolver +from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver DEFAULT_PORT = 9420 diff --git a/bot_bottle/policy_resolver.py b/bot_bottle/gateway/policy_resolver.py similarity index 100% rename from bot_bottle/policy_resolver.py rename to bot_bottle/gateway/policy_resolver.py diff --git a/bot_bottle/supervise_server.py b/bot_bottle/gateway/supervise_server.py similarity index 99% rename from bot_bottle/supervise_server.py rename to bot_bottle/gateway/supervise_server.py index aad06d1..26495a3 100644 --- a/bot_bottle/supervise_server.py +++ b/bot_bottle/gateway/supervise_server.py @@ -58,10 +58,10 @@ import typing from dataclasses import dataclass from bot_bottle.constants import IDENTITY_HEADER -from bot_bottle.egress_addon_core import ( +from bot_bottle.gateway.egress_addon_core import ( LOG_OFF, load_config, resolve_client_context, route_to_yaml_dict, ) -from bot_bottle.policy_resolver import PolicyResolveError, PolicyResolver +from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver from bot_bottle import supervise as _sv diff --git a/bot_bottle/git_gate.py b/bot_bottle/git_gate.py index d7c0f96..cfb8213 100644 --- a/bot_bottle/git_gate.py +++ b/bot_bottle/git_gate.py @@ -39,7 +39,7 @@ from .manifest import ManifestBottle # Rendering and the deploy-key lifecycle live in sibling modules; the # names are re-exported here (see __all__) so existing # `from bot_bottle.git_gate import …` callers are unchanged. -from .git_gate_render import ( +from .gateway.git_gate_render import ( GIT_GATE_HOSTNAME, GIT_GATE_TIMEOUT_SECS, GitGateUpstream, diff --git a/bot_bottle/git_gate_provision.py b/bot_bottle/git_gate_provision.py index 00f42fc..ac9eebd 100644 --- a/bot_bottle/git_gate_provision.py +++ b/bot_bottle/git_gate_provision.py @@ -17,7 +17,7 @@ from .bottle_state import globalize_slug from .errors import MissingEnvVarError from .log import info from .manifest import ManifestBottle, ManifestGitEntry -from .git_gate_render import GitGateUpstream +from .gateway.git_gate_render import GitGateUpstream if TYPE_CHECKING: from .git_gate import GitGatePlan diff --git a/bot_bottle/orchestrator/__init__.py b/bot_bottle/orchestrator/__init__.py index 3f03a7e..5346c00 100644 --- a/bot_bottle/orchestrator/__init__.py +++ b/bot_bottle/orchestrator/__init__.py @@ -38,7 +38,7 @@ from .broker import ( verify_request, ) from .docker_broker import DockerBroker, DockerBrokerError -from .gateway import Gateway, GatewayError +from ..gateway import Gateway, GatewayError from .service import Orchestrator from .control_plane import ControlPlaneServer, dispatch, make_server diff --git a/bot_bottle/orchestrator/lifecycle.py b/bot_bottle/orchestrator/lifecycle.py index b75e87d..5c7b9c6 100644 --- a/bot_bottle/orchestrator/lifecycle.py +++ b/bot_bottle/orchestrator/lifecycle.py @@ -31,7 +31,7 @@ from ..paths import ( host_control_plane_token, host_gateway_ca_dir, ) -from .gateway import ( +from ..gateway import ( GATEWAY_DOCKERFILE, GATEWAY_IMAGE, GATEWAY_NETWORK, diff --git a/bot_bottle/orchestrator/rotate_ca.py b/bot_bottle/orchestrator/rotate_ca.py index 5495c77..5ae7d2d 100644 --- a/bot_bottle/orchestrator/rotate_ca.py +++ b/bot_bottle/orchestrator/rotate_ca.py @@ -25,7 +25,7 @@ from pathlib import Path from ..docker_cmd import run_docker from ..paths import host_gateway_ca_dir -from .gateway import GATEWAY_NAME, rotate_gateway_ca +from ..gateway import GATEWAY_NAME, rotate_gateway_ca from .lifecycle import INFRA_NAME # The containers whose mitmproxy would still be serving the old CA from memory: diff --git a/bot_bottle/supervise.py b/bot_bottle/supervise.py index 96d25a3..d71253c 100644 --- a/bot_bottle/supervise.py +++ b/bot_bottle/supervise.py @@ -1,7 +1,7 @@ """Per-bottle supervise plane (PRD 0013). The supervise plane is the per-bottle MCP daemon plus its host-side -queue/audit support. The daemon (bot_bottle.supervise_server) +queue/audit support. The daemon (bot_bottle.gateway.supervise_server) sits on the bottle's internal network and exposes MCP tools the agent calls when it needs an operator-reviewed egress change: @@ -18,7 +18,7 @@ the response and returns `{status, notes}` to the agent. This module defines the host-side library: dataclasses for the queue record shapes, queue read/write helpers, the audit log writer, and the diff renderer. The in-gateway daemon lives in -bot_bottle/supervise_server.py; the supervise daemon's container +bot_bottle/gateway/supervise_server.py; the supervise daemon's container lifecycle is owned by the gateway (PRD 0024). For 0013 the supervisor's approval handlers are deliberately no-ops: diff --git a/tests/integration/test_gateway_image.py b/tests/integration/test_gateway_image.py index 87e5d97..624bda9 100644 --- a/tests/integration/test_gateway_image.py +++ b/tests/integration/test_gateway_image.py @@ -91,7 +91,7 @@ class TestGatewayImage(unittest.TestCase): # Probe that the package imports resolve inside the image. rc, out = self._run_in_image( "python3", "-c", - "from bot_bottle import supervise, supervise_server; print('ok')", + "from bot_bottle import supervise; from bot_bottle.gateway import supervise_server; print('ok')", ) self.assertEqual(0, rc, msg=out) self.assertIn("ok", out) diff --git a/tests/integration/test_multitenant_isolation.py b/tests/integration/test_multitenant_isolation.py index 7ed3031..fdd8ea5 100644 --- a/tests/integration/test_multitenant_isolation.py +++ b/tests/integration/test_multitenant_isolation.py @@ -29,7 +29,7 @@ from bot_bottle.backend.docker.consolidated_launch import ( from bot_bottle.backend.docker.egress import EGRESS_PORT from bot_bottle.backend.docker.gateway_net import next_free_ip from bot_bottle.orchestrator.client import OrchestratorClient -from bot_bottle.orchestrator.gateway import GATEWAY_IMAGE, GATEWAY_NAME, GATEWAY_NETWORK +from bot_bottle.gateway import GATEWAY_IMAGE, GATEWAY_NAME, GATEWAY_NETWORK from bot_bottle.orchestrator.lifecycle import OrchestratorService from tests._docker import skip_unless_docker diff --git a/tests/unit/test_dlp_detectors.py b/tests/unit/test_dlp_detectors.py index 0cbd54f..d62fb4d 100644 --- a/tests/unit/test_dlp_detectors.py +++ b/tests/unit/test_dlp_detectors.py @@ -7,7 +7,7 @@ import base64 import gzip import unittest -from bot_bottle.dlp_detectors import ( +from bot_bottle.gateway.dlp_detectors import ( ENTROPY_BLOCK_THRESHOLD, PARTIAL_MATCH_MIN_LEN, REDACT, @@ -465,17 +465,17 @@ class TestMatchedAndSafeTokens(unittest.TestCase): class TestStripCrlf(unittest.TestCase): def test_removes_url_encoded_crlf(self): - from bot_bottle.dlp_detectors import strip_crlf + from bot_bottle.gateway.dlp_detectors import strip_crlf out = strip_crlf("next=%0d%0aX-Injected: evil") self.assertNotRegex(out, r"%0[dD]%0[aA]") def test_removes_literal_header_injection(self): - from bot_bottle.dlp_detectors import strip_crlf + from bot_bottle.gateway.dlp_detectors import strip_crlf out = strip_crlf("value\r\nX-Injected: evil") self.assertIsNone(scan_crlf_injection(out)) def test_leaves_clean_text_unchanged(self): - from bot_bottle.dlp_detectors import strip_crlf + from bot_bottle.gateway.dlp_detectors import strip_crlf self.assertEqual("/api/v1/data?q=hello", strip_crlf("/api/v1/data?q=hello")) class TestAlnumProjection(unittest.TestCase): diff --git a/tests/unit/test_egress.py b/tests/unit/test_egress.py index 6d3e134..027df82 100644 --- a/tests/unit/test_egress.py +++ b/tests/unit/test_egress.py @@ -344,7 +344,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertEqual([], parse_yaml_subset(rendered)["routes"]) def test_round_trip_through_addon_core(self): - from bot_bottle.egress_addon_core import load_config + from bot_bottle.gateway.egress_addon_core import load_config b = _bottle([ {"host": "api.github.com", "auth": {"scheme": "Bearer", "token_ref": "GH_PAT"}, @@ -363,7 +363,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertEqual("", addon_routes[2].auth_scheme) def test_dlp_round_trips(self): - from bot_bottle.egress_addon_core import load_config + from bot_bottle.gateway.egress_addon_core import load_config b = _bottle([{"host": "x.example", "dlp": { "outbound_detectors": ["token_patterns"], "inbound_detectors": False, @@ -375,7 +375,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertEqual((), addon_routes[0].inbound_detectors) def test_outbound_on_match_round_trips(self): - from bot_bottle.egress_addon_core import load_config + from bot_bottle.gateway.egress_addon_core import load_config b = _bottle([{"host": "logs.example", "dlp": { "outbound_on_match": "redact", }}]) @@ -392,7 +392,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertNotIn("outbound_on_match", rendered) def test_git_fetch_policy_round_trips(self): - from bot_bottle.egress_addon_core import load_config + from bot_bottle.gateway.egress_addon_core import load_config b = _bottle([{"host": "github.com", "git": {"fetch": True}}]) routes = egress_routes_for_bottle(b) rendered = egress_render_routes(routes) @@ -405,7 +405,7 @@ class TestRenderRoutes(unittest.TestCase): it, but the renderer in between dropped it — so the flag never reached the proxy and registry pulls kept failing with "unauthorized" while the config looked correct everywhere it was inspected.""" - from bot_bottle.egress_addon_core import load_config + from bot_bottle.gateway.egress_addon_core import load_config b = _bottle([{"host": "registry-1.docker.io", "preserve_auth": True}]) routes = egress_routes_for_bottle(b) rendered = egress_render_routes(routes) @@ -416,7 +416,7 @@ class TestRenderRoutes(unittest.TestCase): b = _bottle([{"host": "x.example"}]) rendered = egress_render_routes(egress_routes_for_bottle(b)) self.assertNotIn("preserve_auth", rendered) - from bot_bottle.egress_addon_core import load_config + from bot_bottle.gateway.egress_addon_core import load_config self.assertFalse(load_config(rendered).routes[0].preserve_auth) def test_log_zero_omitted_from_render(self): @@ -434,7 +434,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertTrue(rendered.startswith(f"log: {level}\n")) def test_log_level_round_trips_to_addon_core(self): - from bot_bottle.egress_addon_core import load_config, LOG_FULL + from bot_bottle.gateway.egress_addon_core import load_config, LOG_FULL b = _bottle([{"host": "x.example"}]) routes = egress_routes_for_bottle(b) rendered = egress_render_routes(routes, log=LOG_FULL) @@ -444,7 +444,7 @@ class TestRenderRoutes(unittest.TestCase): def test_log_via_manifest_flows_to_render(self): from bot_bottle.manifest import ManifestIndex - from bot_bottle.egress_addon_core import load_config, LOG_BLOCKS + from bot_bottle.gateway.egress_addon_core import load_config, LOG_BLOCKS m = ManifestIndex.from_json_obj({ "bottles": {"dev": {"egress": { "log": 1, @@ -512,7 +512,7 @@ class TestRenderRoutesEscaping(unittest.TestCase): self.assertEqual('Bear"er', parsed[0]["inspect"]["auth_scheme"]) def test_path_value_with_double_quote_round_trips(self): - from bot_bottle.egress_addon_core import PathMatch, MatchEntry + from bot_bottle.gateway.egress_addon_core import PathMatch, MatchEntry routes = (EgressRoute( host="api.example", matches=(MatchEntry(paths=(PathMatch(type="prefix", value='/v1/"quoted"/'),)),), @@ -521,7 +521,7 @@ class TestRenderRoutesEscaping(unittest.TestCase): self.assertEqual('/v1/"quoted"/', parsed[0]["inspect"]["matches"][0]["paths"][0]["value"]) def test_header_value_with_double_quote_round_trips(self): - from bot_bottle.egress_addon_core import HeaderMatch, MatchEntry + from bot_bottle.gateway.egress_addon_core import HeaderMatch, MatchEntry routes = (EgressRoute( host="api.example", matches=(MatchEntry(headers=(HeaderMatch(name="x-h", value='val"ue'),)),), @@ -598,7 +598,7 @@ class TestCanaryGeneration(unittest.TestCase): self.assertNotEqual(plan_a.canary, plan_b.canary) def test_canary_detected_by_scan_known_secrets(self): - from bot_bottle.dlp_detectors import scan_known_secrets + from bot_bottle.gateway.dlp_detectors import scan_known_secrets plan = self._make_plan() env = {plan.canary_env: plan.canary} diff --git a/tests/unit/test_egress_addon_core.py b/tests/unit/test_egress_addon_core.py index a88d1ba..593d6ca 100644 --- a/tests/unit/test_egress_addon_core.py +++ b/tests/unit/test_egress_addon_core.py @@ -12,7 +12,7 @@ import unittest from pathlib import Path from urllib.parse import urlsplit -from bot_bottle.egress_addon_core import ( +from bot_bottle.gateway.egress_addon_core import ( LOG_BLOCKS, LOG_FULL, LOG_OFF, @@ -1377,15 +1377,15 @@ class TestScanOutboundEnhanced(unittest.TestCase): class TestOutboundDetectorNames(unittest.TestCase): def test_entropy_in_outbound_detector_names(self): - from bot_bottle.egress_addon_core import OUTBOUND_DETECTOR_NAMES + from bot_bottle.gateway.egress_addon_core import OUTBOUND_DETECTOR_NAMES self.assertIn("entropy", OUTBOUND_DETECTOR_NAMES) def test_known_secrets_in_outbound_detector_names(self): - from bot_bottle.egress_addon_core import OUTBOUND_DETECTOR_NAMES + from bot_bottle.gateway.egress_addon_core import OUTBOUND_DETECTOR_NAMES self.assertIn("known_secrets", OUTBOUND_DETECTOR_NAMES) def test_token_patterns_in_outbound_detector_names(self): - from bot_bottle.egress_addon_core import OUTBOUND_DETECTOR_NAMES + from bot_bottle.gateway.egress_addon_core import OUTBOUND_DETECTOR_NAMES self.assertIn("token_patterns", OUTBOUND_DETECTOR_NAMES) diff --git a/tests/unit/test_egress_addon_log_redaction.py b/tests/unit/test_egress_addon_log_redaction.py index 8b64f67..44af815 100644 --- a/tests/unit/test_egress_addon_log_redaction.py +++ b/tests/unit/test_egress_addon_log_redaction.py @@ -36,7 +36,7 @@ def _ensure_shims() -> None: _ensure_shims() -from bot_bottle.egress_addon import EgressAddon # noqa: E402 (import after shims) +from bot_bottle.gateway.egress_addon import EgressAddon # noqa: E402 (import after shims) # --------------------------------------------------------------------------- diff --git a/tests/unit/test_egress_addon_request_flow.py b/tests/unit/test_egress_addon_request_flow.py index 6896e9d..a8a06d2 100644 --- a/tests/unit/test_egress_addon_request_flow.py +++ b/tests/unit/test_egress_addon_request_flow.py @@ -194,22 +194,22 @@ def _ensure_shims() -> None: _ensure_shims() -import bot_bottle.egress_addon as _ea_mod # noqa: E402 (after shims) -from bot_bottle.egress_addon import EgressAddon # noqa: E402 (after shims) -from bot_bottle.egress_addon import ( # noqa: E402 +import bot_bottle.gateway.egress_addon as _ea_mod # noqa: E402 (after shims) +from bot_bottle.gateway.egress_addon import EgressAddon # noqa: E402 (after shims) +from bot_bottle.gateway.egress_addon import ( # noqa: E402 DEFAULT_INBOUND_SCAN_LIMIT_BYTES, DEFAULT_TOKEN_ALLOW_TIMEOUT_SECONDS, _inbound_scan_limit_from_env, _token_allow_timeout_from_env, ) -from bot_bottle.egress_addon_core import ( # noqa: E402 +from bot_bottle.gateway.egress_addon_core import ( # noqa: E402 Config, LOG_BLOCKS, LOG_FULL, Route, route_to_yaml_dict, ) -from bot_bottle.policy_resolver import PolicyResolveError # noqa: E402 +from bot_bottle.gateway.policy_resolver import PolicyResolveError # noqa: E402 # --------------------------------------------------------------------------- diff --git a/tests/unit/test_egress_core_parsing.py b/tests/unit/test_egress_core_parsing.py index 7762a93..6a56e90 100644 --- a/tests/unit/test_egress_core_parsing.py +++ b/tests/unit/test_egress_core_parsing.py @@ -8,7 +8,7 @@ from __future__ import annotations import unittest -from bot_bottle.egress_addon_core import ( +from bot_bottle.gateway.egress_addon_core import ( HeaderMatch, MatchEntry, PathMatch, diff --git a/tests/unit/test_egress_multitenant.py b/tests/unit/test_egress_multitenant.py index 6d99e0d..adabb3e 100644 --- a/tests/unit/test_egress_multitenant.py +++ b/tests/unit/test_egress_multitenant.py @@ -4,7 +4,7 @@ from __future__ import annotations import unittest -from bot_bottle.egress_addon_core import ( +from bot_bottle.gateway.egress_addon_core import ( DENY_RESOLVER_ERROR, DENY_UNATTRIBUTED, DENY_UNPARSEABLE, @@ -12,7 +12,7 @@ from bot_bottle.egress_addon_core import ( resolve_client_config, resolve_client_context, ) -from bot_bottle.policy_resolver import PolicyResolveError +from bot_bottle.gateway.policy_resolver import PolicyResolveError class _FakeResolver: diff --git a/tests/unit/test_firecracker_infra_vm.py b/tests/unit/test_firecracker_infra_vm.py index a809344..e0ae9bb 100644 --- a/tests/unit/test_firecracker_infra_vm.py +++ b/tests/unit/test_firecracker_infra_vm.py @@ -72,7 +72,7 @@ class TestBuildInfraRootfs(unittest.TestCase): self.assertIn("bot_bottle.orchestrator", init) # Gateway launches via the installed package (there is no # /app/gateway_init.py file since the daemons moved into bot_bottle). - self.assertIn("bot_bottle.gateway_init", init) + self.assertIn("bot_bottle.gateway.gateway_init", init) self.assertIn("export PATH=", init) # Persistent registry volume mounted at the DB dir before the CP starts. self.assertIn("/dev/vdb", init) diff --git a/tests/unit/test_gateway_init.py b/tests/unit/test_gateway_init.py index 17b5213..bb987ba 100644 --- a/tests/unit/test_gateway_init.py +++ b/tests/unit/test_gateway_init.py @@ -1,6 +1,6 @@ """Unit: gateway data-plane init supervisor (PRD 0070; PRD 0024 bundle shape). -Tests both the helper functions in `bot_bottle.gateway_init` +Tests both the helper functions in `bot_bottle.gateway.gateway_init` and the supervisor's end-to-end signal / exit-code behavior. The end-to-end tests use real subprocesses (`sleep`, `/bin/sh -c '...'`) — short-lived, no docker required — so they run under `tests/unit/` @@ -18,7 +18,7 @@ import warnings from pathlib import Path from unittest.mock import patch -from bot_bottle.gateway_init import ( +from bot_bottle.gateway.gateway_init import ( _DaemonSpec, _Supervisor, _argv_for_daemon, @@ -489,7 +489,7 @@ class TestSupervisor(unittest.TestCase): time.sleep(0.3) # let `trap` register sup.request_shutdown(reason="test") - with patch("bot_bottle.gateway_init._GRACE_SECONDS", 0.3): + with patch("bot_bottle.gateway.gateway_init._GRACE_SECONDS", 0.3): rc = self._drive(sup, max_wait_s=4.0) # Process was SIGKILL'd → returncode -9 on POSIX. @@ -531,7 +531,7 @@ class TestMainEndToEnd(unittest.TestCase): helper = ( "import os, runpy, sys\n" - "from bot_bottle import gateway_init as si\n" + "from bot_bottle.gateway import gateway_init as si\n" "si._DAEMONS = (\n" f" si._DaemonSpec('alpha', ({SLEEP!r},'30')),\n" f" si._DaemonSpec('beta', ({SLEEP!r},'30')),\n" diff --git a/tests/unit/test_git_gate.py b/tests/unit/test_git_gate.py index fd899f0..4befb5d 100644 --- a/tests/unit/test_git_gate.py +++ b/tests/unit/test_git_gate.py @@ -230,7 +230,7 @@ class TestHookRender(unittest.TestCase): # execute from the bare repo directory, so the embedded Python must # include /app and support both import layouts. self.assertIn('PYTHONPATH="/app${PYTHONPATH:+:$PYTHONPATH}"', hook) - self.assertIn("from bot_bottle.policy_resolver import PolicyResolver", hook) + self.assertIn("from bot_bottle.gateway.policy_resolver import PolicyResolver", hook) self.assertIn("from policy_resolver import PolicyResolver", hook) def test_inline_gitleaks_allow_fails_closed_without_supervisor(self): diff --git a/tests/unit/test_git_gate_provision_render.py b/tests/unit/test_git_gate_provision_render.py index 3fc1a07..b9bce15 100644 --- a/tests/unit/test_git_gate_provision_render.py +++ b/tests/unit/test_git_gate_provision_render.py @@ -4,7 +4,7 @@ from __future__ import annotations import unittest -from bot_bottle.git_gate_render import ( +from bot_bottle.gateway.git_gate_render import ( GitGateUpstream, git_gate_render_entrypoint, git_gate_render_provision, diff --git a/tests/unit/test_git_http_backend.py b/tests/unit/test_git_http_backend.py index 405f0b0..6c58772 100644 --- a/tests/unit/test_git_http_backend.py +++ b/tests/unit/test_git_http_backend.py @@ -10,7 +10,7 @@ from pathlib import Path from unittest import mock from bot_bottle.git_gate import GIT_GATE_TIMEOUT_SECS -from bot_bottle.git_http_backend import GitHttpHandler, MAX_BODY_BYTES +from bot_bottle.gateway.git_http_backend import GitHttpHandler, MAX_BODY_BYTES # The git-http backend is resolver-only: every request is attributed to a @@ -199,7 +199,7 @@ class TestGitHttpBackend(unittest.TestCase): subprocess.CompletedProcess(["git"], 0, backend_response, b""), ] with mock.patch( - "bot_bottle.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_http_backend.subprocess.run", side_effect=calls, ) as run: request = urllib.request.Request( @@ -265,7 +265,7 @@ class TestGitHttpBackend(unittest.TestCase): subprocess.CompletedProcess(["git"], 0, backend_response, b""), ] with mock.patch( - "bot_bottle.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_http_backend.subprocess.run", side_effect=calls, ) as run: req = urllib.request.Request( @@ -309,7 +309,7 @@ class TestGitHttpBackend(unittest.TestCase): denial = b"git-gate: upstream fetch failed; refusing to serve stale data\n" with mock.patch( - "bot_bottle.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_http_backend.subprocess.run", return_value=subprocess.CompletedProcess( ["hook"], 1, b"", denial, ), @@ -355,7 +355,7 @@ class TestGitHttpBackend(unittest.TestCase): self.addCleanup(server.server_close) with mock.patch( - "bot_bottle.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_http_backend.subprocess.run", return_value=subprocess.CompletedProcess( ["hook"], 2, b"", b"", ), @@ -402,7 +402,7 @@ class TestGitHttpBackend(unittest.TestCase): self.addCleanup(server.server_close) with mock.patch( - "bot_bottle.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_http_backend.subprocess.run", side_effect=PermissionError(13, "Permission denied"), ): buf = io.StringIO() @@ -461,7 +461,7 @@ class TestMalformedStatusHeader(unittest.TestCase): def _get_with_backend_response(self, cgi_response: bytes) -> int: with mock.patch( - "bot_bottle.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_http_backend.subprocess.run", return_value=mock.Mock(returncode=0, stdout=cgi_response), ): req = urllib.request.Request( @@ -545,7 +545,7 @@ class TestContentLengthBounds(unittest.TestCase): # With a valid Content-Length the handler proceeds into # git http-backend; that will fail (no real git repo) but the # status won't be 400 or 413. - with mock.patch("bot_bottle.git_http_backend.subprocess.run") as run: + with mock.patch("bot_bottle.gateway.git_http_backend.subprocess.run") as run: run.return_value = mock.Mock( returncode=0, stdout=( diff --git a/tests/unit/test_git_http_multitenant.py b/tests/unit/test_git_http_multitenant.py index b425f5f..16acff1 100644 --- a/tests/unit/test_git_http_multitenant.py +++ b/tests/unit/test_git_http_multitenant.py @@ -10,8 +10,8 @@ from __future__ import annotations import unittest from pathlib import Path -from bot_bottle.git_http_backend import resolve_sandbox_root -from bot_bottle.policy_resolver import PolicyResolveError +from bot_bottle.gateway.git_http_backend import resolve_sandbox_root +from bot_bottle.gateway.policy_resolver import PolicyResolveError _BASE = Path("/git") diff --git a/tests/unit/test_macos_infra.py b/tests/unit/test_macos_infra.py index cf746dc..7d80840 100644 --- a/tests/unit/test_macos_infra.py +++ b/tests/unit/test_macos_infra.py @@ -48,7 +48,7 @@ class TestInfraRun(unittest.TestCase): self.assertIn("bot_bottle.orchestrator", script) # Gateway launches via the installed package (there is no # /app/gateway_init.py file since the daemons moved into bot_bottle). - self.assertIn("bot_bottle.gateway_init", script) + self.assertIn("bot_bottle.gateway.gateway_init", script) self.assertIn("127.0.0.1", script) # they reach each other on loopback def test_db_is_a_container_only_volume(self) -> None: diff --git a/tests/unit/test_orchestrator_gateway.py b/tests/unit/test_orchestrator_gateway.py index f712fae..b23162e 100644 --- a/tests/unit/test_orchestrator_gateway.py +++ b/tests/unit/test_orchestrator_gateway.py @@ -8,7 +8,7 @@ from pathlib import Path from unittest.mock import Mock, patch from bot_bottle.backend.docker.gateway import DockerGateway -from bot_bottle.orchestrator.gateway import ( +from bot_bottle.gateway import ( GATEWAY_CA_CERT, GATEWAY_NAME, GatewayError, diff --git a/tests/unit/test_orchestrator_lifecycle.py b/tests/unit/test_orchestrator_lifecycle.py index 2672853..1a65908 100644 --- a/tests/unit/test_orchestrator_lifecycle.py +++ b/tests/unit/test_orchestrator_lifecycle.py @@ -8,7 +8,7 @@ import urllib.error from pathlib import Path from unittest.mock import MagicMock, Mock, patch -from bot_bottle.orchestrator.gateway import GatewayError +from bot_bottle.gateway import GatewayError from bot_bottle.orchestrator.lifecycle import ( INFRA_NAME, INFRA_SOURCE_HASH_LABEL, diff --git a/tests/unit/test_orchestrator_registration.py b/tests/unit/test_orchestrator_registration.py index 846da4e..ffb3752 100644 --- a/tests/unit/test_orchestrator_registration.py +++ b/tests/unit/test_orchestrator_registration.py @@ -7,7 +7,7 @@ import unittest from pathlib import Path from bot_bottle.egress import EgressPlan, EgressRoute -from bot_bottle.egress_addon_core import LOG_BLOCKS, load_config +from bot_bottle.gateway.egress_addon_core import LOG_BLOCKS, load_config from bot_bottle.orchestrator.registration import ( RegistrationInputs, egress_policy, diff --git a/tests/unit/test_orchestrator_rotate_ca.py b/tests/unit/test_orchestrator_rotate_ca.py index 22199b8..3c2db89 100644 --- a/tests/unit/test_orchestrator_rotate_ca.py +++ b/tests/unit/test_orchestrator_rotate_ca.py @@ -8,7 +8,7 @@ from pathlib import Path from unittest.mock import Mock, patch from bot_bottle.orchestrator import rotate_ca -from bot_bottle.orchestrator.gateway import GATEWAY_NAME +from bot_bottle.gateway import GATEWAY_NAME from bot_bottle.orchestrator.lifecycle import INFRA_NAME from bot_bottle.paths import host_gateway_ca_dir from tests.unit import use_bottle_root diff --git a/tests/unit/test_policy_resolver.py b/tests/unit/test_policy_resolver.py index 83544ff..44178a3 100644 --- a/tests/unit/test_policy_resolver.py +++ b/tests/unit/test_policy_resolver.py @@ -7,7 +7,7 @@ import unittest import urllib.error from unittest.mock import MagicMock, patch -from bot_bottle.policy_resolver import ( +from bot_bottle.gateway.policy_resolver import ( CONTROL_AUTH_HEADER, CONTROL_AUTH_JWT_ENV, PolicyResolveError, @@ -15,7 +15,7 @@ from bot_bottle.policy_resolver import ( _control_auth_headers, ) -_URLOPEN = "bot_bottle.policy_resolver.urllib.request.urlopen" +_URLOPEN = "bot_bottle.gateway.policy_resolver.urllib.request.urlopen" def _resp(payload: object) -> MagicMock: diff --git a/tests/unit/test_supervise_server.py b/tests/unit/test_supervise_server.py index 3f596b0..7f0f21d 100644 --- a/tests/unit/test_supervise_server.py +++ b/tests/unit/test_supervise_server.py @@ -22,8 +22,8 @@ from bot_bottle import supervise as _sv from bot_bottle.store import queue_store as _qs from bot_bottle.store import audit_store as _as -from bot_bottle import supervise_server # noqa: E402 -from bot_bottle.supervise_server import ( +from bot_bottle.gateway import supervise_server # noqa: E402 +from bot_bottle.gateway.supervise_server import ( ERR_INTERNAL, ERR_INVALID_PARAMS, ERR_INVALID_REQUEST, -- 2.52.0 From 44e2b5a897ef5c29f6905c1aaaa942f2c6313a67 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 14:15:04 -0400 Subject: [PATCH 10/39] refactor(supervise): split the supervise plane by tier; per-service store managers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Give each service its own store package + manager, and cut the supervise module along the control/data-plane boundary so nothing in the shared layer reaches up into the orchestrator. Stores, by owner: - bot_bottle/store/ keeps only the shared base (DbStore, migrations) and the concrete stores that aren't service-owned (audit_store, config_store). - bot_bottle/orchestrator/store/ now houses the orchestrator-owned stores — queue_store (supervise queue), secret_store, config_store — plus a new orchestrator store_manager that migrates them (composing audit/config downward from the base). The old shared store_manager is gone. Supervise plane, by tier: - bot_bottle/supervisor/ (NEUTRAL, importable by every tier including the gateway): types.py (the Proposal/Response/AuditEntry wire types + the tool/ status/poll constants + the shared daemon constants moved out of supervise.py) and plan.py (SupervisePlan, a pure DTO). - bot_bottle/orchestrator/supervisor/ (orchestrator-only): queue.py (the queue/audit I/O wrappers + render_diff + sha256_hex) and supervise.py (the Supervise lifecycle that stages the DB via the store manager). Its __init__ re-exports the neutral vocabulary so orchestrator-side callers import from one place. The gateway now imports only bot_bottle.supervisor.types (never bot_bottle.supervise), so the data plane holds no code dependency on the orchestrator — it reaches the queue over the control-plane RPC. This removes the circular import that moving queue_store under orchestrator introduced (supervise -> orchestrator -> service -> supervise). supervise_types.py -> supervisor/types.py; supervise.py deleted (split). Full unit suite green (2251). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/backend/__init__.py | 2 +- bot_bottle/backend/consolidated_util.py | 4 +- bot_bottle/backend/docker/backend.py | 4 +- .../backend/docker/consolidated_compose.py | 2 +- .../backend/docker/consolidated_launch.py | 2 +- bot_bottle/backend/docker/launch.py | 4 +- bot_bottle/backend/docker/resolve_plan.py | 2 +- bot_bottle/backend/firecracker/backend.py | 2 +- .../firecracker/consolidated_launch.py | 2 +- bot_bottle/backend/firecracker/launch.py | 6 +- .../backend/firecracker/resolve_plan.py | 2 +- bot_bottle/backend/macos_container/backend.py | 2 +- .../macos_container/consolidated_launch.py | 2 +- bot_bottle/backend/macos_container/launch.py | 6 +- .../backend/macos_container/resolve_plan.py | 2 +- bot_bottle/backend/resolve_common.py | 3 +- bot_bottle/cli/__init__.py | 2 +- bot_bottle/cli/supervise.py | 2 +- bot_bottle/gateway/egress_addon.py | 2 +- bot_bottle/gateway/git_gate_render.py | 4 +- bot_bottle/gateway/supervise_server.py | 2 +- bot_bottle/orchestrator/__main__.py | 2 +- bot_bottle/orchestrator/control_plane.py | 2 +- bot_bottle/orchestrator/service.py | 6 +- bot_bottle/orchestrator/store/__init__.py | 10 + .../orchestrator/{ => store}/config_store.py | 6 +- .../{ => orchestrator}/store/queue_store.py | 10 +- .../orchestrator/{ => store}/secret_store.py | 0 .../{ => orchestrator}/store/store_manager.py | 32 +- .../orchestrator/supervisor/__init__.py | 34 ++ bot_bottle/orchestrator/supervisor/queue.py | 141 +++++++++ .../orchestrator/supervisor/supervise.py | 36 +++ bot_bottle/store/__init__.py | 14 +- bot_bottle/store/audit_store.py | 4 +- bot_bottle/supervise.py | 298 ------------------ bot_bottle/supervisor/__init__.py | 16 + bot_bottle/supervisor/plan.py | 26 ++ .../types.py} | 25 ++ tests/integration/test_gateway_image.py | 2 +- tests/unit/_docker_bottle_plan.py | 2 +- tests/unit/test_cli_dispatch.py | 2 +- tests/unit/test_config_store.py | 2 +- tests/unit/test_contrib_claude_provider.py | 2 +- tests/unit/test_contrib_codex_provider.py | 2 +- tests/unit/test_orchestrator_config_store.py | 2 +- tests/unit/test_orchestrator_control_plane.py | 4 +- tests/unit/test_orchestrator_secret_store.py | 2 +- tests/unit/test_orchestrator_service.py | 6 +- tests/unit/test_supervise.py | 6 +- tests/unit/test_supervise_cli.py | 2 +- tests/unit/test_supervise_edge.py | 6 +- tests/unit/test_supervise_server.py | 4 +- 52 files changed, 380 insertions(+), 385 deletions(-) create mode 100644 bot_bottle/orchestrator/store/__init__.py rename bot_bottle/orchestrator/{ => store}/config_store.py (96%) rename bot_bottle/{ => orchestrator}/store/queue_store.py (96%) rename bot_bottle/orchestrator/{ => store}/secret_store.py (100%) rename bot_bottle/{ => orchestrator}/store/store_manager.py (54%) create mode 100644 bot_bottle/orchestrator/supervisor/__init__.py create mode 100644 bot_bottle/orchestrator/supervisor/queue.py create mode 100644 bot_bottle/orchestrator/supervisor/supervise.py delete mode 100644 bot_bottle/supervise.py create mode 100644 bot_bottle/supervisor/__init__.py create mode 100644 bot_bottle/supervisor/plan.py rename bot_bottle/{supervise_types.py => supervisor/types.py} (84%) diff --git a/bot_bottle/backend/__init__.py b/bot_bottle/backend/__init__.py index ff5223e..79cd7af 100644 --- a/bot_bottle/backend/__init__.py +++ b/bot_bottle/backend/__init__.py @@ -48,7 +48,7 @@ from ..git_gate import GitGatePlan from ..log import die, info, warn from ..util import read_tty_line from ..manifest import Manifest, ManifestIndex -from ..supervise import SupervisePlan +from ..supervisor.plan import SupervisePlan from ..util import expand_tilde from ..env import resolve_env, ResolvedEnv from ..workspace import WorkspacePlan, workspace_plan diff --git a/bot_bottle/backend/consolidated_util.py b/bot_bottle/backend/consolidated_util.py index 520ca0a..3aa8590 100644 --- a/bot_bottle/backend/consolidated_util.py +++ b/bot_bottle/backend/consolidated_util.py @@ -13,7 +13,7 @@ from ..egress import EgressPlan from ..git_gate import GitGatePlan from ..orchestrator.client import OrchestratorClient, RegisteredBottle from ..orchestrator.registration import registration_inputs -from ..orchestrator.secret_store import new_env_var_secret +from ..orchestrator.store.secret_store import new_env_var_secret from .docker.gateway_provision import GatewayTransport, deprovision_git_gate, provision_git_gate @@ -58,7 +58,7 @@ def teardown_consolidated( ) -> None: """Deregister the bottle and remove its git-gate state. Both steps are idempotent so this is safe from a cleanup trap.""" - from ..orchestrator.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS + from ..orchestrator.store.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS OrchestratorClient( orchestrator_url, timeout=timeout if timeout is not None else DEFAULT_TEARDOWN_TIMEOUT_SECONDS, diff --git a/bot_bottle/backend/docker/backend.py b/bot_bottle/backend/docker/backend.py index b2ae45b..8cab6d7 100644 --- a/bot_bottle/backend/docker/backend.py +++ b/bot_bottle/backend/docker/backend.py @@ -24,12 +24,12 @@ from contextlib import contextmanager from pathlib import Path from typing import Generator, Sequence -from ...supervise import SUPERVISE_HOSTNAME, SUPERVISE_PORT +from ...supervisor.types import SUPERVISE_HOSTNAME, SUPERVISE_PORT from ...agent_provider import AgentProvisionPlan from ...egress import EgressPlan from ...env import ResolvedEnv from ...git_gate import GitGatePlan -from ...supervise import SupervisePlan +from ...supervisor.plan import SupervisePlan from ...manifest import Manifest from .. import ActiveAgent, BottleBackend, BottleImages, BottleSpec from . import cleanup as _cleanup diff --git a/bot_bottle/backend/docker/consolidated_compose.py b/bot_bottle/backend/docker/consolidated_compose.py index 0065224..45296b5 100644 --- a/bot_bottle/backend/docker/consolidated_compose.py +++ b/bot_bottle/backend/docker/consolidated_compose.py @@ -17,7 +17,7 @@ from __future__ import annotations from typing import Any from ...egress import egress_agent_env_entries -from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME +from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH from .bottle_plan import DockerBottlePlan from .egress import EGRESS_PORT diff --git a/bot_bottle/backend/docker/consolidated_launch.py b/bot_bottle/backend/docker/consolidated_launch.py index 482a30f..2f57efb 100644 --- a/bot_bottle/backend/docker/consolidated_launch.py +++ b/bot_bottle/backend/docker/consolidated_launch.py @@ -22,7 +22,7 @@ from ...git_gate import GitGatePlan from ...orchestrator.client import OrchestratorClient from ...gateway import GATEWAY_NETWORK from ...orchestrator.lifecycle import INFRA_NAME, OrchestratorService -from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME +from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME from ...orchestrator.reprovision import reprovision_bottles from ..consolidated_util import provision_bottle from ..consolidated_util import teardown_consolidated as _teardown_util diff --git a/bot_bottle/backend/docker/launch.py b/bot_bottle/backend/docker/launch.py index 47e2f64..78c8785 100644 --- a/bot_bottle/backend/docker/launch.py +++ b/bot_bottle/backend/docker/launch.py @@ -64,7 +64,7 @@ from .compose import ( write_compose_file, ) from .consolidated_compose import consolidated_agent_compose -from ...orchestrator.config_store import resolve_teardown_timeout +from ...orchestrator.store.config_store import resolve_teardown_timeout from .consolidated_launch import launch_consolidated, teardown_consolidated from ...orchestrator.lifecycle import INFRA_NAME from .gateway import DockerGateway @@ -207,7 +207,7 @@ def launch( # spec, value only in the subprocess env so it is never written to disk. compose_env: dict[str, str] = {**os.environ, **plan.forwarded_env} if plan.env_var_secret: - from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME + from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME compose_env[ENV_VAR_SECRET_NAME] = plan.env_var_secret info( f"docker compose up -d (project {project}, agent on shared " diff --git a/bot_bottle/backend/docker/resolve_plan.py b/bot_bottle/backend/docker/resolve_plan.py index c48e3ac..eac5d2a 100644 --- a/bot_bottle/backend/docker/resolve_plan.py +++ b/bot_bottle/backend/docker/resolve_plan.py @@ -19,7 +19,7 @@ from ...env import ResolvedEnv from ...agent_provider import AgentProvisionPlan from ...egress import EgressPlan from ...manifest import Manifest -from ...supervise import SupervisePlan +from ...supervisor.plan import SupervisePlan from ...git_gate import GitGatePlan def preflight() -> None: diff --git a/bot_bottle/backend/firecracker/backend.py b/bot_bottle/backend/firecracker/backend.py index a8a9df9..a528182 100644 --- a/bot_bottle/backend/firecracker/backend.py +++ b/bot_bottle/backend/firecracker/backend.py @@ -17,7 +17,7 @@ from ...egress import EgressPlan from ...env import ResolvedEnv from ...git_gate import GitGatePlan from ...manifest import Manifest -from ...supervise import SupervisePlan +from ...supervisor.plan import SupervisePlan from .. import ActiveAgent, BottleBackend, BottleImages, BottleSpec from . import cleanup as _cleanup from . import enumerate as _enumerate diff --git a/bot_bottle/backend/firecracker/consolidated_launch.py b/bot_bottle/backend/firecracker/consolidated_launch.py index cb72690..503d0af 100644 --- a/bot_bottle/backend/firecracker/consolidated_launch.py +++ b/bot_bottle/backend/firecracker/consolidated_launch.py @@ -38,7 +38,7 @@ from ...orchestrator.lifecycle import ( OrchestratorStartError, # re-exported so callers can catch it ) from ...orchestrator.reprovision import reprovision_bottles -from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME +from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME from ..consolidated_util import ( provision_bottle, teardown_consolidated as _teardown_util, diff --git a/bot_bottle/backend/firecracker/launch.py b/bot_bottle/backend/firecracker/launch.py index d72f40c..37bfb2d 100644 --- a/bot_bottle/backend/firecracker/launch.py +++ b/bot_bottle/backend/firecracker/launch.py @@ -48,14 +48,14 @@ from ...git_gate import ( ) from ...image_cache import check_stale_path from ...log import die, info, warn -from ...supervise import SUPERVISE_PORT +from ...supervisor.types import SUPERVISE_PORT from ..docker.egress import EGRESS_PORT from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH from . import firecracker_vm, image_builder, isolation_probe, netpool, util from .bottle import FirecrackerBottle from .bottle_plan import FirecrackerBottlePlan -from ...orchestrator.config_store import resolve_teardown_timeout -from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME +from ...orchestrator.store.config_store import resolve_teardown_timeout +from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME from .consolidated_launch import ( launch_consolidated, persist_env_var_secret, diff --git a/bot_bottle/backend/firecracker/resolve_plan.py b/bot_bottle/backend/firecracker/resolve_plan.py index 6292175..f5b0dd6 100644 --- a/bot_bottle/backend/firecracker/resolve_plan.py +++ b/bot_bottle/backend/firecracker/resolve_plan.py @@ -9,7 +9,7 @@ from ...egress import EgressPlan from ...env import ResolvedEnv from ...git_gate import GitGatePlan from ...manifest import Manifest -from ...supervise import SupervisePlan +from ...supervisor.plan import SupervisePlan from .. import BottleSpec from . import util from .bottle_plan import FirecrackerBottlePlan diff --git a/bot_bottle/backend/macos_container/backend.py b/bot_bottle/backend/macos_container/backend.py index 2448c00..71b4d5e 100644 --- a/bot_bottle/backend/macos_container/backend.py +++ b/bot_bottle/backend/macos_container/backend.py @@ -10,7 +10,7 @@ from ...agent_provider import AgentProvisionPlan from ...egress import EgressPlan from ...env import ResolvedEnv from ...git_gate import GitGatePlan -from ...supervise import SupervisePlan +from ...supervisor.plan import SupervisePlan from ...manifest import Manifest from .. import ActiveAgent, BottleBackend, BottleImages, BottleSpec from . import cleanup as _cleanup diff --git a/bot_bottle/backend/macos_container/consolidated_launch.py b/bot_bottle/backend/macos_container/consolidated_launch.py index 6c25f5e..94976fb 100644 --- a/bot_bottle/backend/macos_container/consolidated_launch.py +++ b/bot_bottle/backend/macos_container/consolidated_launch.py @@ -39,7 +39,7 @@ from ...git_gate import GitGatePlan from ...log import info from ...orchestrator.client import OrchestratorClient, OrchestratorClientError from ...orchestrator.reprovision import reprovision_bottles -from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME +from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME from ..consolidated_util import ( provision_bottle, teardown_consolidated as _teardown_util, diff --git a/bot_bottle/backend/macos_container/launch.py b/bot_bottle/backend/macos_container/launch.py index d54a20a..95f5d61 100644 --- a/bot_bottle/backend/macos_container/launch.py +++ b/bot_bottle/backend/macos_container/launch.py @@ -56,7 +56,7 @@ from ...gateway.git_http_backend import DEFAULT_PORT as _GIT_HTTP_PORT from ...image_cache import check_stale from ...log import die, info, warn from .. import BottleImages -from ...supervise import SUPERVISE_PORT +from ...supervisor.types import SUPERVISE_PORT from ..docker.egress import EGRESS_PORT from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH from . import util as container_mod @@ -68,8 +68,8 @@ from .gateway_hosts import ( ) from . import nested_containers as nested_containers_mod from .bottle_plan import MacosContainerBottlePlan -from ...orchestrator.config_store import resolve_teardown_timeout -from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME, new_env_var_secret +from ...orchestrator.store.config_store import resolve_teardown_timeout +from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME, new_env_var_secret from .consolidated_launch import ( GatewayEndpoint, ensure_gateway, diff --git a/bot_bottle/backend/macos_container/resolve_plan.py b/bot_bottle/backend/macos_container/resolve_plan.py index 4c9883e..f8fb975 100644 --- a/bot_bottle/backend/macos_container/resolve_plan.py +++ b/bot_bottle/backend/macos_container/resolve_plan.py @@ -8,7 +8,7 @@ from ...agent_provider import AgentProvisionPlan from ...egress import EgressPlan from ...env import ResolvedEnv from ...git_gate import GitGatePlan -from ...supervise import SupervisePlan +from ...supervisor.plan import SupervisePlan from ...manifest import Manifest from .. import BottleSpec from . import util as container_mod diff --git a/bot_bottle/backend/resolve_common.py b/bot_bottle/backend/resolve_common.py index 88f94ff..612291b 100644 --- a/bot_bottle/backend/resolve_common.py +++ b/bot_bottle/backend/resolve_common.py @@ -28,7 +28,8 @@ from ..egress import Egress, EgressPlan from ..git_gate import GitGate, GitGatePlan from ..log import die from ..manifest import Manifest, ManifestBottle -from ..supervise import Supervise, SupervisePlan +from ..supervisor.plan import SupervisePlan +from ..orchestrator.supervisor.supervise import Supervise from . import BottleSpec diff --git a/bot_bottle/cli/__init__.py b/bot_bottle/cli/__init__.py index c1618b6..d1d550a 100644 --- a/bot_bottle/cli/__init__.py +++ b/bot_bottle/cli/__init__.py @@ -10,7 +10,7 @@ import sys from ..errors import MissingEnvVarError from ..log import Die, die, error from ..manifest import ManifestError -from ..store.store_manager import StoreManager +from ..orchestrator.store.store_manager import StoreManager from ._common import PROG from . import list as _list_mod from .backend import cmd_backend diff --git a/bot_bottle/cli/supervise.py b/bot_bottle/cli/supervise.py index d87726c..abf7d70 100644 --- a/bot_bottle/cli/supervise.py +++ b/bot_bottle/cli/supervise.py @@ -27,7 +27,7 @@ from ..orchestrator.client import ( discover_orchestrator_url, ) -from ..supervise import ( +from ..supervisor.types import ( Proposal, TOOL_EGRESS_ALLOW, TOOL_EGRESS_BLOCK, diff --git a/bot_bottle/gateway/egress_addon.py b/bot_bottle/gateway/egress_addon.py index 112de8b..eda9402 100644 --- a/bot_bottle/gateway/egress_addon.py +++ b/bot_bottle/gateway/egress_addon.py @@ -41,7 +41,7 @@ from bot_bottle.gateway.egress_addon_core import ( scan_outbound, ) from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver -from bot_bottle.supervise_types import ( +from bot_bottle.supervisor.types import ( STATUS_APPROVED, STATUS_MODIFIED, STATUSES, diff --git a/bot_bottle/gateway/git_gate_render.py b/bot_bottle/gateway/git_gate_render.py index 294d675..d6fe825 100644 --- a/bot_bottle/gateway/git_gate_render.py +++ b/bot_bottle/gateway/git_gate_render.py @@ -283,10 +283,10 @@ from pathlib import Path # calling bottle exactly as a direct write once did. try: from bot_bottle.gateway.policy_resolver import PolicyResolver, PolicyResolveError - from bot_bottle.supervise_types import TOOL_GITLEAKS_ALLOW + from bot_bottle.supervisor.types import TOOL_GITLEAKS_ALLOW except ImportError: from policy_resolver import PolicyResolver, PolicyResolveError - from supervise_types import TOOL_GITLEAKS_ALLOW + from supervisor.types import TOOL_GITLEAKS_ALLOW report_path = Path(sys.argv[1]) source_ip = os.environ.get("SUPERVISE_SOURCE_IP", "") diff --git a/bot_bottle/gateway/supervise_server.py b/bot_bottle/gateway/supervise_server.py index 26495a3..520f1da 100644 --- a/bot_bottle/gateway/supervise_server.py +++ b/bot_bottle/gateway/supervise_server.py @@ -62,7 +62,7 @@ from bot_bottle.gateway.egress_addon_core import ( LOG_OFF, load_config, resolve_client_context, route_to_yaml_dict, ) from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver -from bot_bottle import supervise as _sv +from bot_bottle.supervisor import types as _sv # --- JSON-RPC / MCP plumbing ---------------------------------------------- diff --git a/bot_bottle/orchestrator/__main__.py b/bot_bottle/orchestrator/__main__.py index 38d4e0c..cd532f9 100644 --- a/bot_bottle/orchestrator/__main__.py +++ b/bot_bottle/orchestrator/__main__.py @@ -16,7 +16,7 @@ import secrets from pathlib import Path from .. import log -from ..store.store_manager import StoreManager +from .store.store_manager import StoreManager from .broker import LaunchBroker, StubBroker from .control_plane import make_server from .docker_broker import DockerBroker diff --git a/bot_bottle/orchestrator/control_plane.py b/bot_bottle/orchestrator/control_plane.py index 5760923..4daed70 100644 --- a/bot_bottle/orchestrator/control_plane.py +++ b/bot_bottle/orchestrator/control_plane.py @@ -65,7 +65,7 @@ from urllib.parse import urlsplit from ..control_auth import ROLE_CLI, ROLES, verify from ..paths import CONTROL_PLANE_TOKEN_ENV -from ..supervise_types import TOOLS +from ..supervisor.types import TOOLS from .service import Orchestrator # JSON body payload type (parsed request / rendered response). diff --git a/bot_bottle/orchestrator/service.py b/bot_bottle/orchestrator/service.py index d521937..bf6a14d 100644 --- a/bot_bottle/orchestrator/service.py +++ b/bot_bottle/orchestrator/service.py @@ -27,7 +27,7 @@ from datetime import datetime, timezone from .broker import LaunchBroker, LaunchRequest, sign_request from .registry import DEFAULT_REAP_GRACE_SECONDS, BottleRecord, RegistryStore -from ..supervise import ( +from .supervisor import ( AuditEntry, COMPONENT_FOR_TOOL, POLL_STATUS_PENDING, @@ -104,7 +104,7 @@ class Orchestrator: if tokens: self._tokens[rec.bottle_id] = dict(tokens) if env_var_secret: - from .secret_store import encrypt_value + from .store.secret_store import encrypt_value encrypted = {k: encrypt_value(env_var_secret, v) for k, v in tokens.items()} self.registry.store_agent_secrets(rec.bottle_id, encrypted) req = LaunchRequest( @@ -367,7 +367,7 @@ class Orchestrator: value with *env_var_secret*, and restores ``_tokens[bottle_id]``. Returns True on success, False when no stored secrets exist for this bottle or decryption fails (wrong key / corrupt data).""" - from .secret_store import decrypt_value + from .store.secret_store import decrypt_value encrypted = self.registry.get_agent_secrets(bottle_id) if not encrypted: return False diff --git a/bot_bottle/orchestrator/store/__init__.py b/bot_bottle/orchestrator/store/__init__.py new file mode 100644 index 0000000..1c4a81d --- /dev/null +++ b/bot_bottle/orchestrator/store/__init__.py @@ -0,0 +1,10 @@ +"""Orchestrator-owned SQLite stores. + +The stores whose tables only the orchestrator (control plane) opens: the +supervise proposal/response `queue_store`, the agent-secret `secret_store`, and +the orchestrator `config_store`. They build on the shared `DbStore` / +`migrations` base in `bot_bottle.store`. + +Callers import the concrete module directly, e.g. +`from bot_bottle.orchestrator.store.queue_store import QueueStore`. +""" diff --git a/bot_bottle/orchestrator/config_store.py b/bot_bottle/orchestrator/store/config_store.py similarity index 96% rename from bot_bottle/orchestrator/config_store.py rename to bot_bottle/orchestrator/store/config_store.py index 7779b68..251f0ef 100644 --- a/bot_bottle/orchestrator/config_store.py +++ b/bot_bottle/orchestrator/store/config_store.py @@ -11,9 +11,9 @@ import os import sqlite3 from pathlib import Path -from ..store.db_store import DbStore -from ..store.migrations import TableMigrations -from ..paths import host_db_path +from ...store.db_store import DbStore +from ...store.migrations import TableMigrations +from ...paths import host_db_path TEARDOWN_TIMEOUT_ENV = "BOT_BOTTLE_ORCHESTRATOR_TEARDOWN_TIMEOUT_SECONDS" DEFAULT_TEARDOWN_TIMEOUT_SECONDS = 30.0 diff --git a/bot_bottle/store/queue_store.py b/bot_bottle/orchestrator/store/queue_store.py similarity index 96% rename from bot_bottle/store/queue_store.py rename to bot_bottle/orchestrator/store/queue_store.py index 09ef1cf..16637f9 100644 --- a/bot_bottle/store/queue_store.py +++ b/bot_bottle/orchestrator/store/queue_store.py @@ -7,12 +7,12 @@ import sqlite3 from pathlib import Path try: - from ..supervise_types import Proposal, Response - from ..paths import host_db_path - from .db_store import DbStore - from .migrations import TableMigrations + from ...supervisor.types import Proposal, Response + from ...paths import host_db_path + from ...store.db_store import DbStore + from ...store.migrations import TableMigrations except ImportError: - from supervise_types import Proposal, Response # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module + from supervisor.types import Proposal, Response # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module from paths import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module from db_store import DbStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module diff --git a/bot_bottle/orchestrator/secret_store.py b/bot_bottle/orchestrator/store/secret_store.py similarity index 100% rename from bot_bottle/orchestrator/secret_store.py rename to bot_bottle/orchestrator/store/secret_store.py diff --git a/bot_bottle/store/store_manager.py b/bot_bottle/orchestrator/store/store_manager.py similarity index 54% rename from bot_bottle/store/store_manager.py rename to bot_bottle/orchestrator/store/store_manager.py index e7729cc..2e40e0f 100644 --- a/bot_bottle/store/store_manager.py +++ b/bot_bottle/orchestrator/store/store_manager.py @@ -1,34 +1,34 @@ -"""Singleton manager for all bot-bottle SQLite stores (PRD 0013).""" +"""The orchestrator's store manager (PRD 0013 / 0070). + +One store manager per service. Post-PRD-0070 the orchestrator is the sole +opener of `bot-bottle.db`, so it owns migrating every table in it: the supervise +`queue_store` (local to `orchestrator.store`) plus the `audit_store` and +`config_store` from the shared `bot_bottle.store` base. The data plane never +touches these — it reaches state over the control-plane RPC. +""" from __future__ import annotations from pathlib import Path -try: - from .audit_store import AuditStore - from .config_store import ConfigStore - from .queue_store import QueueStore -except ImportError: - from audit_store import AuditStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from config_store import ConfigStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from queue_store import QueueStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module +from .queue_store import QueueStore +from ...store.audit_store import AuditStore +from ...store.config_store import ConfigStore _instance: StoreManager | None = None class StoreManager: - """Owns db_path and delegates migrate/is_migrated across all stores. + """Owns db_path and delegates migrate/is_migrated across the orchestrator's + stores. - Use instance() for normal access. Call reset(db_path) in tests to swap - the singleton to a temp path, then reset() with no args to restore the + Use instance() for normal access. Call reset(db_path) in tests to swap the + singleton to a temp path, then reset() with no args to restore the default.""" def __init__(self, db_path: Path | None = None) -> None: if db_path is None: - try: - from ..paths import host_db_path - except ImportError: - from paths import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module + from ...paths import host_db_path db_path = host_db_path() self.db_path = db_path diff --git a/bot_bottle/orchestrator/supervisor/__init__.py b/bot_bottle/orchestrator/supervisor/__init__.py new file mode 100644 index 0000000..c6d903e --- /dev/null +++ b/bot_bottle/orchestrator/supervisor/__init__.py @@ -0,0 +1,34 @@ +"""The orchestrator's view of the supervise plane. + +Bundles the orchestrator-owned supervise surface: the queue/audit I/O and diff +rendering (`queue`) and the `Supervise` lifecycle (`supervise`). For +convenience it re-exports the neutral vocabulary (`bot_bottle.supervisor`'s +types + `SupervisePlan`) so orchestrator-side callers — service, CLI, tests — +import from one place. + +The data plane must NOT import this package; it imports `bot_bottle.supervisor` +(neutral) directly and reaches the queue over the control-plane RPC. +""" + +from __future__ import annotations + +from ...supervisor.types import * # noqa: F401,F403 — re-export the neutral vocabulary +from ...supervisor.plan import SupervisePlan +from .queue import ( + archive_all_proposals, + archive_proposal, + audit_dir, + audit_log_path, + list_all_pending_proposals, + list_pending_proposals, + read_audit_entries, + read_proposal, + read_response, + render_diff, + sha256_hex, + wait_for_response, + write_audit_entry, + write_proposal, + write_response, +) +from .supervise import Supervise diff --git a/bot_bottle/orchestrator/supervisor/queue.py b/bot_bottle/orchestrator/supervisor/queue.py new file mode 100644 index 0000000..fed4132 --- /dev/null +++ b/bot_bottle/orchestrator/supervisor/queue.py @@ -0,0 +1,141 @@ +"""Orchestrator-side supervise queue + audit I/O and diff rendering. + +The host-side library the orchestrator uses to drive the supervise flow: read / +write proposals and responses against the queue store, append audit entries, +and render operator diffs. Only the orchestrator opens `bot-bottle.db` (PRD +0070), so this lives under `orchestrator/` — the data plane reaches it over the +control-plane RPC, never by importing this module. +""" + +from __future__ import annotations + +import difflib +import hashlib +import time +from pathlib import Path + +from ...supervisor.types import ( + AuditEntry, + DEFAULT_POLL_INTERVAL_SEC, + Proposal, + Response, +) +from ...paths import bot_bottle_root +from ..store.queue_store import QueueStore +from ...store.audit_store import AuditStore + + +# --- Paths ----------------------------------------------------------------- + + +def audit_dir() -> Path: + return bot_bottle_root() / "audit" + + +def audit_log_path(component: str, slug: str) -> Path: + return audit_dir() / f"{component}-{slug}.log" + + +# --- Queue I/O ------------------------------------------------------------- + + +def write_proposal(proposal: Proposal) -> Path: + """Persist `proposal` in the queue database, mode 0o600. + Directory is created if missing.""" + return QueueStore(proposal.bottle_slug).write_proposal(proposal) + + +def read_proposal(bottle_slug: str, proposal_id: str) -> Proposal: + return QueueStore(bottle_slug).read_proposal(proposal_id) + + +def list_pending_proposals(bottle_slug: str) -> list[Proposal]: + """All proposals for `bottle_slug` that do not yet have a matching + response. Sorted by `arrival_timestamp` so the operator sees the queue + FIFO.""" + return QueueStore(bottle_slug).list_pending_proposals() + + +def list_all_pending_proposals() -> list[Proposal]: + """All pending proposals across bottles, sorted FIFO.""" + return QueueStore("").list_all_pending_proposals() + + +def write_response(bottle_slug: str, response: Response) -> Path: + return QueueStore(bottle_slug).write_response(response) + + +def read_response(bottle_slug: str, proposal_id: str) -> Response: + return QueueStore(bottle_slug).read_response(proposal_id) + + +def wait_for_response( + bottle_slug: str, + proposal_id: str, + *, + poll_interval: float = DEFAULT_POLL_INTERVAL_SEC, + deadline: float | None = None, +) -> Response: + """Block until a response file appears for `proposal_id`, then return it. + `deadline` is an absolute time.monotonic() value after which the wait raises + TimeoutError. None waits forever — the natural shape, since the operator's + response time is unbounded. + + Polls SQLite so the implementation stays portable and stdlib-only.""" + store = QueueStore(bottle_slug) + while True: + try: + return store.read_response(proposal_id) + except FileNotFoundError: + pass + if deadline is not None and time.monotonic() >= deadline: + raise TimeoutError(f"no response for proposal {proposal_id!r}") + time.sleep(poll_interval) + + +def archive_proposal(bottle_slug: str, proposal_id: str) -> None: + """Mark both proposal and response rows processed. + Idempotent — missing rows are silently skipped.""" + QueueStore(bottle_slug).archive_proposal(proposal_id) + + +def archive_all_proposals(bottle_slug: str) -> None: + """Archive every proposal + response for `bottle_slug` (bottle teardown / + reconcile cleanup). Idempotent.""" + QueueStore(bottle_slug).archive_all() + + +# --- Audit log ------------------------------------------------------------- + + +def write_audit_entry(entry: AuditEntry) -> Path: + """Append `entry` to the host supervise audit table.""" + return AuditStore().write_audit_entry(entry) + + +def read_audit_entries(component: str, slug: str) -> list[AuditEntry]: + """Load all audit entries for the given component+slug.""" + return AuditStore().read_audit_entries(component, slug) + + +# --- Diff rendering -------------------------------------------------------- + + +def render_diff(before: str, after: str, *, label: str = "config") -> str: + """Unified diff suitable for the audit log + TUI. Empty diff (no changes) + renders as the empty string.""" + diff = difflib.unified_diff( + before.splitlines(keepends=True), + after.splitlines(keepends=True), + fromfile=f"{label} (current)", + tofile=f"{label} (proposed)", + lineterm="", + ) + parts = list(diff) + if not parts: + return "" + return "".join(p if p.endswith("\n") else p + "\n" for p in parts).rstrip("\n") + + +def sha256_hex(content: str) -> str: + return hashlib.sha256(content.encode("utf-8")).hexdigest() diff --git a/bot_bottle/orchestrator/supervisor/supervise.py b/bot_bottle/orchestrator/supervisor/supervise.py new file mode 100644 index 0000000..f2e51fe --- /dev/null +++ b/bot_bottle/orchestrator/supervisor/supervise.py @@ -0,0 +1,36 @@ +"""The `Supervise` lifecycle — host-side database staging (PRD 0013 / 0070). + +`Supervise.prepare` migrates the orchestrator's `bot-bottle.db` and returns the +neutral `SupervisePlan`. It touches the orchestrator store manager, so it lives +here rather than in the neutral `bot_bottle.supervisor` package; the backend +(which drives launch) may import it — backend → orchestrator is an allowed +direction, unlike gateway → orchestrator. +""" + +from __future__ import annotations + +from abc import ABC +from pathlib import Path + +from ..store.store_manager import StoreManager +from ...supervisor.plan import SupervisePlan + + +class Supervise(ABC): + """Per-bottle supervise daemon. Encapsulates host-side database staging; + the gateway's start/stop lifecycle is backend-specific.""" + + def prepare( + self, + slug: str, + stage_dir: Path, + ) -> SupervisePlan: + """Stage the host database. Returns the plan; `internal_network` must be + set by the launch step before .start runs.""" + del stage_dir + mgr = StoreManager.instance() + mgr.migrate() + return SupervisePlan( + slug=slug, + db_path=mgr.db_path, + ) diff --git a/bot_bottle/store/__init__.py b/bot_bottle/store/__init__.py index ec5e39b..0a7f1f3 100644 --- a/bot_bottle/store/__init__.py +++ b/bot_bottle/store/__init__.py @@ -1,9 +1,13 @@ -"""SQLite-backed persistence for bot-bottle. +"""SQLite-backed persistence base for bot-bottle. -The store family: a shared `DbStore` base (versioned `migrations`) and the -concrete stores built on it — `ConfigStore`, `AuditStore`, `QueueStore` — plus -`StoreManager`, which migrates them together against the one per-host DB. +The shared store base: the `DbStore` class (versioned `migrations`) and the +concrete stores built on it that aren't owned by a single service — +`ConfigStore` and `AuditStore`. + +Service-owned stores live under their service: the supervise `queue_store`, the +`secret_store`, the orchestrator `config_store`, and the orchestrator +`store_manager` are under `bot_bottle.orchestrator.store`. Callers import the concrete module they need directly, e.g. -`from bot_bottle.store.store_manager import StoreManager`. +`from bot_bottle.store.db_store import DbStore`. """ diff --git a/bot_bottle/store/audit_store.py b/bot_bottle/store/audit_store.py index 13de470..3a6738b 100644 --- a/bot_bottle/store/audit_store.py +++ b/bot_bottle/store/audit_store.py @@ -6,12 +6,12 @@ import sqlite3 from pathlib import Path try: - from ..supervise_types import AuditEntry + from ..supervisor.types import AuditEntry from ..paths import host_db_path from .db_store import DbStore from .migrations import TableMigrations except ImportError: - from supervise_types import AuditEntry # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module + from supervisor.types import AuditEntry # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module from paths import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module from db_store import DbStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module diff --git a/bot_bottle/supervise.py b/bot_bottle/supervise.py deleted file mode 100644 index d71253c..0000000 --- a/bot_bottle/supervise.py +++ /dev/null @@ -1,298 +0,0 @@ -"""Per-bottle supervise plane (PRD 0013). - -The supervise plane is the per-bottle MCP daemon plus its host-side -queue/audit support. The daemon (bot_bottle.gateway.supervise_server) -sits on the bottle's internal network and exposes MCP tools the agent -calls when it needs an operator-reviewed egress change: - - * egress-block / allow — agent proposes a new routes.yaml - -Each tool call: the agent passes the full proposed file plus a -justification text. The gateway validates the proposal syntactically, -writes it to the host SQLite queue table, and holds the tool-call -connection open. The operator's supervise TUI -(bot_bottle.cli.supervise) sees the proposal, accepts -approve / modify / reject, and writes a response row. The gateway sees -the response and returns `{status, notes}` to the agent. - -This module defines the host-side library: dataclasses for the queue -record shapes, queue read/write helpers, the audit log writer, and the -diff renderer. The in-gateway daemon lives in -bot_bottle/gateway/supervise_server.py; the supervise daemon's container -lifecycle is owned by the gateway (PRD 0024). - -For 0013 the supervisor's approval handlers are deliberately no-ops: -on approval the audit log is written and the response file is -delivered to the agent, but no host-side config change happens. The -remediation engines that wire real config changes land in PRDs 0014, -0015, and 0016. -""" - -from __future__ import annotations - -import difflib -import hashlib -import time -from abc import ABC -from dataclasses import dataclass -from pathlib import Path - -from .supervise_types import ( - ACTION_OPERATOR_EDIT, - AuditEntry, - POLL_STATUS_PENDING, - POLL_STATUS_UNKNOWN, - Proposal, - Response, - STATUSES, - STATUS_APPROVED, - STATUS_MODIFIED, - STATUS_REJECTED, - TOOLS, - TOOL_CHECK_PROPOSAL, - TOOL_EGRESS_ALLOW, - TOOL_EGRESS_BLOCK, - TOOL_EGRESS_TOKEN_ALLOW, - TOOL_GITLEAKS_ALLOW, - TOOL_LIST_EGRESS_ROUTES, -) - - -try: - from .paths import bot_bottle_root -except ImportError: # flat imports inside the gateway - from paths import bot_bottle_root # type: ignore[import-not-found,no-redef] # pylint: disable=import-error,no-name-in-module - - -SUPERVISE_HOSTNAME = "supervise" -SUPERVISE_PORT = 9100 - -# The supervise daemon uses these to query egress's -# introspection endpoint for the `list-egress-routes` MCP -# tool. The hostname + port match egress's docker network -# listen port (see backend.docker.egress.EGRESS_PORT). The supervise -# daemon runs inside the gateway alongside egress, so loopback -# is the stable address across docker, firecracker, and Apple -# Container backends. -EGRESS_FORWARD_PROXY = "http://127.0.0.1:9099" -EGRESS_INTROSPECT_URL = "http://_egress.local/allowlist" - -COMPONENT_FOR_TOOL: dict[str, str] = { - TOOL_EGRESS_ALLOW: "egress", - TOOL_EGRESS_BLOCK: "egress", -} - -DB_PATH_IN_CONTAINER = "/run/supervise/bot-bottle.db" -DEFAULT_POLL_INTERVAL_SEC = 0.5 - - -# --- Paths ----------------------------------------------------------------- - - -def audit_dir() -> Path: - return bot_bottle_root() / "audit" - - -def audit_log_path(component: str, slug: str) -> Path: - return audit_dir() / f"{component}-{slug}.log" - - -try: - from .store.queue_store import QueueStore - from .store.audit_store import AuditStore - from .store.store_manager import StoreManager -except ImportError: - # Gateway: files are flat-copied under /app, not a package. - from queue_store import QueueStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from audit_store import AuditStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from store_manager import StoreManager # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - - -# --- Queue I/O ------------------------------------------------------------- - - -def write_proposal(proposal: Proposal) -> Path: - """Persist `proposal` in the queue database, mode 0o600. - Directory is created if missing.""" - return QueueStore(proposal.bottle_slug).write_proposal(proposal) - - -def read_proposal(bottle_slug: str, proposal_id: str) -> Proposal: - return QueueStore(bottle_slug).read_proposal(proposal_id) - - -def list_pending_proposals(bottle_slug: str) -> list[Proposal]: - """All proposals for `bottle_slug` that do not yet have a matching - response. Sorted by `arrival_timestamp` so the operator - sees the queue FIFO.""" - return QueueStore(bottle_slug).list_pending_proposals() - - -def list_all_pending_proposals() -> list[Proposal]: - """All pending proposals across bottles, sorted FIFO.""" - return QueueStore("").list_all_pending_proposals() - - -def write_response(bottle_slug: str, response: Response) -> Path: - return QueueStore(bottle_slug).write_response(response) - - -def read_response(bottle_slug: str, proposal_id: str) -> Response: - return QueueStore(bottle_slug).read_response(proposal_id) - - -def wait_for_response( - bottle_slug: str, - proposal_id: str, - *, - poll_interval: float = DEFAULT_POLL_INTERVAL_SEC, - deadline: float | None = None, -) -> Response: - """Block until a response file appears for `proposal_id`, then - return it. `deadline` is an absolute time.monotonic() value after - which the wait raises TimeoutError. None waits forever — the - natural shape, since the operator's response time is unbounded. - - Polls SQLite so the implementation stays portable and stdlib-only.""" - store = QueueStore(bottle_slug) - while True: - try: - return store.read_response(proposal_id) - except FileNotFoundError: - pass - if deadline is not None and time.monotonic() >= deadline: - raise TimeoutError(f"no response for proposal {proposal_id!r}") - time.sleep(poll_interval) - - -def archive_proposal(bottle_slug: str, proposal_id: str) -> None: - """Mark both proposal and response rows processed. - Idempotent — missing rows are silently skipped.""" - QueueStore(bottle_slug).archive_proposal(proposal_id) - - -def archive_all_proposals(bottle_slug: str) -> None: - """Archive every proposal + response for `bottle_slug` (bottle teardown / - reconcile cleanup). Idempotent.""" - QueueStore(bottle_slug).archive_all() - - -# --- Audit log ------------------------------------------------------------- - - -def write_audit_entry(entry: AuditEntry) -> Path: - """Append `entry` to the host supervise audit table.""" - return AuditStore().write_audit_entry(entry) - - -def read_audit_entries(component: str, slug: str) -> list[AuditEntry]: - """Load all audit entries for the given component+slug.""" - return AuditStore().read_audit_entries(component, slug) - - -# --- Diff rendering -------------------------------------------------------- - - -def render_diff(before: str, after: str, *, label: str = "config") -> str: - """Unified diff suitable for the audit log + TUI. Empty diff (no - changes) renders as the empty string.""" - diff = difflib.unified_diff( - before.splitlines(keepends=True), - after.splitlines(keepends=True), - fromfile=f"{label} (current)", - tofile=f"{label} (proposed)", - lineterm="", - ) - parts = list(diff) - if not parts: - return "" - return "".join(p if p.endswith("\n") else p + "\n" for p in parts).rstrip("\n") - - -def sha256_hex(content: str) -> str: - return hashlib.sha256(content.encode("utf-8")).hexdigest() - - -# --- Gateway plan + abstract lifecycle ------------------------------------- - - -@dataclass(frozen=True) -class SupervisePlan: - """Output of Supervise.prepare; consumed by .start. - - `db_path` is the host database bind-mounted into the gateway at - /run/supervise/bot-bottle.db. `internal_network` is empty at - prepare time; the backend's launch step fills it via - dataclasses.replace before calling .start.""" - - slug: str - db_path: Path - internal_network: str = "" - - -class Supervise(ABC): - """Per-bottle supervise daemon. Encapsulates host-side database - staging; the gateway's start/stop lifecycle is backend-specific.""" - - def prepare( - self, - slug: str, - stage_dir: Path, - ) -> SupervisePlan: - """Stage the host database. Returns the plan; `internal_network` - must be set by the launch step before .start runs.""" - del stage_dir - mgr = StoreManager.instance() - mgr.migrate() - return SupervisePlan( - slug=slug, - db_path=mgr.db_path, - ) - - -__all__ = [ - "ACTION_OPERATOR_EDIT", - "AuditEntry", - "AuditStore", - "COMPONENT_FOR_TOOL", - "DEFAULT_POLL_INTERVAL_SEC", - "DB_PATH_IN_CONTAINER", - "Proposal", - "QueueStore", - "Response", - "StoreManager", - "STATUSES", - "STATUS_APPROVED", - "STATUS_MODIFIED", - "STATUS_REJECTED", - "POLL_STATUS_PENDING", - "POLL_STATUS_UNKNOWN", - "SUPERVISE_HOSTNAME", - "SUPERVISE_PORT", - "Supervise", - "SupervisePlan", - "TOOLS", - "EGRESS_FORWARD_PROXY", - "EGRESS_INTROSPECT_URL", - "TOOL_CHECK_PROPOSAL", - "TOOL_EGRESS_ALLOW", - "TOOL_EGRESS_BLOCK", - "TOOL_GITLEAKS_ALLOW", - "TOOL_EGRESS_TOKEN_ALLOW", - "TOOL_LIST_EGRESS_ROUTES", - "archive_proposal", - "archive_all_proposals", - "audit_dir", - "audit_log_path", - "list_pending_proposals", - "list_all_pending_proposals", - "read_audit_entries", - "read_proposal", - "read_response", - "render_diff", - "sha256_hex", - "wait_for_response", - "write_audit_entry", - "write_proposal", - "write_response", -] diff --git a/bot_bottle/supervisor/__init__.py b/bot_bottle/supervisor/__init__.py new file mode 100644 index 0000000..d76c1e9 --- /dev/null +++ b/bot_bottle/supervisor/__init__.py @@ -0,0 +1,16 @@ +"""Neutral supervise vocabulary — importable by every tier. + +The supervise plane spans the control plane (orchestrator) and the data plane +(gateway `supervise_server`): the gateway proposes/polls over RPC, the +orchestrator queues and applies operator decisions. Both sides must agree on +the wire types and constants, so those live here — a neutral package neither +tier's private code, importable by `bot_bottle.gateway` **and** +`bot_bottle.orchestrator` without either depending on the other. + + * `types` — the `Proposal`/`Response`/`AuditEntry` dataclasses, the tool / + status / poll-status constants, and the shared daemon constants. + * `plan` — `SupervisePlan`, the launch-time staging DTO. + +The orchestrator-only half (queue I/O, diff rendering, the `Supervise` +lifecycle) lives under `bot_bottle.orchestrator.supervisor`. +""" diff --git a/bot_bottle/supervisor/plan.py b/bot_bottle/supervisor/plan.py new file mode 100644 index 0000000..2962d82 --- /dev/null +++ b/bot_bottle/supervisor/plan.py @@ -0,0 +1,26 @@ +"""`SupervisePlan` — the launch-time supervise staging DTO. + +A pure value type (no store access), so it lives in the neutral package: the +backend builds it at launch and the orchestrator's `Supervise.prepare` returns +it. The behaviour that fills it in — staging the host database — is the +orchestrator's, in `bot_bottle.orchestrator.supervisor.supervise`. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class SupervisePlan: + """Output of `Supervise.prepare`; consumed by `.start`. + + `db_path` is the host database bind-mounted into the gateway at + /run/supervise/bot-bottle.db. `internal_network` is empty at prepare time; + the backend's launch step fills it via `dataclasses.replace` before calling + `.start`.""" + + slug: str + db_path: Path + internal_network: str = "" diff --git a/bot_bottle/supervise_types.py b/bot_bottle/supervisor/types.py similarity index 84% rename from bot_bottle/supervise_types.py rename to bot_bottle/supervisor/types.py index 6bb65e5..e09fbde 100644 --- a/bot_bottle/supervise_types.py +++ b/bot_bottle/supervisor/types.py @@ -48,6 +48,24 @@ POLL_STATUS_UNKNOWN = "unknown" ACTION_OPERATOR_EDIT = "operator-edit" +# Shared supervise-daemon constants. The gateway `supervise_server` and the +# backend read these, so they live with the wire types in the neutral package +# (never in the orchestrator's private code the data plane can't import). +SUPERVISE_HOSTNAME = "supervise" +SUPERVISE_PORT = 9100 +# The supervise daemon queries egress's introspection endpoint for the +# `list-egress-routes` tool over loopback (egress runs alongside it in the +# gateway), stable across docker / firecracker / Apple Container backends. +EGRESS_FORWARD_PROXY = "http://127.0.0.1:9099" +EGRESS_INTROSPECT_URL = "http://_egress.local/allowlist" +DB_PATH_IN_CONTAINER = "/run/supervise/bot-bottle.db" +DEFAULT_POLL_INTERVAL_SEC = 0.5 + +COMPONENT_FOR_TOOL: dict[str, str] = { + TOOL_EGRESS_ALLOW: "egress", + TOOL_EGRESS_BLOCK: "egress", +} + def _require_str(raw: dict[str, object], key: str) -> str: value = raw.get(key) @@ -175,4 +193,11 @@ __all__ = [ "TOOL_EGRESS_TOKEN_ALLOW", "TOOL_GITLEAKS_ALLOW", "TOOL_LIST_EGRESS_ROUTES", + "SUPERVISE_HOSTNAME", + "SUPERVISE_PORT", + "EGRESS_FORWARD_PROXY", + "EGRESS_INTROSPECT_URL", + "DB_PATH_IN_CONTAINER", + "DEFAULT_POLL_INTERVAL_SEC", + "COMPONENT_FOR_TOOL", ] diff --git a/tests/integration/test_gateway_image.py b/tests/integration/test_gateway_image.py index 624bda9..213c93b 100644 --- a/tests/integration/test_gateway_image.py +++ b/tests/integration/test_gateway_image.py @@ -91,7 +91,7 @@ class TestGatewayImage(unittest.TestCase): # Probe that the package imports resolve inside the image. rc, out = self._run_in_image( "python3", "-c", - "from bot_bottle import supervise; from bot_bottle.gateway import supervise_server; print('ok')", + "from bot_bottle.supervisor import types; from bot_bottle.gateway import supervise_server; print('ok')", ) self.assertEqual(0, rc, msg=out) self.assertIn("ok", out) diff --git a/tests/unit/_docker_bottle_plan.py b/tests/unit/_docker_bottle_plan.py index c3a16c2..78f4b19 100644 --- a/tests/unit/_docker_bottle_plan.py +++ b/tests/unit/_docker_bottle_plan.py @@ -16,7 +16,7 @@ from bot_bottle.backend.docker.bottle_plan import DockerBottlePlan from bot_bottle.egress import EgressPlan, EgressRoute from bot_bottle.git_gate import GitGatePlan, GitGateUpstream from bot_bottle.manifest import ManifestIndex -from bot_bottle.supervise import SupervisePlan +from bot_bottle.supervisor.plan import SupervisePlan SLUG = "demo-abc12" diff --git a/tests/unit/test_cli_dispatch.py b/tests/unit/test_cli_dispatch.py index e5e10d5..bdc37bf 100644 --- a/tests/unit/test_cli_dispatch.py +++ b/tests/unit/test_cli_dispatch.py @@ -15,7 +15,7 @@ from bot_bottle.cli import main from bot_bottle.store.db_store import DbStore from bot_bottle.log import Die from bot_bottle.manifest import ManifestError -from bot_bottle.store.store_manager import StoreManager +from bot_bottle.orchestrator.store.store_manager import StoreManager class TestMainDispatch(unittest.TestCase): diff --git a/tests/unit/test_config_store.py b/tests/unit/test_config_store.py index 0f40576..b47cb6d 100644 --- a/tests/unit/test_config_store.py +++ b/tests/unit/test_config_store.py @@ -11,7 +11,7 @@ from bot_bottle.store.config_store import ( DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS, ConfigStore, ) -from bot_bottle.store.store_manager import StoreManager +from bot_bottle.orchestrator.store.store_manager import StoreManager class TestConfigStore(unittest.TestCase): diff --git a/tests/unit/test_contrib_claude_provider.py b/tests/unit/test_contrib_claude_provider.py index 0e8589b..1dc5db3 100644 --- a/tests/unit/test_contrib_claude_provider.py +++ b/tests/unit/test_contrib_claude_provider.py @@ -25,7 +25,7 @@ from bot_bottle.contrib.claude.agent_provider import ClaudeAgentProvider from bot_bottle.egress import EgressPlan from bot_bottle.git_gate import GitGatePlan from bot_bottle.manifest import ManifestIndex -from bot_bottle.supervise import SupervisePlan +from bot_bottle.supervisor.plan import SupervisePlan _URL = "http://supervise:9100/" diff --git a/tests/unit/test_contrib_codex_provider.py b/tests/unit/test_contrib_codex_provider.py index c53d2fb..4b2ca1f 100644 --- a/tests/unit/test_contrib_codex_provider.py +++ b/tests/unit/test_contrib_codex_provider.py @@ -25,7 +25,7 @@ from bot_bottle.contrib.codex.agent_provider import CodexAgentProvider from bot_bottle.egress import EgressPlan from bot_bottle.git_gate import GitGatePlan from bot_bottle.manifest import ManifestIndex -from bot_bottle.supervise import SupervisePlan +from bot_bottle.supervisor.plan import SupervisePlan _URL = "http://supervise:9100/" diff --git a/tests/unit/test_orchestrator_config_store.py b/tests/unit/test_orchestrator_config_store.py index ff43987..538a475 100644 --- a/tests/unit/test_orchestrator_config_store.py +++ b/tests/unit/test_orchestrator_config_store.py @@ -15,7 +15,7 @@ import unittest from pathlib import Path from types import ModuleType -from bot_bottle.orchestrator.config_store import ( +from bot_bottle.orchestrator.store.config_store import ( DEFAULT_TEARDOWN_TIMEOUT_SECONDS, TEARDOWN_TIMEOUT_ENV, OrchestratorConfigStore, diff --git a/tests/unit/test_orchestrator_control_plane.py b/tests/unit/test_orchestrator_control_plane.py index 1cd06b8..0bacd90 100644 --- a/tests/unit/test_orchestrator_control_plane.py +++ b/tests/unit/test_orchestrator_control_plane.py @@ -24,8 +24,8 @@ from bot_bottle.orchestrator.broker import StubBroker from bot_bottle.orchestrator.control_plane import dispatch, make_server from bot_bottle.orchestrator.registry import BottleRecord, RegistryStore from bot_bottle.orchestrator.service import Orchestrator -from bot_bottle.store.store_manager import StoreManager -from bot_bottle.supervise import ( +from bot_bottle.orchestrator.store.store_manager import StoreManager +from bot_bottle.orchestrator.supervisor import ( Proposal, TOOL_EGRESS_ALLOW, sha256_hex, diff --git a/tests/unit/test_orchestrator_secret_store.py b/tests/unit/test_orchestrator_secret_store.py index 64dd453..2b0e0d5 100644 --- a/tests/unit/test_orchestrator_secret_store.py +++ b/tests/unit/test_orchestrator_secret_store.py @@ -4,7 +4,7 @@ from __future__ import annotations import unittest -from bot_bottle.orchestrator.secret_store import ( +from bot_bottle.orchestrator.store.secret_store import ( ENV_VAR_SECRET_NAME, decrypt_value, encrypt_value, diff --git a/tests/unit/test_orchestrator_service.py b/tests/unit/test_orchestrator_service.py index 9441654..a83f8c1 100644 --- a/tests/unit/test_orchestrator_service.py +++ b/tests/unit/test_orchestrator_service.py @@ -14,9 +14,9 @@ from unittest.mock import patch from bot_bottle.orchestrator.broker import LaunchBroker, LaunchRequest, StubBroker from bot_bottle.orchestrator.registry import RegistryStore from bot_bottle.orchestrator.service import Orchestrator -from bot_bottle.orchestrator.secret_store import new_env_var_secret -from bot_bottle.store.store_manager import StoreManager -from bot_bottle.supervise import ( +from bot_bottle.orchestrator.store.secret_store import new_env_var_secret +from bot_bottle.orchestrator.store.store_manager import StoreManager +from bot_bottle.orchestrator.supervisor import ( Proposal, STATUS_APPROVED, TOOL_EGRESS_ALLOW, diff --git a/tests/unit/test_supervise.py b/tests/unit/test_supervise.py index e3360a7..f0f41be 100644 --- a/tests/unit/test_supervise.py +++ b/tests/unit/test_supervise.py @@ -7,12 +7,12 @@ import unittest from datetime import datetime, timezone from pathlib import Path -from bot_bottle import supervise +from bot_bottle.orchestrator import supervisor as supervise from bot_bottle.paths import host_db_path from tests.unit import use_bottle_root from bot_bottle.store.audit_store import AuditStore -from bot_bottle.store.queue_store import QueueStore -from bot_bottle.supervise import ( +from bot_bottle.orchestrator.store.queue_store import QueueStore +from bot_bottle.orchestrator.supervisor import ( AuditEntry, Proposal, Response, diff --git a/tests/unit/test_supervise_cli.py b/tests/unit/test_supervise_cli.py index b785a43..f35105b 100644 --- a/tests/unit/test_supervise_cli.py +++ b/tests/unit/test_supervise_cli.py @@ -13,7 +13,7 @@ from datetime import datetime, timezone from unittest.mock import MagicMock, patch from bot_bottle.cli import supervise as supervise_cli -from bot_bottle.supervise import ( +from bot_bottle.orchestrator.supervisor import ( Proposal, TOOL_EGRESS_ALLOW, TOOL_EGRESS_BLOCK, diff --git a/tests/unit/test_supervise_edge.py b/tests/unit/test_supervise_edge.py index acbd420..c56d9c5 100644 --- a/tests/unit/test_supervise_edge.py +++ b/tests/unit/test_supervise_edge.py @@ -10,11 +10,11 @@ import unittest from pathlib import Path from unittest.mock import patch -from bot_bottle import supervise +from bot_bottle.orchestrator import supervisor as supervise from bot_bottle.store.audit_store import AuditStore from bot_bottle.paths import bot_bottle_root -from bot_bottle.store.queue_store import QueueStore -from bot_bottle.supervise import ( +from bot_bottle.orchestrator.store.queue_store import QueueStore +from bot_bottle.orchestrator.supervisor import ( AuditEntry, Proposal, STATUS_APPROVED, diff --git a/tests/unit/test_supervise_server.py b/tests/unit/test_supervise_server.py index 7f0f21d..a082998 100644 --- a/tests/unit/test_supervise_server.py +++ b/tests/unit/test_supervise_server.py @@ -18,8 +18,8 @@ from pathlib import Path from tests.unit import use_bottle_root -from bot_bottle import supervise as _sv -from bot_bottle.store import queue_store as _qs +from bot_bottle.orchestrator import supervisor as _sv +from bot_bottle.orchestrator.store import queue_store as _qs from bot_bottle.store import audit_store as _as from bot_bottle.gateway import supervise_server # noqa: E402 -- 2.52.0 From 3e2cbcab881a8df0c5b5772d7a257dc6aaa0235c Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 14:24:06 -0400 Subject: [PATCH 11/39] refactor(supervise): fold Supervise into the facade as Supervisor; drop dead flat-fallbacks Move the `Supervise` lifecycle out of its own `orchestrator/supervisor/ supervise.py` and into the package `__init__`, renaming the class to `Supervisor`. Callers now import it from `bot_bottle.orchestrator.supervisor` alongside the queue surface it belongs with. Remove the dead `try/except ImportError` flat-import fallbacks from the package-only store modules (db_store, audit_store, config_store, queue_store) and image_cache. Those fallbacks existed for when the store files were flat-copied into the gateway; post-PRD-0070 the data plane never opens the DB, so these modules are only ever imported as part of the package. The two gateway data-plane files that may still be loaded flat (egress_addon_core, git_gate_render) keep their fallbacks. Full unit suite green (2251). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/backend/resolve_common.py | 4 +-- bot_bottle/image_cache.py | 5 +-- bot_bottle/orchestrator/store/queue_store.py | 14 +++----- .../orchestrator/supervisor/__init__.py | 32 ++++++++++++++--- .../orchestrator/supervisor/supervise.py | 36 ------------------- bot_bottle/store/audit_store.py | 14 +++----- bot_bottle/store/config_store.py | 11 ++---- bot_bottle/store/db_store.py | 5 +-- bot_bottle/supervisor/__init__.py | 2 +- bot_bottle/supervisor/plan.py | 6 ++-- tests/unit/test_supervise.py | 2 +- 11 files changed, 47 insertions(+), 84 deletions(-) delete mode 100644 bot_bottle/orchestrator/supervisor/supervise.py diff --git a/bot_bottle/backend/resolve_common.py b/bot_bottle/backend/resolve_common.py index 612291b..7ea9c14 100644 --- a/bot_bottle/backend/resolve_common.py +++ b/bot_bottle/backend/resolve_common.py @@ -29,7 +29,7 @@ from ..git_gate import GitGate, GitGatePlan from ..log import die from ..manifest import Manifest, ManifestBottle from ..supervisor.plan import SupervisePlan -from ..orchestrator.supervisor.supervise import Supervise +from ..orchestrator.supervisor import Supervisor from . import BottleSpec @@ -102,7 +102,7 @@ def prepare_supervise(bottle: ManifestBottle, slug: str) -> SupervisePlan | None return None supervise_dir = supervise_state_dir(slug) supervise_dir.mkdir(parents=True, exist_ok=True) - return Supervise().prepare(slug, supervise_dir) + return Supervisor().prepare(slug, supervise_dir) def merge_provision_env_vars(provision: AgentProvisionPlan) -> AgentProvisionPlan: diff --git a/bot_bottle/image_cache.py b/bot_bottle/image_cache.py index ff16544..65538e3 100644 --- a/bot_bottle/image_cache.py +++ b/bot_bottle/image_cache.py @@ -5,10 +5,7 @@ from __future__ import annotations from datetime import datetime, timezone from pathlib import Path -try: - from .store.config_store import ConfigStore -except ImportError: - from config_store import ConfigStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module +from .store.config_store import ConfigStore class StaleImageError(Exception): diff --git a/bot_bottle/orchestrator/store/queue_store.py b/bot_bottle/orchestrator/store/queue_store.py index 16637f9..a8428e3 100644 --- a/bot_bottle/orchestrator/store/queue_store.py +++ b/bot_bottle/orchestrator/store/queue_store.py @@ -6,16 +6,10 @@ import os import sqlite3 from pathlib import Path -try: - from ...supervisor.types import Proposal, Response - from ...paths import host_db_path - from ...store.db_store import DbStore - from ...store.migrations import TableMigrations -except ImportError: - from supervisor.types import Proposal, Response # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from paths import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from db_store import DbStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module +from ...supervisor.types import Proposal, Response +from ...paths import host_db_path +from ...store.db_store import DbStore +from ...store.migrations import TableMigrations class QueueStore(DbStore): diff --git a/bot_bottle/orchestrator/supervisor/__init__.py b/bot_bottle/orchestrator/supervisor/__init__.py index c6d903e..e222677 100644 --- a/bot_bottle/orchestrator/supervisor/__init__.py +++ b/bot_bottle/orchestrator/supervisor/__init__.py @@ -1,10 +1,10 @@ """The orchestrator's view of the supervise plane. Bundles the orchestrator-owned supervise surface: the queue/audit I/O and diff -rendering (`queue`) and the `Supervise` lifecycle (`supervise`). For -convenience it re-exports the neutral vocabulary (`bot_bottle.supervisor`'s -types + `SupervisePlan`) so orchestrator-side callers — service, CLI, tests — -import from one place. +rendering (`queue`), and the `Supervisor` lifecycle (host-side database +staging). For convenience it re-exports the neutral vocabulary +(`bot_bottle.supervisor`'s types + `SupervisePlan`) so orchestrator-side +callers — service, CLI, tests — import from one place. The data plane must NOT import this package; it imports `bot_bottle.supervisor` (neutral) directly and reaches the queue over the control-plane RPC. @@ -12,8 +12,12 @@ The data plane must NOT import this package; it imports `bot_bottle.supervisor` from __future__ import annotations +from abc import ABC +from pathlib import Path + from ...supervisor.types import * # noqa: F401,F403 — re-export the neutral vocabulary from ...supervisor.plan import SupervisePlan +from ..store.store_manager import StoreManager from .queue import ( archive_all_proposals, archive_proposal, @@ -31,4 +35,22 @@ from .queue import ( write_proposal, write_response, ) -from .supervise import Supervise + + +class Supervisor(ABC): + """Per-bottle supervise lifecycle. Encapsulates host-side database staging; + the gateway's start/stop lifecycle is backend-specific. + + `prepare` migrates the orchestrator's `bot-bottle.db` and returns the + neutral `SupervisePlan`. It touches the orchestrator store manager, so it + lives here rather than in the neutral `bot_bottle.supervisor` package; the + backend (which drives launch) may import it — backend → orchestrator is an + allowed direction, unlike gateway → orchestrator.""" + + def prepare(self, slug: str, stage_dir: Path) -> SupervisePlan: + """Stage the host database. Returns the plan; `internal_network` must be + set by the launch step before .start runs.""" + del stage_dir + mgr = StoreManager.instance() + mgr.migrate() + return SupervisePlan(slug=slug, db_path=mgr.db_path) diff --git a/bot_bottle/orchestrator/supervisor/supervise.py b/bot_bottle/orchestrator/supervisor/supervise.py deleted file mode 100644 index f2e51fe..0000000 --- a/bot_bottle/orchestrator/supervisor/supervise.py +++ /dev/null @@ -1,36 +0,0 @@ -"""The `Supervise` lifecycle — host-side database staging (PRD 0013 / 0070). - -`Supervise.prepare` migrates the orchestrator's `bot-bottle.db` and returns the -neutral `SupervisePlan`. It touches the orchestrator store manager, so it lives -here rather than in the neutral `bot_bottle.supervisor` package; the backend -(which drives launch) may import it — backend → orchestrator is an allowed -direction, unlike gateway → orchestrator. -""" - -from __future__ import annotations - -from abc import ABC -from pathlib import Path - -from ..store.store_manager import StoreManager -from ...supervisor.plan import SupervisePlan - - -class Supervise(ABC): - """Per-bottle supervise daemon. Encapsulates host-side database staging; - the gateway's start/stop lifecycle is backend-specific.""" - - def prepare( - self, - slug: str, - stage_dir: Path, - ) -> SupervisePlan: - """Stage the host database. Returns the plan; `internal_network` must be - set by the launch step before .start runs.""" - del stage_dir - mgr = StoreManager.instance() - mgr.migrate() - return SupervisePlan( - slug=slug, - db_path=mgr.db_path, - ) diff --git a/bot_bottle/store/audit_store.py b/bot_bottle/store/audit_store.py index 3a6738b..d80451b 100644 --- a/bot_bottle/store/audit_store.py +++ b/bot_bottle/store/audit_store.py @@ -5,16 +5,10 @@ from __future__ import annotations import sqlite3 from pathlib import Path -try: - from ..supervisor.types import AuditEntry - from ..paths import host_db_path - from .db_store import DbStore - from .migrations import TableMigrations -except ImportError: - from supervisor.types import AuditEntry # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from paths import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from db_store import DbStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module +from ..supervisor.types import AuditEntry +from ..paths import host_db_path +from .db_store import DbStore +from .migrations import TableMigrations class AuditStore(DbStore): diff --git a/bot_bottle/store/config_store.py b/bot_bottle/store/config_store.py index d282d71..8d7811e 100644 --- a/bot_bottle/store/config_store.py +++ b/bot_bottle/store/config_store.py @@ -4,14 +4,9 @@ from __future__ import annotations from pathlib import Path -try: - from .db_store import DbStore - from .migrations import TableMigrations - from ..paths import host_db_path -except ImportError: - from db_store import DbStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from paths import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module +from .db_store import DbStore +from .migrations import TableMigrations +from ..paths import host_db_path DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS = 1 diff --git a/bot_bottle/store/db_store.py b/bot_bottle/store/db_store.py index 256f29e..49bcced 100644 --- a/bot_bottle/store/db_store.py +++ b/bot_bottle/store/db_store.py @@ -6,10 +6,7 @@ import sqlite3 from contextlib import contextmanager from pathlib import Path -try: - from .migrations import TableMigrations -except ImportError: - from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module +from .migrations import TableMigrations class DbVersionError(Exception): diff --git a/bot_bottle/supervisor/__init__.py b/bot_bottle/supervisor/__init__.py index d76c1e9..7d002f9 100644 --- a/bot_bottle/supervisor/__init__.py +++ b/bot_bottle/supervisor/__init__.py @@ -11,6 +11,6 @@ tier's private code, importable by `bot_bottle.gateway` **and** status / poll-status constants, and the shared daemon constants. * `plan` — `SupervisePlan`, the launch-time staging DTO. -The orchestrator-only half (queue I/O, diff rendering, the `Supervise` +The orchestrator-only half (queue I/O, diff rendering, the `Supervisor` lifecycle) lives under `bot_bottle.orchestrator.supervisor`. """ diff --git a/bot_bottle/supervisor/plan.py b/bot_bottle/supervisor/plan.py index 2962d82..e61cf91 100644 --- a/bot_bottle/supervisor/plan.py +++ b/bot_bottle/supervisor/plan.py @@ -1,9 +1,9 @@ """`SupervisePlan` — the launch-time supervise staging DTO. A pure value type (no store access), so it lives in the neutral package: the -backend builds it at launch and the orchestrator's `Supervise.prepare` returns +backend builds it at launch and the orchestrator's `Supervisor.prepare` returns it. The behaviour that fills it in — staging the host database — is the -orchestrator's, in `bot_bottle.orchestrator.supervisor.supervise`. +orchestrator's, in `bot_bottle.orchestrator.supervisor.Supervisor`. """ from __future__ import annotations @@ -14,7 +14,7 @@ from pathlib import Path @dataclass(frozen=True) class SupervisePlan: - """Output of `Supervise.prepare`; consumed by `.start`. + """Output of `Supervisor.prepare`; consumed by `.start`. `db_path` is the host database bind-mounted into the gateway at /run/supervise/bot-bottle.db. `internal_network` is empty at prepare time; diff --git a/tests/unit/test_supervise.py b/tests/unit/test_supervise.py index f0f41be..7dbdb68 100644 --- a/tests/unit/test_supervise.py +++ b/tests/unit/test_supervise.py @@ -352,7 +352,7 @@ class TestToolConstants(unittest.TestCase): ) -class _StubSupervise(supervise.Supervise): +class _StubSupervise(supervise.Supervisor): """Concrete Supervise subclass for testing the prepare template.""" def start(self, plan): # type: ignore -- 2.52.0 From 27a122e24be1817b52dfbeaed2dd534d129c3c61 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 14:36:32 -0400 Subject: [PATCH 12/39] refactor(supervise): drop obsolete queue helpers (archive_proposal, wait_for_response, audit_dir/path) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the supervise queue helpers that no live path uses anymore — they're mechanisms the PRD-0070 / idempotent-poll migration superseded: - archive_proposal (+ QueueStore.archive_proposal): the poll stopped archiving on read (#469 review), so single-proposal archiving has no caller; decided proposals drop off the pending list via their response row and are reaped in bulk by archive_all_proposals on teardown/reconcile. - wait_for_response: the data plane polls the queue over the control-plane RPC now, so nothing block-waits on it. - audit_dir / audit_log_path: file-based audit paths, replaced by the SQLite AuditStore. Also drops the tests that covered those obsolete helpers. Kept the prod-unused read accessors list_pending_proposals and read_audit_entries: those aren't dead mechanisms — they're the read side of live write paths that many tests use to verify real queue/audit behavior. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/orchestrator/store/queue_store.py | 19 ------- .../orchestrator/supervisor/__init__.py | 4 -- bot_bottle/orchestrator/supervisor/queue.py | 50 +------------------ tests/unit/test_supervise.py | 46 ----------------- tests/unit/test_supervise_edge.py | 39 --------------- 5 files changed, 1 insertion(+), 157 deletions(-) diff --git a/bot_bottle/orchestrator/store/queue_store.py b/bot_bottle/orchestrator/store/queue_store.py index a8428e3..b9d8b1a 100644 --- a/bot_bottle/orchestrator/store/queue_store.py +++ b/bot_bottle/orchestrator/store/queue_store.py @@ -167,25 +167,6 @@ class QueueStore(DbStore): raise FileNotFoundError(proposal_id) return self._row_to_response(row) - def archive_proposal(self, proposal_id: str) -> None: - if not self.db_path.is_file(): - return - with self._connection() as conn: - conn.execute( - """ - UPDATE supervise_proposals SET archived = 1 - WHERE queue_key = ? AND id = ? - """, - (self.queue_key, proposal_id), - ) - conn.execute( - """ - UPDATE supervise_responses SET archived = 1 - WHERE queue_key = ? AND proposal_id = ? - """, - (self.queue_key, proposal_id), - ) - def archive_all(self) -> None: """Archive every proposal + response for this queue key. Used to reap a bottle's supervise records when it is torn down or reconciled away — diff --git a/bot_bottle/orchestrator/supervisor/__init__.py b/bot_bottle/orchestrator/supervisor/__init__.py index e222677..48b96b0 100644 --- a/bot_bottle/orchestrator/supervisor/__init__.py +++ b/bot_bottle/orchestrator/supervisor/__init__.py @@ -20,9 +20,6 @@ from ...supervisor.plan import SupervisePlan from ..store.store_manager import StoreManager from .queue import ( archive_all_proposals, - archive_proposal, - audit_dir, - audit_log_path, list_all_pending_proposals, list_pending_proposals, read_audit_entries, @@ -30,7 +27,6 @@ from .queue import ( read_response, render_diff, sha256_hex, - wait_for_response, write_audit_entry, write_proposal, write_response, diff --git a/bot_bottle/orchestrator/supervisor/queue.py b/bot_bottle/orchestrator/supervisor/queue.py index fed4132..17b0c7f 100644 --- a/bot_bottle/orchestrator/supervisor/queue.py +++ b/bot_bottle/orchestrator/supervisor/queue.py @@ -11,31 +11,13 @@ from __future__ import annotations import difflib import hashlib -import time from pathlib import Path -from ...supervisor.types import ( - AuditEntry, - DEFAULT_POLL_INTERVAL_SEC, - Proposal, - Response, -) -from ...paths import bot_bottle_root +from ...supervisor.types import AuditEntry, Proposal, Response from ..store.queue_store import QueueStore from ...store.audit_store import AuditStore -# --- Paths ----------------------------------------------------------------- - - -def audit_dir() -> Path: - return bot_bottle_root() / "audit" - - -def audit_log_path(component: str, slug: str) -> Path: - return audit_dir() / f"{component}-{slug}.log" - - # --- Queue I/O ------------------------------------------------------------- @@ -69,36 +51,6 @@ def read_response(bottle_slug: str, proposal_id: str) -> Response: return QueueStore(bottle_slug).read_response(proposal_id) -def wait_for_response( - bottle_slug: str, - proposal_id: str, - *, - poll_interval: float = DEFAULT_POLL_INTERVAL_SEC, - deadline: float | None = None, -) -> Response: - """Block until a response file appears for `proposal_id`, then return it. - `deadline` is an absolute time.monotonic() value after which the wait raises - TimeoutError. None waits forever — the natural shape, since the operator's - response time is unbounded. - - Polls SQLite so the implementation stays portable and stdlib-only.""" - store = QueueStore(bottle_slug) - while True: - try: - return store.read_response(proposal_id) - except FileNotFoundError: - pass - if deadline is not None and time.monotonic() >= deadline: - raise TimeoutError(f"no response for proposal {proposal_id!r}") - time.sleep(poll_interval) - - -def archive_proposal(bottle_slug: str, proposal_id: str) -> None: - """Mark both proposal and response rows processed. - Idempotent — missing rows are silently skipped.""" - QueueStore(bottle_slug).archive_proposal(proposal_id) - - def archive_all_proposals(bottle_slug: str) -> None: """Archive every proposal + response for `bottle_slug` (bottle teardown / reconcile cleanup). Idempotent.""" diff --git a/tests/unit/test_supervise.py b/tests/unit/test_supervise.py index 7dbdb68..da2b5f0 100644 --- a/tests/unit/test_supervise.py +++ b/tests/unit/test_supervise.py @@ -1,8 +1,6 @@ """Unit: supervise queue + audit log + diff helpers (PRD 0013).""" import tempfile -import threading -import time import unittest from datetime import datetime, timezone from pathlib import Path @@ -21,14 +19,12 @@ from bot_bottle.orchestrator.supervisor import ( STATUS_REJECTED, TOOL_EGRESS_ALLOW, TOOL_GITLEAKS_ALLOW, - archive_proposal, list_pending_proposals, read_audit_entries, read_proposal, read_response, render_diff, sha256_hex, - wait_for_response, write_audit_entry, write_proposal, write_response, @@ -173,48 +169,6 @@ class TestQueueIO(unittest.TestCase): write_response(self.slug, r) self.assertEqual(r, read_response(self.slug, "xyz")) - def test_wait_for_response_returns_when_file_appears(self): - p = _proposal() - write_proposal(p) - - def write_after_delay(): - time.sleep(0.05) - write_response(self.slug, Response( - proposal_id=p.id, status=STATUS_APPROVED, notes="ok", - )) - - t = threading.Thread(target=write_after_delay) - t.start() - try: - r = wait_for_response(self.slug, p.id, poll_interval=0.01) - finally: - t.join() - self.assertEqual(STATUS_APPROVED, r.status) - self.assertEqual("ok", r.notes) - - def test_wait_for_response_times_out(self): - deadline = time.monotonic() + 0.05 - with self.assertRaises(TimeoutError): - wait_for_response( - self.slug, "never", - poll_interval=0.01, deadline=deadline, - ) - - def test_archive_proposal_hides_rows(self): - p = _proposal() - write_proposal(p) - write_response(self.slug, Response( - proposal_id=p.id, status=STATUS_APPROVED, notes="", - )) - archive_proposal(self.slug, p.id) - self.assertEqual([], list_pending_proposals(self.slug)) - with self.assertRaises(FileNotFoundError): - read_response(self.slug, p.id) - - def test_archive_is_idempotent_on_missing_files(self): - # Should not raise. - archive_proposal(self.slug, "nope") - class TestAuditLog(unittest.TestCase): def setUp(self): diff --git a/tests/unit/test_supervise_edge.py b/tests/unit/test_supervise_edge.py index c56d9c5..8548040 100644 --- a/tests/unit/test_supervise_edge.py +++ b/tests/unit/test_supervise_edge.py @@ -5,7 +5,6 @@ fallback paths.""" from __future__ import annotations import tempfile -import time import unittest from pathlib import Path from unittest.mock import patch @@ -23,7 +22,6 @@ from bot_bottle.orchestrator.supervisor import ( read_audit_entries, read_proposal, read_response, - wait_for_response, write_audit_entry, ) @@ -81,22 +79,6 @@ class TestReadMalformed(unittest.TestCase): self.assertEqual([], list_pending_proposals("slug")) -class TestWaitForResponse(unittest.TestCase): - def test_missing_response_times_out(self) -> None: - with tempfile.TemporaryDirectory() as d: - with patch.dict("os.environ", {"HOME": d}): - QueueStore("slug").migrate() - with self.assertRaises(TimeoutError): - wait_for_response("slug", "p", deadline=time.monotonic()) - - def test_empty_db_response_does_not_count(self) -> None: - with tempfile.TemporaryDirectory() as d: - with patch.dict("os.environ", {"HOME": d}): - QueueStore("slug").migrate() - with self.assertRaises(TimeoutError): - wait_for_response("slug", "p", deadline=time.monotonic()) - - class TestReadAuditEntries(unittest.TestCase): def test_missing_log_returns_empty(self) -> None: with tempfile.TemporaryDirectory() as home, \ @@ -129,19 +111,6 @@ class TestReadAuditEntries(unittest.TestCase): self.assertEqual(1, len(entries)) self.assertEqual("approve", entries[0].operator_action) - def test_legacy_audit_log_file_does_not_count(self) -> None: - with tempfile.TemporaryDirectory() as home, \ - patch.dict("os.environ", {"HOME": home}): - path = supervise.audit_log_path("egress", "slug") - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text( - '{"timestamp": "t", "bottle_slug": "slug", "component": "egress",' - ' "operator_action": "approve", "operator_notes": "",' - ' "justification": "", "diff": ""}\n' - ) - entries = read_audit_entries("egress", "slug") - self.assertEqual([], entries) - class TestStoreGuardBranches(unittest.TestCase): """Direct QueueStore / AuditStore construction and early-return guard branches.""" @@ -170,14 +139,6 @@ class TestStoreGuardBranches(unittest.TestCase): db.unlink() self.assertEqual([], store.list_all_pending_proposals()) - def test_queue_store_missing_db_archive_is_noop(self): - with tempfile.TemporaryDirectory() as d: - db = Path(d) / "q.db" - store = QueueStore("key", db_path=db) - store.migrate() - db.unlink() - store.archive_proposal("anything") # must not raise - def test_queue_store_chmod_oserror_is_swallowed(self): with tempfile.TemporaryDirectory() as d: db = Path(d) / "q.db" -- 2.52.0 From 8abccf7ffecab3cbfae3d1026d5acc5d1505a310 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 14:52:17 -0400 Subject: [PATCH 13/39] refactor(supervise): make Supervisor a service class the orchestrator calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turn the loose queue/audit functions in orchestrator/supervisor/queue.py into methods on a concrete Supervisor service. The Orchestrator now owns an injectable `self._supervisor` and calls `self._supervisor.write_proposal(...)` etc., instead of module-level free functions — the supervise dependency is explicit and mockable, and a Supervisor can be scoped to a `db_path` (defaults to the host DB) so tests can point it at a temp database. - Supervisor (in the package __init__) drops the ABC and gains write_proposal, read_proposal, list_pending_proposals, list_all_pending_proposals, write_response, read_response, archive_all_proposals, write_audit_entry, read_audit_entries, plus the launch-time prepare(). - render_diff / sha256_hex are pure and stateless, so they move to orchestrator/supervisor/util.py (re-exported from the facade) rather than becoming methods. - queue.py is deleted; service.py + the supervise tests call through a Supervisor instance. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/orchestrator/service.py | 30 +++-- .../orchestrator/supervisor/__init__.py | 105 ++++++++++++------ bot_bottle/orchestrator/supervisor/queue.py | 93 ---------------- bot_bottle/orchestrator/supervisor/util.py | 31 ++++++ tests/unit/test_orchestrator_control_plane.py | 4 +- tests/unit/test_orchestrator_service.py | 9 +- tests/unit/test_supervise.py | 56 +++++----- tests/unit/test_supervise_edge.py | 30 +++-- tests/unit/test_supervise_server.py | 35 +++--- 9 files changed, 183 insertions(+), 210 deletions(-) delete mode 100644 bot_bottle/orchestrator/supervisor/queue.py create mode 100644 bot_bottle/orchestrator/supervisor/util.py diff --git a/bot_bottle/orchestrator/service.py b/bot_bottle/orchestrator/service.py index bf6a14d..452af25 100644 --- a/bot_bottle/orchestrator/service.py +++ b/bot_bottle/orchestrator/service.py @@ -39,15 +39,9 @@ from .supervisor import ( STATUS_REJECTED, TOOL_EGRESS_ALLOW, TOOL_EGRESS_BLOCK, - archive_all_proposals, - list_all_pending_proposals, - read_proposal, - read_response, + Supervisor, render_diff, sha256_hex, - write_audit_entry, - write_proposal, - write_response, ) @@ -71,10 +65,14 @@ class Orchestrator: registry: RegistryStore, broker: LaunchBroker, sign_secret: bytes, + supervisor: Supervisor | None = None, ) -> None: self.registry = registry self._broker = broker self._secret = sign_secret + # The supervise service (queue + audit I/O). Injectable so tests can + # point it at a temp DB; defaults to the host DB. + self._supervisor = supervisor or Supervisor() # Per-bottle egress auth tokens (env_name -> value), keyed by bottle_id. # Held **in memory only** — never written to the registry DB — so the # gateway can inject each bottle's upstream credential without secrets @@ -134,7 +132,7 @@ class Orchestrator: self.registry.deregister(bottle_id) self._tokens.pop(bottle_id, None) # Reap any supervise proposals the gone bottle never acknowledged. - archive_all_proposals(bottle_id) + self._supervisor.archive_all_proposals(bottle_id) return True def reconcile( @@ -160,7 +158,7 @@ class Orchestrator: for rec in reaped: self._tokens.pop(rec.bottle_id, None) # Reap any supervise proposals the gone bottle never acknowledged. - archive_all_proposals(rec.bottle_id) + self._supervisor.archive_all_proposals(rec.bottle_id) return [rec.bottle_id for rec in reaped] def tokens_for(self, bottle_id: str) -> dict[str, str]: @@ -220,7 +218,7 @@ class Orchestrator: justification=justification, current_file_hash=sha256_hex(proposed_file), ) - write_proposal(proposal) + self._supervisor.write_proposal(proposal) return proposal.id def supervise_poll_response(self, bottle_id: str, proposal_id: str) -> dict[str, object]: @@ -241,10 +239,10 @@ class Orchestrator: Reads are scoped to `bottle_id` (the queue key), so a caller can never read another bottle's proposal even with a guessed id.""" try: - response = read_response(bottle_id, proposal_id) + response = self._supervisor.read_response(bottle_id, proposal_id) except FileNotFoundError: try: - read_proposal(bottle_id, proposal_id) + self._supervisor.read_proposal(bottle_id, proposal_id) except FileNotFoundError: return {"status": POLL_STATUS_UNKNOWN} return {"status": POLL_STATUS_PENDING} @@ -263,7 +261,7 @@ class Orchestrator: orchestrator-assigned bottle_id, which is opaque to an operator). The CLI renders the label but still responds against `bottle_slug`.""" out: list[dict[str, object]] = [] - for p in list_all_pending_proposals(): + for p in self._supervisor.list_all_pending_proposals(): d = p.to_dict() d["bottle_label"] = self._label_for(p.bottle_slug) out.append(d) @@ -326,7 +324,7 @@ class Orchestrator: if status is None: return False, f"unknown decision {decision!r}" try: - proposal = read_proposal(bottle_slug, proposal_id) + proposal = self._supervisor.read_proposal(bottle_slug, proposal_id) except FileNotFoundError: return False, "no such proposal" @@ -342,12 +340,12 @@ class Orchestrator: diff_before, diff_after = rec.policy, new_policy self.set_policy(rec.bottle_id, new_policy) - write_response(bottle_slug, Response( + self._supervisor.write_response(bottle_slug, Response( proposal_id=proposal_id, status=status, notes=notes, final_file=final_file, )) component = COMPONENT_FOR_TOOL.get(proposal.tool) if component is not None: - write_audit_entry(AuditEntry( + self._supervisor.write_audit_entry(AuditEntry( timestamp=datetime.now(timezone.utc).isoformat(), bottle_slug=bottle_slug, component=component, diff --git a/bot_bottle/orchestrator/supervisor/__init__.py b/bot_bottle/orchestrator/supervisor/__init__.py index 48b96b0..8a2d912 100644 --- a/bot_bottle/orchestrator/supervisor/__init__.py +++ b/bot_bottle/orchestrator/supervisor/__init__.py @@ -1,52 +1,91 @@ -"""The orchestrator's view of the supervise plane. +"""The orchestrator's supervise service. -Bundles the orchestrator-owned supervise surface: the queue/audit I/O and diff -rendering (`queue`), and the `Supervisor` lifecycle (host-side database -staging). For convenience it re-exports the neutral vocabulary -(`bot_bottle.supervisor`'s types + `SupervisePlan`) so orchestrator-side -callers — service, CLI, tests — import from one place. +`Supervisor` is the service object the `Orchestrator` owns and calls: it wraps +the supervise queue + audit stores (queue proposals, record operator responses, +write audit entries, stage the DB at launch). Binding those operations to one +object — optionally scoped to a `db_path` — makes the orchestrator's supervise +dependency explicit and injectable in tests. -The data plane must NOT import this package; it imports `bot_bottle.supervisor` -(neutral) directly and reaches the queue over the control-plane RPC. +For convenience this package also re-exports the neutral vocabulary +(`bot_bottle.supervisor`'s types + `SupervisePlan`) and the pure `render_diff` +/ `sha256_hex` helpers (`util`), so orchestrator-side callers import from one +place. The data plane must NOT import this package — it imports +`bot_bottle.supervisor` (neutral) directly and reaches the queue over the +control-plane RPC. """ from __future__ import annotations -from abc import ABC from pathlib import Path from ...supervisor.types import * # noqa: F401,F403 — re-export the neutral vocabulary from ...supervisor.plan import SupervisePlan from ..store.store_manager import StoreManager -from .queue import ( - archive_all_proposals, - list_all_pending_proposals, - list_pending_proposals, - read_audit_entries, - read_proposal, - read_response, - render_diff, - sha256_hex, - write_audit_entry, - write_proposal, - write_response, -) +from ..store.queue_store import QueueStore +from ...store.audit_store import AuditStore +from ...supervisor.types import AuditEntry, Proposal, Response +from .util import render_diff, sha256_hex # noqa: F401 — re-exported for callers -class Supervisor(ABC): - """Per-bottle supervise lifecycle. Encapsulates host-side database staging; - the gateway's start/stop lifecycle is backend-specific. +class Supervisor: + """Host-side supervise service: the queue + audit I/O and launch-time DB + staging the orchestrator drives. Stateless apart from an optional `db_path` + (None → the host DB, resolved per store as before), so a test can point a + `Supervisor` at a temp database.""" - `prepare` migrates the orchestrator's `bot-bottle.db` and returns the - neutral `SupervisePlan`. It touches the orchestrator store manager, so it - lives here rather than in the neutral `bot_bottle.supervisor` package; the - backend (which drives launch) may import it — backend → orchestrator is an - allowed direction, unlike gateway → orchestrator.""" + def __init__(self, db_path: Path | None = None) -> None: + self._db_path = db_path + + def _queue(self, queue_key: str) -> QueueStore: + return QueueStore(queue_key, self._db_path) + + def _audit(self) -> AuditStore: + return AuditStore(self._db_path) + + # --- Queue I/O --------------------------------------------------------- + + def write_proposal(self, proposal: Proposal) -> Path: + """Persist `proposal` in the queue database, mode 0o600.""" + return self._queue(proposal.bottle_slug).write_proposal(proposal) + + def read_proposal(self, bottle_slug: str, proposal_id: str) -> Proposal: + return self._queue(bottle_slug).read_proposal(proposal_id) + + def list_pending_proposals(self, bottle_slug: str) -> list[Proposal]: + """A single bottle's undecided proposals, FIFO by arrival.""" + return self._queue(bottle_slug).list_pending_proposals() + + def list_all_pending_proposals(self) -> list[Proposal]: + """All pending proposals across bottles, sorted FIFO.""" + return self._queue("").list_all_pending_proposals() + + def write_response(self, bottle_slug: str, response: Response) -> Path: + return self._queue(bottle_slug).write_response(response) + + def read_response(self, bottle_slug: str, proposal_id: str) -> Response: + return self._queue(bottle_slug).read_response(proposal_id) + + def archive_all_proposals(self, bottle_slug: str) -> None: + """Archive every proposal + response for `bottle_slug` (bottle teardown + / reconcile cleanup). Idempotent.""" + self._queue(bottle_slug).archive_all() + + # --- Audit log --------------------------------------------------------- + + def write_audit_entry(self, entry: AuditEntry) -> Path: + """Append `entry` to the host supervise audit table.""" + return self._audit().write_audit_entry(entry) + + def read_audit_entries(self, component: str, slug: str) -> list[AuditEntry]: + """Load all audit entries for the given component+slug.""" + return self._audit().read_audit_entries(component, slug) + + # --- Launch-time staging ---------------------------------------------- def prepare(self, slug: str, stage_dir: Path) -> SupervisePlan: - """Stage the host database. Returns the plan; `internal_network` must be - set by the launch step before .start runs.""" + """Stage the host database and return the plan. Called by the backend at + launch; `internal_network` is set by the launch step before `.start`.""" del stage_dir - mgr = StoreManager.instance() + mgr = StoreManager(self._db_path) if self._db_path else StoreManager.instance() mgr.migrate() return SupervisePlan(slug=slug, db_path=mgr.db_path) diff --git a/bot_bottle/orchestrator/supervisor/queue.py b/bot_bottle/orchestrator/supervisor/queue.py deleted file mode 100644 index 17b0c7f..0000000 --- a/bot_bottle/orchestrator/supervisor/queue.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Orchestrator-side supervise queue + audit I/O and diff rendering. - -The host-side library the orchestrator uses to drive the supervise flow: read / -write proposals and responses against the queue store, append audit entries, -and render operator diffs. Only the orchestrator opens `bot-bottle.db` (PRD -0070), so this lives under `orchestrator/` — the data plane reaches it over the -control-plane RPC, never by importing this module. -""" - -from __future__ import annotations - -import difflib -import hashlib -from pathlib import Path - -from ...supervisor.types import AuditEntry, Proposal, Response -from ..store.queue_store import QueueStore -from ...store.audit_store import AuditStore - - -# --- Queue I/O ------------------------------------------------------------- - - -def write_proposal(proposal: Proposal) -> Path: - """Persist `proposal` in the queue database, mode 0o600. - Directory is created if missing.""" - return QueueStore(proposal.bottle_slug).write_proposal(proposal) - - -def read_proposal(bottle_slug: str, proposal_id: str) -> Proposal: - return QueueStore(bottle_slug).read_proposal(proposal_id) - - -def list_pending_proposals(bottle_slug: str) -> list[Proposal]: - """All proposals for `bottle_slug` that do not yet have a matching - response. Sorted by `arrival_timestamp` so the operator sees the queue - FIFO.""" - return QueueStore(bottle_slug).list_pending_proposals() - - -def list_all_pending_proposals() -> list[Proposal]: - """All pending proposals across bottles, sorted FIFO.""" - return QueueStore("").list_all_pending_proposals() - - -def write_response(bottle_slug: str, response: Response) -> Path: - return QueueStore(bottle_slug).write_response(response) - - -def read_response(bottle_slug: str, proposal_id: str) -> Response: - return QueueStore(bottle_slug).read_response(proposal_id) - - -def archive_all_proposals(bottle_slug: str) -> None: - """Archive every proposal + response for `bottle_slug` (bottle teardown / - reconcile cleanup). Idempotent.""" - QueueStore(bottle_slug).archive_all() - - -# --- Audit log ------------------------------------------------------------- - - -def write_audit_entry(entry: AuditEntry) -> Path: - """Append `entry` to the host supervise audit table.""" - return AuditStore().write_audit_entry(entry) - - -def read_audit_entries(component: str, slug: str) -> list[AuditEntry]: - """Load all audit entries for the given component+slug.""" - return AuditStore().read_audit_entries(component, slug) - - -# --- Diff rendering -------------------------------------------------------- - - -def render_diff(before: str, after: str, *, label: str = "config") -> str: - """Unified diff suitable for the audit log + TUI. Empty diff (no changes) - renders as the empty string.""" - diff = difflib.unified_diff( - before.splitlines(keepends=True), - after.splitlines(keepends=True), - fromfile=f"{label} (current)", - tofile=f"{label} (proposed)", - lineterm="", - ) - parts = list(diff) - if not parts: - return "" - return "".join(p if p.endswith("\n") else p + "\n" for p in parts).rstrip("\n") - - -def sha256_hex(content: str) -> str: - return hashlib.sha256(content.encode("utf-8")).hexdigest() diff --git a/bot_bottle/orchestrator/supervisor/util.py b/bot_bottle/orchestrator/supervisor/util.py new file mode 100644 index 0000000..3776b56 --- /dev/null +++ b/bot_bottle/orchestrator/supervisor/util.py @@ -0,0 +1,31 @@ +"""Pure supervise helpers — diff rendering and content hashing. + +Stateless utilities used alongside the `Supervisor` service. Kept as plain +functions (not methods) because they carry no state and are also handy to +callers that only need to hash or diff. +""" + +from __future__ import annotations + +import difflib +import hashlib + + +def render_diff(before: str, after: str, *, label: str = "config") -> str: + """Unified diff suitable for the audit log + TUI. Empty diff (no changes) + renders as the empty string.""" + diff = difflib.unified_diff( + before.splitlines(keepends=True), + after.splitlines(keepends=True), + fromfile=f"{label} (current)", + tofile=f"{label} (proposed)", + lineterm="", + ) + parts = list(diff) + if not parts: + return "" + return "".join(p if p.endswith("\n") else p + "\n" for p in parts).rstrip("\n") + + +def sha256_hex(content: str) -> str: + return hashlib.sha256(content.encode("utf-8")).hexdigest() diff --git a/tests/unit/test_orchestrator_control_plane.py b/tests/unit/test_orchestrator_control_plane.py index 0bacd90..66119c3 100644 --- a/tests/unit/test_orchestrator_control_plane.py +++ b/tests/unit/test_orchestrator_control_plane.py @@ -29,7 +29,7 @@ from bot_bottle.orchestrator.supervisor import ( Proposal, TOOL_EGRESS_ALLOW, sha256_hex, - write_proposal, + Supervisor, ) @@ -421,7 +421,7 @@ class TestDispatchSupervise(unittest.TestCase): p = Proposal.new( bottle_slug=slug, tool=TOOL_EGRESS_ALLOW, proposed_file=proposed, justification="need it", current_file_hash=sha256_hex(proposed)) - write_proposal(p) + Supervisor().write_proposal(p) return p.id def test_list_pending(self) -> None: diff --git a/tests/unit/test_orchestrator_service.py b/tests/unit/test_orchestrator_service.py index a83f8c1..d090f3a 100644 --- a/tests/unit/test_orchestrator_service.py +++ b/tests/unit/test_orchestrator_service.py @@ -20,9 +20,8 @@ from bot_bottle.orchestrator.supervisor import ( Proposal, STATUS_APPROVED, TOOL_EGRESS_ALLOW, - read_response, sha256_hex, - write_proposal, + Supervisor, ) @@ -185,7 +184,7 @@ class TestOrchestratorSupervise(unittest.TestCase): p = Proposal.new( bottle_slug=slug, tool=TOOL_EGRESS_ALLOW, proposed_file=proposed, justification="need it", current_file_hash=sha256_hex(proposed)) - write_proposal(p) + Supervisor().write_proposal(p) return p.id def test_pending_lists_queued_proposal(self) -> None: @@ -208,7 +207,7 @@ class TestOrchestratorSupervise(unittest.TestCase): assert rec is not None self.assertEqual(new_routes, rec.policy) # response written -> agent unblocks, proposal no longer pending - self.assertEqual(STATUS_APPROVED, read_response("demo", pid).status) + self.assertEqual(STATUS_APPROVED, Supervisor().read_response("demo", pid).status) self.assertEqual([], self.orch.supervise_pending()) def test_pending_carries_human_label(self) -> None: @@ -261,7 +260,7 @@ class TestOrchestratorSupervise(unittest.TestCase): rec = self.store.get(bottle_id) assert rec is not None self.assertEqual("routes:\n - host: existing.com\n", rec.policy) - self.assertEqual("rejected", read_response("demo", pid).status) + self.assertEqual("rejected", Supervisor().read_response("demo", pid).status) def test_unknown_proposal_is_error(self) -> None: ok, err = self.orch.supervise_respond( diff --git a/tests/unit/test_supervise.py b/tests/unit/test_supervise.py index da2b5f0..c609dbe 100644 --- a/tests/unit/test_supervise.py +++ b/tests/unit/test_supervise.py @@ -17,19 +17,15 @@ from bot_bottle.orchestrator.supervisor import ( STATUS_APPROVED, STATUS_MODIFIED, STATUS_REJECTED, + Supervisor, TOOL_EGRESS_ALLOW, TOOL_GITLEAKS_ALLOW, - list_pending_proposals, - read_audit_entries, - read_proposal, - read_response, render_diff, sha256_hex, - write_audit_entry, - write_proposal, - write_response, ) +_SV = Supervisor() + FIXED_TS = datetime(2026, 5, 25, 12, 0, 0, tzinfo=timezone.utc) @@ -123,26 +119,26 @@ class TestQueueIO(unittest.TestCase): def test_write_and_read_proposal(self): p = _proposal() - path = write_proposal(p) + path = _SV.write_proposal(p) self.assertTrue(path.exists()) self.assertEqual(host_db_path(), path) self.assertEqual(0o600, path.stat().st_mode & 0o777) - loaded = read_proposal(self.slug, p.id) + loaded = _SV.read_proposal(self.slug, p.id) self.assertEqual(p, loaded) def test_list_pending_excludes_responded(self): a = _proposal(justification="first") b = _proposal(justification="second") - write_proposal(a) - write_proposal(b) - write_response(self.slug, Response( + _SV.write_proposal(a) + _SV.write_proposal(b) + _SV.write_response(self.slug, Response( proposal_id=a.id, status=STATUS_APPROVED, notes="", )) - pending = list_pending_proposals(self.slug) + pending = _SV.list_pending_proposals(self.slug) self.assertEqual([b.id], [p.id for p in pending]) def test_list_pending_returns_empty_for_missing_slug(self): - self.assertEqual([], list_pending_proposals("nope")) + self.assertEqual([], _SV.list_pending_proposals("nope")) def test_list_pending_sorted_by_arrival(self): # Fabricate two with explicit timestamps. @@ -159,15 +155,15 @@ class TestQueueIO(unittest.TestCase): now=datetime(2026, 5, 25, 14, 0, 0, tzinfo=timezone.utc), ) # Write in reverse order. - write_proposal(b) - write_proposal(a) - ordered = list_pending_proposals(self.slug) + _SV.write_proposal(b) + _SV.write_proposal(a) + ordered = _SV.list_pending_proposals(self.slug) self.assertEqual([a.id, b.id], [p.id for p in ordered]) def test_write_and_read_response(self): r = Response(proposal_id="xyz", status=STATUS_REJECTED, notes="no") - write_response(self.slug, r) - self.assertEqual(r, read_response(self.slug, "xyz")) + _SV.write_response(self.slug, r) + self.assertEqual(r, _SV.read_response(self.slug, "xyz")) class TestAuditLog(unittest.TestCase): @@ -193,15 +189,15 @@ class TestAuditLog(unittest.TestCase): justification="agent needed gh-api token", diff="--- before\n+++ after\n", ) - path = write_audit_entry(e) + path = _SV.write_audit_entry(e) self.assertEqual(host_db_path(), path) self.assertEqual(0o600, path.stat().st_mode & 0o777) - loaded = read_audit_entries("cred-proxy", "dev") + loaded = _SV.read_audit_entries("cred-proxy", "dev") self.assertEqual([e], loaded) def test_appends_one_line_per_entry(self): for i in range(3): - write_audit_entry(AuditEntry( + _SV.write_audit_entry(AuditEntry( timestamp=f"2026-05-25T12:00:0{i}+00:00", bottle_slug="dev", component="egress", @@ -210,7 +206,7 @@ class TestAuditLog(unittest.TestCase): justification="", diff="", )) - entries = read_audit_entries("egress", "dev") + entries = _SV.read_audit_entries("egress", "dev") self.assertEqual(3, len(entries)) self.assertEqual( ["2026-05-25T12:00:00+00:00", "2026-05-25T12:00:01+00:00", @@ -219,7 +215,7 @@ class TestAuditLog(unittest.TestCase): ) def test_separate_logs_per_component_slug(self): - write_audit_entry(AuditEntry( + _SV.write_audit_entry(AuditEntry( timestamp="t", bottle_slug="dev", component="cred-proxy", @@ -228,7 +224,7 @@ class TestAuditLog(unittest.TestCase): justification="", diff="", )) - write_audit_entry(AuditEntry( + _SV.write_audit_entry(AuditEntry( timestamp="t", bottle_slug="dev", component="egress", @@ -237,7 +233,7 @@ class TestAuditLog(unittest.TestCase): justification="", diff="", )) - write_audit_entry(AuditEntry( + _SV.write_audit_entry(AuditEntry( timestamp="t", bottle_slug="other", component="cred-proxy", @@ -246,12 +242,12 @@ class TestAuditLog(unittest.TestCase): justification="", diff="", )) - self.assertEqual(1, len(read_audit_entries("cred-proxy", "dev"))) - self.assertEqual(1, len(read_audit_entries("egress", "dev"))) - self.assertEqual(1, len(read_audit_entries("cred-proxy", "other"))) + self.assertEqual(1, len(_SV.read_audit_entries("cred-proxy", "dev"))) + self.assertEqual(1, len(_SV.read_audit_entries("egress", "dev"))) + self.assertEqual(1, len(_SV.read_audit_entries("cred-proxy", "other"))) def test_read_audit_entries_missing_log_returns_empty(self): - self.assertEqual([], read_audit_entries("cred-proxy", "no-such-bottle")) + self.assertEqual([], _SV.read_audit_entries("cred-proxy", "no-such-bottle")) class TestDiffAndHash(unittest.TestCase): diff --git a/tests/unit/test_supervise_edge.py b/tests/unit/test_supervise_edge.py index 8548040..d29135b 100644 --- a/tests/unit/test_supervise_edge.py +++ b/tests/unit/test_supervise_edge.py @@ -17,14 +17,12 @@ from bot_bottle.orchestrator.supervisor import ( AuditEntry, Proposal, STATUS_APPROVED, + Supervisor, TOOL_EGRESS_ALLOW, - list_pending_proposals, - read_audit_entries, - read_proposal, - read_response, - write_audit_entry, ) +_SV = Supervisor() + def _proposal() -> Proposal: return Proposal.new( @@ -47,21 +45,21 @@ class TestReadMalformed(unittest.TestCase): with patch.dict("os.environ", {"HOME": d}): QueueStore("slug").migrate() with self.assertRaises(FileNotFoundError): - read_proposal("slug", "p") + _SV.read_proposal("slug", "p") def test_read_response_missing_row(self) -> None: with tempfile.TemporaryDirectory() as d: with patch.dict("os.environ", {"HOME": d}): QueueStore("slug").migrate() with self.assertRaises(FileNotFoundError): - read_response("slug", "p") + _SV.read_response("slug", "p") def test_list_pending_reads_db_only(self) -> None: with tempfile.TemporaryDirectory() as d: with patch.dict("os.environ", {"HOME": d}): QueueStore("slug").migrate() - supervise.write_proposal(_proposal()) - pending = list_pending_proposals("slug") + _SV.write_proposal(_proposal()) + pending = _SV.list_pending_proposals("slug") self.assertEqual(1, len(pending)) self.assertEqual("slug", pending[0].bottle_slug) @@ -70,26 +68,26 @@ class TestReadMalformed(unittest.TestCase): with patch.dict("os.environ", {"HOME": d}): QueueStore("slug").migrate() p = _proposal() - supervise.write_proposal(p) - supervise.write_response("slug", supervise.Response( + _SV.write_proposal(p) + _SV.write_response("slug", supervise.Response( proposal_id=p.id, status=STATUS_APPROVED, notes="", )) - self.assertEqual([], list_pending_proposals("slug")) + self.assertEqual([], _SV.list_pending_proposals("slug")) class TestReadAuditEntries(unittest.TestCase): def test_missing_log_returns_empty(self) -> None: with tempfile.TemporaryDirectory() as home, \ patch.dict("os.environ", {"HOME": home}): - self.assertEqual([], read_audit_entries("egress", "nope")) + self.assertEqual([], _SV.read_audit_entries("egress", "nope")) def test_reads_entries_from_db(self) -> None: with tempfile.TemporaryDirectory() as home, \ patch.dict("os.environ", {"HOME": home}): AuditStore().migrate() - write_audit_entry(AuditEntry( + _SV.write_audit_entry(AuditEntry( timestamp="t", bottle_slug="slug", component="egress", @@ -98,7 +96,7 @@ class TestReadAuditEntries(unittest.TestCase): justification="", diff="", )) - write_audit_entry(AuditEntry( + _SV.write_audit_entry(AuditEntry( timestamp="t", bottle_slug="other", component="egress", @@ -107,7 +105,7 @@ class TestReadAuditEntries(unittest.TestCase): justification="", diff="", )) - entries = read_audit_entries("egress", "slug") + entries = _SV.read_audit_entries("egress", "slug") self.assertEqual(1, len(entries)) self.assertEqual("approve", entries[0].operator_action) diff --git a/tests/unit/test_supervise_server.py b/tests/unit/test_supervise_server.py index a082998..fdaff90 100644 --- a/tests/unit/test_supervise_server.py +++ b/tests/unit/test_supervise_server.py @@ -50,6 +50,11 @@ from bot_bottle.gateway.supervise_server import ( validate_proposed_file, ) +# The orchestrator's supervise service, backing the fake resolver's queue reads +# and writes (the fake stands in for the RPC; state still goes through the real +# queue store). +_SV = _sv.Supervisor() + # Fixed caller identity for the handler tests. The control plane attributes by # (source_ip, identity_token); the fake resolver ignores them and answers for a # fixed bottle, since attribution itself is covered by the orchestrator tests. @@ -89,7 +94,7 @@ class _FakeSuperviseResolver: bottle_slug=self.bottle_id, tool=tool, proposed_file=proposed_file, justification=justification, current_file_hash=_sv.sha256_hex(proposed_file), ) - _sv.write_proposal(proposal) + _SV.write_proposal(proposal) return proposal.id def poll_supervise( @@ -101,10 +106,10 @@ class _FakeSuperviseResolver: if self.bottle_id is None or self.poll_none: return None try: - response = _sv.read_response(self.bottle_id, proposal_id) + response = _SV.read_response(self.bottle_id, proposal_id) except FileNotFoundError: try: - _sv.read_proposal(self.bottle_id, proposal_id) + _SV.read_proposal(self.bottle_id, proposal_id) except FileNotFoundError: return {"status": _sv.POLL_STATUS_UNKNOWN} return {"status": _sv.POLL_STATUS_PENDING} @@ -377,10 +382,10 @@ class TestHandleToolsCall(unittest.TestCase): matching response — the operator half, out of band.""" def runner(): for _ in range(200): - pending = _sv.list_pending_proposals("dev") + pending = _SV.list_pending_proposals("dev") if pending: p = pending[0] - _sv.write_response("dev", _sv.Response( + _SV.write_response("dev", _sv.Response( proposal_id=p.id, status=status, notes=notes, )) return @@ -487,7 +492,7 @@ class TestHandleToolsCall(unittest.TestCase): responder.join() # A decided proposal drops off the operator's pending list (a response # row exists) — poll itself no longer archives (issue #469 review). - self.assertEqual([], _sv.list_pending_proposals("dev")) + self.assertEqual([], _SV.list_pending_proposals("dev")) def test_pending_response_times_out_without_archive(self): result = _tools_call( @@ -505,7 +510,7 @@ class TestHandleToolsCall(unittest.TestCase): text = result["content"][0]["text"] # type: ignore[index] self.assertIn("status: pending", text) self.assertIn("proposal remains queued", text) - self.assertEqual(1, len(_sv.list_pending_proposals("dev"))) + self.assertEqual(1, len(_SV.list_pending_proposals("dev"))) _ALLOW: dict[str, object] = { "name": _sv.TOOL_EGRESS_ALLOW, @@ -747,7 +752,7 @@ class TestNonBlockingSupervise(unittest.TestCase): justification="need example.com", current_file_hash=_sv.sha256_hex(self._ROUTES), ) - _sv.write_proposal(p) + _SV.write_proposal(p) return p # --- pending response carries the id --- @@ -771,7 +776,7 @@ class TestNonBlockingSupervise(unittest.TestCase): self.assertFalse(result["isError"]) # type: ignore[index] text = result["content"][0]["text"] # type: ignore[index] self.assertIn("status: pending", text) - pending = _sv.list_pending_proposals("dev") + pending = _SV.list_pending_proposals("dev") self.assertEqual(1, len(pending)) # still queued, not archived self.assertIn(pending[0].id, text) # agent got the id to poll @@ -779,7 +784,7 @@ class TestNonBlockingSupervise(unittest.TestCase): def test_check_returns_approved_idempotently(self): p = self._seed_proposal() - _sv.write_response("dev", _sv.Response(proposal_id=p.id, status=_sv.STATUS_APPROVED, notes="ok")) + _SV.write_response("dev", _sv.Response(proposal_id=p.id, status=_sv.STATUS_APPROVED, notes="ok")) result = _check(self.resolver, {"arguments": {"proposal_id": p.id}}) self.assertFalse(result["isError"]) text = result["content"][0]["text"] # type: ignore[index] @@ -792,7 +797,7 @@ class TestNonBlockingSupervise(unittest.TestCase): def test_check_rejected_sets_isError(self): p = self._seed_proposal() - _sv.write_response("dev", _sv.Response(proposal_id=p.id, status=_sv.STATUS_REJECTED, notes="no")) + _SV.write_response("dev", _sv.Response(proposal_id=p.id, status=_sv.STATUS_REJECTED, notes="no")) result = _check(self.resolver, {"arguments": {"proposal_id": p.id}}) self.assertTrue(result["isError"]) self.assertIn("status: rejected", result["content"][0]["text"]) # type: ignore[index] @@ -804,7 +809,7 @@ class TestNonBlockingSupervise(unittest.TestCase): text = result["content"][0]["text"] # type: ignore[index] self.assertIn("status: pending", text) self.assertIn(p.id, text) - self.assertEqual(1, len(_sv.list_pending_proposals("dev"))) # not archived + self.assertEqual(1, len(_SV.list_pending_proposals("dev"))) # not archived def test_check_unknown_id_is_error(self): result = _check(self.resolver, {"arguments": {"proposal_id": "no-such-proposal"}}) @@ -848,15 +853,15 @@ class TestNonBlockingSupervise(unittest.TestCase): }, ServerConfig(response_timeout_seconds=0.05), ) - pid = _sv.list_pending_proposals("dev")[0].id + pid = _SV.list_pending_proposals("dev")[0].id self.assertIn(pid, result["content"][0]["text"]) # type: ignore[index] # 2. operator decides out-of-band - _sv.write_response("dev", _sv.Response(proposal_id=pid, status=_sv.STATUS_APPROVED, notes="ok")) + _SV.write_response("dev", _sv.Response(proposal_id=pid, status=_sv.STATUS_APPROVED, notes="ok")) # 3. agent resumes by polling — no re-proposing poll = _check(self.resolver, {"arguments": {"proposal_id": pid}}) self.assertFalse(poll["isError"]) self.assertIn("status: approved", poll["content"][0]["text"]) # type: ignore[index] - self.assertEqual([], _sv.list_pending_proposals("dev")) # resolved (response exists) + self.assertEqual([], _SV.list_pending_proposals("dev")) # resolved (response exists) if __name__ == "__main__": -- 2.52.0 From db6a151803b0b61239f84e72bad14792a3178d83 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 15:03:37 -0400 Subject: [PATCH 14/39] =?UTF-8?q?refactor(supervise):=20hash=20the=20propo?= =?UTF-8?q?sed=20file=20inside=20Proposal.new;=20sha256=5Fhex=20=E2=86=92?= =?UTF-8?q?=20root=20util?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move sha256_hex out of orchestrator/supervisor/util.py to the root bot_bottle.util (it's a generic string hash, not supervise-specific), and have Proposal.new take the proposed file contents and compute current_file_hash itself instead of every caller passing sha256_hex(proposed_file). Every call site set current_file_hash to the hash of the proposed file, and nothing reads the field except the DB round-trip (schema + _row_to_proposal + from_dict, all untouched), so folding the hash into the factory is behavior-preserving and removes the boilerplate. Callers now pass just the file; the orchestrator no longer imports sha256_hex at all. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/orchestrator/service.py | 2 -- bot_bottle/orchestrator/supervisor/__init__.py | 6 +++--- bot_bottle/orchestrator/supervisor/util.py | 13 ++++--------- bot_bottle/supervisor/types.py | 5 +++-- bot_bottle/util.py | 6 ++++++ tests/unit/test_orchestrator_control_plane.py | 3 +-- tests/unit/test_orchestrator_service.py | 3 +-- tests/unit/test_supervise.py | 6 +----- tests/unit/test_supervise_cli.py | 3 +-- tests/unit/test_supervise_edge.py | 1 - tests/unit/test_supervise_server.py | 3 +-- 11 files changed, 21 insertions(+), 30 deletions(-) diff --git a/bot_bottle/orchestrator/service.py b/bot_bottle/orchestrator/service.py index 452af25..d35169e 100644 --- a/bot_bottle/orchestrator/service.py +++ b/bot_bottle/orchestrator/service.py @@ -41,7 +41,6 @@ from .supervisor import ( TOOL_EGRESS_BLOCK, Supervisor, render_diff, - sha256_hex, ) @@ -216,7 +215,6 @@ class Orchestrator: tool=tool, proposed_file=proposed_file, justification=justification, - current_file_hash=sha256_hex(proposed_file), ) self._supervisor.write_proposal(proposal) return proposal.id diff --git a/bot_bottle/orchestrator/supervisor/__init__.py b/bot_bottle/orchestrator/supervisor/__init__.py index 8a2d912..40d4c50 100644 --- a/bot_bottle/orchestrator/supervisor/__init__.py +++ b/bot_bottle/orchestrator/supervisor/__init__.py @@ -8,8 +8,8 @@ dependency explicit and injectable in tests. For convenience this package also re-exports the neutral vocabulary (`bot_bottle.supervisor`'s types + `SupervisePlan`) and the pure `render_diff` -/ `sha256_hex` helpers (`util`), so orchestrator-side callers import from one -place. The data plane must NOT import this package — it imports +helper (`util`), so orchestrator-side callers import from one place. The data +plane must NOT import this package — it imports `bot_bottle.supervisor` (neutral) directly and reaches the queue over the control-plane RPC. """ @@ -24,7 +24,7 @@ from ..store.store_manager import StoreManager from ..store.queue_store import QueueStore from ...store.audit_store import AuditStore from ...supervisor.types import AuditEntry, Proposal, Response -from .util import render_diff, sha256_hex # noqa: F401 — re-exported for callers +from .util import render_diff # noqa: F401 — re-exported for callers class Supervisor: diff --git a/bot_bottle/orchestrator/supervisor/util.py b/bot_bottle/orchestrator/supervisor/util.py index 3776b56..4c02aaa 100644 --- a/bot_bottle/orchestrator/supervisor/util.py +++ b/bot_bottle/orchestrator/supervisor/util.py @@ -1,14 +1,13 @@ -"""Pure supervise helpers — diff rendering and content hashing. +"""Pure supervise helpers — diff rendering. -Stateless utilities used alongside the `Supervisor` service. Kept as plain -functions (not methods) because they carry no state and are also handy to -callers that only need to hash or diff. +Stateless utility used alongside the `Supervisor` service. Kept as a plain +function (not a method) because it carries no state. (Content hashing is a +generic helper — `bot_bottle.util.sha256_hex`.) """ from __future__ import annotations import difflib -import hashlib def render_diff(before: str, after: str, *, label: str = "config") -> str: @@ -25,7 +24,3 @@ def render_diff(before: str, after: str, *, label: str = "config") -> str: if not parts: return "" return "".join(p if p.endswith("\n") else p + "\n" for p in parts).rstrip("\n") - - -def sha256_hex(content: str) -> str: - return hashlib.sha256(content.encode("utf-8")).hexdigest() diff --git a/bot_bottle/supervisor/types.py b/bot_bottle/supervisor/types.py index e09fbde..cd6c391 100644 --- a/bot_bottle/supervisor/types.py +++ b/bot_bottle/supervisor/types.py @@ -15,6 +15,8 @@ import uuid from dataclasses import dataclass from datetime import datetime, timezone +from ..util import sha256_hex + TOOL_EGRESS_BLOCK = "egress-block" TOOL_EGRESS_ALLOW = "egress-allow" TOOL_GITLEAKS_ALLOW = "gitleaks-allow" @@ -94,7 +96,6 @@ class Proposal: tool: str, proposed_file: str, justification: str, - current_file_hash: str, now: datetime | None = None, ) -> "Proposal": ts = (now or datetime.now(timezone.utc)).isoformat() @@ -105,7 +106,7 @@ class Proposal: proposed_file=proposed_file, justification=justification, arrival_timestamp=ts, - current_file_hash=current_file_hash, + current_file_hash=sha256_hex(proposed_file), ) def to_dict(self) -> dict[str, object]: diff --git a/bot_bottle/util.py b/bot_bottle/util.py index 8cc0920..6274163 100644 --- a/bot_bottle/util.py +++ b/bot_bottle/util.py @@ -5,11 +5,17 @@ level deeper, under their backend package.""" from __future__ import annotations +import hashlib import ipaddress import os import sys +def sha256_hex(content: str) -> str: + """Hex SHA-256 of a UTF-8 string.""" + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + def is_ip_literal(value: str) -> bool: try: ipaddress.ip_address(value) diff --git a/tests/unit/test_orchestrator_control_plane.py b/tests/unit/test_orchestrator_control_plane.py index 66119c3..8210df5 100644 --- a/tests/unit/test_orchestrator_control_plane.py +++ b/tests/unit/test_orchestrator_control_plane.py @@ -28,7 +28,6 @@ from bot_bottle.orchestrator.store.store_manager import StoreManager from bot_bottle.orchestrator.supervisor import ( Proposal, TOOL_EGRESS_ALLOW, - sha256_hex, Supervisor, ) @@ -420,7 +419,7 @@ class TestDispatchSupervise(unittest.TestCase): "10.243.0.1", metadata=json.dumps({"slug": slug}), policy="routes: []\n") p = Proposal.new( bottle_slug=slug, tool=TOOL_EGRESS_ALLOW, proposed_file=proposed, - justification="need it", current_file_hash=sha256_hex(proposed)) + justification="need it") Supervisor().write_proposal(p) return p.id diff --git a/tests/unit/test_orchestrator_service.py b/tests/unit/test_orchestrator_service.py index d090f3a..a203af3 100644 --- a/tests/unit/test_orchestrator_service.py +++ b/tests/unit/test_orchestrator_service.py @@ -20,7 +20,6 @@ from bot_bottle.orchestrator.supervisor import ( Proposal, STATUS_APPROVED, TOOL_EGRESS_ALLOW, - sha256_hex, Supervisor, ) @@ -183,7 +182,7 @@ class TestOrchestratorSupervise(unittest.TestCase): def _queue(self, slug: str, proposed: str) -> str: p = Proposal.new( bottle_slug=slug, tool=TOOL_EGRESS_ALLOW, proposed_file=proposed, - justification="need it", current_file_hash=sha256_hex(proposed)) + justification="need it") Supervisor().write_proposal(p) return p.id diff --git a/tests/unit/test_supervise.py b/tests/unit/test_supervise.py index c609dbe..8edfe44 100644 --- a/tests/unit/test_supervise.py +++ b/tests/unit/test_supervise.py @@ -7,6 +7,7 @@ from pathlib import Path from bot_bottle.orchestrator import supervisor as supervise from bot_bottle.paths import host_db_path +from bot_bottle.util import sha256_hex from tests.unit import use_bottle_root from bot_bottle.store.audit_store import AuditStore from bot_bottle.orchestrator.store.queue_store import QueueStore @@ -21,7 +22,6 @@ from bot_bottle.orchestrator.supervisor import ( TOOL_EGRESS_ALLOW, TOOL_GITLEAKS_ALLOW, render_diff, - sha256_hex, ) _SV = Supervisor() @@ -40,7 +40,6 @@ def _proposal( tool=tool, proposed_file=proposed, justification=justification, - current_file_hash=sha256_hex(proposed), now=FIXED_TS, ) @@ -145,13 +144,11 @@ class TestQueueIO(unittest.TestCase): a = Proposal.new( bottle_slug="dev", tool=TOOL_EGRESS_ALLOW, proposed_file="routes:\n - host: early.example.com\n", justification="early", - current_file_hash="x", now=datetime(2026, 5, 25, 10, 0, 0, tzinfo=timezone.utc), ) b = Proposal.new( bottle_slug="dev", tool=TOOL_EGRESS_ALLOW, proposed_file="routes:\n - host: late.example.com\n", justification="late", - current_file_hash="x", now=datetime(2026, 5, 25, 14, 0, 0, tzinfo=timezone.utc), ) # Write in reverse order. @@ -288,7 +285,6 @@ class TestToolConstants(unittest.TestCase): tool=supervise.TOOL_EGRESS_TOKEN_ALLOW, proposed_file="host: api.example.com\n", justification="false positive", - current_file_hash="h", ) self.assertEqual(p, Proposal.from_dict(p.to_dict())) diff --git a/tests/unit/test_supervise_cli.py b/tests/unit/test_supervise_cli.py index f35105b..f3f4477 100644 --- a/tests/unit/test_supervise_cli.py +++ b/tests/unit/test_supervise_cli.py @@ -19,7 +19,6 @@ from bot_bottle.orchestrator.supervisor import ( TOOL_EGRESS_BLOCK, TOOL_GITLEAKS_ALLOW, TOOL_EGRESS_TOKEN_ALLOW, - sha256_hex, ) @@ -37,7 +36,7 @@ def _proposal(slug: str = "dev", tool: str = TOOL_EGRESS_ALLOW, payload = payloads.get(tool, "") return Proposal.new( bottle_slug=slug, tool=tool, proposed_file=payload, - justification=f"needed for {slug}", current_file_hash=sha256_hex(payload), + justification=f"needed for {slug}", now=now, ) diff --git a/tests/unit/test_supervise_edge.py b/tests/unit/test_supervise_edge.py index d29135b..82929e4 100644 --- a/tests/unit/test_supervise_edge.py +++ b/tests/unit/test_supervise_edge.py @@ -30,7 +30,6 @@ def _proposal() -> Proposal: tool=TOOL_EGRESS_ALLOW, proposed_file="x", justification="j", - current_file_hash="h", ) diff --git a/tests/unit/test_supervise_server.py b/tests/unit/test_supervise_server.py index fdaff90..0ae9ce5 100644 --- a/tests/unit/test_supervise_server.py +++ b/tests/unit/test_supervise_server.py @@ -92,7 +92,7 @@ class _FakeSuperviseResolver: return None proposal = _sv.Proposal.new( bottle_slug=self.bottle_id, tool=tool, proposed_file=proposed_file, - justification=justification, current_file_hash=_sv.sha256_hex(proposed_file), + justification=justification, ) _SV.write_proposal(proposal) return proposal.id @@ -750,7 +750,6 @@ class TestNonBlockingSupervise(unittest.TestCase): tool=_sv.TOOL_EGRESS_ALLOW, proposed_file=self._ROUTES, justification="need example.com", - current_file_hash=_sv.sha256_hex(self._ROUTES), ) _SV.write_proposal(p) return p -- 2.52.0 From d41236c376e7e7f324ea9a6cb43f746c0404be52 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 15:09:27 -0400 Subject: [PATCH 15/39] refactor(util): make render_diff generic (caller supplies side titles); move to root util MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generalize render_diff so it isn't supervise-flavored: the "(current)" / "(proposed)" side titles are no longer baked in — callers pass `before_title` and `after_label` (required). Move it from orchestrator/supervisor/util.py to the base bot_bottle.util alongside sha256_hex, and delete the now-empty supervisor/util.py. The supervise callsite (Orchestrator.supervise_respond) assigns before_title="current", after_label="proposed"; the facade no longer re-exports render_diff (callers import from bot_bottle.util). Behavior at the supervise callsite is unchanged. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/orchestrator/service.py | 7 +++-- .../orchestrator/supervisor/__init__.py | 8 +++--- bot_bottle/orchestrator/supervisor/util.py | 26 ------------------- bot_bottle/util.py | 26 +++++++++++++++++++ tests/unit/test_supervise.py | 7 +++-- 5 files changed, 38 insertions(+), 36 deletions(-) delete mode 100644 bot_bottle/orchestrator/supervisor/util.py diff --git a/bot_bottle/orchestrator/service.py b/bot_bottle/orchestrator/service.py index d35169e..740a6fa 100644 --- a/bot_bottle/orchestrator/service.py +++ b/bot_bottle/orchestrator/service.py @@ -40,8 +40,8 @@ from .supervisor import ( TOOL_EGRESS_ALLOW, TOOL_EGRESS_BLOCK, Supervisor, - render_diff, ) +from ..util import render_diff # Operator decision → Response.status. The apply half (egress tools) runs @@ -350,7 +350,10 @@ class Orchestrator: operator_action=status, operator_notes=notes, justification=proposal.justification, - diff=render_diff(diff_before, diff_after, label=component), + diff=render_diff( + diff_before, diff_after, label=component, + before_title="current", after_label="proposed", + ), )) return True, "" diff --git a/bot_bottle/orchestrator/supervisor/__init__.py b/bot_bottle/orchestrator/supervisor/__init__.py index 40d4c50..de4cf24 100644 --- a/bot_bottle/orchestrator/supervisor/__init__.py +++ b/bot_bottle/orchestrator/supervisor/__init__.py @@ -7,9 +7,10 @@ object — optionally scoped to a `db_path` — makes the orchestrator's supervi dependency explicit and injectable in tests. For convenience this package also re-exports the neutral vocabulary -(`bot_bottle.supervisor`'s types + `SupervisePlan`) and the pure `render_diff` -helper (`util`), so orchestrator-side callers import from one place. The data -plane must NOT import this package — it imports +(`bot_bottle.supervisor`'s types + `SupervisePlan`), so orchestrator-side +callers import from one place. (Generic helpers like `render_diff` / +`sha256_hex` live in `bot_bottle.util`.) The data plane must NOT import this +package — it imports `bot_bottle.supervisor` (neutral) directly and reaches the queue over the control-plane RPC. """ @@ -24,7 +25,6 @@ from ..store.store_manager import StoreManager from ..store.queue_store import QueueStore from ...store.audit_store import AuditStore from ...supervisor.types import AuditEntry, Proposal, Response -from .util import render_diff # noqa: F401 — re-exported for callers class Supervisor: diff --git a/bot_bottle/orchestrator/supervisor/util.py b/bot_bottle/orchestrator/supervisor/util.py deleted file mode 100644 index 4c02aaa..0000000 --- a/bot_bottle/orchestrator/supervisor/util.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Pure supervise helpers — diff rendering. - -Stateless utility used alongside the `Supervisor` service. Kept as a plain -function (not a method) because it carries no state. (Content hashing is a -generic helper — `bot_bottle.util.sha256_hex`.) -""" - -from __future__ import annotations - -import difflib - - -def render_diff(before: str, after: str, *, label: str = "config") -> str: - """Unified diff suitable for the audit log + TUI. Empty diff (no changes) - renders as the empty string.""" - diff = difflib.unified_diff( - before.splitlines(keepends=True), - after.splitlines(keepends=True), - fromfile=f"{label} (current)", - tofile=f"{label} (proposed)", - lineterm="", - ) - parts = list(diff) - if not parts: - return "" - return "".join(p if p.endswith("\n") else p + "\n" for p in parts).rstrip("\n") diff --git a/bot_bottle/util.py b/bot_bottle/util.py index 6274163..767e365 100644 --- a/bot_bottle/util.py +++ b/bot_bottle/util.py @@ -5,6 +5,7 @@ level deeper, under their backend package.""" from __future__ import annotations +import difflib import hashlib import ipaddress import os @@ -16,6 +17,31 @@ def sha256_hex(content: str) -> str: return hashlib.sha256(content.encode("utf-8")).hexdigest() +def render_diff( + before: str, + after: str, + *, + label: str = "config", + before_title: str, + after_label: str, +) -> str: + """Unified diff of `before` vs `after`, with the two sides headed + `{label} ({before_title})` and `{label} ({after_label})`. Empty diff (no + changes) renders as the empty string. The side titles are the caller's — + e.g. "current"/"proposed" for a supervise proposal.""" + diff = difflib.unified_diff( + before.splitlines(keepends=True), + after.splitlines(keepends=True), + fromfile=f"{label} ({before_title})", + tofile=f"{label} ({after_label})", + lineterm="", + ) + parts = list(diff) + if not parts: + return "" + return "".join(p if p.endswith("\n") else p + "\n" for p in parts).rstrip("\n") + + def is_ip_literal(value: str) -> bool: try: ipaddress.ip_address(value) diff --git a/tests/unit/test_supervise.py b/tests/unit/test_supervise.py index 8edfe44..1395705 100644 --- a/tests/unit/test_supervise.py +++ b/tests/unit/test_supervise.py @@ -7,7 +7,7 @@ from pathlib import Path from bot_bottle.orchestrator import supervisor as supervise from bot_bottle.paths import host_db_path -from bot_bottle.util import sha256_hex +from bot_bottle.util import render_diff, sha256_hex from tests.unit import use_bottle_root from bot_bottle.store.audit_store import AuditStore from bot_bottle.orchestrator.store.queue_store import QueueStore @@ -21,7 +21,6 @@ from bot_bottle.orchestrator.supervisor import ( Supervisor, TOOL_EGRESS_ALLOW, TOOL_GITLEAKS_ALLOW, - render_diff, ) _SV = Supervisor() @@ -249,10 +248,10 @@ class TestAuditLog(unittest.TestCase): class TestDiffAndHash(unittest.TestCase): def test_render_diff_returns_empty_when_unchanged(self): - self.assertEqual("", render_diff("a\nb\n", "a\nb\n")) + self.assertEqual("", render_diff("a\nb\n", "a\nb\n", before_title="current", after_label="proposed")) def test_render_diff_shows_changes(self): - diff = render_diff("a\nb\nc\n", "a\nB\nc\n", label="routes.yaml") + diff = render_diff("a\nb\nc\n", "a\nB\nc\n", label="routes.yaml", before_title="current", after_label="proposed") self.assertIn("routes.yaml (current)", diff) self.assertIn("routes.yaml (proposed)", diff) self.assertIn("-b", diff) -- 2.52.0 From b96a8b44e0829164548ed77ebc2b83a5aab0fc33 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 15:26:52 -0400 Subject: [PATCH 16/39] refactor(docker): rename OrchestratorService -> DockerInfraService, move to backend/docker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OrchestratorService wasn't the orchestrator — it's the host-side lifecycle of the docker infra *container* (the one that runs the Orchestrator). It read as "the orchestrator as a service" and lived in orchestrator/lifecycle.py, while its siblings (macOS MacosInfraService, Firecracker infra_vm) live under their backend package. Rename it DockerInfraService and move it to backend/docker/infra.py alongside the docker backend, with its docker-only constants (INFRA_*/ORCHESTRATOR_* image + container names, daemon list, mount paths). orchestrator/lifecycle.py keeps only the backend-neutral pieces the other infra services share — DEFAULT_PORT, DEFAULT_STARTUP_TIMEOUT_SECONDS, OrchestratorStartError, source_hash — which macOS / firecracker / client still import from there. backend/docker/infra.py imports those (backend -> orchestrator is an allowed direction). Renamed the unit test to test_docker_infra.py. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/backend/docker/backend.py | 4 +- .../backend/docker/consolidated_launch.py | 6 +- bot_bottle/backend/docker/infra.py | 261 ++++++++++++++++++ bot_bottle/backend/docker/launch.py | 2 +- bot_bottle/orchestrator/lifecycle.py | 253 +---------------- bot_bottle/orchestrator/rotate_ca.py | 2 +- .../integration/test_multitenant_isolation.py | 4 +- ..._orchestrator_docker_control_plane_auth.py | 6 +- tests/unit/test_backend_selection.py | 4 +- ...ator_lifecycle.py => test_docker_infra.py} | 20 +- tests/unit/test_orchestrator_rotate_ca.py | 2 +- 11 files changed, 296 insertions(+), 268 deletions(-) create mode 100644 bot_bottle/backend/docker/infra.py rename tests/unit/{test_orchestrator_lifecycle.py => test_docker_infra.py} (95%) diff --git a/bot_bottle/backend/docker/backend.py b/bot_bottle/backend/docker/backend.py index 8cab6d7..eff4d5d 100644 --- a/bot_bottle/backend/docker/backend.py +++ b/bot_bottle/backend/docker/backend.py @@ -112,8 +112,8 @@ class DockerBottleBackend(BottleBackend["DockerBottlePlan", "DockerBottleCleanup yield bottle def ensure_orchestrator(self) -> str: - from ...orchestrator.lifecycle import OrchestratorService - return OrchestratorService().ensure_running() + from .infra import DockerInfraService + return DockerInfraService().ensure_running() def supervise_mcp_url(self, plan: DockerBottlePlan) -> str: """Docker bottles reach the supervise daemon via the diff --git a/bot_bottle/backend/docker/consolidated_launch.py b/bot_bottle/backend/docker/consolidated_launch.py index 2f57efb..2b7097f 100644 --- a/bot_bottle/backend/docker/consolidated_launch.py +++ b/bot_bottle/backend/docker/consolidated_launch.py @@ -21,7 +21,7 @@ from ...egress import EgressPlan from ...git_gate import GitGatePlan from ...orchestrator.client import OrchestratorClient from ...gateway import GATEWAY_NETWORK -from ...orchestrator.lifecycle import INFRA_NAME, OrchestratorService +from .infra import INFRA_NAME, DockerInfraService from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME from ...orchestrator.reprovision import reprovision_bottles from ..consolidated_util import provision_bottle @@ -144,7 +144,7 @@ def launch_consolidated( *, image_ref: str = "", tokens: dict[str, str] | None = None, - service: OrchestratorService | None = None, + service: DockerInfraService | None = None, infra_name: str = INFRA_NAME, network: str = GATEWAY_NETWORK, ) -> LaunchContext: @@ -154,7 +154,7 @@ def launch_consolidated( Also reprovisiones egress tokens for any already-running bottles that lost their in-memory credentials (e.g. after an infra container restart), so they regain egress access before the new bottle is registered.""" - service = service or OrchestratorService() + service = service or DockerInfraService() url = service.ensure_running() _reprovision_running_bottles(url, network=network, infra_name=infra_name) client = OrchestratorClient(url) diff --git a/bot_bottle/backend/docker/infra.py b/bot_bottle/backend/docker/infra.py new file mode 100644 index 0000000..2471082 --- /dev/null +++ b/bot_bottle/backend/docker/infra.py @@ -0,0 +1,261 @@ +"""The per-host infra container for the docker backend (PRD 0070). + +Runs both the orchestrator control plane and the gateway data plane inside a +single `bot-bottle-infra` container on the shared gateway network — the docker +analogue of the macOS infra container (`backend/macos_container/infra.py`) and +the Firecracker infra VM (`backend/firecracker/infra_vm.py`). `gateway_init` is +PID 1 and supervises both; the infra container is an idempotent per-host +singleton. + +The combined container replaces the prior two-container split +(bot-bottle-orchestrator + bot-bottle-orch-gateway). The host CLI reaches the +control plane via a published loopback port; gateway daemons reach it over +127.0.0.1 (same container). + +The shared, backend-neutral pieces (control-plane port, startup timeout, +`OrchestratorStartError`, `source_hash`) live in `orchestrator.lifecycle`. +""" + +from __future__ import annotations + +import os +import time +import urllib.error +import urllib.request +from pathlib import Path + +from ... import log +from ...control_auth import ROLE_GATEWAY, mint +from ...docker_cmd import run_docker +from ...paths import ( + CONTROL_AUTH_JWT_ENV, + CONTROL_PLANE_TOKEN_ENV, + bot_bottle_root, + host_control_plane_token, + host_gateway_ca_dir, +) +from ...gateway import ( + GATEWAY_DOCKERFILE, + GATEWAY_IMAGE, + GATEWAY_NETWORK, + GatewayError, + MITMPROXY_HOME, +) +from ...orchestrator.lifecycle import ( + DEFAULT_PORT, + DEFAULT_STARTUP_TIMEOUT_SECONDS, + OrchestratorStartError, + source_hash, +) + +INFRA_NAME = "bot-bottle-infra" +INFRA_LABEL = "bot-bottle-infra=1" +# The combined infra image: gateway data plane + orchestrator content. +# Built from Dockerfile.infra (FROM gateway + COPY --from orchestrator). +INFRA_IMAGE = os.environ.get("BOT_BOTTLE_INFRA_IMAGE", "bot-bottle-infra:latest") +INFRA_DOCKERFILE = "Dockerfile.infra" +# Baked as a container label so `ensure_running` can detect whether the +# running container is executing the current bind-mounted source. +INFRA_SOURCE_HASH_LABEL = "bot-bottle-infra-source-hash" + +# Orchestrator image: the single canonical definition of the control-plane +# content (lean: python:3.12-slim + bot_bottle package, no mitmproxy/git). +# Used as a build intermediate: `Dockerfile.infra` COPY --from this image. +ORCHESTRATOR_IMAGE = os.environ.get( + "BOT_BOTTLE_ORCHESTRATOR_IMAGE", "bot-bottle-orchestrator:latest" +) +ORCHESTRATOR_DOCKERFILE = "Dockerfile.orchestrator" + +# The gateway daemons + orchestrator the infra container runs. +# BOT_BOTTLE_GATEWAY_DAEMONS listing `orchestrator` opts it in to +# gateway_init's supervise tree (see gateway_init._OPT_IN_DAEMONS). +_INFRA_DAEMONS = "egress,git-http,supervise,orchestrator" + +# The bind-mount path for the live control-plane source inside the +# container. Separate from /app so the gateway's baked scripts +# (egress_addon.py, egress-entrypoint.sh) are not overlaid. +_SRC_IN_CONTAINER = "/bot-bottle-src" +# Bot-bottle host-root bind-mount inside the container (DB + state). The +# orchestrator control plane opens bot-bottle.db under here (via BOT_BOTTLE_ROOT +# -> host_db_path()); it is the ONLY process in the container with a file +# handle on it (PRD 0070 / issue #469). +_ROOT_IN_CONTAINER = "/bot-bottle-root" + +_HEALTH_POLL_SECONDS = 0.25 +_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0 + +_REPO_ROOT = Path(__file__).resolve().parents[3] + + +class DockerInfraService: + """Manages the single per-host docker infra container (control plane + + gateway). Callers only need `ensure_running()` + `url`. + + `infra_name` / `infra_label` let callers run independent infra containers + on the same host without name collisions (e.g. isolated integration tests + that can't share the production INFRA_NAME singleton).""" + + def __init__( + self, + *, + port: int = DEFAULT_PORT, + network: str = GATEWAY_NETWORK, + image: str = INFRA_IMAGE, + repo_root: Path = _REPO_ROOT, + host_root: Path | None = None, + infra_name: str = INFRA_NAME, + infra_label: str = INFRA_LABEL, + ) -> None: + self.port = port + self.network = network + self.image = image + self._repo_root = repo_root + self._host_root = host_root or bot_bottle_root() + self._infra_name = infra_name + self._infra_label = infra_label + + @property + def url(self) -> str: + """Host-side control-plane URL (published loopback port).""" + return f"http://127.0.0.1:{self.port}" + + def is_healthy(self, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS) -> bool: + try: + with urllib.request.urlopen(f"{self.url}/health", timeout=timeout) as resp: + return resp.status == 200 + except (urllib.error.URLError, TimeoutError, OSError): + return False + + def _container_running(self, name: str) -> bool: + proc = run_docker(["docker", "ps", "--filter", f"name=^/{name}$", "--format", "{{.Names}}"]) + return name in proc.stdout.split() + + def _infra_source_current(self, current_hash: str) -> bool: + """True iff the running infra container was started from the current + bind-mounted source. Mirrors the macOS backend's `_source_current`.""" + if not self._container_running(self._infra_name): + return False + proc = run_docker([ + "docker", "inspect", "--format", + "{{ index .Config.Labels \"" + INFRA_SOURCE_HASH_LABEL + "\" }}", + self._infra_name, + ]) + if proc.returncode != 0: + return True # can't compare → don't churn a working container + return proc.stdout.strip() == current_hash + + def _ensure_network(self) -> None: + if run_docker(["docker", "network", "inspect", self.network]).returncode == 0: + return + proc = run_docker(["docker", "network", "create", self.network]) + if proc.returncode != 0 and "already exists" not in proc.stderr: + raise GatewayError( + f"gateway network {self.network} failed to create: {proc.stderr.strip()}" + ) + + def _build_images(self) -> None: + """Build the gateway base, the orchestrator intermediate, then the + infra image. All are cache-aware: a no-op when nothing changed.""" + for tag, dockerfile in ( + (GATEWAY_IMAGE, GATEWAY_DOCKERFILE), + (ORCHESTRATOR_IMAGE, ORCHESTRATOR_DOCKERFILE), + (self.image, INFRA_DOCKERFILE), + ): + argv = ["docker", "build", "-t", tag, + "-f", str(self._repo_root / dockerfile), + str(self._repo_root)] + if os.environ.get("BOT_BOTTLE_NO_CACHE"): + argv.insert(2, "--no-cache") + proc = run_docker(argv) + if proc.returncode != 0: + raise GatewayError(f"{dockerfile} build failed: {proc.stderr.strip()}") + + def _run_infra_container(self, current_hash: str) -> None: + """Start the combined infra container (idempotent: clears a stale + fixed-name container first). Labels the container with `current_hash` + so a later `ensure_running` can detect a real code change.""" + self._ensure_network() + run_docker(["docker", "rm", "--force", self._infra_name]) + _signing_key = host_control_plane_token() + proc = run_docker([ + "docker", "run", "--detach", + "--name", self._infra_name, + "--label", self._infra_label, + "--label", f"{INFRA_SOURCE_HASH_LABEL}={current_hash}", + "--network", self.network, + # Host CLI reaches the control plane here (loopback only). + # gateway_init always starts the orchestrator on DEFAULT_PORT (8099) + # inside the container; self.port is the host-side published port. + "--publish", f"127.0.0.1:{self.port}:{DEFAULT_PORT}", + # Persist the mitmproxy CA on the host so it survives container + # recreation AND docker volume pruning (issue #450): every agent + # trusts this one CA, so a fresh one would break all running bottles. + "--volume", f"{host_gateway_ca_dir()}:{MITMPROXY_HOME}", + # Live control-plane source, mounted to a path that does not + # overlay the gateway's baked /app scripts. + "--volume", f"{self._repo_root}:{_SRC_IN_CONTAINER}:ro", + # PYTHONPATH lets the orchestrator (and other Python daemons) + # import the live source ahead of the installed package. + "--env", f"PYTHONPATH={_SRC_IN_CONTAINER}", + # Orchestrator registry DB on the host (sole writer: control plane). + "--volume", f"{self._host_root}:{_ROOT_IN_CONTAINER}", + "--env", f"BOT_BOTTLE_ROOT={_ROOT_IN_CONTAINER}", + # Control-plane signing key (orchestrator: verifies tokens) + the + # pre-minted `gateway` JWT (data-plane daemons: present it). gateway_init + # scopes each to its process, so a compromised data-plane daemon never + # sees the key and can't mint a `cli` token (issue #469 review). + "--env", CONTROL_PLANE_TOKEN_ENV, + "--env", CONTROL_AUTH_JWT_ENV, + # Gateway daemons reach the orchestrator over loopback at its + # fixed internal port (DEFAULT_PORT), independent of self.port. + "--env", f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{DEFAULT_PORT}", + # Opt the orchestrator into gateway_init's supervise tree. + "--env", f"BOT_BOTTLE_GATEWAY_DAEMONS={_INFRA_DAEMONS}", + self.image, + ], env={ + **os.environ, + CONTROL_PLANE_TOKEN_ENV: _signing_key, + CONTROL_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key), + }) + if proc.returncode != 0: + raise OrchestratorStartError( + f"infra container failed to start: {proc.stderr.strip()}" + ) + + def ensure_running( + self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS, + ) -> str: + """Ensure the infra container (control plane + gateway) is up; return + the host control-plane URL. Idempotent — a healthy container on current + source is left untouched. Raises `OrchestratorStartError` on timeout.""" + self._build_images() + + current_hash = source_hash(self._repo_root) + if self.is_healthy() and self._infra_source_current(current_hash): + return self.url + + log.info("starting infra container", context={"name": self._infra_name}) + self._run_infra_container(current_hash) + + deadline = time.monotonic() + startup_timeout + while time.monotonic() < deadline: + if self.is_healthy(): + log.info("infra container healthy", context={"url": self.url}) + return self.url + time.sleep(_HEALTH_POLL_SECONDS) + raise OrchestratorStartError( + f"infra container at {self.url} did not become healthy within {startup_timeout:g}s" + ) + + def stop(self) -> None: + """Remove the infra container (idempotent).""" + run_docker(["docker", "rm", "--force", self._infra_name]) + + +__all__ = [ + "DockerInfraService", + "INFRA_NAME", + "INFRA_IMAGE", + "INFRA_SOURCE_HASH_LABEL", + "ORCHESTRATOR_IMAGE", +] diff --git a/bot_bottle/backend/docker/launch.py b/bot_bottle/backend/docker/launch.py index 78c8785..068b7c4 100644 --- a/bot_bottle/backend/docker/launch.py +++ b/bot_bottle/backend/docker/launch.py @@ -66,7 +66,7 @@ from .compose import ( from .consolidated_compose import consolidated_agent_compose from ...orchestrator.store.config_store import resolve_teardown_timeout from .consolidated_launch import launch_consolidated, teardown_consolidated -from ...orchestrator.lifecycle import INFRA_NAME +from .infra import INFRA_NAME from .gateway import DockerGateway diff --git a/bot_bottle/orchestrator/lifecycle.py b/bot_bottle/orchestrator/lifecycle.py index 5c7b9c6..d57b14d 100644 --- a/bot_bottle/orchestrator/lifecycle.py +++ b/bot_bottle/orchestrator/lifecycle.py @@ -1,94 +1,29 @@ -"""Orchestrator + gateway lifecycle (PRD 0070, docker slice). +"""Shared orchestrator-process constants + helpers (PRD 0070). -Runs both the orchestrator control plane and the gateway data plane inside -a single `bot-bottle-infra` container on the shared gateway network — -matching the structure already used by the macOS and Firecracker backends. -`gateway_init` is PID 1 and supervises both; the infra container is an -idempotent per-host singleton. - -The combined container replaces the prior two-container split -(bot-bottle-orchestrator + bot-bottle-orch-gateway). The host CLI reaches -the control plane via a published loopback port; gateway daemons reach it -over 127.0.0.1 (same container). +Backend-neutral pieces the per-backend infra services build on: the +control-plane port, the startup timeout, the start-error, and the source hash +used to detect a code change. The per-backend infra lifecycle lives with each +backend — docker: `backend.docker.infra.DockerInfraService`; macOS: +`backend.macos_container.infra`; firecracker: `backend.firecracker.infra_vm`. """ from __future__ import annotations import hashlib -import os -import time -import urllib.error -import urllib.request from pathlib import Path -from .. import log -from ..control_auth import ROLE_GATEWAY, mint -from ..docker_cmd import run_docker -from ..paths import ( - CONTROL_AUTH_JWT_ENV, - CONTROL_PLANE_TOKEN_ENV, - bot_bottle_root, - host_control_plane_token, - host_gateway_ca_dir, -) -from ..gateway import ( - GATEWAY_DOCKERFILE, - GATEWAY_IMAGE, - GATEWAY_NETWORK, - GatewayError, - MITMPROXY_HOME, -) - DEFAULT_PORT = 8099 DEFAULT_STARTUP_TIMEOUT_SECONDS = 45.0 -INFRA_NAME = "bot-bottle-infra" -INFRA_LABEL = "bot-bottle-infra=1" -# The combined infra image: gateway data plane + orchestrator content. -# Built from Dockerfile.infra (FROM gateway + COPY --from orchestrator). -INFRA_IMAGE = os.environ.get("BOT_BOTTLE_INFRA_IMAGE", "bot-bottle-infra:latest") -INFRA_DOCKERFILE = "Dockerfile.infra" -# Baked as a container label so `ensure_running` can detect whether the -# running container is executing the current bind-mounted source. -INFRA_SOURCE_HASH_LABEL = "bot-bottle-infra-source-hash" - -# Orchestrator image: the single canonical definition of the control-plane -# content (lean: python:3.12-slim + bot_bottle package, no mitmproxy/git). -# Used as a build intermediate: `Dockerfile.infra` COPY --from this image. -ORCHESTRATOR_IMAGE = os.environ.get( - "BOT_BOTTLE_ORCHESTRATOR_IMAGE", "bot-bottle-orchestrator:latest" -) -ORCHESTRATOR_DOCKERFILE = "Dockerfile.orchestrator" - -# The gateway daemons + orchestrator the infra container runs. -# BOT_BOTTLE_GATEWAY_DAEMONS listing `orchestrator` opts it in to -# gateway_init's supervise tree (see gateway_init._OPT_IN_DAEMONS). -_INFRA_DAEMONS = "egress,git-http,supervise,orchestrator" - -# The bind-mount path for the live control-plane source inside the -# container. Separate from /app so the gateway's baked scripts -# (egress_addon.py, egress-entrypoint.sh) are not overlaid. -_SRC_IN_CONTAINER = "/bot-bottle-src" -# Bot-bottle host-root bind-mount inside the container (DB + state). The -# orchestrator control plane opens bot-bottle.db under here (via BOT_BOTTLE_ROOT -# -> host_db_path()); it is the ONLY process in the container with a file -# handle on it (PRD 0070 / issue #469). -_ROOT_IN_CONTAINER = "/bot-bottle-root" - -_HEALTH_POLL_SECONDS = 0.25 -_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0 - -_REPO_ROOT = Path(__file__).resolve().parents[2] - class OrchestratorStartError(RuntimeError): - """The infra container did not become healthy within the timeout.""" + """The infra container/VM did not become healthy within the timeout.""" def source_hash(repo_root: Path) -> str: """Content hash of the orchestrator's bind-mounted Python source (the `bot_bottle` package the control-plane process imports). Changes only - when the code that would actually run changes — `ensure_running` + when the code that would actually run changes — a backend's `ensure_running` recreates the container on a mismatch so a code change takes effect, but leaves a healthy up-to-date container alone to preserve in-memory egress tokens.""" @@ -99,179 +34,9 @@ def source_hash(repo_root: Path) -> str: return h.hexdigest() -class OrchestratorService: - """Manages the single per-host infra container (control plane + gateway). - Callers only need `ensure_running()` + `url`. - - `infra_name` / `infra_label` let backends run independent infra containers - on the same host without name collisions (e.g. isolated integration tests - that can't share the production INFRA_NAME singleton).""" - - def __init__( - self, - *, - port: int = DEFAULT_PORT, - network: str = GATEWAY_NETWORK, - image: str = INFRA_IMAGE, - repo_root: Path = _REPO_ROOT, - host_root: Path | None = None, - infra_name: str = INFRA_NAME, - infra_label: str = INFRA_LABEL, - ) -> None: - self.port = port - self.network = network - self.image = image - self._repo_root = repo_root - self._host_root = host_root or bot_bottle_root() - self._infra_name = infra_name - self._infra_label = infra_label - - @property - def url(self) -> str: - """Host-side control-plane URL (published loopback port).""" - return f"http://127.0.0.1:{self.port}" - - def is_healthy(self, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS) -> bool: - try: - with urllib.request.urlopen(f"{self.url}/health", timeout=timeout) as resp: - return resp.status == 200 - except (urllib.error.URLError, TimeoutError, OSError): - return False - - def _container_running(self, name: str) -> bool: - proc = run_docker(["docker", "ps", "--filter", f"name=^/{name}$", "--format", "{{.Names}}"]) - return name in proc.stdout.split() - - def _infra_source_current(self, current_hash: str) -> bool: - """True iff the running infra container was started from the current - bind-mounted source. Mirrors the macOS backend's `_source_current`.""" - if not self._container_running(self._infra_name): - return False - proc = run_docker([ - "docker", "inspect", "--format", - "{{ index .Config.Labels \"" + INFRA_SOURCE_HASH_LABEL + "\" }}", - self._infra_name, - ]) - if proc.returncode != 0: - return True # can't compare → don't churn a working container - return proc.stdout.strip() == current_hash - - def _ensure_network(self) -> None: - if run_docker(["docker", "network", "inspect", self.network]).returncode == 0: - return - proc = run_docker(["docker", "network", "create", self.network]) - if proc.returncode != 0 and "already exists" not in proc.stderr: - raise GatewayError( - f"gateway network {self.network} failed to create: {proc.stderr.strip()}" - ) - - def _build_images(self) -> None: - """Build the gateway base, the orchestrator intermediate, then the - infra image. All are cache-aware: a no-op when nothing changed.""" - for tag, dockerfile in ( - (GATEWAY_IMAGE, GATEWAY_DOCKERFILE), - (ORCHESTRATOR_IMAGE, ORCHESTRATOR_DOCKERFILE), - (self.image, INFRA_DOCKERFILE), - ): - argv = ["docker", "build", "-t", tag, - "-f", str(self._repo_root / dockerfile), - str(self._repo_root)] - if os.environ.get("BOT_BOTTLE_NO_CACHE"): - argv.insert(2, "--no-cache") - proc = run_docker(argv) - if proc.returncode != 0: - raise GatewayError(f"{dockerfile} build failed: {proc.stderr.strip()}") - - def _run_infra_container(self, current_hash: str) -> None: - """Start the combined infra container (idempotent: clears a stale - fixed-name container first). Labels the container with `current_hash` - so a later `ensure_running` can detect a real code change.""" - self._ensure_network() - run_docker(["docker", "rm", "--force", self._infra_name]) - _signing_key = host_control_plane_token() - proc = run_docker([ - "docker", "run", "--detach", - "--name", self._infra_name, - "--label", self._infra_label, - "--label", f"{INFRA_SOURCE_HASH_LABEL}={current_hash}", - "--network", self.network, - # Host CLI reaches the control plane here (loopback only). - # gateway_init always starts the orchestrator on DEFAULT_PORT (8099) - # inside the container; self.port is the host-side published port. - "--publish", f"127.0.0.1:{self.port}:{DEFAULT_PORT}", - # Persist the mitmproxy CA on the host so it survives container - # recreation AND docker volume pruning (issue #450): every agent - # trusts this one CA, so a fresh one would break all running bottles. - "--volume", f"{host_gateway_ca_dir()}:{MITMPROXY_HOME}", - # Live control-plane source, mounted to a path that does not - # overlay the gateway's baked /app scripts. - "--volume", f"{self._repo_root}:{_SRC_IN_CONTAINER}:ro", - # PYTHONPATH lets the orchestrator (and other Python daemons) - # import the live source ahead of the installed package. - "--env", f"PYTHONPATH={_SRC_IN_CONTAINER}", - # Orchestrator registry DB on the host (sole writer: control plane). - "--volume", f"{self._host_root}:{_ROOT_IN_CONTAINER}", - "--env", f"BOT_BOTTLE_ROOT={_ROOT_IN_CONTAINER}", - # Control-plane signing key (orchestrator: verifies tokens) + the - # pre-minted `gateway` JWT (data-plane daemons: present it). gateway_init - # scopes each to its process, so a compromised data-plane daemon never - # sees the key and can't mint a `cli` token (issue #469 review). - "--env", CONTROL_PLANE_TOKEN_ENV, - "--env", CONTROL_AUTH_JWT_ENV, - # Gateway daemons reach the orchestrator over loopback at its - # fixed internal port (DEFAULT_PORT), independent of self.port. - "--env", f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{DEFAULT_PORT}", - # Opt the orchestrator into gateway_init's supervise tree. - "--env", f"BOT_BOTTLE_GATEWAY_DAEMONS={_INFRA_DAEMONS}", - self.image, - ], env={ - **os.environ, - CONTROL_PLANE_TOKEN_ENV: _signing_key, - CONTROL_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key), - }) - if proc.returncode != 0: - raise OrchestratorStartError( - f"infra container failed to start: {proc.stderr.strip()}" - ) - - def ensure_running( - self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS, - ) -> str: - """Ensure the infra container (control plane + gateway) is up; return - the host control-plane URL. Idempotent — a healthy container on current - source is left untouched. Raises `OrchestratorStartError` on timeout.""" - self._build_images() - - current_hash = source_hash(self._repo_root) - if self.is_healthy() and self._infra_source_current(current_hash): - return self.url - - log.info("starting infra container", context={"name": self._infra_name}) - self._run_infra_container(current_hash) - - deadline = time.monotonic() + startup_timeout - while time.monotonic() < deadline: - if self.is_healthy(): - log.info("infra container healthy", context={"url": self.url}) - return self.url - time.sleep(_HEALTH_POLL_SECONDS) - raise OrchestratorStartError( - f"infra container at {self.url} did not become healthy within {startup_timeout:g}s" - ) - - def stop(self) -> None: - """Remove the infra container (idempotent).""" - run_docker(["docker", "rm", "--force", self._infra_name]) - - __all__ = [ - "OrchestratorService", - "OrchestratorStartError", - "INFRA_NAME", - "INFRA_IMAGE", - "INFRA_SOURCE_HASH_LABEL", - "ORCHESTRATOR_IMAGE", "DEFAULT_PORT", "DEFAULT_STARTUP_TIMEOUT_SECONDS", + "OrchestratorStartError", "source_hash", ] diff --git a/bot_bottle/orchestrator/rotate_ca.py b/bot_bottle/orchestrator/rotate_ca.py index 5ae7d2d..0cae300 100644 --- a/bot_bottle/orchestrator/rotate_ca.py +++ b/bot_bottle/orchestrator/rotate_ca.py @@ -26,7 +26,7 @@ from pathlib import Path from ..docker_cmd import run_docker from ..paths import host_gateway_ca_dir from ..gateway import GATEWAY_NAME, rotate_gateway_ca -from .lifecycle import INFRA_NAME +from ..backend.docker.infra import INFRA_NAME # The containers whose mitmproxy would still be serving the old CA from memory: # the consolidated infra container and the standalone per-host gateway. diff --git a/tests/integration/test_multitenant_isolation.py b/tests/integration/test_multitenant_isolation.py index fdd8ea5..e32f9e2 100644 --- a/tests/integration/test_multitenant_isolation.py +++ b/tests/integration/test_multitenant_isolation.py @@ -30,7 +30,7 @@ from bot_bottle.backend.docker.egress import EGRESS_PORT from bot_bottle.backend.docker.gateway_net import next_free_ip from bot_bottle.orchestrator.client import OrchestratorClient from bot_bottle.gateway import GATEWAY_IMAGE, GATEWAY_NAME, GATEWAY_NETWORK -from bot_bottle.orchestrator.lifecycle import OrchestratorService +from bot_bottle.backend.docker.infra import DockerInfraService from tests._docker import skip_unless_docker # One upstream reachable under two names; reflects the Authorization header so @@ -85,7 +85,7 @@ class TestMultitenantIsolation(unittest.TestCase): self._tmp = tempfile.TemporaryDirectory() self.addCleanup(self._tmp.cleanup) # Throwaway root → a clean registry DB, independent of the host's. - self.svc = OrchestratorService(host_root=Path(self._tmp.name)) + self.svc = DockerInfraService(host_root=Path(self._tmp.name)) self.addCleanup(self._teardown_docker) # ensure_running builds the bundle image (slow on a cold cache) and # brings up the shared network + gateway + orchestrator. diff --git a/tests/integration/test_orchestrator_docker_control_plane_auth.py b/tests/integration/test_orchestrator_docker_control_plane_auth.py index aedb8ca..08a6c62 100644 --- a/tests/integration/test_orchestrator_docker_control_plane_auth.py +++ b/tests/integration/test_orchestrator_docker_control_plane_auth.py @@ -27,7 +27,7 @@ from pathlib import Path from bot_bottle.control_auth import ROLE_CLI, ROLE_GATEWAY, mint from bot_bottle.orchestrator.client import OrchestratorClient -from bot_bottle.orchestrator.lifecycle import OrchestratorService +from bot_bottle.backend.docker.infra import DockerInfraService from bot_bottle.paths import host_control_plane_token from tests._docker import skip_unless_docker @@ -54,7 +54,7 @@ class TestDockerControlPlaneAuthIntegration(unittest.TestCase): cls.addClassCleanup(cls._tmp.cleanup) # host_control_plane_token() — both the token read below and the one - # OrchestratorService injects into the container's env — resolves its + # DockerInfraService injects into the container's env — resolves its # path via the *ambient* BOT_BOTTLE_ROOT env var, not the host_root # kwarg passed to the constructor (that kwarg only controls the DB # bind-mount destination). Without pointing the env var at the same @@ -78,7 +78,7 @@ class TestDockerControlPlaneAuthIntegration(unittest.TestCase): cls._teardown_docker, infra_name, network, host_root ) - cls.svc = OrchestratorService( + cls.svc = DockerInfraService( infra_name=infra_name, network=network, image=_TEST_INFRA_IMAGE, diff --git a/tests/unit/test_backend_selection.py b/tests/unit/test_backend_selection.py index a5a4c18..bb549d1 100644 --- a/tests/unit/test_backend_selection.py +++ b/tests/unit/test_backend_selection.py @@ -390,10 +390,10 @@ class TestEnsureOrchestrator(unittest.TestCase): the orchestrator + gateway containers; firecracker boots the infra VM; macos-container starts the infra container.""" - def test_docker_delegates_to_orchestrator_service(self): + def test_docker_delegates_to_infra_service(self): b = get_bottle_backend("docker") with patch( - "bot_bottle.orchestrator.lifecycle.OrchestratorService" + "bot_bottle.backend.docker.infra.DockerInfraService" ) as service_cls: service_cls.return_value.ensure_running.return_value = ( "http://127.0.0.1:8099" diff --git a/tests/unit/test_orchestrator_lifecycle.py b/tests/unit/test_docker_infra.py similarity index 95% rename from tests/unit/test_orchestrator_lifecycle.py rename to tests/unit/test_docker_infra.py index 1a65908..bf85eb4 100644 --- a/tests/unit/test_orchestrator_lifecycle.py +++ b/tests/unit/test_docker_infra.py @@ -9,20 +9,22 @@ from pathlib import Path from unittest.mock import MagicMock, Mock, patch from bot_bottle.gateway import GatewayError -from bot_bottle.orchestrator.lifecycle import ( +from bot_bottle.backend.docker.infra import ( INFRA_NAME, INFRA_SOURCE_HASH_LABEL, - OrchestratorService, + DockerInfraService, +) +from bot_bottle.orchestrator.lifecycle import ( OrchestratorStartError, source_hash, ) from bot_bottle.paths import GATEWAY_CA_DIRNAME from tests.unit import use_bottle_root -_URLOPEN = "bot_bottle.orchestrator.lifecycle.urllib.request.urlopen" -_RUN = "bot_bottle.orchestrator.lifecycle.run_docker" -_SLEEP = "bot_bottle.orchestrator.lifecycle.time.sleep" -_MONOTONIC = "bot_bottle.orchestrator.lifecycle.time.monotonic" +_URLOPEN = "bot_bottle.backend.docker.infra.urllib.request.urlopen" +_RUN = "bot_bottle.backend.docker.infra.run_docker" +_SLEEP = "bot_bottle.backend.docker.infra.time.sleep" +_MONOTONIC = "bot_bottle.backend.docker.infra.time.monotonic" def _health(status: int) -> MagicMock: @@ -35,12 +37,12 @@ def _proc(returncode: int = 0, stdout: str = "", stderr: str = "") -> Mock: return Mock(returncode=returncode, stdout=stdout, stderr=stderr) -class TestOrchestratorService(unittest.TestCase): +class TestDockerInfraService(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() self.addCleanup(self._tmp.cleanup) self.addCleanup(use_bottle_root(Path(self._tmp.name))) - self.svc = OrchestratorService(port=8099) + self.svc = DockerInfraService(port=8099) def test_url(self) -> None: self.assertEqual("http://127.0.0.1:8099", self.svc.url) @@ -159,7 +161,7 @@ class TestOrchestratorService(unittest.TestCase): return _proc(stdout="") return _proc() - svc = OrchestratorService(port=20001) + svc = DockerInfraService(port=20001) with patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \ patch(_RUN, side_effect=fake), patch(_SLEEP): svc.ensure_running() diff --git a/tests/unit/test_orchestrator_rotate_ca.py b/tests/unit/test_orchestrator_rotate_ca.py index 3c2db89..e3ee60b 100644 --- a/tests/unit/test_orchestrator_rotate_ca.py +++ b/tests/unit/test_orchestrator_rotate_ca.py @@ -9,7 +9,7 @@ from unittest.mock import Mock, patch from bot_bottle.orchestrator import rotate_ca from bot_bottle.gateway import GATEWAY_NAME -from bot_bottle.orchestrator.lifecycle import INFRA_NAME +from bot_bottle.backend.docker.infra import INFRA_NAME from bot_bottle.paths import host_gateway_ca_dir from tests.unit import use_bottle_root -- 2.52.0 From 3e62f31d8be8f27cf4c12d0b0a3c751cf903fe7e Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 15:47:29 -0400 Subject: [PATCH 17/39] refactor(backend): split the heavy backend/__init__ into base + selection, thin the package init MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit backend/__init__.py eagerly imported the whole framework (manifest, egress, git-gate, env, workspace, agent-provider), so importing ANY backend.* submodule paid ~32 modules just to run the package init. Split it: - backend/base.py — the abstract contract (BottleSpec, BottlePlan, BottleCleanupPlan, ExecResult, ActiveAgent, Bottle, BottleImages, BottleBackend). Carries the framework imports; loaded only when a caller needs the contract. - backend/selection.py — backend registry / selection / enumeration (get_bottle_backend, _get_backends, _auto_select_backend, has_backend, known_backend_names, enumerate_active_agents). Concrete backends still imported lazily inside. - backend/__init__.py — thin: a __getattr__ that resolves the public names from those submodules on first access (+ a TYPE_CHECKING block for checkers). Existing `from bot_bottle.backend import X` and patch.object call-sites keep working. `import bot_bottle.backend` drops from 32 -> 2 bot_bottle modules. Repointed the backend-selection test's patch targets to the `selection` module (the functions reference each other there now; the FirecrackerBottleBackend class-method patches are unchanged — same class object). Note: backend.docker.util is still heavy because backend/docker/__init__ eagerly imports DockerBottleBackend — slimming the three sub-package inits is the follow-up that makes the leaves cheap. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/backend/__init__.py | 893 +++------------------------ bot_bottle/backend/base.py | 607 ++++++++++++++++++ bot_bottle/backend/selection.py | 188 ++++++ tests/unit/test_backend_selection.py | 51 +- 4 files changed, 895 insertions(+), 844 deletions(-) create mode 100644 bot_bottle/backend/base.py create mode 100644 bot_bottle/backend/selection.py diff --git a/bot_bottle/backend/__init__.py b/bot_bottle/backend/__init__.py index 79cd7af..0b74d80 100644 --- a/bot_bottle/backend/__init__.py +++ b/bot_bottle/backend/__init__.py @@ -1,836 +1,87 @@ -"""Per-backend bottle factories. +"""The bottle-backend package: abstract contract + backend selection. -A bottle is a running, isolated environment with claude inside. Each -backend exposes five methods: +Thin by design — nothing framework-heavy is imported at package init, so +importing any `backend.*` submodule (e.g. `backend.docker.util`) doesn't drag +the manifest / egress / git-gate framework into memory. The public names are +re-exported lazily: - prepare(spec, stage_dir=...) -> BottlePlan - Resolves names, validates host-side prerequisites, and writes - scratch files. No remote/runtime resources are created yet. - Safe to call before the y/N preflight. + * the abstract contract (`BottleBackend`, `BottleSpec`, `Bottle`, …) lives in + `backend.base`; + * backend selection / enumeration (`get_bottle_backend`, + `enumerate_active_agents`, …) in `backend.selection`; + * the concrete backends and the freeze helpers in their own submodules. - launch(plan) -> ContextManager[Bottle] - Brings up the container (or VM, or remote machine), provisions - it, yields a Bottle handle, and tears everything down on exit. - - prepare_cleanup() -> BottleCleanupPlan - Enumerates orphaned resources left behind by previous bottles - (containers, networks, ...). Idempotent; no side effects. - - cleanup(plan) -> None - Actually removes everything described by the cleanup plan. - - enumerate_active() -> Sequence[ActiveAgent] - Return every currently-running bottle on this backend, with - enough metadata for callers (CLI `list active`, dashboard - agents pane) to render a row. - -Selection is driven by `--backend` on `start` or BOT_BOTTLE_BACKEND -(env var). When neither is set, compatible macOS hosts default to -`macos-container`; Linux hosts with KVM default to `firecracker`; -otherwise `docker`. Per PRD 0003 the manifest does not carry a -backend field; the host picks. +`from bot_bottle.backend import X` resolves X on first access via `__getattr__` +and caches it at package level, so existing call-sites (and +`patch.object(backend_mod, X, …)`) keep working. """ from __future__ import annotations -import os -import shlex -import sys -from abc import ABC, abstractmethod -from contextlib import AbstractContextManager, contextmanager -from dataclasses import dataclass -from pathlib import Path -from typing import TYPE_CHECKING, Any, Generator, Generic, Sequence, TypeVar - -from ..agent_provider import AgentProvisionPlan, get_provider, build_agent_provision_plan -from ..egress import EgressPlan -from ..git_gate import GitGatePlan -from ..log import die, info, warn -from ..util import read_tty_line -from ..manifest import Manifest, ManifestIndex -from ..supervisor.plan import SupervisePlan -from ..util import expand_tilde -from ..env import resolve_env, ResolvedEnv -from ..workspace import WorkspacePlan, workspace_plan -from .print_util import print_multi, visible_agent_env_names -from .util import host_skill_dir +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: + from .base import ( + ActiveAgent, + Bottle, + BottleBackend, + BottleCleanupPlan, + BottleImages, + BottlePlan, + BottleSpec, + ExecResult, + ) + from .selection import ( + enumerate_active_agents, + get_bottle_backend, + has_backend, + known_backend_names, + ) + from .docker import DockerBottleBackend + from .firecracker import FirecrackerBottleBackend + from .macos_container import MacosContainerBottleBackend from .freeze import CommitCancelled, Freezer, get_freezer -@dataclass(frozen=True) -class BottleSpec: - """CLI-supplied intent. Backend-agnostic — each backend's prepare - step consumes it and produces its own backend-specific plan. - Resolved values (image names, container name, scratch paths, runsc - availability) live on the plan, not the spec.""" - - manifest: ManifestIndex - agent_name: str - copy_cwd: bool - user_cwd: str - # PRD 0016 follow-up: when set, the backend's prepare step uses - # this identity instead of minting a fresh one — the resume path - # (`cli.py resume `) sets this to continue an existing - # bottle's state. Empty string for a fresh `start`. - identity: str = "" - label: str = "" - color: str = "" - # Ordered bottle names selected at launch (issue #269). When non-empty - # they are merged in order and replace the agent's `bottle:` field. - bottle_names: tuple[str, ...] = () - # True when launched via --headless (no TTY, no interactive prompts). - # The git-gate host-key preflight uses this to error rather than prompt. - headless: bool = False - # Image startup policy. "fresh" preserves the normal build path; - # "cached" reuses the current local image/artifact without rebuilding. - image_policy: str = "fresh" - - -@dataclass(frozen=True) -class BottlePlan(ABC): - """Base output of a backend's prepare step. Concrete subclasses - (e.g. DockerBottlePlan) add backend-specific resolved fields.""" - - spec: BottleSpec - manifest: Manifest - stage_dir: Path - git_gate_plan: GitGatePlan - - @property - def guest_home(self) -> str: - return self.agent_provision.guest_home - - @property - def git_gate_insteadof_host(self) -> str: - """Host (and optional port) used in git-gate insteadOf URLs. - Docker uses the compose-network DNS alias; VM backends may - override with an IP:port when the guest has no DNS.""" - return "git-gate" - - @property - def git_gate_insteadof_scheme(self) -> str: - """URL scheme for git-gate insteadOf rewrites. 'git' for - Docker (git daemon); VM backends may override (e.g. 'http' - over a published host port).""" - return "git" - egress_plan: EgressPlan - supervise_plan: SupervisePlan | None - agent_provision: AgentProvisionPlan - - @property - def workspace_plan(self) -> WorkspacePlan: - return workspace_plan(self.spec, guest_home=self.guest_home) - - def print(self) -> None: - """Render the y/N preflight summary to stderr.""" - spec = self.spec - manifest = self.manifest - agent = manifest.agent - bottle = manifest.bottle - - env_names = visible_agent_env_names( - sorted( - set(bottle.env.keys()) - | set(self.agent_provision.guest_env.keys()) - ), - hidden_env_names=self.agent_provision.hidden_env_names, - ) - - print(file=sys.stderr) - info(f"agent : {spec.agent_name}") - info(f"provider : {self.agent_provision.template}") - print_multi("env ", env_names) - print_multi("skills ", list(agent.skills)) - effective_bottles = ( - list(spec.bottle_names) if spec.bottle_names - else ([agent.bottle] if agent.bottle else []) - ) - print_multi("bottle ", effective_bottles) - - identity = manifest.git_identity_summary() - if identity: - info(f" git identity : {identity}") - - git_lines = [ - f"{u.name} → {u.upstream_host}:{u.upstream_port}" - for u in self.git_gate_plan.upstreams - ] - if git_lines: - print_multi(" git gate ", git_lines) - - if self.egress_plan.routes: - egress_lines = [] - for r in self.egress_plan.routes: - auth = f" [auth:{r.auth_scheme}]" if r.auth_scheme else "" - egress_lines.append(f"{r.host}{auth}") - print_multi(" egress ", egress_lines) - print(file=sys.stderr) - - -@dataclass(frozen=True) -class BottleCleanupPlan(ABC): - """Base output of a backend's prepare_cleanup step. Concrete - subclasses (e.g. DockerBottleCleanupPlan) carry backend-specific - lists of resources to be removed and implement `print` + `empty`.""" - - @abstractmethod - def print(self) -> None: - """Render the cleanup y/N summary to stderr.""" - - @property - @abstractmethod - def empty(self) -> bool: - """True iff there is nothing to clean up; the CLI uses this to - short-circuit before showing the y/N.""" - - -@dataclass(frozen=True) -class ExecResult: - """Captured result of `Bottle.exec`. Backend-neutral: the Docker - impl populates it from a `subprocess.CompletedProcess`, but a - VM backend could populate it from any source that produces a - returncode + captured streams.""" - - returncode: int - stdout: str - stderr: str - - -@dataclass(frozen=True) -class ActiveAgent: - """One currently-running agent, as the CLI `list active` and - dashboard agents pane render it. ("Agent" is the project's - consistent name for the thing running inside a bottle — the - bottle is the container, the agent is what runs in it.) - - Fields are deliberately backend-neutral. `services` is the set - of gateway daemons currently up for this bottle (`egress`, - `git-gate`, `supervise`); the dashboard uses it to - gate edit verbs. `backend_name` is the matching key in - `_BACKENDS` (`docker` / `firecracker` / `macos-container`) — used by the active- - list rendering to disambiguate and by the dashboard's - re-attach path.""" - - backend_name: str - slug: str - agent_name: str # from metadata.json; "?" if missing - started_at: str # ISO 8601 from metadata.json; "" if missing - services: tuple[str, ...] # alphabetical - label: str = "" - color: str = "" - - -class Bottle(ABC): - """Handle to a running bottle. Yielded by a backend's launch step. - - `exec_agent` runs the selected agent CLI inside the bottle and - blocks until the session ends. `exec` runs a POSIX shell script inside the bottle - and returns the captured result. `cp_in` copies a host path into - the bottle. `close` is an idempotent alias for context-manager - teardown. - """ - - name: str - - @abstractmethod - def agent_argv( - self, argv: list[str], *, tty: bool = True, - ) -> list[str]: - """Return the host-side argv that runs the selected agent - inside the bottle. Used by `exec_agent` for foreground - handoffs and by the dashboard's tmux `respawn-pane` flow, - which needs the argv up front (it spawns claude in a tmux - pane rather than as a child of the current process). - - Implementations transparently inject - `--append-system-prompt-file` when the bottle was launched - with a provisioned prompt path.""" - ... - - @abstractmethod - def exec_agent(self, argv: list[str], *, tty: bool = True) -> int: ... - - @abstractmethod - def exec(self, script: str, *, user: str = "node") -> ExecResult: - """Run `script` as a POSIX shell script inside the bottle as - `user` (default `node`, matching the agent image's USER - directive) and return the captured stdout/stderr/returncode. - The bottle's environment (including HTTPS_PROXY pointing at - the egress daemon) is inherited by the child. Non-zero - exit does not raise — callers inspect `returncode` - themselves. - - Pass `user="root"` for shell-outs that need privileged file - writes / package install — provisioning calls that need root - bypass `Bottle.exec` and use the backend-specific raw - machine-exec helper, but the tests have a legitimate use - case for arbitrary-user runs.""" - - @abstractmethod - def cp_in(self, host_path: str, container_path: str) -> None: ... - - @abstractmethod - def close(self) -> None: ... - - - - -PlanT = TypeVar("PlanT", bound=BottlePlan) -CleanupT = TypeVar("CleanupT", bound=BottleCleanupPlan) - - -@dataclass(frozen=True) -class BottleImages: - """Resolved image references (or artifact paths) for a bottle launch. - - For Docker/macOS-container backends, `agent` and `sidecar` are string - image refs. For the smolmachines backend they are Path objects pointing - to pre-built `.smolmachine` artifacts.""" - - agent: str | Path - sidecar: str | Path = "" - - -class BottleBackend(ABC, Generic[PlanT, CleanupT]): - """Abstract base for selectable bottle backends. Concrete subclasses - (e.g. DockerBottleBackend) own their own prepare/launch impls. - Parameterized over the backend's concrete plan + cleanup-plan types - so subclass methods get the narrow type without isinstance - boilerplate.""" - - name: str - - # Whether this backend can run a container engine *inside* the bottle. - # Backends that cannot must reject `nested_containers: true` rather than - # reach for a host daemon socket (issue #392). - supports_nested_containers: bool = False - - def prepare(self, spec: BottleSpec, stage_dir: Path) -> PlanT: - """Template method: run cross-backend host-side validation, then - delegate to the subclass's `_resolve_plan` for the - backend-specific resolution (names, scratch files, etc.). The - validation step is enforced here so a future backend cannot - accidentally skip it. No remote/runtime resources are created.""" - from .resolve_common import ( - merge_provision_env_vars, - mint_slug, - prepare_agent_state_dir, - prepare_egress, - prepare_git_gate, - prepare_supervise, - reject_nested_containers, - resolve_manifest_dockerfile, - write_launch_metadata, - ) - - manifest = self._validate(spec) - - if not self.supports_nested_containers: - reject_nested_containers(self.name, manifest) - - self._preflight() - - from ..git_gate_host_key import preflight_host_keys - manifest = preflight_host_keys( - manifest, - headless=spec.headless, - home_md=spec.manifest.home_md, - ) - - manifest_bottle = manifest.bottle - manifest_agent_provider = manifest_bottle.agent_provider - agent_provider = get_provider(manifest_agent_provider.template) - resolved_env = resolve_env(manifest) - workspace = workspace_plan(spec, guest_home=agent_provider.guest_home) - - slug = mint_slug(spec) - write_launch_metadata(slug, spec, compose_project="", backend=self.name) - - # Manifest may override the Dockerfile per-bottle; otherwise fall - # back to the provider plugin's bundled Dockerfile (next to its - # agent_provider.py module). - if manifest_agent_provider.dockerfile: - agent_dockerfile_path = resolve_manifest_dockerfile( - manifest_agent_provider.dockerfile, spec, - ) - else: - agent_dockerfile_path = str(agent_provider.dockerfile) - - agent_dir, prompt_file = prepare_agent_state_dir(slug, manifest) - - agent_provision_plan = build_agent_provision_plan( - template=manifest_agent_provider.template, - dockerfile=agent_dockerfile_path, - state_dir=agent_dir, - instance_name=f"bot-bottle-{slug}", - prompt_file=prompt_file, - guest_env=self._build_guest_env(resolved_env), - forward_host_credentials=manifest_agent_provider.forward_host_credentials, - auth_token=manifest_agent_provider.auth_token, - host_env=dict(os.environ), - trusted_project_path=workspace.workdir, - label=spec.label, - color=spec.color, - provider_settings=manifest_agent_provider.settings, - ) - agent_provision_plan = merge_provision_env_vars(agent_provision_plan) - egress_plan = prepare_egress(manifest_bottle, slug, agent_provision_plan) - supervise_plan = prepare_supervise(manifest_bottle, slug) - git_gate_plan = prepare_git_gate(manifest_bottle, slug) - - return self._resolve_plan( - spec, - manifest=manifest, - slug=slug, - resolved_env=resolved_env, - agent_provision_plan=agent_provision_plan, - egress_plan=egress_plan, - supervise_plan=supervise_plan, - git_gate_plan=git_gate_plan, - stage_dir=stage_dir, - ) - - def _build_guest_env(self, resolved_env: ResolvedEnv) -> dict[str, str]: - return {} - - def _preflight(self) -> None: - """ - tasks to do before resolving a plan - """ - pass - - def _validate(self, spec: BottleSpec) -> Manifest: - """Cross-backend pre-launch checks. Parses the selected agent and - its bottle (raising ManifestError on invalid content), confirms - skills are present on the host, and every git IdentityFile resolves. - - Returns the loaded Manifest for the selected agent. Subclasses with - additional preconditions should override and call - `super()._validate(spec)` first.""" - manifest = spec.manifest.load_for_agent(spec.agent_name, spec.bottle_names) - self._validate_skills(manifest.agent.skills) - self._validate_agent_provider_dockerfile(spec, manifest) - return manifest - - def _validate_skills(self, skills: Sequence[str]) -> None: - """Each named skill must be a directory under the host's - `~/.claude/skills/`. The check is purely host-side, so the - default impl covers every backend.""" - for name in skills: - path = host_skill_dir(name) - if not os.path.isdir(path): - die( - f"skill '{name}' not found on host at {path}. " - f"Create it under ~/.claude/skills/, then re-run." - ) - - def _validate_agent_provider_dockerfile(self, spec: BottleSpec, manifest: Manifest) -> None: - bottle = manifest.bottle - dockerfile = bottle.agent_provider.dockerfile - if not dockerfile: - return - path = Path(expand_tilde(dockerfile)) - if not path.is_absolute(): - path = Path(spec.user_cwd) / path - if not path.is_file(): - effective = ( - ", ".join(spec.bottle_names) if spec.bottle_names else manifest.agent.bottle - ) - die( - f"agent_provider.dockerfile for bottle " - f"'{effective}' not found: {path}" - ) - - @abstractmethod - def _resolve_plan(self, - spec: BottleSpec, - *, - manifest: Manifest, - slug: str, - resolved_env: ResolvedEnv, - agent_provision_plan: AgentProvisionPlan, - egress_plan: EgressPlan, - git_gate_plan: GitGatePlan, - supervise_plan: SupervisePlan | None, - stage_dir: Path) -> PlanT: - """Backend-specific plan resolution: image/container names, - env-file, prompt-file, proxy plan, runtime detection. Called by - `prepare` after `_validate` succeeds. Instance name, image, - prompt file, Dockerfile path, and guest home all live on - `agent_provision_plan` — the source of truth.""" - - def prelaunch_checks(self, plan: PlanT) -> None: - """Raise StaleImageError if any cached image used by this plan is stale. - No-op default; backends override to call the shared check_stale* - helpers on their image/artifact timestamps. Called by the CLI before - launch so the operator can be prompted outside the launch context.""" - - @contextmanager - def launch(self, plan: PlanT) -> Generator[Bottle, None, None]: - """Template: build or load images, then delegate to _launch_impl.""" - images = self._build_or_load_images(plan) - with self._launch_impl(plan, images) as bottle: - yield bottle - - @abstractmethod - def _build_or_load_images(self, plan: PlanT) -> BottleImages: - """Return the agent and sidecar image references (or artifact paths) - for this plan, building fresh images when the policy requires it.""" - - @abstractmethod - def _launch_impl(self, plan: PlanT, images: BottleImages) -> AbstractContextManager[Bottle]: - """Bring up the bottle using pre-resolved images; yield a handle; tear down on exit.""" - - def provision(self, plan: PlanT, bottle: "Bottle") -> str | None: - """Copy host-side files (CA cert, prompt, skills, .git) into - the running bottle. Called from `launch` after the container - / machine is up. Returns the in-container prompt path if a - prompt was provisioned, else None — the Bottle handle uses it - to decide whether to add provider-specific prompt args to the - agent's argv. - - Default orchestration: ca → prompt → provider apply → skills - → workspace → git → supervise-mcp. CA install runs first so - the agent's trust store is rebuilt before anything inside the - agent makes a TLS call. - - Per PRD 0050 the per-provider steps (prompt, skills, - declarative provision-plan apply, supervise MCP registration) - live on the `AgentProvider` plugin. The backend only owns the - steps that are about backend infrastructure (CA, workspace, - git) and surfaces the supervise daemon URL its launch step - knows about via `supervise_mcp_url`. - - PRD 0017: cred-proxy's agent-side dotfile rewrites (~/.npmrc, - ~/.gitconfig insteadOf, tea config) are gone. Egress-proxy is - on the agent's HTTP_PROXY path so every tool that respects - HTTPS_PROXY (claude-code, git over HTTPS, npm, curl) is - intercepted without per-tool reconfiguration.""" - provider = get_provider(plan.agent_provision.template) - provider.provision_ca(bottle, plan) - prompt_path = provider.provision_prompt(plan, bottle) - provider.provision(plan, bottle) - provider.provision_skills(plan, bottle) - self.provision_workspace(plan, bottle) - provider.provision_git(bottle, plan) - provider.provision_supervise_mcp( - plan, bottle, self.supervise_mcp_url(plan), - ) - return prompt_path - - def provision_workspace(self, plan: PlanT, bottle: "Bottle") -> None: - """Copy the operator workspace into the running bottle. - - This is the only supported workspace-provisioning path: Docker - does not build a derived image containing the current - workspace.""" - workspace = plan.workspace_plan - if not (workspace.enabled and workspace.copy_contents): - return - - guest_parent = workspace.guest_path.rsplit("/", 1)[0] or "/" - guest_path = shlex.quote(workspace.guest_path) - guest_parent = shlex.quote(guest_parent) - owner = shlex.quote(workspace.owner) - mode = shlex.quote(workspace.mode) - info(f"copying {workspace.host_path} -> {bottle.name}:{workspace.guest_path}") - bottle.exec( - f"rm -rf {guest_path} && mkdir -p {guest_parent}", - user="root", - ) - bottle.cp_in(str(workspace.host_path), workspace.guest_path) - bottle.exec( - f"chown -R {owner} {guest_path} && chmod {mode} {guest_path}", - user="root", - ) - - def supervise_mcp_url(self, plan: PlanT) -> str: - """Return the agent-side URL of the per-bottle supervise - gateway, or "" when this bottle has no gateway. The provider - plugin's `provision_supervise_mcp` uses it to register the - MCP entry inside the guest. - - Default returns "" so backends without supervise support - don't have to implement it. Docker and firecracker override.""" - del plan - return "" - - def ensure_orchestrator(self) -> str: - """Bring up this backend's per-host orchestrator + shared gateway - (idempotent) and return the host-reachable control-plane URL. - - This is the backend-agnostic bring-up entry point: `launch` calls - it as part of starting a bottle, and operator tools (`supervise`) - call it to start the control plane on demand when none is running - yet. Docker starts the orchestrator + gateway containers; - firecracker boots the infra VM. Backends with no orchestrator - (macos-container) die with a pointer — the default here.""" - die(f"backend {self.name!r} has no orchestrator control plane") - - @abstractmethod - def prepare_cleanup(self) -> CleanupT: - """Enumerate orphaned resources from previous bottles. No side - effects; safe to call before the y/N.""" - - @abstractmethod - def cleanup(self, plan: CleanupT) -> None: - """Remove everything described by the cleanup plan.""" - - @abstractmethod - def enumerate_active(self) -> Sequence[ActiveAgent]: - """Return every currently-running agent on this backend. - Empty when none. Backend-specific: docker queries `docker - compose ls`; firecracker cross-references its running gateway - containers against per-bottle metadata.""" - - @classmethod - @abstractmethod - def is_available(cls) -> bool: - """Whether this backend's runtime prerequisites are satisfied - on the current host. Docker → `docker` on PATH; firecracker → - Linux + KVM. Used by the cross-backend - `enumerate_active_agents` / `cmd_cleanup` to skip backends - the operator hasn't installed, so a docker-only host - doesn't fail when `cli.py list active` walks past - firecracker.""" - - @classmethod - @abstractmethod - def setup(cls) -> int: - """Emit this backend's one-time host setup — privileged network - pool, daemon bring-up, install pointers, etc. — as - host-appropriate config or commands. Prints to stdout/stderr and - returns a shell exit code (0 = nothing to report / success). A - backend that needs no host setup prints a short note and returns - 0. Invoked generically by `./cli.py backend setup [--backend=…]` - so operators can provision any backend without a - backend-specific command. Classmethod (like `is_available`) — - it's a host query, not per-bottle state.""" - - @classmethod - @abstractmethod - def status(cls) -> int: - """Report whether this backend's prerequisites are satisfied on - the host — binaries, daemon reachability, network pool, range - conflicts, etc. Prints a human-readable summary; returns 0 when - the backend is ready to launch and non-zero when something is - missing. Invoked by `./cli.py backend status [--backend=…]`.""" - - @classmethod - @abstractmethod - def teardown(cls) -> int: - """Undo `setup()` — the inverse operation, surfaced as - `./cli.py backend teardown [--backend=…]` (uninstall). Symmetric - with setup: where setup is advisory (prints the privileged - commands / declarative config to apply), teardown prints the - commands / config change to remove the host prerequisites. A - backend with no host setup prints a short note and returns 0. - Not called by the launch path or the test suite.""" - - -# _backends is None until the first call to _get_backends(), at which -# point all three concrete backend classes are imported and instantiated. -# Keeping the imports out of module scope means that importing any -# backend sub-module (e.g. `backend.docker.util`) no longer drags the -# firecracker and macos-container implementations into memory. -# -# Tests may replace _backends with a {name: fake} dict via patch.object; -# _get_backends() returns the current module-level value as-is when it -# is not None, so test fakes take effect without triggering real imports. -_backends: dict[str, BottleBackend[Any, Any]] | None = None - - -def _get_backends() -> dict[str, BottleBackend[Any, Any]]: - """Return the registry of all backend instances, loading lazily on first call.""" - global _backends # pylint: disable=global-statement - if _backends is None: - from .docker import DockerBottleBackend - from .firecracker import FirecrackerBottleBackend - from .macos_container import MacosContainerBottleBackend - _backends = { - "docker": DockerBottleBackend(), - "firecracker": FirecrackerBottleBackend(), - "macos-container": MacosContainerBottleBackend(), - } - return _backends +# Public name -> submodule that defines it. Contract types resolve from `base`, +# selection/enumeration from `selection`, the concrete backends + freeze helpers +# from their own subpackages. +_LAZY_MODULES: dict[str, str] = { + "BottleSpec": "base", + "BottlePlan": "base", + "BottleCleanupPlan": "base", + "ExecResult": "base", + "ActiveAgent": "base", + "Bottle": "base", + "BottleImages": "base", + "BottleBackend": "base", + "get_bottle_backend": "selection", + "known_backend_names": "selection", + "has_backend": "selection", + "enumerate_active_agents": "selection", + "_print_vm_install_instructions": "selection", + "DockerBottleBackend": "docker", + "FirecrackerBottleBackend": "firecracker", + "MacosContainerBottleBackend": "macos_container", + "CommitCancelled": "freeze", + "Freezer": "freeze", + "get_freezer": "freeze", +} def __getattr__(name: str) -> Any: - """Lazily surface concrete backend classes and freeze symbols at the - package level so existing `from bot_bottle.backend import X` and - `patch.object(backend_mod, X, ...)` call-sites keep working without - forcing an import of every backend at module-init time.""" - if name == "DockerBottleBackend": - from .docker import DockerBottleBackend - globals()[name] = DockerBottleBackend - return DockerBottleBackend - if name == "FirecrackerBottleBackend": - from .firecracker import FirecrackerBottleBackend - globals()[name] = FirecrackerBottleBackend - return FirecrackerBottleBackend - if name == "MacosContainerBottleBackend": - from .macos_container import MacosContainerBottleBackend - globals()[name] = MacosContainerBottleBackend - return MacosContainerBottleBackend - if name == "CommitCancelled": - from .freeze import CommitCancelled - globals()[name] = CommitCancelled - return CommitCancelled - if name == "Freezer": - from .freeze import Freezer - globals()[name] = Freezer - return Freezer - if name == "get_freezer": - from .freeze import get_freezer - globals()[name] = get_freezer - return get_freezer - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + """Lazily surface the package's public names from their submodules and + cache them at package level — so `from bot_bottle.backend import X` and + `patch.object(backend_mod, X, …)` keep working without importing the + framework (or every backend) at package-init time.""" + mod = _LAZY_MODULES.get(name) + if mod is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module - -def get_bottle_backend( - name: str | None = None, - *, - prompt: bool = True, -) -> BottleBackend[Any, Any]: - """Resolve the bottle backend. - - `name` precedence: - 1. explicit arg (e.g. resume passes the recorded backend name) - 2. BOT_BOTTLE_BACKEND env var - 3. auto-selection: VM backend first, docker fallback with prompt - - `prompt` controls whether auto-selection may block on an interactive - [i/d/q] prompt when falling back to docker. Pass `prompt=False` in - non-interactive contexts (headless launches, CI) so the call dies - with an actionable message instead of hanging. - - Dies with a pointer at the known backends if the chosen name - isn't implemented.""" - resolved = name or os.environ.get("BOT_BOTTLE_BACKEND") - if resolved is None: - resolved = _auto_select_backend(prompt=prompt) - backends = _get_backends() - if resolved not in backends: - known = ", ".join(sorted(backends)) - die(f"unknown backend {resolved!r}; known backends: {known}") - return backends[resolved] - - -def _platform_vm_suggestion() -> str: - """Platform-appropriate VM backend name for install suggestions.""" - return "macos-container" if sys.platform == "darwin" else "firecracker" - - -def _print_vm_install_instructions() -> None: - """Print platform-appropriate VM backend install instructions to stderr.""" - vm = _platform_vm_suggestion() - if vm == "macos-container": - info("Install Apple Container: https://github.com/apple/container/releases") - info("Then start the service: container system start") - else: - info("Install Firecracker: https://github.com/firecracker-microvm/firecracker/releases") - info("Configure the host: ./cli.py backend setup") - - -def _auto_select_backend(prompt: bool = True) -> str: - """Tier-1 / tier-2 backend auto-selection. - - Tier 1: VM backend — macos-container on macOS when Apple Container is - installed; firecracker on KVM-capable Linux even before the binary is - present (its preflight prints an install pointer). - - Tier 2: docker, with a security warning and an interactive prompt. - When `prompt=False` (headless / CI), dies with an actionable message - instead of blocking on a TTY read. When docker is also absent, prints - VM install instructions and exits. - """ - # --- Tier 1: VM backend ----------------------------------------- - if has_backend("macos-container"): - return "macos-container" - # A KVM-capable Linux host defaults to firecracker even when the - # `firecracker` binary isn't installed yet: selecting it here routes - # start through firecracker's preflight, which prints an install - # pointer, instead of silently falling back to docker. - from .firecracker import FirecrackerBottleBackend - if FirecrackerBottleBackend.is_host_capable(): - return "firecracker" - - # --- Tier 2: docker fallback ------------------------------------ - if not has_backend("docker"): - info("No backend available on this host.") - _print_vm_install_instructions() - die("no backend available; install a VM backend and re-run") - - vm = _platform_vm_suggestion() - warn( - "docker is less secure than VM backends — " - "containers share the host kernel." - ) - if not prompt: - die( - f"no VM backend available; set BOT_BOTTLE_BACKEND=docker to proceed " - f"with docker, or install the {vm!r} backend." - ) - sys.stderr.write( - f"bot-bottle: For better isolation, install the {vm!r} backend.\n" - f" [i] show {vm} install instructions and exit\n" - " [d] use docker anyway\n" - " [q] quit\n" - "bot-bottle: choice [i/d/q]: " - ) - sys.stderr.flush() - reply = read_tty_line().strip().lower() - if reply == "d": - return "docker" - if reply == "i": - _print_vm_install_instructions() - die("not proceeding with docker; install a VM backend or set BOT_BOTTLE_BACKEND=docker") - - -def known_backend_names() -> tuple[str, ...]: - """Sorted tuple of all backend keys in `_get_backends()`. Used by - argparse (`--backend` choices) and the dashboard's backend - picker.""" - return tuple(sorted(_get_backends())) - - -def has_backend(name: str) -> bool: - """Whether the named backend's runtime prerequisites are - available on the current host. Cross-backend callers (list, - cleanup) skip unavailable backends so a docker-only host - doesn't fail when the firecracker backend isn't usable, - and vice versa. - - Returns False for unknown names so callers can pass - arbitrary input without separate validation.""" - backends = _get_backends() - if name not in backends: - return False - return backends[name].is_available() - - -def enumerate_active_agents() -> list[ActiveAgent]: - """All currently-running agents, across every available - backend. Used by CLI `list active` and the dashboard's agents - pane so neither has to know which backends exist. Skips - backends whose `is_available()` reports False. - - Sorted by `(started_at, slug)` so the list is stable across - dashboard refresh ticks — agents don't shift position while - the operator navigates with arrow keys. ISO 8601 timestamps - sort lexicographically in chronological order; `slug` is the - deterministic tiebreaker. Agents with missing metadata - (`started_at == ""`) sort first.""" - out: list[ActiveAgent] = [] - backends = _get_backends() - for name in sorted(backends): - if not backends[name].is_available(): - continue - out.extend(backends[name].enumerate_active()) - out.sort(key=lambda a: (a.started_at, a.slug)) - return out + value = getattr(import_module(f"{__name__}.{mod}"), name) + globals()[name] = value + return value __all__ = [ @@ -838,14 +89,18 @@ __all__ = [ "Bottle", "BottleBackend", "BottleCleanupPlan", + "BottleImages", "BottlePlan", "BottleSpec", - "CommitCancelled", "ExecResult", + "CommitCancelled", "Freezer", + "get_freezer", + "DockerBottleBackend", + "FirecrackerBottleBackend", + "MacosContainerBottleBackend", "enumerate_active_agents", "get_bottle_backend", - "get_freezer", "has_backend", "known_backend_names", ] diff --git a/bot_bottle/backend/base.py b/bot_bottle/backend/base.py new file mode 100644 index 0000000..bb8a2ab --- /dev/null +++ b/bot_bottle/backend/base.py @@ -0,0 +1,607 @@ +"""The abstract backend contract (PRD 0018 / 0070). + +The backend-neutral types every bottle backend implements: the launch +`BottleSpec`, the `BottlePlan` / `BottleCleanupPlan` ABCs, the running-`Bottle` ++ `ExecResult` shapes, `BottleImages`, and the `BottleBackend` ABC itself. + +This carries the framework imports (manifest, egress, git-gate, env, workspace, +agent-provider) the contract's signatures and helpers need — which is why it +lives here rather than in `backend/__init__.py`: touching an unrelated +`backend.*` module then doesn't drag the whole framework into memory. The +thin package `__init__` re-exports these names lazily. +""" + +from __future__ import annotations + +import os +import shlex +import sys +from abc import ABC, abstractmethod +from contextlib import AbstractContextManager, contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Generator, Generic, Sequence, TypeVar + +from ..agent_provider import AgentProvisionPlan, get_provider, build_agent_provision_plan +from ..egress import EgressPlan +from ..git_gate import GitGatePlan +from ..log import die, info +from ..util import expand_tilde +from ..manifest import Manifest, ManifestIndex +from ..supervisor.plan import SupervisePlan +from ..env import resolve_env, ResolvedEnv +from ..workspace import WorkspacePlan, workspace_plan +from .print_util import print_multi, visible_agent_env_names +from .util import host_skill_dir + + + +@dataclass(frozen=True) +class BottleSpec: + """CLI-supplied intent. Backend-agnostic — each backend's prepare + step consumes it and produces its own backend-specific plan. + Resolved values (image names, container name, scratch paths, runsc + availability) live on the plan, not the spec.""" + + manifest: ManifestIndex + agent_name: str + copy_cwd: bool + user_cwd: str + # PRD 0016 follow-up: when set, the backend's prepare step uses + # this identity instead of minting a fresh one — the resume path + # (`cli.py resume `) sets this to continue an existing + # bottle's state. Empty string for a fresh `start`. + identity: str = "" + label: str = "" + color: str = "" + # Ordered bottle names selected at launch (issue #269). When non-empty + # they are merged in order and replace the agent's `bottle:` field. + bottle_names: tuple[str, ...] = () + # True when launched via --headless (no TTY, no interactive prompts). + # The git-gate host-key preflight uses this to error rather than prompt. + headless: bool = False + # Image startup policy. "fresh" preserves the normal build path; + # "cached" reuses the current local image/artifact without rebuilding. + image_policy: str = "fresh" + + +@dataclass(frozen=True) +class BottlePlan(ABC): + """Base output of a backend's prepare step. Concrete subclasses + (e.g. DockerBottlePlan) add backend-specific resolved fields.""" + + spec: BottleSpec + manifest: Manifest + stage_dir: Path + git_gate_plan: GitGatePlan + + @property + def guest_home(self) -> str: + return self.agent_provision.guest_home + + @property + def git_gate_insteadof_host(self) -> str: + """Host (and optional port) used in git-gate insteadOf URLs. + Docker uses the compose-network DNS alias; VM backends may + override with an IP:port when the guest has no DNS.""" + return "git-gate" + + @property + def git_gate_insteadof_scheme(self) -> str: + """URL scheme for git-gate insteadOf rewrites. 'git' for + Docker (git daemon); VM backends may override (e.g. 'http' + over a published host port).""" + return "git" + egress_plan: EgressPlan + supervise_plan: SupervisePlan | None + agent_provision: AgentProvisionPlan + + @property + def workspace_plan(self) -> WorkspacePlan: + return workspace_plan(self.spec, guest_home=self.guest_home) + + def print(self) -> None: + """Render the y/N preflight summary to stderr.""" + spec = self.spec + manifest = self.manifest + agent = manifest.agent + bottle = manifest.bottle + + env_names = visible_agent_env_names( + sorted( + set(bottle.env.keys()) + | set(self.agent_provision.guest_env.keys()) + ), + hidden_env_names=self.agent_provision.hidden_env_names, + ) + + print(file=sys.stderr) + info(f"agent : {spec.agent_name}") + info(f"provider : {self.agent_provision.template}") + print_multi("env ", env_names) + print_multi("skills ", list(agent.skills)) + effective_bottles = ( + list(spec.bottle_names) if spec.bottle_names + else ([agent.bottle] if agent.bottle else []) + ) + print_multi("bottle ", effective_bottles) + + identity = manifest.git_identity_summary() + if identity: + info(f" git identity : {identity}") + + git_lines = [ + f"{u.name} → {u.upstream_host}:{u.upstream_port}" + for u in self.git_gate_plan.upstreams + ] + if git_lines: + print_multi(" git gate ", git_lines) + + if self.egress_plan.routes: + egress_lines = [] + for r in self.egress_plan.routes: + auth = f" [auth:{r.auth_scheme}]" if r.auth_scheme else "" + egress_lines.append(f"{r.host}{auth}") + print_multi(" egress ", egress_lines) + print(file=sys.stderr) + + +@dataclass(frozen=True) +class BottleCleanupPlan(ABC): + """Base output of a backend's prepare_cleanup step. Concrete + subclasses (e.g. DockerBottleCleanupPlan) carry backend-specific + lists of resources to be removed and implement `print` + `empty`.""" + + @abstractmethod + def print(self) -> None: + """Render the cleanup y/N summary to stderr.""" + + @property + @abstractmethod + def empty(self) -> bool: + """True iff there is nothing to clean up; the CLI uses this to + short-circuit before showing the y/N.""" + + +@dataclass(frozen=True) +class ExecResult: + """Captured result of `Bottle.exec`. Backend-neutral: the Docker + impl populates it from a `subprocess.CompletedProcess`, but a + VM backend could populate it from any source that produces a + returncode + captured streams.""" + + returncode: int + stdout: str + stderr: str + + +@dataclass(frozen=True) +class ActiveAgent: + """One currently-running agent, as the CLI `list active` and + dashboard agents pane render it. ("Agent" is the project's + consistent name for the thing running inside a bottle — the + bottle is the container, the agent is what runs in it.) + + Fields are deliberately backend-neutral. `services` is the set + of gateway daemons currently up for this bottle (`egress`, + `git-gate`, `supervise`); the dashboard uses it to + gate edit verbs. `backend_name` is the matching key in + `_BACKENDS` (`docker` / `firecracker` / `macos-container`) — used by the active- + list rendering to disambiguate and by the dashboard's + re-attach path.""" + + backend_name: str + slug: str + agent_name: str # from metadata.json; "?" if missing + started_at: str # ISO 8601 from metadata.json; "" if missing + services: tuple[str, ...] # alphabetical + label: str = "" + color: str = "" + + +class Bottle(ABC): + """Handle to a running bottle. Yielded by a backend's launch step. + + `exec_agent` runs the selected agent CLI inside the bottle and + blocks until the session ends. `exec` runs a POSIX shell script inside the bottle + and returns the captured result. `cp_in` copies a host path into + the bottle. `close` is an idempotent alias for context-manager + teardown. + """ + + name: str + + @abstractmethod + def agent_argv( + self, argv: list[str], *, tty: bool = True, + ) -> list[str]: + """Return the host-side argv that runs the selected agent + inside the bottle. Used by `exec_agent` for foreground + handoffs and by the dashboard's tmux `respawn-pane` flow, + which needs the argv up front (it spawns claude in a tmux + pane rather than as a child of the current process). + + Implementations transparently inject + `--append-system-prompt-file` when the bottle was launched + with a provisioned prompt path.""" + ... + + @abstractmethod + def exec_agent(self, argv: list[str], *, tty: bool = True) -> int: ... + + @abstractmethod + def exec(self, script: str, *, user: str = "node") -> ExecResult: + """Run `script` as a POSIX shell script inside the bottle as + `user` (default `node`, matching the agent image's USER + directive) and return the captured stdout/stderr/returncode. + The bottle's environment (including HTTPS_PROXY pointing at + the egress daemon) is inherited by the child. Non-zero + exit does not raise — callers inspect `returncode` + themselves. + + Pass `user="root"` for shell-outs that need privileged file + writes / package install — provisioning calls that need root + bypass `Bottle.exec` and use the backend-specific raw + machine-exec helper, but the tests have a legitimate use + case for arbitrary-user runs.""" + + @abstractmethod + def cp_in(self, host_path: str, container_path: str) -> None: ... + + @abstractmethod + def close(self) -> None: ... + + + + +PlanT = TypeVar("PlanT", bound=BottlePlan) +CleanupT = TypeVar("CleanupT", bound=BottleCleanupPlan) + + +@dataclass(frozen=True) +class BottleImages: + """Resolved image references (or artifact paths) for a bottle launch. + + For Docker/macOS-container backends, `agent` and `sidecar` are string + image refs. For the smolmachines backend they are Path objects pointing + to pre-built `.smolmachine` artifacts.""" + + agent: str | Path + sidecar: str | Path = "" + + +class BottleBackend(ABC, Generic[PlanT, CleanupT]): + """Abstract base for selectable bottle backends. Concrete subclasses + (e.g. DockerBottleBackend) own their own prepare/launch impls. + Parameterized over the backend's concrete plan + cleanup-plan types + so subclass methods get the narrow type without isinstance + boilerplate.""" + + name: str + + # Whether this backend can run a container engine *inside* the bottle. + # Backends that cannot must reject `nested_containers: true` rather than + # reach for a host daemon socket (issue #392). + supports_nested_containers: bool = False + + def prepare(self, spec: BottleSpec, stage_dir: Path) -> PlanT: + """Template method: run cross-backend host-side validation, then + delegate to the subclass's `_resolve_plan` for the + backend-specific resolution (names, scratch files, etc.). The + validation step is enforced here so a future backend cannot + accidentally skip it. No remote/runtime resources are created.""" + from .resolve_common import ( + merge_provision_env_vars, + mint_slug, + prepare_agent_state_dir, + prepare_egress, + prepare_git_gate, + prepare_supervise, + reject_nested_containers, + resolve_manifest_dockerfile, + write_launch_metadata, + ) + + manifest = self._validate(spec) + + if not self.supports_nested_containers: + reject_nested_containers(self.name, manifest) + + self._preflight() + + from ..git_gate_host_key import preflight_host_keys + manifest = preflight_host_keys( + manifest, + headless=spec.headless, + home_md=spec.manifest.home_md, + ) + + manifest_bottle = manifest.bottle + manifest_agent_provider = manifest_bottle.agent_provider + agent_provider = get_provider(manifest_agent_provider.template) + resolved_env = resolve_env(manifest) + workspace = workspace_plan(spec, guest_home=agent_provider.guest_home) + + slug = mint_slug(spec) + write_launch_metadata(slug, spec, compose_project="", backend=self.name) + + # Manifest may override the Dockerfile per-bottle; otherwise fall + # back to the provider plugin's bundled Dockerfile (next to its + # agent_provider.py module). + if manifest_agent_provider.dockerfile: + agent_dockerfile_path = resolve_manifest_dockerfile( + manifest_agent_provider.dockerfile, spec, + ) + else: + agent_dockerfile_path = str(agent_provider.dockerfile) + + agent_dir, prompt_file = prepare_agent_state_dir(slug, manifest) + + agent_provision_plan = build_agent_provision_plan( + template=manifest_agent_provider.template, + dockerfile=agent_dockerfile_path, + state_dir=agent_dir, + instance_name=f"bot-bottle-{slug}", + prompt_file=prompt_file, + guest_env=self._build_guest_env(resolved_env), + forward_host_credentials=manifest_agent_provider.forward_host_credentials, + auth_token=manifest_agent_provider.auth_token, + host_env=dict(os.environ), + trusted_project_path=workspace.workdir, + label=spec.label, + color=spec.color, + provider_settings=manifest_agent_provider.settings, + ) + agent_provision_plan = merge_provision_env_vars(agent_provision_plan) + egress_plan = prepare_egress(manifest_bottle, slug, agent_provision_plan) + supervise_plan = prepare_supervise(manifest_bottle, slug) + git_gate_plan = prepare_git_gate(manifest_bottle, slug) + + return self._resolve_plan( + spec, + manifest=manifest, + slug=slug, + resolved_env=resolved_env, + agent_provision_plan=agent_provision_plan, + egress_plan=egress_plan, + supervise_plan=supervise_plan, + git_gate_plan=git_gate_plan, + stage_dir=stage_dir, + ) + + def _build_guest_env(self, resolved_env: ResolvedEnv) -> dict[str, str]: + return {} + + def _preflight(self) -> None: + """ + tasks to do before resolving a plan + """ + pass + + def _validate(self, spec: BottleSpec) -> Manifest: + """Cross-backend pre-launch checks. Parses the selected agent and + its bottle (raising ManifestError on invalid content), confirms + skills are present on the host, and every git IdentityFile resolves. + + Returns the loaded Manifest for the selected agent. Subclasses with + additional preconditions should override and call + `super()._validate(spec)` first.""" + manifest = spec.manifest.load_for_agent(spec.agent_name, spec.bottle_names) + self._validate_skills(manifest.agent.skills) + self._validate_agent_provider_dockerfile(spec, manifest) + return manifest + + def _validate_skills(self, skills: Sequence[str]) -> None: + """Each named skill must be a directory under the host's + `~/.claude/skills/`. The check is purely host-side, so the + default impl covers every backend.""" + for name in skills: + path = host_skill_dir(name) + if not os.path.isdir(path): + die( + f"skill '{name}' not found on host at {path}. " + f"Create it under ~/.claude/skills/, then re-run." + ) + + def _validate_agent_provider_dockerfile(self, spec: BottleSpec, manifest: Manifest) -> None: + bottle = manifest.bottle + dockerfile = bottle.agent_provider.dockerfile + if not dockerfile: + return + path = Path(expand_tilde(dockerfile)) + if not path.is_absolute(): + path = Path(spec.user_cwd) / path + if not path.is_file(): + effective = ( + ", ".join(spec.bottle_names) if spec.bottle_names else manifest.agent.bottle + ) + die( + f"agent_provider.dockerfile for bottle " + f"'{effective}' not found: {path}" + ) + + @abstractmethod + def _resolve_plan(self, + spec: BottleSpec, + *, + manifest: Manifest, + slug: str, + resolved_env: ResolvedEnv, + agent_provision_plan: AgentProvisionPlan, + egress_plan: EgressPlan, + git_gate_plan: GitGatePlan, + supervise_plan: SupervisePlan | None, + stage_dir: Path) -> PlanT: + """Backend-specific plan resolution: image/container names, + env-file, prompt-file, proxy plan, runtime detection. Called by + `prepare` after `_validate` succeeds. Instance name, image, + prompt file, Dockerfile path, and guest home all live on + `agent_provision_plan` — the source of truth.""" + + def prelaunch_checks(self, plan: PlanT) -> None: + """Raise StaleImageError if any cached image used by this plan is stale. + No-op default; backends override to call the shared check_stale* + helpers on their image/artifact timestamps. Called by the CLI before + launch so the operator can be prompted outside the launch context.""" + + @contextmanager + def launch(self, plan: PlanT) -> Generator[Bottle, None, None]: + """Template: build or load images, then delegate to _launch_impl.""" + images = self._build_or_load_images(plan) + with self._launch_impl(plan, images) as bottle: + yield bottle + + @abstractmethod + def _build_or_load_images(self, plan: PlanT) -> BottleImages: + """Return the agent and sidecar image references (or artifact paths) + for this plan, building fresh images when the policy requires it.""" + + @abstractmethod + def _launch_impl(self, plan: PlanT, images: BottleImages) -> AbstractContextManager[Bottle]: + """Bring up the bottle using pre-resolved images; yield a handle; tear down on exit.""" + + def provision(self, plan: PlanT, bottle: "Bottle") -> str | None: + """Copy host-side files (CA cert, prompt, skills, .git) into + the running bottle. Called from `launch` after the container + / machine is up. Returns the in-container prompt path if a + prompt was provisioned, else None — the Bottle handle uses it + to decide whether to add provider-specific prompt args to the + agent's argv. + + Default orchestration: ca → prompt → provider apply → skills + → workspace → git → supervise-mcp. CA install runs first so + the agent's trust store is rebuilt before anything inside the + agent makes a TLS call. + + Per PRD 0050 the per-provider steps (prompt, skills, + declarative provision-plan apply, supervise MCP registration) + live on the `AgentProvider` plugin. The backend only owns the + steps that are about backend infrastructure (CA, workspace, + git) and surfaces the supervise daemon URL its launch step + knows about via `supervise_mcp_url`. + + PRD 0017: cred-proxy's agent-side dotfile rewrites (~/.npmrc, + ~/.gitconfig insteadOf, tea config) are gone. Egress-proxy is + on the agent's HTTP_PROXY path so every tool that respects + HTTPS_PROXY (claude-code, git over HTTPS, npm, curl) is + intercepted without per-tool reconfiguration.""" + provider = get_provider(plan.agent_provision.template) + provider.provision_ca(bottle, plan) + prompt_path = provider.provision_prompt(plan, bottle) + provider.provision(plan, bottle) + provider.provision_skills(plan, bottle) + self.provision_workspace(plan, bottle) + provider.provision_git(bottle, plan) + provider.provision_supervise_mcp( + plan, bottle, self.supervise_mcp_url(plan), + ) + return prompt_path + + def provision_workspace(self, plan: PlanT, bottle: "Bottle") -> None: + """Copy the operator workspace into the running bottle. + + This is the only supported workspace-provisioning path: Docker + does not build a derived image containing the current + workspace.""" + workspace = plan.workspace_plan + if not (workspace.enabled and workspace.copy_contents): + return + + guest_parent = workspace.guest_path.rsplit("/", 1)[0] or "/" + guest_path = shlex.quote(workspace.guest_path) + guest_parent = shlex.quote(guest_parent) + owner = shlex.quote(workspace.owner) + mode = shlex.quote(workspace.mode) + info(f"copying {workspace.host_path} -> {bottle.name}:{workspace.guest_path}") + bottle.exec( + f"rm -rf {guest_path} && mkdir -p {guest_parent}", + user="root", + ) + bottle.cp_in(str(workspace.host_path), workspace.guest_path) + bottle.exec( + f"chown -R {owner} {guest_path} && chmod {mode} {guest_path}", + user="root", + ) + + def supervise_mcp_url(self, plan: PlanT) -> str: + """Return the agent-side URL of the per-bottle supervise + gateway, or "" when this bottle has no gateway. The provider + plugin's `provision_supervise_mcp` uses it to register the + MCP entry inside the guest. + + Default returns "" so backends without supervise support + don't have to implement it. Docker and firecracker override.""" + del plan + return "" + + def ensure_orchestrator(self) -> str: + """Bring up this backend's per-host orchestrator + shared gateway + (idempotent) and return the host-reachable control-plane URL. + + This is the backend-agnostic bring-up entry point: `launch` calls + it as part of starting a bottle, and operator tools (`supervise`) + call it to start the control plane on demand when none is running + yet. Docker starts the orchestrator + gateway containers; + firecracker boots the infra VM. Backends with no orchestrator + (macos-container) die with a pointer — the default here.""" + die(f"backend {self.name!r} has no orchestrator control plane") + + @abstractmethod + def prepare_cleanup(self) -> CleanupT: + """Enumerate orphaned resources from previous bottles. No side + effects; safe to call before the y/N.""" + + @abstractmethod + def cleanup(self, plan: CleanupT) -> None: + """Remove everything described by the cleanup plan.""" + + @abstractmethod + def enumerate_active(self) -> Sequence[ActiveAgent]: + """Return every currently-running agent on this backend. + Empty when none. Backend-specific: docker queries `docker + compose ls`; firecracker cross-references its running gateway + containers against per-bottle metadata.""" + + @classmethod + @abstractmethod + def is_available(cls) -> bool: + """Whether this backend's runtime prerequisites are satisfied + on the current host. Docker → `docker` on PATH; firecracker → + Linux + KVM. Used by the cross-backend + `enumerate_active_agents` / `cmd_cleanup` to skip backends + the operator hasn't installed, so a docker-only host + doesn't fail when `cli.py list active` walks past + firecracker.""" + + @classmethod + @abstractmethod + def setup(cls) -> int: + """Emit this backend's one-time host setup — privileged network + pool, daemon bring-up, install pointers, etc. — as + host-appropriate config or commands. Prints to stdout/stderr and + returns a shell exit code (0 = nothing to report / success). A + backend that needs no host setup prints a short note and returns + 0. Invoked generically by `./cli.py backend setup [--backend=…]` + so operators can provision any backend without a + backend-specific command. Classmethod (like `is_available`) — + it's a host query, not per-bottle state.""" + + @classmethod + @abstractmethod + def status(cls) -> int: + """Report whether this backend's prerequisites are satisfied on + the host — binaries, daemon reachability, network pool, range + conflicts, etc. Prints a human-readable summary; returns 0 when + the backend is ready to launch and non-zero when something is + missing. Invoked by `./cli.py backend status [--backend=…]`.""" + + @classmethod + @abstractmethod + def teardown(cls) -> int: + """Undo `setup()` — the inverse operation, surfaced as + `./cli.py backend teardown [--backend=…]` (uninstall). Symmetric + with setup: where setup is advisory (prints the privileged + commands / declarative config to apply), teardown prints the + commands / config change to remove the host prerequisites. A + backend with no host setup prints a short note and returns 0. + Not called by the launch path or the test suite.""" diff --git a/bot_bottle/backend/selection.py b/bot_bottle/backend/selection.py new file mode 100644 index 0000000..60e2fa5 --- /dev/null +++ b/bot_bottle/backend/selection.py @@ -0,0 +1,188 @@ +"""Backend registry, selection, and active-agent enumeration. + +Resolves which bottle backend to use (explicit name / `BOT_BOTTLE_BACKEND` / +auto-select), and enumerates running agents across every available backend. The +three concrete backends are imported lazily inside `_get_backends` so this +module — and anything that only needs to *select* a backend — stays cheap. +""" + +from __future__ import annotations + +import os +import sys +from typing import Any + +from ..log import die, info, warn +from ..util import read_tty_line +from .base import ActiveAgent, BottleBackend + + +# _backends is None until the first call to _get_backends(), at which +# point all three concrete backend classes are imported and instantiated. +# Keeping the imports out of module scope means that importing any +# backend sub-module (e.g. `backend.docker.util`) no longer drags the +# firecracker and macos-container implementations into memory. +# +# Tests may replace _backends with a {name: fake} dict via patch.object; +# _get_backends() returns the current module-level value as-is when it +# is not None, so test fakes take effect without triggering real imports. +_backends: dict[str, BottleBackend[Any, Any]] | None = None + + +def _get_backends() -> dict[str, BottleBackend[Any, Any]]: + """Return the registry of all backend instances, loading lazily on first call.""" + global _backends # pylint: disable=global-statement + if _backends is None: + from .docker import DockerBottleBackend + from .firecracker import FirecrackerBottleBackend + from .macos_container import MacosContainerBottleBackend + _backends = { + "docker": DockerBottleBackend(), + "firecracker": FirecrackerBottleBackend(), + "macos-container": MacosContainerBottleBackend(), + } + return _backends + + +def get_bottle_backend( + name: str | None = None, + *, + prompt: bool = True, +) -> BottleBackend[Any, Any]: + """Resolve the bottle backend. + + `name` precedence: + 1. explicit arg (e.g. resume passes the recorded backend name) + 2. BOT_BOTTLE_BACKEND env var + 3. auto-selection: VM backend first, docker fallback with prompt + + `prompt` controls whether auto-selection may block on an interactive + [i/d/q] prompt when falling back to docker. Pass `prompt=False` in + non-interactive contexts (headless launches, CI) so the call dies + with an actionable message instead of hanging. + + Dies with a pointer at the known backends if the chosen name + isn't implemented.""" + resolved = name or os.environ.get("BOT_BOTTLE_BACKEND") + if resolved is None: + resolved = _auto_select_backend(prompt=prompt) + backends = _get_backends() + if resolved not in backends: + known = ", ".join(sorted(backends)) + die(f"unknown backend {resolved!r}; known backends: {known}") + return backends[resolved] + + +def _platform_vm_suggestion() -> str: + """Platform-appropriate VM backend name for install suggestions.""" + return "macos-container" if sys.platform == "darwin" else "firecracker" + + +def _print_vm_install_instructions() -> None: + """Print platform-appropriate VM backend install instructions to stderr.""" + vm = _platform_vm_suggestion() + if vm == "macos-container": + info("Install Apple Container: https://github.com/apple/container/releases") + info("Then start the service: container system start") + else: + info("Install Firecracker: https://github.com/firecracker-microvm/firecracker/releases") + info("Configure the host: ./cli.py backend setup") + + +def _auto_select_backend(prompt: bool = True) -> str: + """Tier-1 / tier-2 backend auto-selection. + + Tier 1: VM backend — macos-container on macOS when Apple Container is + installed; firecracker on KVM-capable Linux even before the binary is + present (its preflight prints an install pointer). + + Tier 2: docker, with a security warning and an interactive prompt. + When `prompt=False` (headless / CI), dies with an actionable message + instead of blocking on a TTY read. When docker is also absent, prints + VM install instructions and exits. + """ + # --- Tier 1: VM backend ----------------------------------------- + if has_backend("macos-container"): + return "macos-container" + # A KVM-capable Linux host defaults to firecracker even when the + # `firecracker` binary isn't installed yet: selecting it here routes + # start through firecracker's preflight, which prints an install + # pointer, instead of silently falling back to docker. + from .firecracker import FirecrackerBottleBackend + if FirecrackerBottleBackend.is_host_capable(): + return "firecracker" + + # --- Tier 2: docker fallback ------------------------------------ + if not has_backend("docker"): + info("No backend available on this host.") + _print_vm_install_instructions() + die("no backend available; install a VM backend and re-run") + + vm = _platform_vm_suggestion() + warn( + "docker is less secure than VM backends — " + "containers share the host kernel." + ) + if not prompt: + die( + f"no VM backend available; set BOT_BOTTLE_BACKEND=docker to proceed " + f"with docker, or install the {vm!r} backend." + ) + sys.stderr.write( + f"bot-bottle: For better isolation, install the {vm!r} backend.\n" + f" [i] show {vm} install instructions and exit\n" + " [d] use docker anyway\n" + " [q] quit\n" + "bot-bottle: choice [i/d/q]: " + ) + sys.stderr.flush() + reply = read_tty_line().strip().lower() + if reply == "d": + return "docker" + if reply == "i": + _print_vm_install_instructions() + die("not proceeding with docker; install a VM backend or set BOT_BOTTLE_BACKEND=docker") + + +def known_backend_names() -> tuple[str, ...]: + """Sorted tuple of all backend keys in `_get_backends()`. Used by + argparse (`--backend` choices) and the dashboard's backend + picker.""" + return tuple(sorted(_get_backends())) + + +def has_backend(name: str) -> bool: + """Whether the named backend's runtime prerequisites are + available on the current host. Cross-backend callers (list, + cleanup) skip unavailable backends so a docker-only host + doesn't fail when the firecracker backend isn't usable, + and vice versa. + + Returns False for unknown names so callers can pass + arbitrary input without separate validation.""" + backends = _get_backends() + if name not in backends: + return False + return backends[name].is_available() + + +def enumerate_active_agents() -> list[ActiveAgent]: + """All currently-running agents, across every available + backend. Used by CLI `list active` and the dashboard's agents + pane so neither has to know which backends exist. Skips + backends whose `is_available()` reports False. + + Sorted by `(started_at, slug)` so the list is stable across + dashboard refresh ticks — agents don't shift position while + the operator navigates with arrow keys. ISO 8601 timestamps + sort lexicographically in chronological order; `slug` is the + deterministic tiebreaker. Agents with missing metadata + (`started_at == ""`) sort first.""" + out: list[ActiveAgent] = [] + backends = _get_backends() + for name in sorted(backends): + if not backends[name].is_available(): + continue + out.extend(backends[name].enumerate_active()) + out.sort(key=lambda a: (a.started_at, a.slug)) + return out diff --git a/tests/unit/test_backend_selection.py b/tests/unit/test_backend_selection.py index bb549d1..1a2304d 100644 --- a/tests/unit/test_backend_selection.py +++ b/tests/unit/test_backend_selection.py @@ -13,6 +13,7 @@ import unittest from unittest.mock import patch from bot_bottle import backend as backend_mod +from bot_bottle.backend import selection as _sel from bot_bottle.backend import ( ActiveAgent, enumerate_active_agents, @@ -40,7 +41,7 @@ class TestGetBottleBackend(unittest.TestCase): return True with patch.dict(os.environ, {}, clear=True), \ - patch.object(backend_mod, "_backends", { + patch.object(_sel, "_backends", { "macos-container": _FakeBackend(), "docker": _FakeBackend(), }): @@ -62,11 +63,11 @@ class TestGetBottleBackend(unittest.TestCase): with patch.dict(os.environ, {}, clear=True), \ patch.object(backend_mod.FirecrackerBottleBackend, "is_host_capable", classmethod(lambda cls: False)), \ - patch.object(backend_mod, "_backends", { + patch.object(_sel, "_backends", { "macos-container": _FakeBackend("macos-container", False), "docker": _FakeBackend("docker", True), }), \ - patch.object(backend_mod, "read_tty_line", return_value="d"): + patch.object(_sel, "read_tty_line", return_value="d"): b = get_bottle_backend() self.assertEqual("docker", b.name) @@ -85,7 +86,7 @@ class TestGetBottleBackend(unittest.TestCase): with patch.dict(os.environ, {}, clear=True), \ patch.object(backend_mod.FirecrackerBottleBackend, "is_host_capable", classmethod(lambda cls: True)), \ - patch.object(backend_mod, "_backends", { + patch.object(_sel, "_backends", { "macos-container": _FakeBackend("macos-container", False), "firecracker": _FakeBackend("firecracker", False), "docker": _FakeBackend("docker", True), @@ -94,7 +95,7 @@ class TestGetBottleBackend(unittest.TestCase): self.assertEqual("firecracker", b.name) def test_unknown_dies(self): - with patch.object(backend_mod, "die", side_effect=SystemExit("die")): + with patch.object(_sel, "die", side_effect=SystemExit("die")): with self.assertRaises(SystemExit): get_bottle_backend("nonexistent") @@ -111,11 +112,11 @@ class TestGetBottleBackend(unittest.TestCase): with patch.dict(os.environ, {}, clear=True), \ patch.object(backend_mod.FirecrackerBottleBackend, "is_host_capable", classmethod(lambda cls: False)), \ - patch.object(backend_mod, "_backends", { + patch.object(_sel, "_backends", { "macos-container": _FakeBackend("macos-container", False), "docker": _FakeBackend("docker", False), }), \ - patch.object(backend_mod, "die", side_effect=SystemExit("die")): + patch.object(_sel, "die", side_effect=SystemExit("die")): with self.assertRaises(SystemExit): get_bottle_backend() @@ -132,12 +133,12 @@ class TestGetBottleBackend(unittest.TestCase): with patch.dict(os.environ, {}, clear=True), \ patch.object(backend_mod.FirecrackerBottleBackend, "is_host_capable", classmethod(lambda cls: False)), \ - patch.object(backend_mod, "_backends", { + patch.object(_sel, "_backends", { "macos-container": _FakeBackend("macos-container", False), "docker": _FakeBackend("docker", True), }), \ - patch.object(backend_mod, "read_tty_line", return_value="q"), \ - patch.object(backend_mod, "die", side_effect=SystemExit("die")): + patch.object(_sel, "read_tty_line", return_value="q"), \ + patch.object(_sel, "die", side_effect=SystemExit("die")): with self.assertRaises(SystemExit): get_bottle_backend() @@ -155,11 +156,11 @@ class TestGetBottleBackend(unittest.TestCase): with patch.dict(os.environ, {}, clear=True), \ patch.object(backend_mod.FirecrackerBottleBackend, "is_host_capable", classmethod(lambda cls: False)), \ - patch.object(backend_mod, "_backends", { + patch.object(_sel, "_backends", { "macos-container": _FakeBackend("macos-container", False), "docker": _FakeBackend("docker", True), }), \ - patch.object(backend_mod, "die", side_effect=SystemExit("die")): + patch.object(_sel, "die", side_effect=SystemExit("die")): with self.assertRaises(SystemExit): get_bottle_backend(prompt=False) @@ -176,13 +177,13 @@ class TestGetBottleBackend(unittest.TestCase): with patch.dict(os.environ, {}, clear=True), \ patch.object(backend_mod.FirecrackerBottleBackend, "is_host_capable", classmethod(lambda cls: False)), \ - patch.object(backend_mod, "_backends", { + patch.object(_sel, "_backends", { "macos-container": _FakeBackend("macos-container", False), "docker": _FakeBackend("docker", True), }), \ - patch.object(backend_mod, "read_tty_line", return_value="i"), \ - patch.object(backend_mod, "_print_vm_install_instructions") as mock_inst, \ - patch.object(backend_mod, "die", side_effect=SystemExit("die")): + patch.object(_sel, "read_tty_line", return_value="i"), \ + patch.object(_sel, "_print_vm_install_instructions") as mock_inst, \ + patch.object(_sel, "die", side_effect=SystemExit("die")): with self.assertRaises(SystemExit): get_bottle_backend() mock_inst.assert_called_once() @@ -218,9 +219,9 @@ class TestPrintVmInstallInstructions(unittest.TestCase): def test_linux_prints_firecracker_instructions(self): from bot_bottle.backend import _print_vm_install_instructions - with patch.object(backend_mod, "_platform_vm_suggestion", + with patch.object(_sel, "_platform_vm_suggestion", return_value="firecracker"), \ - patch.object(backend_mod, "info") as mock_info: + patch.object(_sel, "info") as mock_info: _print_vm_install_instructions() messages = [str(c[0][0]) for c in mock_info.call_args_list] @@ -229,9 +230,9 @@ class TestPrintVmInstallInstructions(unittest.TestCase): def test_macos_prints_apple_container_instructions(self): from bot_bottle.backend import _print_vm_install_instructions - with patch.object(backend_mod, "_platform_vm_suggestion", + with patch.object(_sel, "_platform_vm_suggestion", return_value="macos-container"), \ - patch.object(backend_mod, "info") as mock_info: + patch.object(_sel, "info") as mock_info: _print_vm_install_instructions() messages = [str(c[0][0]) for c in mock_info.call_args_list] @@ -274,7 +275,7 @@ class TestEnumerateActiveAgents(unittest.TestCase): return self._items with patch.object( - backend_mod, "_backends", + _sel, "_backends", {"docker": _FakeBackend([a]), "firecracker": _FakeBackend([b])}, ): self.assertEqual([a, b], enumerate_active_agents()) @@ -308,7 +309,7 @@ class TestEnumerateActiveAgents(unittest.TestCase): return self._items with patch.object( - backend_mod, "_backends", + _sel, "_backends", { "docker": _FakeBackend([newer, tie_b]), "firecracker": _FakeBackend([missing_metadata, tie_a]), @@ -328,7 +329,7 @@ class TestEnumerateActiveAgents(unittest.TestCase): return [] with patch.object( - backend_mod, "_backends", + _sel, "_backends", {"docker": _FakeBackend(), "firecracker": _FakeBackend()}, ): self.assertEqual([], enumerate_active_agents()) @@ -359,7 +360,7 @@ class TestEnumerateActiveAgents(unittest.TestCase): return self._items with patch.object( - backend_mod, "_backends", + _sel, "_backends", { "docker": _FakeBackend([present], available=True), "firecracker": _FakeBackend([hidden], available=False), @@ -375,7 +376,7 @@ class TestHasBackend(unittest.TestCase): return False with patch.object( - backend_mod, "_backends", {"docker": _FakeBackend()}, + _sel, "_backends", {"docker": _FakeBackend()}, ): from bot_bottle.backend import has_backend self.assertFalse(has_backend("docker")) -- 2.52.0 From 923d44bc096099fb778d90f47f9dd8603b8c10c2 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 15:53:48 -0400 Subject: [PATCH 18/39] refactor(backend): thin the docker/firecracker/macos-container package inits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three concrete-backend package inits eagerly imported their `*BottleBackend` (which pulls the whole framework), so importing any leaf under them (e.g. `backend.docker.util`) paid ~78 modules. Give each the same thin `__getattr__` treatment as the top-level backend package: re-export the public class(es) lazily on first access, with a TYPE_CHECKING block for checkers. `import bot_bottle.backend.docker.util` drops from 78 -> 6 bot_bottle modules; the docker/firecracker/macos-container packages themselves are ~3 each. All `from bot_bottle.backend.docker import DockerBottleBackend` call-sites and the lazy selection path keep working. This is what makes backend leaves cheap — and unblocks moving docker_cmd into backend/docker/util if we want. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/backend/docker/__init__.py | 38 +++++++++++++++---- bot_bottle/backend/firecracker/__init__.py | 22 ++++++++++- .../backend/macos_container/__init__.py | 27 +++++++++++-- 3 files changed, 74 insertions(+), 13 deletions(-) diff --git a/bot_bottle/backend/docker/__init__.py b/bot_bottle/backend/docker/__init__.py index dc2340b..8da7dc8 100644 --- a/bot_bottle/backend/docker/__init__.py +++ b/bot_bottle/backend/docker/__init__.py @@ -11,18 +11,42 @@ The bulk of the implementation lives in sibling modules: - launch: bring-up + teardown context manager - cleanup: orphan enumeration, removal, active listing - backend: DockerBottleBackend façade wiring the above + - infra: DockerInfraService (the per-host infra container) -This file only re-exports the public names so -`from bot_bottle.backend.docker import DockerBottleBackend` keeps -working. +Thin by design: the public names are re-exported lazily via `__getattr__`, so +importing a leaf like `backend.docker.util` doesn't drag `DockerBottleBackend` +(and the whole framework it pulls) into memory. """ from __future__ import annotations -from .backend import DockerBottleBackend -from .bottle import DockerBottle -from .bottle_cleanup_plan import DockerBottleCleanupPlan -from .bottle_plan import DockerBottlePlan +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .backend import DockerBottleBackend + from .bottle import DockerBottle + from .bottle_cleanup_plan import DockerBottleCleanupPlan + from .bottle_plan import DockerBottlePlan + + +_LAZY_MODULES: dict[str, str] = { + "DockerBottleBackend": "backend", + "DockerBottle": "bottle", + "DockerBottleCleanupPlan": "bottle_cleanup_plan", + "DockerBottlePlan": "bottle_plan", +} + + +def __getattr__(name: str) -> Any: + mod = _LAZY_MODULES.get(name) + if mod is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module + + value = getattr(import_module(f"{__name__}.{mod}"), name) + globals()[name] = value + return value + __all__ = [ "DockerBottle", diff --git a/bot_bottle/backend/firecracker/__init__.py b/bot_bottle/backend/firecracker/__init__.py index 57b5840..cae1248 100644 --- a/bot_bottle/backend/firecracker/__init__.py +++ b/bot_bottle/backend/firecracker/__init__.py @@ -1,7 +1,25 @@ -"""Firecracker backend: Linux KVM microVM isolation (issue #342).""" +"""Firecracker backend: Linux KVM microVM isolation (issue #342). + +Thin by design: `FirecrackerBottleBackend` is re-exported lazily via +`__getattr__`, so importing a leaf module under this package doesn't drag the +backend (and the framework it pulls) into memory. +""" from __future__ import annotations -from .backend import FirecrackerBottleBackend +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .backend import FirecrackerBottleBackend + + +def __getattr__(name: str) -> Any: + if name == "FirecrackerBottleBackend": + from .backend import FirecrackerBottleBackend + + globals()[name] = FirecrackerBottleBackend + return FirecrackerBottleBackend + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + __all__ = ["FirecrackerBottleBackend"] diff --git a/bot_bottle/backend/macos_container/__init__.py b/bot_bottle/backend/macos_container/__init__.py index 6d56f95..c655d15 100644 --- a/bot_bottle/backend/macos_container/__init__.py +++ b/bot_bottle/backend/macos_container/__init__.py @@ -1,10 +1,29 @@ """macOS Apple Container backend. -Selectable via `BOT_BOTTLE_BACKEND=macos-container`. This package owns -the Apple `container` CLI integration; launch remains gated until the -gateway network enforcement shape is implemented. +Selectable via `BOT_BOTTLE_BACKEND=macos-container`. This package owns the Apple +`container` CLI integration; launch remains gated until the gateway network +enforcement shape is implemented. + +Thin by design: `MacosContainerBottleBackend` is re-exported lazily via +`__getattr__`, so importing a leaf module under this package doesn't drag the +backend (and the framework it pulls) into memory. """ -from .backend import MacosContainerBottleBackend +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .backend import MacosContainerBottleBackend + + +def __getattr__(name: str) -> Any: + if name == "MacosContainerBottleBackend": + from .backend import MacosContainerBottleBackend + + globals()[name] = MacosContainerBottleBackend + return MacosContainerBottleBackend + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + __all__ = ["MacosContainerBottleBackend"] -- 2.52.0 From eab9d15130d8f579ec94960d4b98fc29027c62b3 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 15:58:23 -0400 Subject: [PATCH 19/39] refactor(docker): fold run_docker into backend/docker/util; drop the docker_cmd shim docker_cmd.py existed as a top-level module solely so the orchestrator's docker components could share run_docker without dragging the (then-heavy) backend layer in. Now that backend/__init__ and backend/docker/__init__ are thin, importing backend.docker.util costs 6 modules instead of 76, so the shim's whole reason to exist is gone. Move run_docker into backend/docker/util.py (where the other docker subprocess primitives live) and delete docker_cmd.py. Backend siblings import it via `from .util import run_docker`; the two docker-specific orchestrator modules (docker_broker, rotate_ca) import `from ..backend.docker.util import run_docker`. No import cycle (backend.docker.util pulls nothing from orchestrator); orchestrator.__main__ stays lean at 28 modules. run_docker patch targets are unchanged (tests patch it in the importing module). Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- .../backend/docker/consolidated_launch.py | 2 +- bot_bottle/backend/docker/gateway.py | 2 +- .../backend/docker/gateway_provision.py | 2 +- bot_bottle/backend/docker/infra.py | 2 +- bot_bottle/backend/docker/util.py | 25 +++++++++++--- bot_bottle/docker_cmd.py | 34 ------------------- bot_bottle/orchestrator/docker_broker.py | 2 +- bot_bottle/orchestrator/rotate_ca.py | 2 +- 8 files changed, 26 insertions(+), 45 deletions(-) delete mode 100644 bot_bottle/docker_cmd.py diff --git a/bot_bottle/backend/docker/consolidated_launch.py b/bot_bottle/backend/docker/consolidated_launch.py index 2b7097f..7efabc6 100644 --- a/bot_bottle/backend/docker/consolidated_launch.py +++ b/bot_bottle/backend/docker/consolidated_launch.py @@ -16,7 +16,7 @@ from __future__ import annotations from dataclasses import dataclass from ... import log -from ...docker_cmd import run_docker +from .util import run_docker from ...egress import EgressPlan from ...git_gate import GitGatePlan from ...orchestrator.client import OrchestratorClient diff --git a/bot_bottle/backend/docker/gateway.py b/bot_bottle/backend/docker/gateway.py index edea8e7..c45ff8e 100644 --- a/bot_bottle/backend/docker/gateway.py +++ b/bot_bottle/backend/docker/gateway.py @@ -5,7 +5,7 @@ import time from pathlib import Path from ...control_auth import ROLE_GATEWAY, mint -from ...docker_cmd import run_docker +from .util import run_docker from ...paths import ( CONTROL_AUTH_JWT_ENV, host_control_plane_token, diff --git a/bot_bottle/backend/docker/gateway_provision.py b/bot_bottle/backend/docker/gateway_provision.py index d1094dc..3eaadcb 100644 --- a/bot_bottle/backend/docker/gateway_provision.py +++ b/bot_bottle/backend/docker/gateway_provision.py @@ -17,7 +17,7 @@ from __future__ import annotations import re from typing import Protocol -from ...docker_cmd import run_docker +from .util import run_docker from ...git_gate import GitGatePlan, git_gate_render_provision # bottle ids index the gateway's per-bottle repo + creds dirs; they land in diff --git a/bot_bottle/backend/docker/infra.py b/bot_bottle/backend/docker/infra.py index 2471082..5a1709e 100644 --- a/bot_bottle/backend/docker/infra.py +++ b/bot_bottle/backend/docker/infra.py @@ -26,7 +26,7 @@ from pathlib import Path from ... import log from ...control_auth import ROLE_GATEWAY, mint -from ...docker_cmd import run_docker +from .util import run_docker from ...paths import ( CONTROL_AUTH_JWT_ENV, CONTROL_PLANE_TOKEN_ENV, diff --git a/bot_bottle/backend/docker/util.py b/bot_bottle/backend/docker/util.py index fb2091b..b05cbd6 100644 --- a/bot_bottle/backend/docker/util.py +++ b/bot_bottle/backend/docker/util.py @@ -1,6 +1,6 @@ -"""Docker host-side primitives used by DockerBottleBackend: probing -for docker on PATH, slugifying agent names, checking image/container -existence, and building images.""" +"""Docker host-side primitives used by DockerBottleBackend: the lean +`run_docker` subprocess wrapper, probing for docker on PATH, slugifying agent +names, checking image/container existence, and building images.""" from __future__ import annotations @@ -11,9 +11,24 @@ import shutil import subprocess from typing import Iterator -from ...docker_cmd import run_docker from ...log import die, info -# from ...workspace import WorkspacePlan + + +def run_docker( + argv: list[str], *, env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[str]: + """Run a `docker` command, capturing stdout/stderr as text. Never raises on + a non-zero exit — callers inspect `returncode` / `stderr` so they can stay + fail-closed or tolerate idempotent no-ops (e.g. removing an already-absent + container). + + `env` sets the child process environment — used to hand a secret to a bare + `--env NAME` flag (docker inherits its value from this process) so the value + never lands on argv or in `docker inspect`'s recorded command line.""" + return subprocess.run( + argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + check=False, env=env, + ) # Cap on the suffix the container-name conflict logic will try before diff --git a/bot_bottle/docker_cmd.py b/bot_bottle/docker_cmd.py deleted file mode 100644 index 32a617e..0000000 --- a/bot_bottle/docker_cmd.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Lean, framework-free `docker` subprocess primitive. - -Deliberately a top-level module with a single stdlib import so it can be -reused from anywhere without cost. It is intentionally *not* placed in -`backend.docker.util`: importing that module runs `backend/__init__.py`, -which eagerly loads all three bottle backends (docker + firecracker + -macos) plus the manifest/egress/git-gate/supervise framework — ~76 modules -— which would drag the whole backend layer into the deliberately-lean -orchestrator. This primitive stays free of that so both the orchestrator's -docker components and (in time) `backend.docker.util` can share it.""" - -from __future__ import annotations - -import subprocess - - -def run_docker( - argv: list[str], *, env: dict[str, str] | None = None, -) -> subprocess.CompletedProcess[str]: - """Run a `docker` command, capturing stdout/stderr as text. Never raises - on a non-zero exit — callers inspect `returncode` / `stderr` so they can - stay fail-closed or tolerate idempotent no-ops (e.g. removing an - already-absent container). - - `env` sets the child process environment — used to hand a secret to a bare - `--env NAME` flag (docker inherits its value from this process) so the - value never lands on argv or in `docker inspect`'s recorded command line.""" - return subprocess.run( - argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, - check=False, env=env, - ) - - -__all__ = ["run_docker"] diff --git a/bot_bottle/orchestrator/docker_broker.py b/bot_bottle/orchestrator/docker_broker.py index e5af91c..2e22845 100644 --- a/bot_bottle/orchestrator/docker_broker.py +++ b/bot_bottle/orchestrator/docker_broker.py @@ -15,7 +15,7 @@ from __future__ import annotations import subprocess -from ..docker_cmd import run_docker +from ..backend.docker.util import run_docker from .broker import LaunchBroker, LaunchRequest CONTAINER_PREFIX = "bot-bottle-orch-" diff --git a/bot_bottle/orchestrator/rotate_ca.py b/bot_bottle/orchestrator/rotate_ca.py index 0cae300..9303005 100644 --- a/bot_bottle/orchestrator/rotate_ca.py +++ b/bot_bottle/orchestrator/rotate_ca.py @@ -23,7 +23,7 @@ from __future__ import annotations import sys from pathlib import Path -from ..docker_cmd import run_docker +from ..backend.docker.util import run_docker from ..paths import host_gateway_ca_dir from ..gateway import GATEWAY_NAME, rotate_gateway_ca from ..backend.docker.infra import INFRA_NAME -- 2.52.0 From 14c28946a7f0b0bb105040f83338f00a399ea7bc Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 16:04:15 -0400 Subject: [PATCH 20/39] refactor(manifest): move Manifest/ManifestIndex to manifest.index; thin the package init MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit manifest/__init__.py was a facade that eagerly imported every piece (agent, bottle, egress, git, loader, schema, util) and defined the aggregate Manifest / ManifestIndex, so importing a leaf like `manifest.util` paid 19 modules. Move the aggregate model — Manifest, ManifestIndex, and the bottle-resolution helpers — into manifest/index.py (which keeps the framework imports it needs), and make __init__ a thin __getattr__ facade over the 12 __all__ names (mapping each to its submodule), with a TYPE_CHECKING block. The package docstring is preserved. `import bot_bottle.manifest.util` drops 19 -> 3; the package itself is 2. All `from bot_bottle.manifest import Manifest/ManifestIndex/ManifestError/…` call-sites keep working via the lazy facade; no importer changes needed. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/manifest/__init__.py | 452 ++++---------------------------- bot_bottle/manifest/index.py | 413 +++++++++++++++++++++++++++++ 2 files changed, 459 insertions(+), 406 deletions(-) create mode 100644 bot_bottle/manifest/index.py diff --git a/bot_bottle/manifest/__init__.py b/bot_bottle/manifest/__init__.py index 8289298..1dde028 100644 --- a/bot_bottle/manifest/__init__.py +++ b/bot_bottle/manifest/__init__.py @@ -58,421 +58,61 @@ useful for building manifests without on-disk files. from __future__ import annotations -import os -from dataclasses import dataclass, field, replace -from pathlib import Path -from typing import Mapping +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .index import Manifest, ManifestIndex + from .util import ManifestError + from .agent import ManifestAgent, ManifestAgentProvider + from .bottle import ManifestBottle + from .egress import EGRESS_AUTH_SCHEMES, ManifestEgressConfig, ManifestEgressRoute + from .git import ManifestGitEntry, ManifestGitUser, ManifestKeyConfig + + +# Facade name -> submodule that defines it. The aggregate model (`Manifest`, +# `ManifestIndex`) lives in `index`; the piece types in their own modules. +_LAZY_MODULES: dict[str, str] = { + "Manifest": "index", + "ManifestIndex": "index", + "ManifestError": "util", + "ManifestAgent": "agent", + "ManifestAgentProvider": "agent", + "ManifestBottle": "bottle", + "EGRESS_AUTH_SCHEMES": "egress", + "ManifestEgressRoute": "egress", + "ManifestEgressConfig": "egress", + "ManifestGitEntry": "git", + "ManifestGitUser": "git", + "ManifestKeyConfig": "git", +} + + +def __getattr__(name: str) -> Any: + """Lazily surface the manifest facade names from their submodules and cache + them at package level — so importing a leaf (e.g. `manifest.util`) doesn't + build the whole manifest model, while `from bot_bottle.manifest import + Manifest` keeps working.""" + mod = _LAZY_MODULES.get(name) + if mod is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module + + value = getattr(import_module(f"{__name__}.{mod}"), name) + globals()[name] = value + return value -from ..log import warn -from .util import ManifestError, as_json_object -from .agent import ManifestAgent, ManifestAgentProvider -from .bottle import ManifestBottle -from .egress import ( - EGRESS_AUTH_SCHEMES, - ManifestEgressConfig, - ManifestEgressRoute, -) -from .extends import merge_bottles_runtime, resolve_bottles -from .git import ManifestGitEntry, ManifestGitUser, ManifestKeyConfig -from .loader import ( - check_stale_json, - load_bottle_chain_from_dir, - scan_agent_names, - scan_bottle_names, -) -from .schema import validate_agent_frontmatter_keys -from ..yaml_subset import YamlSubsetError, parse_frontmatter -# Re-export everything that callers currently import from this module. __all__ = [ + "Manifest", + "ManifestIndex", "ManifestError", "ManifestGitEntry", "ManifestGitUser", "ManifestKeyConfig", "ManifestAgentProvider", + "ManifestAgent", + "ManifestBottle", "EGRESS_AUTH_SCHEMES", "ManifestEgressRoute", "ManifestEgressConfig", - "ManifestAgent", - "ManifestBottle", - "ManifestIndex", - "Manifest", ] - - -def _section_dict(value: object, label: str) -> dict[str, object]: - """Like as_json_object but treats absent/null as an empty section.""" - if value is None: - return {} - return as_json_object(value, label) - - -def _merge_git_user( - agent_user: ManifestGitUser, base_user: ManifestGitUser -) -> ManifestGitUser: - """Merge the agent's git.user over the bottle's, agent-wins-on-non-empty.""" - if agent_user.is_empty(): - return base_user - return ManifestGitUser( - name=agent_user.name or base_user.name, - email=agent_user.email or base_user.email, - ) - - -def _manifest_with_merged_git_user( - agent: "ManifestAgent", raw_bottle: "ManifestBottle" -) -> "Manifest": - """Build the single-value Manifest, overlaying the agent's git-gate.user - onto the bottle (agent wins on non-empty, per-field). Shared by the eager - and lazy load_for_agent paths.""" - merged = _merge_git_user(agent.git_user, raw_bottle.git_user) - bottle = ( - raw_bottle if merged == raw_bottle.git_user - else replace(raw_bottle, git_user=merged) - ) - return Manifest(agent=agent, bottle=bottle) - - -def _resolve_effective_bottle_eager( - agent_name: str, - agent: "ManifestAgent", - bottle_names: "tuple[str, ...]", - bottles: "Mapping[str, ManifestBottle]", -) -> "ManifestBottle": - """Return the effective ManifestBottle for the eager (from_json_obj) path. - - When bottle_names is non-empty they are merged in order. When empty, falls - back to agent.bottle. Raises ManifestError when neither is set.""" - if bottle_names: - resolved: list[ManifestBottle] = [] - for bn in bottle_names: - if bn not in bottles: - available = ", ".join(sorted(bottles.keys())) or "(none)" - raise ManifestError( - f"bottle '{bn}' not defined. Available: {available}" - ) - resolved.append(bottles[bn]) - return merge_bottles_runtime(resolved) - - if not agent.bottle: - raise ManifestError( - f"agent '{agent_name}' has no 'bottle' field and no bottles were " - f"selected at launch. Select at least one bottle or add " - f"'bottle: ' to the agent manifest." - ) - return bottles[agent.bottle] - - -def _resolve_effective_bottle_lazy( - agent_name: str, - agent_bottle: str, - bottle_names: "tuple[str, ...]", - bottles_dir: "Path", -) -> "ManifestBottle": - """Return the effective ManifestBottle for the lazy (from_md_dirs) path. - - When bottle_names is non-empty they are resolved from disk and merged in - order. When empty, falls back to agent_bottle. Raises ManifestError when - neither is set.""" - if bottle_names: - resolved = [load_bottle_chain_from_dir(bn, bottles_dir) for bn in bottle_names] - return merge_bottles_runtime(resolved) - - if not agent_bottle: - raise ManifestError( - f"agent '{agent_name}' has no 'bottle' field and no bottles were " - f"selected at launch. Select at least one bottle or add " - f"'bottle: ' to the agent manifest." - ) - return load_bottle_chain_from_dir(agent_bottle, bottles_dir) - - -@dataclass(frozen=True) -class Manifest: - """Single-agent/bottle value type. Returned by ManifestIndex.load_for_agent(). - - `bottle` is the effective bottle with the agent's git-gate.user already - overlaid per-field (agent wins on non-empty). Backends and provisioners - use this directly — no agent_name lookup needed.""" - - agent: ManifestAgent - bottle: ManifestBottle - - def git_identity_summary(self) -> str | None: - """One-line effective git identity with per-field provenance, e.g. - `name=claude (agent), email=eric@dideric.is (bottle)`. - Returns None when neither agent nor bottle sets an identity.""" - over = self.agent.git_user # agent's declared git_user (pre-merge) - merged = self.bottle.git_user # effective git_user (post-merge) - if merged.is_empty(): - return None - parts: list[str] = [] - if merged.name: - parts.append(f"name={merged.name} ({'agent' if over.name else 'bottle'})") - if merged.email: - parts.append(f"email={merged.email} ({'agent' if over.email else 'bottle'})") - return ", ".join(parts) - - -@dataclass(frozen=True) -class ManifestIndex: - """Multi-agent/bottle collection. The pre-preflight form. - - In lazy mode (from resolve()/from_md_dirs()) only filenames are scanned; - no file content is read. In eager mode (from from_json_obj()) all agents - and bottles are pre-parsed. Call load_for_agent() to get a single-value - Manifest ready for backend use.""" - - bottles: Mapping[str, ManifestBottle] - agents: Mapping[str, ManifestAgent] - # Set by from_md_dirs; None in from_json_obj (test/programmatic) mode. - # Stores the manifest root dirs so load_for_agent can locate files later. - home_md: Path | None = field(default=None) - cwd_md: Path | None = field(default=None) - - @classmethod - def resolve(cls, cwd: str, *, missing_ok: bool = False) -> "ManifestIndex": - """Walk the per-file manifest tree and build a ManifestIndex. - - Layout (PRD 0011): - $HOME/.bot-bottle/bottles/.md — bottles (home-only) - $HOME/.bot-bottle/agents/.md — home agents - $CWD/.bot-bottle/agents/.md — cwd agents - - Cwd agents merge into the home agents on the same name - (cwd wins). A bottles/ subdir under $CWD is logged as a - warning and ignored — the filesystem layout IS the trust - boundary. - - If `missing_ok` is true, a missing `$HOME/.bot-bottle/` - returns an empty index instead of dying. This is for - passive UI surfaces like the dashboard, which can still - monitor already-running agents without launch config. - - If `bot-bottle.json` exists alongside a missing - `.bot-bottle/` directory at either side, dies with a - clear pointer at the README's manifest section — the - manifest format changed in PRD 0011 and we don't silently - fall back.""" - home_dir = Path(os.environ["HOME"]) - cwd_dir = Path(cwd) - home_md = home_dir / ".bot-bottle" - cwd_md = cwd_dir / ".bot-bottle" - - check_stale_json(home_dir, home_md, "$HOME") - if cwd_dir.resolve() != home_dir.resolve(): - check_stale_json(cwd_dir, cwd_md, "$CWD") - - if not home_md.is_dir(): - if missing_ok: - return cls.from_json_obj({"bottles": {}, "agents": {}}) - raise ManifestError( - f"no manifest found: {home_md} does not exist. " - f"See README.md for the per-file Markdown layout " - f"(PRD 0011)." - ) - - # When CWD == HOME (running from $HOME directly), pass the - # same dir for both — _load_md_dirs will dedupe. - cwd_md_arg = cwd_md if cwd_md.is_dir() and cwd_dir.resolve() != home_dir.resolve() else None - return cls.from_md_dirs(home_md, cwd_md_arg) - - @classmethod - def from_md_dirs( - cls, - home_dir: Path, - cwd_dir: Path | None, - ) -> "ManifestIndex": - """Return a names-only ManifestIndex. No file content is read; only - filenames are scanned for the agent selector. Full parsing happens - later, per-agent, via `load_for_agent`. - - A `bottles/` subdir under `cwd_dir` is logged as a warning and - ignored — the filesystem layout IS the trust boundary. - - Used by tests to build a ManifestIndex from fixture directories - without touching `os.environ`.""" - if cwd_dir is not None: - stale_bottles = cwd_dir / "bottles" - if stale_bottles.is_dir(): - files = sorted(stale_bottles.glob("*.md")) - if files: - names = ", ".join(p.name for p in files) - warn( - f"ignoring bottle file(s) under " - f"{stale_bottles}: {names}. Bottles can only " - f"live under $HOME/.bot-bottle/bottles/ " - f"(PRD 0011). Move them or delete." - ) - return cls(bottles={}, agents={}, home_md=home_dir, cwd_md=cwd_dir) - - @classmethod - def from_json_obj(cls, obj: object) -> "ManifestIndex": - """Validate and build a ManifestIndex from a raw JSON-like dict.""" - d = as_json_object(obj, "manifest") - raw_bottles_obj = _section_dict(d.get("bottles"), "manifest 'bottles'") - raw_agents = _section_dict(d.get("agents"), "manifest 'agents'") - - # Coerce each bottle's raw to dict[str, object] so the - # PRD 0025 resolver can apply extends-merge rules - # consistently with the md-loader path. - raw_bottles: dict[str, dict[str, object]] = {} - for n, b in raw_bottles_obj.items(): - raw_bottles[n] = as_json_object(b, f"bottle '{n}'") - - bottles = resolve_bottles(raw_bottles) - - bottle_names = set(bottles.keys()) - agents: dict[str, ManifestAgent] = { - n: ManifestAgent.from_dict(n, a, bottle_names) for n, a in raw_agents.items() - } - return cls(bottles=bottles, agents=agents) - - @property - def all_bottle_names(self) -> list[str]: - """Sorted list of all discoverable bottle names. - - In names-only mode (from resolve/from_md_dirs) this scans bottle - filenames without reading their content. In eager mode (from - from_json_obj) it returns the pre-parsed bottles' names.""" - if self.home_md is not None: - return scan_bottle_names(self.home_md / "bottles") - return sorted(self.bottles.keys()) - - @property - def all_agent_names(self) -> list[str]: - """Sorted list of all discoverable agent names. - - In names-only mode (from resolve/from_md_dirs) this scans agent - filenames without reading their content. In eager mode (from - from_json_obj) it returns the pre-parsed agents' names.""" - if self.home_md is not None: - home_names = set(scan_agent_names(self.home_md / "agents").keys()) - cwd_names: set[str] = set() - if self.cwd_md is not None: - cwd_names = set(scan_agent_names(self.cwd_md / "agents").keys()) - return sorted(home_names | cwd_names) - return sorted(self.agents.keys()) - - def load_for_agent( - self, - agent_name: str, - bottle_names: "tuple[str, ...] | None" = None, - ) -> "Manifest": - """Parse the named agent and its bottle; return a single-value Manifest. - - `bottle_names` is an ordered list of bottles selected at launch time. - When non-empty they are resolved and merged in order (index 0 = base; - later entries override). When empty or None, falls back to the agent's - own `bottle:` field. Raises ManifestError when neither is set. - - In lazy mode (from resolve/from_md_dirs) the agent file and its - bottle chain are read from disk for the first time here. In eager - mode (from_json_obj) the data is already parsed; this just filters - down to the requested agent and its bottle. - - The returned Manifest.bottle has the agent's git-gate.user already - overlaid (agent wins on non-empty, per-field). - - Always raises ManifestError if the agent is unknown or invalid. - Backends call this at preflight inside _validate.""" - effective_bottle_names: tuple[str, ...] = bottle_names or () - if self.home_md is None: - return self._load_for_agent_eager(agent_name, effective_bottle_names) - return self._load_for_agent_lazy(agent_name, effective_bottle_names) - - def _load_for_agent_eager( - self, agent_name: str, bottle_names: tuple[str, ...] - ) -> "Manifest": - """Eager path (from_json_obj): data is already parsed; filter to the one - requested agent and its bottle so the returned Manifest always holds - exactly one agent and one bottle regardless of path.""" - if agent_name not in self.agents: - available = ", ".join(sorted(self.agents.keys())) or "(none)" - raise ManifestError( - f"agent '{agent_name}' not defined. Available: {available}" - ) - agent = self.agents[agent_name] - raw_bottle = _resolve_effective_bottle_eager( - agent_name, agent, bottle_names, self.bottles - ) - return _manifest_with_merged_git_user(agent, raw_bottle) - - def _load_for_agent_lazy( - self, agent_name: str, bottle_names: tuple[str, ...] - ) -> "Manifest": - """Lazy path (resolve/from_md_dirs): read and parse the agent file and - its bottle chain from disk for the first time here.""" - assert self.home_md is not None # guaranteed by load_for_agent dispatch - # Locate the agent file; cwd wins over home on name collision. - home_agents = scan_agent_names(self.home_md / "agents") - cwd_agents: dict[str, Path] = {} - if self.cwd_md is not None: - cwd_agents = scan_agent_names(self.cwd_md / "agents") - merged_agents = {**home_agents, **cwd_agents} - - if agent_name not in merged_agents: - available = ", ".join(sorted(merged_agents.keys())) or "(none)" - raise ManifestError( - f"agent '{agent_name}' not defined. Available: {available}" - ) - - agent_path = merged_agents[agent_name] - try: - fm, body = parse_frontmatter(agent_path.read_text()) - except OSError as e: - raise ManifestError(f"could not read {agent_path}: {e}") from e - except YamlSubsetError as e: - raise ManifestError(f"{agent_path}: {e}") from e - - validate_agent_frontmatter_keys(agent_path, fm.keys()) - - # Determine the effective bottle name(s). - agent_bottle = fm.get("bottle") or "" - bottles_dir = self.home_md / "bottles" - raw_bottle = _resolve_effective_bottle_lazy( - agent_name, str(agent_bottle), bottle_names, bottles_dir - ) - effective_bottle_name = ( - bottle_names[-1] if bottle_names else str(agent_bottle) - ) - - # Build and validate the full ManifestAgent. - agent_dict: dict[str, object] = { - "skills": fm.get("skills", []), - "prompt": body.strip(), - } - if agent_bottle: - agent_dict["bottle"] = agent_bottle - if "git-gate" in fm: - agent_dict["git-gate"] = fm["git-gate"] - # Pass the effective bottle name as the known-bottles set so agents - # that have bottle: set are validated; agents without bottle: pass {} - # since bottle_names were already resolved above. - known = {effective_bottle_name} if effective_bottle_name else set() - agent = ManifestAgent.from_dict(agent_name, agent_dict, known) - - return _manifest_with_merged_git_user(agent, raw_bottle) - - def has_agent(self, name: str) -> bool: - return name in self.agents - - def require_agent(self, name: str) -> None: - """Check that `name` is a discoverable agent. In names-only mode - this checks whether the .md file exists; in eager mode it checks - the pre-parsed agents dict. Does NOT parse file content.""" - if self.has_agent(name): - return - if self.home_md is not None: - # Names-only mode: check file existence without parsing. - home_path = self.home_md / "agents" / f"{name}.md" - cwd_path = ( - self.cwd_md / "agents" / f"{name}.md" - if self.cwd_md else None - ) - if home_path.is_file() or (cwd_path and cwd_path.is_file()): - return - available = ", ".join(self.all_agent_names) or "(none)" - raise ManifestError( - f"agent '{name}' not defined. Available: {available}" - ) diff --git a/bot_bottle/manifest/index.py b/bot_bottle/manifest/index.py new file mode 100644 index 0000000..35966b4 --- /dev/null +++ b/bot_bottle/manifest/index.py @@ -0,0 +1,413 @@ +"""The `Manifest` / `ManifestIndex` aggregate model. + +The parsed manifest document (`Manifest`) and the multi-agent index over it +(`ManifestIndex`), plus the bottle-resolution helpers that stitch the pieces +(`agent`, `bottle`, `egress`, `git`, `extends`, `loader`, `schema`) into an +effective manifest. This is the heavy aggregate — the thin package `__init__` +re-exports `Manifest` / `ManifestIndex` lazily. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field, replace +from pathlib import Path +from typing import Mapping + +from ..log import warn +from .util import ManifestError, as_json_object +from .agent import ManifestAgent, ManifestAgentProvider +from .bottle import ManifestBottle +from .egress import ( + EGRESS_AUTH_SCHEMES, + ManifestEgressConfig, + ManifestEgressRoute, +) +from .extends import merge_bottles_runtime, resolve_bottles +from .git import ManifestGitEntry, ManifestGitUser, ManifestKeyConfig +from .loader import ( + check_stale_json, + load_bottle_chain_from_dir, + scan_agent_names, + scan_bottle_names, +) +from .schema import validate_agent_frontmatter_keys +from ..yaml_subset import YamlSubsetError, parse_frontmatter + + +def _section_dict(value: object, label: str) -> dict[str, object]: + """Like as_json_object but treats absent/null as an empty section.""" + if value is None: + return {} + return as_json_object(value, label) + + +def _merge_git_user( + agent_user: ManifestGitUser, base_user: ManifestGitUser +) -> ManifestGitUser: + """Merge the agent's git.user over the bottle's, agent-wins-on-non-empty.""" + if agent_user.is_empty(): + return base_user + return ManifestGitUser( + name=agent_user.name or base_user.name, + email=agent_user.email or base_user.email, + ) + + +def _manifest_with_merged_git_user( + agent: "ManifestAgent", raw_bottle: "ManifestBottle" +) -> "Manifest": + """Build the single-value Manifest, overlaying the agent's git-gate.user + onto the bottle (agent wins on non-empty, per-field). Shared by the eager + and lazy load_for_agent paths.""" + merged = _merge_git_user(agent.git_user, raw_bottle.git_user) + bottle = ( + raw_bottle if merged == raw_bottle.git_user + else replace(raw_bottle, git_user=merged) + ) + return Manifest(agent=agent, bottle=bottle) + + +def _resolve_effective_bottle_eager( + agent_name: str, + agent: "ManifestAgent", + bottle_names: "tuple[str, ...]", + bottles: "Mapping[str, ManifestBottle]", +) -> "ManifestBottle": + """Return the effective ManifestBottle for the eager (from_json_obj) path. + + When bottle_names is non-empty they are merged in order. When empty, falls + back to agent.bottle. Raises ManifestError when neither is set.""" + if bottle_names: + resolved: list[ManifestBottle] = [] + for bn in bottle_names: + if bn not in bottles: + available = ", ".join(sorted(bottles.keys())) or "(none)" + raise ManifestError( + f"bottle '{bn}' not defined. Available: {available}" + ) + resolved.append(bottles[bn]) + return merge_bottles_runtime(resolved) + + if not agent.bottle: + raise ManifestError( + f"agent '{agent_name}' has no 'bottle' field and no bottles were " + f"selected at launch. Select at least one bottle or add " + f"'bottle: ' to the agent manifest." + ) + return bottles[agent.bottle] + + +def _resolve_effective_bottle_lazy( + agent_name: str, + agent_bottle: str, + bottle_names: "tuple[str, ...]", + bottles_dir: "Path", +) -> "ManifestBottle": + """Return the effective ManifestBottle for the lazy (from_md_dirs) path. + + When bottle_names is non-empty they are resolved from disk and merged in + order. When empty, falls back to agent_bottle. Raises ManifestError when + neither is set.""" + if bottle_names: + resolved = [load_bottle_chain_from_dir(bn, bottles_dir) for bn in bottle_names] + return merge_bottles_runtime(resolved) + + if not agent_bottle: + raise ManifestError( + f"agent '{agent_name}' has no 'bottle' field and no bottles were " + f"selected at launch. Select at least one bottle or add " + f"'bottle: ' to the agent manifest." + ) + return load_bottle_chain_from_dir(agent_bottle, bottles_dir) + + +@dataclass(frozen=True) +class Manifest: + """Single-agent/bottle value type. Returned by ManifestIndex.load_for_agent(). + + `bottle` is the effective bottle with the agent's git-gate.user already + overlaid per-field (agent wins on non-empty). Backends and provisioners + use this directly — no agent_name lookup needed.""" + + agent: ManifestAgent + bottle: ManifestBottle + + def git_identity_summary(self) -> str | None: + """One-line effective git identity with per-field provenance, e.g. + `name=claude (agent), email=eric@dideric.is (bottle)`. + Returns None when neither agent nor bottle sets an identity.""" + over = self.agent.git_user # agent's declared git_user (pre-merge) + merged = self.bottle.git_user # effective git_user (post-merge) + if merged.is_empty(): + return None + parts: list[str] = [] + if merged.name: + parts.append(f"name={merged.name} ({'agent' if over.name else 'bottle'})") + if merged.email: + parts.append(f"email={merged.email} ({'agent' if over.email else 'bottle'})") + return ", ".join(parts) + + +@dataclass(frozen=True) +class ManifestIndex: + """Multi-agent/bottle collection. The pre-preflight form. + + In lazy mode (from resolve()/from_md_dirs()) only filenames are scanned; + no file content is read. In eager mode (from from_json_obj()) all agents + and bottles are pre-parsed. Call load_for_agent() to get a single-value + Manifest ready for backend use.""" + + bottles: Mapping[str, ManifestBottle] + agents: Mapping[str, ManifestAgent] + # Set by from_md_dirs; None in from_json_obj (test/programmatic) mode. + # Stores the manifest root dirs so load_for_agent can locate files later. + home_md: Path | None = field(default=None) + cwd_md: Path | None = field(default=None) + + @classmethod + def resolve(cls, cwd: str, *, missing_ok: bool = False) -> "ManifestIndex": + """Walk the per-file manifest tree and build a ManifestIndex. + + Layout (PRD 0011): + $HOME/.bot-bottle/bottles/.md — bottles (home-only) + $HOME/.bot-bottle/agents/.md — home agents + $CWD/.bot-bottle/agents/.md — cwd agents + + Cwd agents merge into the home agents on the same name + (cwd wins). A bottles/ subdir under $CWD is logged as a + warning and ignored — the filesystem layout IS the trust + boundary. + + If `missing_ok` is true, a missing `$HOME/.bot-bottle/` + returns an empty index instead of dying. This is for + passive UI surfaces like the dashboard, which can still + monitor already-running agents without launch config. + + If `bot-bottle.json` exists alongside a missing + `.bot-bottle/` directory at either side, dies with a + clear pointer at the README's manifest section — the + manifest format changed in PRD 0011 and we don't silently + fall back.""" + home_dir = Path(os.environ["HOME"]) + cwd_dir = Path(cwd) + home_md = home_dir / ".bot-bottle" + cwd_md = cwd_dir / ".bot-bottle" + + check_stale_json(home_dir, home_md, "$HOME") + if cwd_dir.resolve() != home_dir.resolve(): + check_stale_json(cwd_dir, cwd_md, "$CWD") + + if not home_md.is_dir(): + if missing_ok: + return cls.from_json_obj({"bottles": {}, "agents": {}}) + raise ManifestError( + f"no manifest found: {home_md} does not exist. " + f"See README.md for the per-file Markdown layout " + f"(PRD 0011)." + ) + + # When CWD == HOME (running from $HOME directly), pass the + # same dir for both — _load_md_dirs will dedupe. + cwd_md_arg = cwd_md if cwd_md.is_dir() and cwd_dir.resolve() != home_dir.resolve() else None + return cls.from_md_dirs(home_md, cwd_md_arg) + + @classmethod + def from_md_dirs( + cls, + home_dir: Path, + cwd_dir: Path | None, + ) -> "ManifestIndex": + """Return a names-only ManifestIndex. No file content is read; only + filenames are scanned for the agent selector. Full parsing happens + later, per-agent, via `load_for_agent`. + + A `bottles/` subdir under `cwd_dir` is logged as a warning and + ignored — the filesystem layout IS the trust boundary. + + Used by tests to build a ManifestIndex from fixture directories + without touching `os.environ`.""" + if cwd_dir is not None: + stale_bottles = cwd_dir / "bottles" + if stale_bottles.is_dir(): + files = sorted(stale_bottles.glob("*.md")) + if files: + names = ", ".join(p.name for p in files) + warn( + f"ignoring bottle file(s) under " + f"{stale_bottles}: {names}. Bottles can only " + f"live under $HOME/.bot-bottle/bottles/ " + f"(PRD 0011). Move them or delete." + ) + return cls(bottles={}, agents={}, home_md=home_dir, cwd_md=cwd_dir) + + @classmethod + def from_json_obj(cls, obj: object) -> "ManifestIndex": + """Validate and build a ManifestIndex from a raw JSON-like dict.""" + d = as_json_object(obj, "manifest") + raw_bottles_obj = _section_dict(d.get("bottles"), "manifest 'bottles'") + raw_agents = _section_dict(d.get("agents"), "manifest 'agents'") + + # Coerce each bottle's raw to dict[str, object] so the + # PRD 0025 resolver can apply extends-merge rules + # consistently with the md-loader path. + raw_bottles: dict[str, dict[str, object]] = {} + for n, b in raw_bottles_obj.items(): + raw_bottles[n] = as_json_object(b, f"bottle '{n}'") + + bottles = resolve_bottles(raw_bottles) + + bottle_names = set(bottles.keys()) + agents: dict[str, ManifestAgent] = { + n: ManifestAgent.from_dict(n, a, bottle_names) for n, a in raw_agents.items() + } + return cls(bottles=bottles, agents=agents) + + @property + def all_bottle_names(self) -> list[str]: + """Sorted list of all discoverable bottle names. + + In names-only mode (from resolve/from_md_dirs) this scans bottle + filenames without reading their content. In eager mode (from + from_json_obj) it returns the pre-parsed bottles' names.""" + if self.home_md is not None: + return scan_bottle_names(self.home_md / "bottles") + return sorted(self.bottles.keys()) + + @property + def all_agent_names(self) -> list[str]: + """Sorted list of all discoverable agent names. + + In names-only mode (from resolve/from_md_dirs) this scans agent + filenames without reading their content. In eager mode (from + from_json_obj) it returns the pre-parsed agents' names.""" + if self.home_md is not None: + home_names = set(scan_agent_names(self.home_md / "agents").keys()) + cwd_names: set[str] = set() + if self.cwd_md is not None: + cwd_names = set(scan_agent_names(self.cwd_md / "agents").keys()) + return sorted(home_names | cwd_names) + return sorted(self.agents.keys()) + + def load_for_agent( + self, + agent_name: str, + bottle_names: "tuple[str, ...] | None" = None, + ) -> "Manifest": + """Parse the named agent and its bottle; return a single-value Manifest. + + `bottle_names` is an ordered list of bottles selected at launch time. + When non-empty they are resolved and merged in order (index 0 = base; + later entries override). When empty or None, falls back to the agent's + own `bottle:` field. Raises ManifestError when neither is set. + + In lazy mode (from resolve/from_md_dirs) the agent file and its + bottle chain are read from disk for the first time here. In eager + mode (from_json_obj) the data is already parsed; this just filters + down to the requested agent and its bottle. + + The returned Manifest.bottle has the agent's git-gate.user already + overlaid (agent wins on non-empty, per-field). + + Always raises ManifestError if the agent is unknown or invalid. + Backends call this at preflight inside _validate.""" + effective_bottle_names: tuple[str, ...] = bottle_names or () + if self.home_md is None: + return self._load_for_agent_eager(agent_name, effective_bottle_names) + return self._load_for_agent_lazy(agent_name, effective_bottle_names) + + def _load_for_agent_eager( + self, agent_name: str, bottle_names: tuple[str, ...] + ) -> "Manifest": + """Eager path (from_json_obj): data is already parsed; filter to the one + requested agent and its bottle so the returned Manifest always holds + exactly one agent and one bottle regardless of path.""" + if agent_name not in self.agents: + available = ", ".join(sorted(self.agents.keys())) or "(none)" + raise ManifestError( + f"agent '{agent_name}' not defined. Available: {available}" + ) + agent = self.agents[agent_name] + raw_bottle = _resolve_effective_bottle_eager( + agent_name, agent, bottle_names, self.bottles + ) + return _manifest_with_merged_git_user(agent, raw_bottle) + + def _load_for_agent_lazy( + self, agent_name: str, bottle_names: tuple[str, ...] + ) -> "Manifest": + """Lazy path (resolve/from_md_dirs): read and parse the agent file and + its bottle chain from disk for the first time here.""" + assert self.home_md is not None # guaranteed by load_for_agent dispatch + # Locate the agent file; cwd wins over home on name collision. + home_agents = scan_agent_names(self.home_md / "agents") + cwd_agents: dict[str, Path] = {} + if self.cwd_md is not None: + cwd_agents = scan_agent_names(self.cwd_md / "agents") + merged_agents = {**home_agents, **cwd_agents} + + if agent_name not in merged_agents: + available = ", ".join(sorted(merged_agents.keys())) or "(none)" + raise ManifestError( + f"agent '{agent_name}' not defined. Available: {available}" + ) + + agent_path = merged_agents[agent_name] + try: + fm, body = parse_frontmatter(agent_path.read_text()) + except OSError as e: + raise ManifestError(f"could not read {agent_path}: {e}") from e + except YamlSubsetError as e: + raise ManifestError(f"{agent_path}: {e}") from e + + validate_agent_frontmatter_keys(agent_path, fm.keys()) + + # Determine the effective bottle name(s). + agent_bottle = fm.get("bottle") or "" + bottles_dir = self.home_md / "bottles" + raw_bottle = _resolve_effective_bottle_lazy( + agent_name, str(agent_bottle), bottle_names, bottles_dir + ) + effective_bottle_name = ( + bottle_names[-1] if bottle_names else str(agent_bottle) + ) + + # Build and validate the full ManifestAgent. + agent_dict: dict[str, object] = { + "skills": fm.get("skills", []), + "prompt": body.strip(), + } + if agent_bottle: + agent_dict["bottle"] = agent_bottle + if "git-gate" in fm: + agent_dict["git-gate"] = fm["git-gate"] + # Pass the effective bottle name as the known-bottles set so agents + # that have bottle: set are validated; agents without bottle: pass {} + # since bottle_names were already resolved above. + known = {effective_bottle_name} if effective_bottle_name else set() + agent = ManifestAgent.from_dict(agent_name, agent_dict, known) + + return _manifest_with_merged_git_user(agent, raw_bottle) + + def has_agent(self, name: str) -> bool: + return name in self.agents + + def require_agent(self, name: str) -> None: + """Check that `name` is a discoverable agent. In names-only mode + this checks whether the .md file exists; in eager mode it checks + the pre-parsed agents dict. Does NOT parse file content.""" + if self.has_agent(name): + return + if self.home_md is not None: + # Names-only mode: check file existence without parsing. + home_path = self.home_md / "agents" / f"{name}.md" + cwd_path = ( + self.cwd_md / "agents" / f"{name}.md" + if self.cwd_md else None + ) + if home_path.is_file() or (cwd_path and cwd_path.is_file()): + return + available = ", ".join(self.all_agent_names) or "(none)" + raise ManifestError( + f"agent '{name}' not defined. Available: {available}" + ) -- 2.52.0 From 450037b7e9c04f3b59f3093e0e10a674f086f8c5 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 16:27:56 -0400 Subject: [PATCH 21/39] refactor(git-gate): make GitGate a service class in a git_gate package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turn the git_gate module into a package with GitGate as a concrete service class (dropping the ABC), mirroring the Supervisor shape. The host-side git-gate operations are now methods the backend drives: git_gate/ __init__.py — thin __getattr__ facade (keeps `from bot_bottle.git_gate import …` working; render/hook names lazily forwarded to gateway.git_gate_render) plan.py — GitGatePlan (the launch DTO the backend contract references) service.py — GitGate: prepare / provision_dynamic_keys / revoke_provisioned_keys / preflight_host_keys provision.py — deploy-key lifecycle (was git_gate_provision.py) host_key.py — host-key preflight (was git_gate_host_key.py) The backend launch + base.py preflight now call GitGate() methods instead of the free functions. The runtime rendering / in-gateway hook execution stay in gateway/git_gate_render.py (data plane) — only the host-side service moved. Because the facade forwards names lazily, the ~25 `from bot_bottle.git_gate import GitGatePlan` call-sites are unchanged, and importing git_gate.plan (the contract's dependency) no longer drags in the provisioning / forge-API code. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/backend/base.py | 4 +- bot_bottle/backend/docker/launch.py | 9 +- bot_bottle/backend/firecracker/launch.py | 9 +- bot_bottle/backend/macos_container/launch.py | 9 +- bot_bottle/gateway/git_gate_render.py | 6 +- bot_bottle/git_gate.py | 169 ------------------ bot_bottle/git_gate/__init__.py | 98 ++++++++++ .../host_key.py} | 6 +- bot_bottle/git_gate/plan.py | 36 ++++ .../provision.py} | 16 +- bot_bottle/git_gate/service.py | 114 ++++++++++++ tests/unit/test_git_gate.py | 6 +- tests/unit/test_git_gate_host_key.py | 38 ++-- 13 files changed, 295 insertions(+), 225 deletions(-) delete mode 100644 bot_bottle/git_gate.py create mode 100644 bot_bottle/git_gate/__init__.py rename bot_bottle/{git_gate_host_key.py => git_gate/host_key.py} (98%) create mode 100644 bot_bottle/git_gate/plan.py rename bot_bottle/{git_gate_provision.py => git_gate/provision.py} (93%) create mode 100644 bot_bottle/git_gate/service.py diff --git a/bot_bottle/backend/base.py b/bot_bottle/backend/base.py index bb8a2ab..a58ad4f 100644 --- a/bot_bottle/backend/base.py +++ b/bot_bottle/backend/base.py @@ -309,8 +309,8 @@ class BottleBackend(ABC, Generic[PlanT, CleanupT]): self._preflight() - from ..git_gate_host_key import preflight_host_keys - manifest = preflight_host_keys( + from ..git_gate import GitGate + manifest = GitGate().preflight_host_keys( manifest, headless=spec.headless, home_md=spec.manifest.home_md, diff --git a/bot_bottle/backend/docker/launch.py b/bot_bottle/backend/docker/launch.py index 068b7c4..665fa5e 100644 --- a/bot_bottle/backend/docker/launch.py +++ b/bot_bottle/backend/docker/launch.py @@ -38,10 +38,7 @@ from typing import Callable, Generator from ...agent_provider import runtime_for from ...egress import egress_resolve_token_values -from ...git_gate import ( - provision_git_gate_dynamic_keys, - revoke_git_gate_provisioned_keys, -) +from ...git_gate import GitGate from ...image_cache import check_stale from ...log import die, info, warn from .. import BottleImages @@ -126,7 +123,7 @@ def launch( f"teardown failed for container {plan.container_name}" f" (compose-down): {exc!r}" ) - revoke_git_gate_provisioned_keys( + GitGate().revoke_provisioned_keys( _bottle_for_revoke, _git_gate_dir_for_revoke ) @@ -135,7 +132,7 @@ def launch( # provisioning the bottle's repos into the shared gateway. git_gate_plan = plan.git_gate_plan if git_gate_plan.upstreams: - git_gate_plan = provision_git_gate_dynamic_keys( + git_gate_plan = GitGate().provision_dynamic_keys( plan.manifest.bottle, git_gate_plan, git_gate_state_dir(plan.slug), ) diff --git a/bot_bottle/backend/firecracker/launch.py b/bot_bottle/backend/firecracker/launch.py index 37bfb2d..b5fc00d 100644 --- a/bot_bottle/backend/firecracker/launch.py +++ b/bot_bottle/backend/firecracker/launch.py @@ -42,10 +42,7 @@ from ...egress import ( egress_agent_env_entries, egress_resolve_token_values, ) -from ...git_gate import ( - provision_git_gate_dynamic_keys, - revoke_git_gate_provisioned_keys, -) +from ...git_gate import GitGate from ...image_cache import check_stale_path from ...log import die, info, warn from ...supervisor.types import SUPERVISE_PORT @@ -86,7 +83,7 @@ def launch( except BaseException as exc: # noqa: W0718 - teardown must continue teardown_exc = exc warn(f"firecracker teardown failed: {exc!r}") - revoke_git_gate_provisioned_keys(bottle_for_revoke, git_gate_dir_for_revoke) + GitGate().revoke_provisioned_keys(bottle_for_revoke, git_gate_dir_for_revoke) if teardown_exc is not None: raise teardown_exc @@ -97,7 +94,7 @@ def launch( # Step 2: mint the git-gate dynamic (gitea) deploy keys, if any. git_gate_plan = plan.git_gate_plan if git_gate_plan.upstreams: - git_gate_plan = provision_git_gate_dynamic_keys( + git_gate_plan = GitGate().provision_dynamic_keys( plan.manifest.bottle, git_gate_plan, git_gate_state_dir(plan.slug), ) diff --git a/bot_bottle/backend/macos_container/launch.py b/bot_bottle/backend/macos_container/launch.py index 95f5d61..bb2fc79 100644 --- a/bot_bottle/backend/macos_container/launch.py +++ b/bot_bottle/backend/macos_container/launch.py @@ -48,10 +48,7 @@ from ...egress import ( egress_agent_env_entries, egress_resolve_token_values, ) -from ...git_gate import ( - provision_git_gate_dynamic_keys, - revoke_git_gate_provisioned_keys, -) +from ...git_gate import GitGate from ...gateway.git_http_backend import DEFAULT_PORT as _GIT_HTTP_PORT from ...image_cache import check_stale from ...log import die, info, warn @@ -151,7 +148,7 @@ def launch( except BaseException as exc: # noqa: W0718 - teardown must continue teardown_exc = exc warn(f"macos-container teardown failed: {exc!r}") - revoke_git_gate_provisioned_keys(bottle_for_revoke, git_gate_dir_for_revoke) + GitGate().revoke_provisioned_keys(bottle_for_revoke, git_gate_dir_for_revoke) if teardown_exc is not None: raise teardown_exc @@ -282,7 +279,7 @@ def _provision_git_gate_keys( ) -> MacosContainerBottlePlan: if not plan.git_gate_plan.upstreams: return plan - git_gate_plan = provision_git_gate_dynamic_keys( + git_gate_plan = GitGate().provision_dynamic_keys( plan.manifest.bottle, plan.git_gate_plan, git_gate_state_dir(plan.slug), diff --git a/bot_bottle/gateway/git_gate_render.py b/bot_bottle/gateway/git_gate_render.py index d6fe825..67f055a 100644 --- a/bot_bottle/gateway/git_gate_render.py +++ b/bot_bottle/gateway/git_gate_render.py @@ -3,9 +3,9 @@ Builds the agent's `.gitconfig` insteadOf rewrites, the known_hosts line, and the entrypoint / pre-receive / access-hook scripts the gateway runs. No docker or forge calls — exposed for tests and reuse across -backends. Split out of `git_gate.py` so the control surface (`GitGate`) -and the deploy-key lifecycle (`git_gate_provision`) each read on their -own; `git_gate` re-exports these names for API stability.""" +backends. The host-side service (`git_gate.GitGate`) and the deploy-key +lifecycle (`git_gate.provision`) each read on their own; `git_gate` +re-exports these names for API stability.""" from __future__ import annotations diff --git a/bot_bottle/git_gate.py b/bot_bottle/git_gate.py deleted file mode 100644 index cfb8213..0000000 --- a/bot_bottle/git_gate.py +++ /dev/null @@ -1,169 +0,0 @@ -"""Per-agent git-gate (PRD 0008). - -A third per-agent daemon that fronts the bottle's declared git -upstreams as a transparent mirror. Each `bottle.git` entry maps to -a bare repo on the gate; `git daemon` serves the bare repos over -`git:///.git`. Two hooks make the mirror bidirectional: - -- **`pre-receive`** (push path) — gitleaks-scans incoming refs and, - on clean, forwards them to the real upstream with the - gate-resident credential. -- **`--access-hook`** (fetch path) — runs `git fetch origin --prune` - against the real upstream before every `upload-pack`, so an - agent fetch returns whatever the upstream has *now*. Fail-closed - if the upstream is unreachable. - -The agent never sees the upstream credential under either path. - -Why a separate daemon (not folded into egress or ssh-gate): the -gate is the only one of the three that holds upstream push -credentials. Mixing it with egress would put push creds in the -same blast radius as internet-facing TLS interception; mixing it -with ssh-gate would force ssh-gate above L4 and into git-protocol -land. See `docs/prds/0008-git-gate.md`. - -This module defines the abstract gate (`GitGate`) and its plan -dataclass (`GitGatePlan`). The gateway's start/stop lifecycle is -backend-specific and lives on concrete subclasses (see -`bot_bottle/backend/docker/git_gate.py`).""" - - -from __future__ import annotations - -from abc import ABC -from dataclasses import dataclass -from pathlib import Path - -from .manifest import ManifestBottle - -# Rendering and the deploy-key lifecycle live in sibling modules; the -# names are re-exported here (see __all__) so existing -# `from bot_bottle.git_gate import …` callers are unchanged. -from .gateway.git_gate_render import ( - GIT_GATE_HOSTNAME, - GIT_GATE_TIMEOUT_SECS, - GitGateUpstream, - git_gate_known_hosts_line, - git_gate_render_access_hook, - git_gate_render_entrypoint, - git_gate_render_provision, - git_gate_render_gitconfig, - git_gate_render_hook, - git_gate_upstreams_for_bottle, - _gitconfig_validate_value, -) -from .git_gate_provision import ( - provision_git_gate_dynamic_keys, - revoke_git_gate_provisioned_keys, - _provision_dynamic_key, - _resolve_identity_file, -) - -@dataclass(frozen=True) -class GitGatePlan: - """Output of GitGate.prepare; consumed by .start. - - The script + slug + upstream fields are filled at prepare time - (host-side, side-effect-free on docker). The network fields are - populated by the backend's launch step via `dataclasses.replace` - once those networks exist. Empty defaults are sentinels meaning - "not yet set"; `.start` validates that they are populated. - - `hook_script` is the shared `pre-receive` for push-time gating; - `access_hook_script` is `git daemon`'s `--access-hook` for the - fetch-time upstream refresh.""" - - slug: str - entrypoint_script: Path - hook_script: Path - access_hook_script: Path - upstreams: tuple[GitGateUpstream, ...] - internal_network: str = "" - egress_network: str = "" - - - - -class GitGate(ABC): - """The per-agent git-gate. Encapsulates the host-side prepare - (upstream lift + entrypoint/hook render); the gateway's - start/stop lifecycle is backend-specific and lives on concrete - subclasses.""" - - def prepare(self, bottle: ManifestBottle, slug: str, stage_dir: Path) -> GitGatePlan: - """Compute the upstream table from `bottle.git` and write the - entrypoint, pre-receive hook, and access-hook scripts (mode - 600) under `stage_dir`. Pure host-side, no docker subprocess. - - For `gitea` key entries, the returned upstream intentionally - has an empty identity file. Backend launch fills that in after - the operator confirms the preflight. - - Returned plan is incomplete: the launch step must fill - `internal_network` / `egress_network` via `dataclasses.replace` - before passing the plan to `.start`.""" - upstreams = git_gate_upstreams_for_bottle(bottle) - entrypoint = stage_dir / "git_gate_entrypoint.sh" - entrypoint.write_text(git_gate_render_entrypoint(upstreams)) - entrypoint.chmod(0o600) - hook = stage_dir / "git_gate_pre_receive.sh" - hook.write_text(git_gate_render_hook()) - hook.chmod(0o600) - access_hook = stage_dir / "git_gate_access_hook.sh" - access_hook.write_text(git_gate_render_access_hook()) - # 0o700 (not 0o600): git daemon execs --access-hook directly, - # not via `sh`, so the script needs the x bit. The gateway copy - # does not necessarily preserve this mode (`docker cp` does, the - # Apple `container cp` does not), so provision_git_gate re-applies - # +x on the gateway side — see backend/docker/gateway_provision.py. - access_hook.chmod(0o700) - upstreams_with_files: list[GitGateUpstream] = [] - for u in upstreams: - known_hosts_file = Path() - if u.known_host_key: - known_hosts_file = stage_dir / f"{u.name}-known_hosts" - known_hosts_file.write_text( - git_gate_known_hosts_line( - u.upstream_host, u.upstream_port, u.known_host_key, - ) - ) - known_hosts_file.chmod(0o600) - upstreams_with_files.append( - GitGateUpstream( - name=u.name, - upstream_url=u.upstream_url, - upstream_host=u.upstream_host, - upstream_port=u.upstream_port, - identity_file=u.identity_file, - known_host_key=u.known_host_key, - known_hosts_file=known_hosts_file, - ) - ) - return GitGatePlan( - slug=slug, - entrypoint_script=entrypoint, - hook_script=hook, - access_hook_script=access_hook, - upstreams=tuple(upstreams_with_files), - ) - - -__all__ = [ - "GIT_GATE_HOSTNAME", - "GIT_GATE_TIMEOUT_SECS", - "GitGateUpstream", - "GitGatePlan", - "GitGate", - "git_gate_upstreams_for_bottle", - "git_gate_render_gitconfig", - "git_gate_known_hosts_line", - "git_gate_render_entrypoint", - "git_gate_render_provision", - "git_gate_render_hook", - "git_gate_render_access_hook", - "provision_git_gate_dynamic_keys", - "revoke_git_gate_provisioned_keys", - "_gitconfig_validate_value", - "_provision_dynamic_key", - "_resolve_identity_file", -] diff --git a/bot_bottle/git_gate/__init__.py b/bot_bottle/git_gate/__init__.py new file mode 100644 index 0000000..c902a1f --- /dev/null +++ b/bot_bottle/git_gate/__init__.py @@ -0,0 +1,98 @@ +"""Per-agent git-gate (PRD 0008). + +The git-gate fronts a bottle's declared git upstreams as a transparent mirror: +a `pre-receive` hook gitleaks-scans pushes and forwards clean refs to the real +upstream with a gate-resident credential; an `--access-hook` refreshes from the +upstream before fetches. The agent never sees the upstream credential. + +Layout: + + * `service` — the `GitGate` host-side service (prepare / provision / revoke / + preflight) the backend drives at launch. + * `plan` — `GitGatePlan`, the launch DTO (in the backend contract). + * `provision`, `host_key` — the deploy-key + host-key host-side helpers the + service delegates to. + +The rendering + the in-gateway hook execution live in +`bot_bottle.gateway.git_gate_render`. The public names are re-exported lazily +via `__getattr__`, so `from bot_bottle.git_gate import …` keeps working and +importing `git_gate.plan` (the contract's dependency) doesn't drag in the +provisioning / forge-API code. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .service import GitGate + from .plan import GitGatePlan + from .provision import ( + provision_git_gate_dynamic_keys, + revoke_git_gate_provisioned_keys, + ) + from ..gateway.git_gate_render import ( + GIT_GATE_HOSTNAME, + GIT_GATE_TIMEOUT_SECS, + GitGateUpstream, + git_gate_known_hosts_line, + git_gate_render_access_hook, + git_gate_render_entrypoint, + git_gate_render_gitconfig, + git_gate_render_hook, + git_gate_render_provision, + git_gate_upstreams_for_bottle, + ) + + +# Public name -> relative module that defines it. Render/hook names come from +# the gateway (where the in-gateway execution lives); the service, plan, and +# provisioning are this package's own submodules. +_LAZY: dict[str, str] = { + "GitGate": ".service", + "GitGatePlan": ".plan", + "provision_git_gate_dynamic_keys": ".provision", + "revoke_git_gate_provisioned_keys": ".provision", + "_provision_dynamic_key": ".provision", + "_resolve_identity_file": ".provision", + "GIT_GATE_HOSTNAME": "..gateway.git_gate_render", + "GIT_GATE_TIMEOUT_SECS": "..gateway.git_gate_render", + "GitGateUpstream": "..gateway.git_gate_render", + "git_gate_upstreams_for_bottle": "..gateway.git_gate_render", + "git_gate_render_gitconfig": "..gateway.git_gate_render", + "git_gate_known_hosts_line": "..gateway.git_gate_render", + "git_gate_render_entrypoint": "..gateway.git_gate_render", + "git_gate_render_provision": "..gateway.git_gate_render", + "git_gate_render_hook": "..gateway.git_gate_render", + "git_gate_render_access_hook": "..gateway.git_gate_render", + "_gitconfig_validate_value": "..gateway.git_gate_render", +} + + +def __getattr__(name: str) -> Any: + src = _LAZY.get(name) + if src is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module + + value = getattr(import_module(src, __name__), name) + globals()[name] = value + return value + + +__all__ = [ + "GitGate", + "GitGatePlan", + "GitGateUpstream", + "GIT_GATE_HOSTNAME", + "GIT_GATE_TIMEOUT_SECS", + "git_gate_upstreams_for_bottle", + "git_gate_render_gitconfig", + "git_gate_known_hosts_line", + "git_gate_render_entrypoint", + "git_gate_render_provision", + "git_gate_render_hook", + "git_gate_render_access_hook", + "provision_git_gate_dynamic_keys", + "revoke_git_gate_provisioned_keys", +] diff --git a/bot_bottle/git_gate_host_key.py b/bot_bottle/git_gate/host_key.py similarity index 98% rename from bot_bottle/git_gate_host_key.py rename to bot_bottle/git_gate/host_key.py index 2210e8e..5c2ba78 100644 --- a/bot_bottle/git_gate_host_key.py +++ b/bot_bottle/git_gate/host_key.py @@ -16,9 +16,9 @@ import sys from pathlib import Path from typing import cast -from .log import die, info -from .manifest import Manifest -from .yaml_subset import YamlSubsetError, parse_frontmatter, serialize_yaml_subset +from ..log import die, info +from ..manifest import Manifest +from ..yaml_subset import YamlSubsetError, parse_frontmatter, serialize_yaml_subset # Preferred key types, most secure first. diff --git a/bot_bottle/git_gate/plan.py b/bot_bottle/git_gate/plan.py new file mode 100644 index 0000000..c5fe3bb --- /dev/null +++ b/bot_bottle/git_gate/plan.py @@ -0,0 +1,36 @@ +"""`GitGatePlan` — the git-gate launch plan (DTO). + +The output of `GitGate.prepare`, consumed by the backend launch step and the +`BottlePlan` contract. A pure value type (its `upstreams` element type +`GitGateUpstream` is the neutral render dataclass). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from ..gateway.git_gate_render import GitGateUpstream + + +@dataclass(frozen=True) +class GitGatePlan: + """Output of GitGate.prepare; consumed by .start. + + The script + slug + upstream fields are filled at prepare time (host-side, + side-effect-free on docker). The network fields are populated by the + backend's launch step via `dataclasses.replace` once those networks exist. + Empty defaults are sentinels meaning "not yet set"; `.start` validates that + they are populated. + + `hook_script` is the shared `pre-receive` for push-time gating; + `access_hook_script` is `git daemon`'s `--access-hook` for the fetch-time + upstream refresh.""" + + slug: str + entrypoint_script: Path + hook_script: Path + access_hook_script: Path + upstreams: tuple[GitGateUpstream, ...] + internal_network: str = "" + egress_network: str = "" diff --git a/bot_bottle/git_gate_provision.py b/bot_bottle/git_gate/provision.py similarity index 93% rename from bot_bottle/git_gate_provision.py rename to bot_bottle/git_gate/provision.py index ac9eebd..007abd3 100644 --- a/bot_bottle/git_gate_provision.py +++ b/bot_bottle/git_gate/provision.py @@ -13,14 +13,14 @@ import dataclasses from pathlib import Path from typing import TYPE_CHECKING -from .bottle_state import globalize_slug -from .errors import MissingEnvVarError -from .log import info -from .manifest import ManifestBottle, ManifestGitEntry -from .gateway.git_gate_render import GitGateUpstream +from ..bottle_state import globalize_slug +from ..errors import MissingEnvVarError +from ..log import info +from ..manifest import ManifestBottle, ManifestGitEntry +from ..gateway.git_gate_render import GitGateUpstream if TYPE_CHECKING: - from .git_gate import GitGatePlan + from .plan import GitGatePlan def _provision_dynamic_key( entry: ManifestGitEntry, @@ -32,7 +32,7 @@ def _provision_dynamic_key( Returns the host-side path to the private key file so the caller can inject it into the GitGateUpstream as `identity_file`.""" - from .deploy_key_provisioner import get_provisioner + from ..deploy_key_provisioner import get_provisioner pk = entry.Key token = os.environ.get(pk.forge_token_env) if token is None: @@ -70,7 +70,7 @@ def revoke_git_gate_provisioned_keys(bottle: ManifestBottle, stage_dir: Path) -> Called at teardown after containers stop. Raises if any revocation fails — a stranded key is a security concern that the operator must address manually.""" - from .deploy_key_provisioner import get_provisioner + from ..deploy_key_provisioner import get_provisioner for entry in bottle.git: if entry.Key.provider != "gitea": continue diff --git a/bot_bottle/git_gate/service.py b/bot_bottle/git_gate/service.py new file mode 100644 index 0000000..edd592b --- /dev/null +++ b/bot_bottle/git_gate/service.py @@ -0,0 +1,114 @@ +"""The `GitGate` host-side service (PRD 0008). + +The git-gate fronts a bottle's declared git upstreams as a transparent mirror: +a `pre-receive` hook gitleaks-scans pushes and forwards clean refs to the real +upstream with a gate-resident credential; an `--access-hook` refreshes from the +upstream before fetches. The agent never sees the upstream credential. + +`GitGate` is the host-side service the backend drives at launch: build the plan +(`prepare`), provision / revoke the per-upstream deploy keys, and preflight the +upstream host keys. The rendering it emits + the in-gateway hook execution live +in `bot_bottle.gateway.git_gate_render`. +""" + +from __future__ import annotations + +from pathlib import Path + +from ..manifest import Manifest, ManifestBottle +from ..gateway.git_gate_render import ( + GitGateUpstream, + git_gate_known_hosts_line, + git_gate_render_access_hook, + git_gate_render_entrypoint, + git_gate_render_hook, + git_gate_upstreams_for_bottle, +) +from .plan import GitGatePlan +from . import provision as _provision +from . import host_key as _host_key + + +class GitGate: + """The per-agent git-gate host-side service. Stateless — the backend's + launch step drives `prepare` → `provision_dynamic_keys` and, at teardown, + `revoke_provisioned_keys`; `preflight_host_keys` gates a launch on the + upstream host keys being known.""" + + def prepare(self, bottle: ManifestBottle, slug: str, stage_dir: Path) -> GitGatePlan: + """Compute the upstream table from `bottle.git` and write the + entrypoint, pre-receive hook, and access-hook scripts (mode 600) under + `stage_dir`. Pure host-side, no docker subprocess. + + For `gitea` key entries, the returned upstream intentionally has an + empty identity file. Backend launch fills that in after the operator + confirms the preflight. + + Returned plan is incomplete: the launch step must fill + `internal_network` / `egress_network` via `dataclasses.replace` before + passing the plan to `.start`.""" + upstreams = git_gate_upstreams_for_bottle(bottle) + entrypoint = stage_dir / "git_gate_entrypoint.sh" + entrypoint.write_text(git_gate_render_entrypoint(upstreams)) + entrypoint.chmod(0o600) + hook = stage_dir / "git_gate_pre_receive.sh" + hook.write_text(git_gate_render_hook()) + hook.chmod(0o600) + access_hook = stage_dir / "git_gate_access_hook.sh" + access_hook.write_text(git_gate_render_access_hook()) + # 0o700 (not 0o600): git daemon execs --access-hook directly, not via + # `sh`, so the script needs the x bit. The gateway copy does not + # necessarily preserve this mode (`docker cp` does, the Apple `container + # cp` does not), so provisioning re-applies +x on the gateway side — see + # backend/docker/gateway_provision.py. + access_hook.chmod(0o700) + upstreams_with_files: list[GitGateUpstream] = [] + for u in upstreams: + known_hosts_file = Path() + if u.known_host_key: + known_hosts_file = stage_dir / f"{u.name}-known_hosts" + known_hosts_file.write_text( + git_gate_known_hosts_line( + u.upstream_host, u.upstream_port, u.known_host_key, + ) + ) + known_hosts_file.chmod(0o600) + upstreams_with_files.append( + GitGateUpstream( + name=u.name, + upstream_url=u.upstream_url, + upstream_host=u.upstream_host, + upstream_port=u.upstream_port, + identity_file=u.identity_file, + known_host_key=u.known_host_key, + known_hosts_file=known_hosts_file, + ) + ) + return GitGatePlan( + slug=slug, + entrypoint_script=entrypoint, + hook_script=hook, + access_hook_script=access_hook, + upstreams=tuple(upstreams_with_files), + ) + + def provision_dynamic_keys( + self, bottle: ManifestBottle, plan: GitGatePlan, stage_dir: Path, + ) -> GitGatePlan: + """Mint the `gitea` upstreams' ephemeral deploy keys via the forge API + and return the plan with their identity files filled in.""" + return _provision.provision_git_gate_dynamic_keys(bottle, plan, stage_dir) + + def revoke_provisioned_keys(self, bottle: ManifestBottle, stage_dir: Path) -> None: + """Revoke every deploy key provisioned for `bottle` (teardown).""" + _provision.revoke_git_gate_provisioned_keys(bottle, stage_dir) + + def preflight_host_keys( + self, manifest: Manifest, *, headless: bool, home_md: Path | None, + ) -> Manifest: + """Ensure every git-gate upstream has a known host key, populating + missing ones (or erroring in headless mode). Returns the manifest, + possibly updated with fetched keys.""" + return _host_key.preflight_host_keys( + manifest, headless=headless, home_md=home_md, + ) diff --git a/tests/unit/test_git_gate.py b/tests/unit/test_git_gate.py index 4befb5d..bf7a891 100644 --- a/tests/unit/test_git_gate.py +++ b/tests/unit/test_git_gate.py @@ -385,13 +385,13 @@ class TestDynamicKeyProvisioning(unittest.TestCase): def test_resolve_identity_file_gitea_provisions_key(self): entry = self._gitea_manifest().bottles["dev"].git[0] - with patch("bot_bottle.git_gate_provision._provision_dynamic_key", return_value="/tmp/provisioned-key") as mock_provision: + with patch("bot_bottle.git_gate.provision._provision_dynamic_key", return_value="/tmp/provisioned-key") as mock_provision: self.assertEqual("/tmp/provisioned-key", _resolve_identity_file(entry, "demo", self.stage)) mock_provision.assert_called_once() def test_prepare_defers_gitea_key_provisioning(self): bottle = self._gitea_manifest().bottles["dev"] - with patch("bot_bottle.git_gate_provision._provision_dynamic_key") as mock_provision: + with patch("bot_bottle.git_gate.provision._provision_dynamic_key") as mock_provision: plan = _StubGate().prepare(bottle, "demo", self.stage) mock_provision.assert_not_called() @@ -402,7 +402,7 @@ class TestDynamicKeyProvisioning(unittest.TestCase): plan = _StubGate().prepare(bottle, "demo", self.stage) with patch( - "bot_bottle.git_gate_provision._provision_dynamic_key", + "bot_bottle.git_gate.provision._provision_dynamic_key", return_value="/tmp/provisioned-key", ) as mock_provision: updated = provision_git_gate_dynamic_keys(bottle, plan, self.stage) diff --git a/tests/unit/test_git_gate_host_key.py b/tests/unit/test_git_gate_host_key.py index 3835d14..4a6541c 100644 --- a/tests/unit/test_git_gate_host_key.py +++ b/tests/unit/test_git_gate_host_key.py @@ -7,7 +7,7 @@ import unittest from pathlib import Path from unittest.mock import MagicMock, patch -from bot_bottle.git_gate_host_key import ( +from bot_bottle.git_gate.host_key import ( add_host_key_to_frontmatter, find_and_update_bottle_file, find_repo_bottle_file, @@ -358,7 +358,7 @@ git-gate: with tempfile.TemporaryDirectory() as d: path = self._write_bottle(d, "dev", content) with patch( - "bot_bottle.git_gate_host_key.find_repo_bottle_file", + "bot_bottle.git_gate.host_key.find_repo_bottle_file", return_value=path, ), patch.object(Path, "read_text", side_effect=OSError("Permission denied")): result = find_and_update_bottle_file(Path(d), "myrepo", "ssh-ed25519 AAAA") @@ -382,7 +382,7 @@ git-gate: with tempfile.TemporaryDirectory() as d: path = self._write_bottle(d, "dev", content) with patch( - "bot_bottle.git_gate_host_key.add_host_key_to_frontmatter", + "bot_bottle.git_gate.host_key.add_host_key_to_frontmatter", return_value=content, # no change ): result = find_and_update_bottle_file(Path(d), "myrepo", "ssh-ed25519 AAAA") @@ -467,10 +467,10 @@ git-gate: mock_prompt = MagicMock(side_effect=["y", "n"]) # confirm key, skip save with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", return_value="ssh-ed25519 FETCHED", ), patch( - "bot_bottle.git_gate_host_key.prompt_tty", mock_prompt, + "bot_bottle.git_gate.host_key.prompt_tty", mock_prompt, ), patch("sys.stderr"): preflight_host_keys(manifest, headless=False, home_md=Path(home)) @@ -520,10 +520,10 @@ class TestPreflightHostKeys(unittest.TestCase): manifest = _manifest_with_git() with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", return_value="ssh-ed25519 FETCHED", ), patch( - "bot_bottle.git_gate_host_key.prompt_tty", + "bot_bottle.git_gate.host_key.prompt_tty", side_effect=["y", "n"], # confirm key, don't persist ), patch("sys.stderr"): result = preflight_host_keys(manifest, headless=False, home_md=None) @@ -534,10 +534,10 @@ class TestPreflightHostKeys(unittest.TestCase): manifest = _manifest_with_git() with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", return_value="ssh-ed25519 FETCHED", ), patch( - "bot_bottle.git_gate_host_key.prompt_tty", + "bot_bottle.git_gate.host_key.prompt_tty", return_value="n", ), patch("sys.stderr"), self.assertRaises(SystemExit): preflight_host_keys(manifest, headless=False, home_md=None) @@ -557,10 +557,10 @@ class TestPreflightHostKeys(unittest.TestCase): ) with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", return_value="ssh-ed25519 FETCHED", ), patch( - "bot_bottle.git_gate_host_key.prompt_tty", + "bot_bottle.git_gate.host_key.prompt_tty", side_effect=["y", "y"], # confirm key, yes persist ), patch("sys.stderr"): result = preflight_host_keys( @@ -586,10 +586,10 @@ class TestPreflightHostKeys(unittest.TestCase): ) with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", return_value="ssh-ed25519 FETCHED", ), patch( - "bot_bottle.git_gate_host_key.prompt_tty", + "bot_bottle.git_gate.host_key.prompt_tty", side_effect=["y", "n"], # confirm key, no persist ), patch("sys.stderr"): result = preflight_host_keys( @@ -604,7 +604,7 @@ class TestPreflightHostKeys(unittest.TestCase): manifest = _manifest_with_git() with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", side_effect=RuntimeError("connection refused"), ), patch("sys.stderr"), self.assertRaises(SystemExit): preflight_host_keys(manifest, headless=False, home_md=None) @@ -620,10 +620,10 @@ class TestPreflightHostKeys(unittest.TestCase): import io buf = io.StringIO() with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", return_value="ssh-ed25519 FETCHED", ), patch( - "bot_bottle.git_gate_host_key.prompt_tty", + "bot_bottle.git_gate.host_key.prompt_tty", side_effect=["y"], # confirm key only; no save prompt when no file found ), patch("sys.stderr", buf): result = preflight_host_keys( @@ -651,13 +651,13 @@ class TestPreflightHostKeys(unittest.TestCase): import io buf = io.StringIO() with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", return_value="ssh-ed25519 FETCHED", ), patch( - "bot_bottle.git_gate_host_key.prompt_tty", + "bot_bottle.git_gate.host_key.prompt_tty", side_effect=["y", "y"], # confirm key, yes persist ), patch( - "bot_bottle.git_gate_host_key.find_and_update_bottle_file", + "bot_bottle.git_gate.host_key.find_and_update_bottle_file", return_value=False, ), patch("sys.stderr", buf): result = preflight_host_keys( -- 2.52.0 From a446551acb0dbacd416b95c813ddf7b11931e603 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 16:43:59 -0400 Subject: [PATCH 22/39] refactor(egress): make Egress a service class in an egress package Split the flat egress.py into an egress/ package: neutral DTOs (EgressRoute, EgressPlan) in plan.py, and the concrete Egress service class plus rendering/env helpers in service.py, behind a thin __getattr__ facade so leaf imports stay cheap. Egress drops ABC and becomes a concrete service the backends call: resolve_token_values / agent_env_entries / prepare are now methods. Backend launch paths (docker, firecracker, macos_container) call Egress().(...) instead of the module-level functions. Co-Authored-By: Claude Opus 4.8 --- .../backend/docker/consolidated_compose.py | 4 +- bot_bottle/backend/docker/launch.py | 4 +- bot_bottle/backend/firecracker/launch.py | 9 +- bot_bottle/backend/macos_container/launch.py | 9 +- bot_bottle/egress/__init__.py | 90 +++++++++++++++++++ bot_bottle/egress/plan.py | 48 ++++++++++ bot_bottle/{egress.py => egress/service.py} | 90 ++++++------------- tests/unit/test_egress.py | 2 +- 8 files changed, 176 insertions(+), 80 deletions(-) create mode 100644 bot_bottle/egress/__init__.py create mode 100644 bot_bottle/egress/plan.py rename bot_bottle/{egress.py => egress/service.py} (86%) diff --git a/bot_bottle/backend/docker/consolidated_compose.py b/bot_bottle/backend/docker/consolidated_compose.py index 45296b5..d3b7996 100644 --- a/bot_bottle/backend/docker/consolidated_compose.py +++ b/bot_bottle/backend/docker/consolidated_compose.py @@ -16,7 +16,7 @@ from __future__ import annotations from typing import Any -from ...egress import egress_agent_env_entries +from ...egress import Egress from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH from .bottle_plan import DockerBottlePlan @@ -63,7 +63,7 @@ def consolidated_agent_compose( # env (set in launch.py) and is never written to the compose file on disk. if getattr(plan, "env_var_secret", ""): env.append(ENV_VAR_SECRET_NAME) - env.extend(egress_agent_env_entries(plan.egress_plan)) + env.extend(Egress().agent_env_entries(plan.egress_plan)) service: dict[str, Any] = { "image": plan.image, diff --git a/bot_bottle/backend/docker/launch.py b/bot_bottle/backend/docker/launch.py index 665fa5e..4f68303 100644 --- a/bot_bottle/backend/docker/launch.py +++ b/bot_bottle/backend/docker/launch.py @@ -37,7 +37,7 @@ from pathlib import Path from typing import Callable, Generator from ...agent_provider import runtime_for -from ...egress import egress_resolve_token_values +from ...egress import Egress from ...git_gate import GitGate from ...image_cache import check_stale from ...log import die, info, warn @@ -143,7 +143,7 @@ def launch( # handed to the orchestrator (in memory) for the gateway to inject — # the agent never sees them. effective_env = {**os.environ, **plan.agent_provision.provisioned_env} - token_values = egress_resolve_token_values( + token_values = Egress().resolve_token_values( plan.egress_plan.token_env_map, effective_env, ) teardown_timeout = resolve_teardown_timeout() diff --git a/bot_bottle/backend/firecracker/launch.py b/bot_bottle/backend/firecracker/launch.py index b5fc00d..007724c 100644 --- a/bot_bottle/backend/firecracker/launch.py +++ b/bot_bottle/backend/firecracker/launch.py @@ -38,10 +38,7 @@ from ...bottle_state import ( git_gate_state_dir, read_committed_image, ) -from ...egress import ( - egress_agent_env_entries, - egress_resolve_token_values, -) +from ...egress import Egress from ...git_gate import GitGate from ...image_cache import check_stale_path from ...log import die, info, warn @@ -110,7 +107,7 @@ def launch( # (in memory) for the gateway to inject — the agent never sees them. # Attribution is by the VM's guest IP (unspoofable via /31 TAP + nft). effective_env = {**os.environ, **plan.agent_provision.provisioned_env} - token_values = egress_resolve_token_values( + token_values = Egress().resolve_token_values( plan.egress_plan.token_env_map, effective_env, ) teardown_timeout = resolve_teardown_timeout() @@ -284,7 +281,7 @@ def _agent_guest_env(plan: FirecrackerBottlePlan, host_ip: str) -> dict[str, str env["MCP_SUPERVISE_URL"] = plan.agent_supervise_url if plan.env_var_secret: env[ENV_VAR_SECRET_NAME] = plan.env_var_secret - for entry in egress_agent_env_entries(plan.egress_plan): + for entry in Egress().agent_env_entries(plan.egress_plan): key, _, value = entry.partition("=") env[key] = value env.update(plan.agent_provision.guest_env) diff --git a/bot_bottle/backend/macos_container/launch.py b/bot_bottle/backend/macos_container/launch.py index bb2fc79..d6e331d 100644 --- a/bot_bottle/backend/macos_container/launch.py +++ b/bot_bottle/backend/macos_container/launch.py @@ -44,10 +44,7 @@ from ...bottle_state import ( git_gate_state_dir, read_committed_image, ) -from ...egress import ( - egress_agent_env_entries, - egress_resolve_token_values, -) +from ...egress import Egress from ...git_gate import GitGate from ...gateway.git_http_backend import DEFAULT_PORT as _GIT_HTTP_PORT from ...image_cache import check_stale @@ -191,7 +188,7 @@ def launch( f"{endpoint.network}" ) effective_env = {**os.environ, **plan.agent_provision.provisioned_env} - token_values = egress_resolve_token_values( + token_values = Egress().resolve_token_values( plan.egress_plan.token_env_map, effective_env, ) teardown_timeout = resolve_teardown_timeout() @@ -451,7 +448,7 @@ def _agent_env_entries( # so the secret value never lands on argv. for name in sorted(plan.forwarded_env.keys()): env.append(name) - env.extend(egress_agent_env_entries(plan.egress_plan)) + env.extend(Egress().agent_env_entries(plan.egress_plan)) return tuple(env) diff --git a/bot_bottle/egress/__init__.py b/bot_bottle/egress/__init__.py new file mode 100644 index 0000000..2e392e3 --- /dev/null +++ b/bot_bottle/egress/__init__.py @@ -0,0 +1,90 @@ +"""Per-agent egress (PRD 0017). + +The egress gateway is a TLS-terminating forward proxy that allow-lists a +bottle's outbound HTTP(S), scans payloads for secret exfil, and injects +per-route upstream credentials the agent never sees. + +Layout: + + * `service` — the `Egress` host-side service (`prepare` the launch plan, + `resolve_token_values`, `agent_env_entries`) + the route-building / + rendering functions. + * `plan` — `EgressPlan` / `EgressRoute`, the launch DTOs (in the backend + contract). + +The runtime enforcement (the mitmproxy addon) lives in +`bot_bottle.gateway.egress_addon*`. Public names are re-exported lazily via +`__getattr__`, so `from bot_bottle.egress import …` keeps working and importing +`egress.plan` (the contract's dependency) stays light. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .plan import EgressPlan, EgressRoute + from .service import ( + CLAUDE_HOST_CREDENTIAL_TOKEN_REF, + CODEX_HOST_CREDENTIAL_TOKEN_REF, + EGRESS_HOSTNAME, + EGRESS_ROUTES_FILENAME, + EGRESS_ROUTES_IN_CONTAINER, + Egress, + egress_agent_env_entries, + egress_gateway_env_entries, + egress_manifest_routes, + egress_render_routes, + egress_resolve_token_values, + egress_routes_for_bottle, + egress_token_env_map, + ) + + +_LAZY: dict[str, str] = { + "EgressPlan": ".plan", + "EgressRoute": ".plan", + "Egress": ".service", + "CLAUDE_HOST_CREDENTIAL_TOKEN_REF": ".service", + "CODEX_HOST_CREDENTIAL_TOKEN_REF": ".service", + "EGRESS_HOSTNAME": ".service", + "EGRESS_ROUTES_FILENAME": ".service", + "EGRESS_ROUTES_IN_CONTAINER": ".service", + "egress_agent_env_entries": ".service", + "egress_gateway_env_entries": ".service", + "egress_manifest_routes": ".service", + "egress_render_routes": ".service", + "egress_resolve_token_values": ".service", + "egress_routes_for_bottle": ".service", + "egress_token_env_map": ".service", +} + + +def __getattr__(name: str) -> Any: + src = _LAZY.get(name) + if src is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module + + value = getattr(import_module(src, __name__), name) + globals()[name] = value + return value + + +__all__ = [ + "CLAUDE_HOST_CREDENTIAL_TOKEN_REF", + "CODEX_HOST_CREDENTIAL_TOKEN_REF", + "EGRESS_HOSTNAME", + "EGRESS_ROUTES_FILENAME", + "EGRESS_ROUTES_IN_CONTAINER", + "Egress", + "EgressPlan", + "EgressRoute", + "egress_manifest_routes", + "egress_render_routes", + "egress_resolve_token_values", + "egress_routes_for_bottle", + "egress_agent_env_entries", + "egress_gateway_env_entries", + "egress_token_env_map", +] diff --git a/bot_bottle/egress/plan.py b/bot_bottle/egress/plan.py new file mode 100644 index 0000000..d62f068 --- /dev/null +++ b/bot_bottle/egress/plan.py @@ -0,0 +1,48 @@ +"""Egress launch DTOs (PRD 0017). + +`EgressRoute` (the host-side extension of the addon's wire `Route`) and +`EgressPlan` (the launch plan the backend contract references). Pure value +types — the route-building / rendering logic and the `Egress` service live in +`egress.service`. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from ..gateway.egress_addon_core import Route + + +@dataclass(frozen=True) +class EgressRoute(Route): + """Host-side extension of the addon's `Route`. + + Inherits `host`, `matches`, `auth_scheme`, and `token_env` + from `egress_addon_core.Route` — those are the fields that cross the + YAML wire into the gateway. The fields below are host-only and + are never serialised to the addon. + + `token_ref` is the host env var the CLI reads at launch and forwards + into the container's environ under `token_env`. + + `roles` carries the manifest route's role tuple (reserved for + future use; always empty today).""" + + token_ref: str = "" + roles: tuple[str, ...] = () + + +@dataclass(frozen=True) +class EgressPlan: + slug: str + routes_path: Path + routes: tuple[EgressRoute, ...] + token_env_map: dict[str, str] + internal_network: str = "" + egress_network: str = "" + mitmproxy_ca_host_path: Path = Path() + mitmproxy_ca_cert_only_host_path: Path = Path() + log: int = 0 + canary: str = "" + canary_env: str = "" diff --git a/bot_bottle/egress.py b/bot_bottle/egress/service.py similarity index 86% rename from bot_bottle/egress.py rename to bot_bottle/egress/service.py index b933bad..c720bf2 100644 --- a/bot_bottle/egress.py +++ b/bot_bottle/egress/service.py @@ -1,33 +1,33 @@ -"""Per-bottle egress proxy (PRD 0017, PRD 0053). +"""The `Egress` host-side service (PRD 0017). -This module defines the abstract proxy (`Egress`), its plan -dataclass (`EgressPlan`), and the resolved per-route shape -(`EgressRoute`). The gateway's start/stop lifecycle is backend- -specific and lives on concrete subclasses (see -`bot_bottle/backend/docker/egress.py`). +`Egress` builds a bottle's egress plan at launch: resolve the manifest's egress +routes, render the gateway's `routes.yaml`, assign per-route token slots, and +plant the exfil canary. The service also resolves the launch-time token values +and the agent/gateway env entries the backend injects. The runtime enforcement +(the mitmproxy addon) lives in `bot_bottle.gateway.egress_addon*`. """ from __future__ import annotations import dataclasses import secrets -from abc import ABC -from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING -from .gateway.egress_addon_core import ( +from ..gateway.egress_addon_core import ( ON_MATCH_REDACT, HeaderMatch as CoreHeaderMatch, MatchEntry as CoreMatchEntry, PathMatch as CorePathMatch, Route, ) -from .errors import MissingEnvVarError -from .log import die +from ..errors import MissingEnvVarError +from ..log import die +from .plan import EgressPlan, EgressRoute if TYPE_CHECKING: - from .manifest import ManifestBottle + from ..manifest import ManifestBottle + CODEX_HOST_CREDENTIAL_TOKEN_REF = "BOT_BOTTLE_CODEX_HOST_ACCESS_TOKEN" CLAUDE_HOST_CREDENTIAL_TOKEN_REF = "BOT_BOTTLE_CLAUDE_HOST_ACCESS_TOKEN" @@ -82,39 +82,6 @@ def egress_agent_env_entries(plan: "EgressPlan") -> tuple[str, ...]: return () -@dataclass(frozen=True) -class EgressRoute(Route): - """Host-side extension of the addon's `Route`. - - Inherits `host`, `matches`, `auth_scheme`, and `token_env` - from `egress_addon_core.Route` — those are the fields that cross the - YAML wire into the gateway. The fields below are host-only and - are never serialised to the addon. - - `token_ref` is the host env var the CLI reads at launch and forwards - into the container's environ under `token_env`. - - `roles` carries the manifest route's role tuple (reserved for - future use; always empty today).""" - - token_ref: str = "" - roles: tuple[str, ...] = () - - -@dataclass(frozen=True) -class EgressPlan: - slug: str - routes_path: Path - routes: tuple[EgressRoute, ...] - token_env_map: dict[str, str] - internal_network: str = "" - egress_network: str = "" - mitmproxy_ca_host_path: Path = Path() - mitmproxy_ca_cert_only_host_path: Path = Path() - log: int = 0 - canary: str = "" - canary_env: str = "" - def egress_manifest_routes( bottle: ManifestBottle, @@ -389,7 +356,11 @@ def egress_resolve_token_values( return out -class Egress(ABC): +class Egress: + """The host-side egress service. The backend drives `prepare` at launch, + then `resolve_token_values` to resolve the routes' upstream credentials and + `agent_env_entries` for the agent's egress env. Stateless.""" + def prepare( self, bottle: ManifestBottle, @@ -416,20 +387,13 @@ class Egress(ABC): canary_env=_random_canary_env(), ) -__all__ = [ - "CLAUDE_HOST_CREDENTIAL_TOKEN_REF", - "CODEX_HOST_CREDENTIAL_TOKEN_REF", - "EGRESS_HOSTNAME", - "EGRESS_ROUTES_FILENAME", - "EGRESS_ROUTES_IN_CONTAINER", - "Egress", - "EgressPlan", - "EgressRoute", - "egress_manifest_routes", - "egress_render_routes", - "egress_resolve_token_values", - "egress_routes_for_bottle", - "egress_agent_env_entries", - "egress_gateway_env_entries", - "egress_token_env_map", -] + def resolve_token_values( + self, token_env_map: dict[str, str], host_env: dict[str, str], + ) -> dict[str, str]: + """Resolve each route's upstream credential from the host env at launch. + Raises `MissingEnvVarError` for an unset/empty referenced host var.""" + return egress_resolve_token_values(token_env_map, host_env) + + def agent_env_entries(self, plan: EgressPlan) -> tuple[str, ...]: + """The agent-visible egress env entries (the exfil canary).""" + return egress_agent_env_entries(plan) diff --git a/tests/unit/test_egress.py b/tests/unit/test_egress.py index 027df82..9447549 100644 --- a/tests/unit/test_egress.py +++ b/tests/unit/test_egress.py @@ -10,7 +10,6 @@ from bot_bottle.egress import ( Egress, EgressPlan, EgressRoute, - _yaml_str_escape, egress_agent_env_entries, egress_manifest_routes, egress_render_routes, @@ -19,6 +18,7 @@ from bot_bottle.egress import ( egress_gateway_env_entries, egress_token_env_map, ) +from bot_bottle.egress.service import _yaml_str_escape from bot_bottle.errors import MissingEnvVarError from bot_bottle.manifest import ManifestIndex from bot_bottle.yaml_subset import parse_yaml_subset -- 2.52.0 From ce744a85c45685159a3bd1bef389b9096188c46d Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 17:00:51 -0400 Subject: [PATCH 23/39] refactor(gateway): split data-plane files into egress/supervisor/git_gate services Group the gateway's data-plane modules into three service sub-packages mirroring the host-side trio (bot_bottle.egress / .supervisor / .git_gate): gateway/egress/ addon_core, addon, dlp_config, dlp_detectors gateway/supervisor/ server (was supervise_server) gateway/git_gate/ render, http_backend Prefix-stripped filenames now that the package namespaces them; each sub-package has a thin docstring __init__ (no eager imports, cheap leaf loads). The two cross-cutting files stay at the gateway root: policy_resolver (shared per-client lookup) and gateway_init, renamed to bootstrap now that gateway/ already namespaces it. Updated all importers (bot_bottle + tests), the in-VM/container `-m` launch strings, the Dockerfile.gateway addon shim + ENTRYPOINT, and the five gateway entries in scripts/critical-modules.txt. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- Dockerfile.gateway | 8 +++--- bot_bottle/backend/egress_apply.py | 2 +- bot_bottle/backend/firecracker/infra_vm.py | 2 +- bot_bottle/backend/macos_container/infra.py | 2 +- bot_bottle/backend/macos_container/launch.py | 2 +- bot_bottle/egress/__init__.py | 2 +- bot_bottle/egress/plan.py | 2 +- bot_bottle/egress/service.py | 4 +-- .../gateway/{gateway_init.py => bootstrap.py} | 4 +-- bot_bottle/gateway/egress/__init__.py | 9 +++++++ .../{egress_addon.py => egress/addon.py} | 4 +-- .../addon_core.py} | 4 +-- .../dlp_config.py} | 0 .../gateway/{ => egress}/dlp_detectors.py | 2 +- bot_bottle/gateway/git_gate/__init__.py | 7 +++++ .../http_backend.py} | 0 .../render.py} | 4 +-- bot_bottle/gateway/supervisor/__init__.py | 7 +++++ .../server.py} | 2 +- bot_bottle/git_gate/__init__.py | 26 +++++++++---------- bot_bottle/git_gate/plan.py | 2 +- bot_bottle/git_gate/provision.py | 2 +- bot_bottle/git_gate/service.py | 4 +-- scripts/critical-modules.txt | 10 +++---- tests/integration/test_gateway_image.py | 2 +- tests/unit/test_dlp_detectors.py | 8 +++--- tests/unit/test_egress.py | 22 ++++++++-------- tests/unit/test_egress_addon_core.py | 8 +++--- tests/unit/test_egress_addon_log_redaction.py | 2 +- tests/unit/test_egress_addon_request_flow.py | 8 +++--- tests/unit/test_egress_core_parsing.py | 2 +- tests/unit/test_egress_multitenant.py | 2 +- tests/unit/test_firecracker_infra_vm.py | 2 +- tests/unit/test_gateway_init.py | 8 +++--- tests/unit/test_git_gate_provision_render.py | 2 +- tests/unit/test_git_http_backend.py | 16 ++++++------ tests/unit/test_git_http_multitenant.py | 2 +- tests/unit/test_macos_infra.py | 2 +- tests/unit/test_orchestrator_registration.py | 2 +- tests/unit/test_supervise_server.py | 4 +-- 40 files changed, 113 insertions(+), 90 deletions(-) rename bot_bottle/gateway/{gateway_init.py => bootstrap.py} (99%) create mode 100644 bot_bottle/gateway/egress/__init__.py rename bot_bottle/gateway/{egress_addon.py => egress/addon.py} (99%) rename bot_bottle/gateway/{egress_addon_core.py => egress/addon_core.py} (99%) rename bot_bottle/gateway/{egress_dlp_config.py => egress/dlp_config.py} (100%) rename bot_bottle/gateway/{ => egress}/dlp_detectors.py (99%) create mode 100644 bot_bottle/gateway/git_gate/__init__.py rename bot_bottle/gateway/{git_http_backend.py => git_gate/http_backend.py} (100%) rename bot_bottle/gateway/{git_gate_render.py => git_gate/render.py} (99%) create mode 100644 bot_bottle/gateway/supervisor/__init__.py rename bot_bottle/gateway/{supervise_server.py => supervisor/server.py} (99%) diff --git a/Dockerfile.gateway b/Dockerfile.gateway index 91d0c52..47ff584 100644 --- a/Dockerfile.gateway +++ b/Dockerfile.gateway @@ -8,8 +8,8 @@ # this image's mitmproxy / git / gitleaks payload. # # Collapses the prior per-daemon images (egress, git-gate, -# supervise) into one. A small stdlib-Python init supervisor at -# /app/gateway_init.py spawns all daemons, forwards SIGTERM, and +# supervise) into one. A small stdlib-Python init supervisor +# (`bot_bottle.gateway.bootstrap`) spawns all daemons, forwards SIGTERM, and # propagates per-daemon stdout/stderr to the container log with a # `[name]` prefix. See PRD 0024 for the rationale. # @@ -102,7 +102,7 @@ RUN pip install --no-cache-dir /src/ # WORKDIR here also creates /app so the shim + COPYs below can write into it # (nothing created /app before this point). WORKDIR /app -RUN printf 'from bot_bottle.gateway.egress_addon import addons\n' > /app/egress_addon.py +RUN printf 'from bot_bottle.gateway.egress.addon import addons\n' > /app/egress_addon.py COPY bot_bottle/egress_entrypoint.sh /app/egress-entrypoint.sh RUN chmod +x /app/egress-entrypoint.sh @@ -123,4 +123,4 @@ EXPOSE 8888 9099 9418 9420 9100 # PID 1 is the supervisor. It owns signal handling and exit-code # propagation; no `exec` chain in the entrypoint itself. -ENTRYPOINT ["python3", "-m", "bot_bottle.gateway.gateway_init"] +ENTRYPOINT ["python3", "-m", "bot_bottle.gateway.bootstrap"] diff --git a/bot_bottle/backend/egress_apply.py b/bot_bottle/backend/egress_apply.py index a198107..e174af3 100644 --- a/bot_bottle/backend/egress_apply.py +++ b/bot_bottle/backend/egress_apply.py @@ -11,7 +11,7 @@ from pathlib import Path from ..bottle_state import egress_state_dir from ..egress import EGRESS_ROUTES_FILENAME -from ..gateway.egress_addon_core import LOG_OFF, load_config +from ..gateway.egress.addon_core import LOG_OFF, load_config class EgressApplyError(RuntimeError): diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py index a7ab12b..1542a60 100644 --- a/bot_bottle/backend/firecracker/infra_vm.py +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -542,7 +542,7 @@ BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_CONTROL_PLANE_TOKEN="$CP_KEY" pyt BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise \\ BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{CONTROL_PLANE_PORT} \\ BOT_BOTTLE_CONTROL_AUTH_JWT="$GW_JWT" \\ - python3 -m bot_bottle.gateway.gateway_init & + python3 -m bot_bottle.gateway.bootstrap & # Reap as PID 1; children are backgrounded, so `wait` blocks. while : ; do wait ; done diff --git a/bot_bottle/backend/macos_container/infra.py b/bot_bottle/backend/macos_container/infra.py index 85bc3f0..5bb7f75 100644 --- a/bot_bottle/backend/macos_container/infra.py +++ b/bot_bottle/backend/macos_container/infra.py @@ -107,7 +107,7 @@ def _init_script(port: int) -> str: # control-plane RPC and never opens bot-bottle.db (PRD 0070 / #469). f"( cd /app && BOT_BOTTLE_GATEWAY_DAEMONS={_GATEWAY_DAEMONS} " f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{port} " - f"python3 -m bot_bottle.gateway.gateway_init ) &\n" + f"python3 -m bot_bottle.gateway.bootstrap ) &\n" "while : ; do wait ; done\n" ) diff --git a/bot_bottle/backend/macos_container/launch.py b/bot_bottle/backend/macos_container/launch.py index d6e331d..ba34ff9 100644 --- a/bot_bottle/backend/macos_container/launch.py +++ b/bot_bottle/backend/macos_container/launch.py @@ -46,7 +46,7 @@ from ...bottle_state import ( ) from ...egress import Egress from ...git_gate import GitGate -from ...gateway.git_http_backend import DEFAULT_PORT as _GIT_HTTP_PORT +from ...gateway.git_gate.http_backend import DEFAULT_PORT as _GIT_HTTP_PORT from ...image_cache import check_stale from ...log import die, info, warn from .. import BottleImages diff --git a/bot_bottle/egress/__init__.py b/bot_bottle/egress/__init__.py index 2e392e3..37756ca 100644 --- a/bot_bottle/egress/__init__.py +++ b/bot_bottle/egress/__init__.py @@ -13,7 +13,7 @@ Layout: contract). The runtime enforcement (the mitmproxy addon) lives in -`bot_bottle.gateway.egress_addon*`. Public names are re-exported lazily via +`bot_bottle.gateway.egress.addon*`. Public names are re-exported lazily via `__getattr__`, so `from bot_bottle.egress import …` keeps working and importing `egress.plan` (the contract's dependency) stays light. """ diff --git a/bot_bottle/egress/plan.py b/bot_bottle/egress/plan.py index d62f068..ecca024 100644 --- a/bot_bottle/egress/plan.py +++ b/bot_bottle/egress/plan.py @@ -11,7 +11,7 @@ from __future__ import annotations from dataclasses import dataclass from pathlib import Path -from ..gateway.egress_addon_core import Route +from ..gateway.egress.addon_core import Route @dataclass(frozen=True) diff --git a/bot_bottle/egress/service.py b/bot_bottle/egress/service.py index c720bf2..76a8829 100644 --- a/bot_bottle/egress/service.py +++ b/bot_bottle/egress/service.py @@ -4,7 +4,7 @@ routes, render the gateway's `routes.yaml`, assign per-route token slots, and plant the exfil canary. The service also resolves the launch-time token values and the agent/gateway env entries the backend injects. The runtime enforcement -(the mitmproxy addon) lives in `bot_bottle.gateway.egress_addon*`. +(the mitmproxy addon) lives in `bot_bottle.gateway.egress.addon*`. """ from __future__ import annotations @@ -14,7 +14,7 @@ import secrets from pathlib import Path from typing import TYPE_CHECKING -from ..gateway.egress_addon_core import ( +from ..gateway.egress.addon_core import ( ON_MATCH_REDACT, HeaderMatch as CoreHeaderMatch, MatchEntry as CoreMatchEntry, diff --git a/bot_bottle/gateway/gateway_init.py b/bot_bottle/gateway/bootstrap.py similarity index 99% rename from bot_bottle/gateway/gateway_init.py rename to bot_bottle/gateway/bootstrap.py index 1e23be7..cd36179 100644 --- a/bot_bottle/gateway/gateway_init.py +++ b/bot_bottle/gateway/bootstrap.py @@ -100,8 +100,8 @@ _DAEMONS: tuple[_DaemonSpec, ...] = ( )), _DaemonSpec("egress", ("/bin/sh", "/app/egress-entrypoint.sh")), _DaemonSpec("git-gate", ("/bin/sh", "/git-gate-entrypoint.sh")), - _DaemonSpec("git-http", ("python3", "-m", "bot_bottle.gateway.git_http_backend")), - _DaemonSpec("supervise", ("python3", "-m", "bot_bottle.gateway.supervise_server")), + _DaemonSpec("git-http", ("python3", "-m", "bot_bottle.gateway.git_gate.http_backend")), + _DaemonSpec("supervise", ("python3", "-m", "bot_bottle.gateway.supervisor.server")), ) diff --git a/bot_bottle/gateway/egress/__init__.py b/bot_bottle/gateway/egress/__init__.py new file mode 100644 index 0000000..e406e60 --- /dev/null +++ b/bot_bottle/gateway/egress/__init__.py @@ -0,0 +1,9 @@ +"""Gateway-side (data-plane) egress service: the mitmproxy addon and its +pure decision core, DLP detectors, and route/DLP config parsing. + +These are the long-running data-plane pieces (loaded by mitmdump inside the +`bot-bottle-gateway` image), distinct from the host-side `bot_bottle.egress` +service that renders routes and prepares env. Import the concrete modules +directly (`from bot_bottle.gateway.egress.addon_core import ...`) — this +package deliberately does no eager work so leaf imports stay cheap. +""" diff --git a/bot_bottle/gateway/egress_addon.py b/bot_bottle/gateway/egress/addon.py similarity index 99% rename from bot_bottle/gateway/egress_addon.py rename to bot_bottle/gateway/egress/addon.py index eda9402..a5292f2 100644 --- a/bot_bottle/gateway/egress_addon.py +++ b/bot_bottle/gateway/egress/addon.py @@ -16,8 +16,8 @@ import typing from mitmproxy import http # type: ignore[import-not-found] # pylint: disable=import-error from bot_bottle.constants import IDENTITY_HEADER -from bot_bottle.gateway.dlp_detectors import redact_tokens, strip_crlf -from bot_bottle.gateway.egress_addon_core import ( +from bot_bottle.gateway.egress.dlp_detectors import redact_tokens, strip_crlf +from bot_bottle.gateway.egress.addon_core import ( LOG_BLOCKS, LOG_FULL, DEFAULT_OUTBOUND_ON_MATCH, diff --git a/bot_bottle/gateway/egress_addon_core.py b/bot_bottle/gateway/egress/addon_core.py similarity index 99% rename from bot_bottle/gateway/egress_addon_core.py rename to bot_bottle/gateway/egress/addon_core.py index e4800d5..3d0bfca 100644 --- a/bot_bottle/gateway/egress_addon_core.py +++ b/bot_bottle/gateway/egress/addon_core.py @@ -16,11 +16,11 @@ import re import typing from dataclasses import dataclass -from ..yaml_subset import YamlSubsetError, parse_yaml_subset +from ...yaml_subset import YamlSubsetError, parse_yaml_subset # DLP detector-config parsing lives in a sibling module. Re-exported below # so existing `from egress_addon_core import ON_MATCH_*` callers keep working. -from .egress_dlp_config import ( +from .dlp_config import ( DEFAULT_OUTBOUND_ON_MATCH, INBOUND_DETECTOR_NAMES, ON_MATCH_BLOCK, diff --git a/bot_bottle/gateway/egress_dlp_config.py b/bot_bottle/gateway/egress/dlp_config.py similarity index 100% rename from bot_bottle/gateway/egress_dlp_config.py rename to bot_bottle/gateway/egress/dlp_config.py diff --git a/bot_bottle/gateway/dlp_detectors.py b/bot_bottle/gateway/egress/dlp_detectors.py similarity index 99% rename from bot_bottle/gateway/dlp_detectors.py rename to bot_bottle/gateway/egress/dlp_detectors.py index 2ca286a..d0c8d1b 100644 --- a/bot_bottle/gateway/dlp_detectors.py +++ b/bot_bottle/gateway/egress/dlp_detectors.py @@ -19,7 +19,7 @@ from math import log2 from collections import Counter from urllib.parse import quote as url_quote -from .egress_addon_core import ScanResult +from .addon_core import ScanResult # --------------------------------------------------------------------------- diff --git a/bot_bottle/gateway/git_gate/__init__.py b/bot_bottle/gateway/git_gate/__init__.py new file mode 100644 index 0000000..67c9fe7 --- /dev/null +++ b/bot_bottle/gateway/git_gate/__init__.py @@ -0,0 +1,7 @@ +"""Gateway-side (data-plane) git-gate service: pure host-side hook rendering +(PRD 0008) and the smart-HTTP backend that fronts git-gate repos. + +The host-side git-gate service (provisioning, preflight) lives in +`bot_bottle.git_gate`; this is the data-plane counterpart. Import the concrete +modules directly (`from bot_bottle.gateway.git_gate.render import ...`). +""" diff --git a/bot_bottle/gateway/git_http_backend.py b/bot_bottle/gateway/git_gate/http_backend.py similarity index 100% rename from bot_bottle/gateway/git_http_backend.py rename to bot_bottle/gateway/git_gate/http_backend.py diff --git a/bot_bottle/gateway/git_gate_render.py b/bot_bottle/gateway/git_gate/render.py similarity index 99% rename from bot_bottle/gateway/git_gate_render.py rename to bot_bottle/gateway/git_gate/render.py index 67f055a..241ed9d 100644 --- a/bot_bottle/gateway/git_gate_render.py +++ b/bot_bottle/gateway/git_gate/render.py @@ -14,8 +14,8 @@ import shlex from dataclasses import dataclass from pathlib import Path -from ..constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER -from ..manifest import ManifestBottle, ManifestGitEntry +from ...constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER +from ...manifest import ManifestBottle, ManifestGitEntry # Short network alias for git-gate inside the gateway. The # agent's `.gitconfig` insteadOf rewrites resolve through this name. diff --git a/bot_bottle/gateway/supervisor/__init__.py b/bot_bottle/gateway/supervisor/__init__.py new file mode 100644 index 0000000..47b48b7 --- /dev/null +++ b/bot_bottle/gateway/supervisor/__init__.py @@ -0,0 +1,7 @@ +"""Gateway-side (data-plane) supervise service: the supervise daemon's HTTP +server (PRD 0013). + +The host-side supervise control lives in `bot_bottle.orchestrator.supervisor`; +this is the in-gateway daemon the agents reach. Import the concrete module +directly (`from bot_bottle.gateway.supervisor.server import ...`). +""" diff --git a/bot_bottle/gateway/supervise_server.py b/bot_bottle/gateway/supervisor/server.py similarity index 99% rename from bot_bottle/gateway/supervise_server.py rename to bot_bottle/gateway/supervisor/server.py index 520f1da..35d53c5 100644 --- a/bot_bottle/gateway/supervise_server.py +++ b/bot_bottle/gateway/supervisor/server.py @@ -58,7 +58,7 @@ import typing from dataclasses import dataclass from bot_bottle.constants import IDENTITY_HEADER -from bot_bottle.gateway.egress_addon_core import ( +from bot_bottle.gateway.egress.addon_core import ( LOG_OFF, load_config, resolve_client_context, route_to_yaml_dict, ) from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver diff --git a/bot_bottle/git_gate/__init__.py b/bot_bottle/git_gate/__init__.py index c902a1f..f6b6256 100644 --- a/bot_bottle/git_gate/__init__.py +++ b/bot_bottle/git_gate/__init__.py @@ -14,7 +14,7 @@ Layout: service delegates to. The rendering + the in-gateway hook execution live in -`bot_bottle.gateway.git_gate_render`. The public names are re-exported lazily +`bot_bottle.gateway.git_gate.render`. The public names are re-exported lazily via `__getattr__`, so `from bot_bottle.git_gate import …` keeps working and importing `git_gate.plan` (the contract's dependency) doesn't drag in the provisioning / forge-API code. @@ -31,7 +31,7 @@ if TYPE_CHECKING: provision_git_gate_dynamic_keys, revoke_git_gate_provisioned_keys, ) - from ..gateway.git_gate_render import ( + from ..gateway.git_gate.render import ( GIT_GATE_HOSTNAME, GIT_GATE_TIMEOUT_SECS, GitGateUpstream, @@ -55,17 +55,17 @@ _LAZY: dict[str, str] = { "revoke_git_gate_provisioned_keys": ".provision", "_provision_dynamic_key": ".provision", "_resolve_identity_file": ".provision", - "GIT_GATE_HOSTNAME": "..gateway.git_gate_render", - "GIT_GATE_TIMEOUT_SECS": "..gateway.git_gate_render", - "GitGateUpstream": "..gateway.git_gate_render", - "git_gate_upstreams_for_bottle": "..gateway.git_gate_render", - "git_gate_render_gitconfig": "..gateway.git_gate_render", - "git_gate_known_hosts_line": "..gateway.git_gate_render", - "git_gate_render_entrypoint": "..gateway.git_gate_render", - "git_gate_render_provision": "..gateway.git_gate_render", - "git_gate_render_hook": "..gateway.git_gate_render", - "git_gate_render_access_hook": "..gateway.git_gate_render", - "_gitconfig_validate_value": "..gateway.git_gate_render", + "GIT_GATE_HOSTNAME": "..gateway.git_gate.render", + "GIT_GATE_TIMEOUT_SECS": "..gateway.git_gate.render", + "GitGateUpstream": "..gateway.git_gate.render", + "git_gate_upstreams_for_bottle": "..gateway.git_gate.render", + "git_gate_render_gitconfig": "..gateway.git_gate.render", + "git_gate_known_hosts_line": "..gateway.git_gate.render", + "git_gate_render_entrypoint": "..gateway.git_gate.render", + "git_gate_render_provision": "..gateway.git_gate.render", + "git_gate_render_hook": "..gateway.git_gate.render", + "git_gate_render_access_hook": "..gateway.git_gate.render", + "_gitconfig_validate_value": "..gateway.git_gate.render", } diff --git a/bot_bottle/git_gate/plan.py b/bot_bottle/git_gate/plan.py index c5fe3bb..edd1bd2 100644 --- a/bot_bottle/git_gate/plan.py +++ b/bot_bottle/git_gate/plan.py @@ -10,7 +10,7 @@ from __future__ import annotations from dataclasses import dataclass from pathlib import Path -from ..gateway.git_gate_render import GitGateUpstream +from ..gateway.git_gate.render import GitGateUpstream @dataclass(frozen=True) diff --git a/bot_bottle/git_gate/provision.py b/bot_bottle/git_gate/provision.py index 007abd3..bae03a7 100644 --- a/bot_bottle/git_gate/provision.py +++ b/bot_bottle/git_gate/provision.py @@ -17,7 +17,7 @@ from ..bottle_state import globalize_slug from ..errors import MissingEnvVarError from ..log import info from ..manifest import ManifestBottle, ManifestGitEntry -from ..gateway.git_gate_render import GitGateUpstream +from ..gateway.git_gate.render import GitGateUpstream if TYPE_CHECKING: from .plan import GitGatePlan diff --git a/bot_bottle/git_gate/service.py b/bot_bottle/git_gate/service.py index edd592b..cfcc8aa 100644 --- a/bot_bottle/git_gate/service.py +++ b/bot_bottle/git_gate/service.py @@ -8,7 +8,7 @@ upstream before fetches. The agent never sees the upstream credential. `GitGate` is the host-side service the backend drives at launch: build the plan (`prepare`), provision / revoke the per-upstream deploy keys, and preflight the upstream host keys. The rendering it emits + the in-gateway hook execution live -in `bot_bottle.gateway.git_gate_render`. +in `bot_bottle.gateway.git_gate.render`. """ from __future__ import annotations @@ -16,7 +16,7 @@ from __future__ import annotations from pathlib import Path from ..manifest import Manifest, ManifestBottle -from ..gateway.git_gate_render import ( +from ..gateway.git_gate.render import ( GitGateUpstream, git_gate_known_hosts_line, git_gate_render_access_hook, diff --git a/scripts/critical-modules.txt b/scripts/critical-modules.txt index fa542f2..845e415 100644 --- a/scripts/critical-modules.txt +++ b/scripts/critical-modules.txt @@ -8,18 +8,18 @@ # # One module path per line, relative to the repo root. Blank lines and # `#` comments are ignored. -bot_bottle/egress_addon.py -bot_bottle/egress_addon_core.py -bot_bottle/dlp_detectors.py +bot_bottle/gateway/egress/addon.py +bot_bottle/gateway/egress/addon_core.py +bot_bottle/gateway/egress/dlp_detectors.py bot_bottle/egress.py bot_bottle/manifest.py bot_bottle/manifest_egress.py bot_bottle/manifest_agent.py bot_bottle/manifest_schema.py bot_bottle/git_gate.py -bot_bottle/git_gate_render.py +bot_bottle/gateway/git_gate/render.py bot_bottle/git_gate_provision.py -bot_bottle/git_http_backend.py +bot_bottle/gateway/git_gate/http_backend.py bot_bottle/supervise.py bot_bottle/yaml_subset.py bot_bottle/bottle_state.py diff --git a/tests/integration/test_gateway_image.py b/tests/integration/test_gateway_image.py index 213c93b..7c02a28 100644 --- a/tests/integration/test_gateway_image.py +++ b/tests/integration/test_gateway_image.py @@ -91,7 +91,7 @@ class TestGatewayImage(unittest.TestCase): # Probe that the package imports resolve inside the image. rc, out = self._run_in_image( "python3", "-c", - "from bot_bottle.supervisor import types; from bot_bottle.gateway import supervise_server; print('ok')", + "from bot_bottle.supervisor import types; from bot_bottle.gateway.supervisor import server as supervise_server; print('ok')", ) self.assertEqual(0, rc, msg=out) self.assertIn("ok", out) diff --git a/tests/unit/test_dlp_detectors.py b/tests/unit/test_dlp_detectors.py index d62fb4d..5310252 100644 --- a/tests/unit/test_dlp_detectors.py +++ b/tests/unit/test_dlp_detectors.py @@ -7,7 +7,7 @@ import base64 import gzip import unittest -from bot_bottle.gateway.dlp_detectors import ( +from bot_bottle.gateway.egress.dlp_detectors import ( ENTROPY_BLOCK_THRESHOLD, PARTIAL_MATCH_MIN_LEN, REDACT, @@ -465,17 +465,17 @@ class TestMatchedAndSafeTokens(unittest.TestCase): class TestStripCrlf(unittest.TestCase): def test_removes_url_encoded_crlf(self): - from bot_bottle.gateway.dlp_detectors import strip_crlf + from bot_bottle.gateway.egress.dlp_detectors import strip_crlf out = strip_crlf("next=%0d%0aX-Injected: evil") self.assertNotRegex(out, r"%0[dD]%0[aA]") def test_removes_literal_header_injection(self): - from bot_bottle.gateway.dlp_detectors import strip_crlf + from bot_bottle.gateway.egress.dlp_detectors import strip_crlf out = strip_crlf("value\r\nX-Injected: evil") self.assertIsNone(scan_crlf_injection(out)) def test_leaves_clean_text_unchanged(self): - from bot_bottle.gateway.dlp_detectors import strip_crlf + from bot_bottle.gateway.egress.dlp_detectors import strip_crlf self.assertEqual("/api/v1/data?q=hello", strip_crlf("/api/v1/data?q=hello")) class TestAlnumProjection(unittest.TestCase): diff --git a/tests/unit/test_egress.py b/tests/unit/test_egress.py index 9447549..2ec776a 100644 --- a/tests/unit/test_egress.py +++ b/tests/unit/test_egress.py @@ -344,7 +344,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertEqual([], parse_yaml_subset(rendered)["routes"]) def test_round_trip_through_addon_core(self): - from bot_bottle.gateway.egress_addon_core import load_config + from bot_bottle.gateway.egress.addon_core import load_config b = _bottle([ {"host": "api.github.com", "auth": {"scheme": "Bearer", "token_ref": "GH_PAT"}, @@ -363,7 +363,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertEqual("", addon_routes[2].auth_scheme) def test_dlp_round_trips(self): - from bot_bottle.gateway.egress_addon_core import load_config + from bot_bottle.gateway.egress.addon_core import load_config b = _bottle([{"host": "x.example", "dlp": { "outbound_detectors": ["token_patterns"], "inbound_detectors": False, @@ -375,7 +375,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertEqual((), addon_routes[0].inbound_detectors) def test_outbound_on_match_round_trips(self): - from bot_bottle.gateway.egress_addon_core import load_config + from bot_bottle.gateway.egress.addon_core import load_config b = _bottle([{"host": "logs.example", "dlp": { "outbound_on_match": "redact", }}]) @@ -392,7 +392,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertNotIn("outbound_on_match", rendered) def test_git_fetch_policy_round_trips(self): - from bot_bottle.gateway.egress_addon_core import load_config + from bot_bottle.gateway.egress.addon_core import load_config b = _bottle([{"host": "github.com", "git": {"fetch": True}}]) routes = egress_routes_for_bottle(b) rendered = egress_render_routes(routes) @@ -405,7 +405,7 @@ class TestRenderRoutes(unittest.TestCase): it, but the renderer in between dropped it — so the flag never reached the proxy and registry pulls kept failing with "unauthorized" while the config looked correct everywhere it was inspected.""" - from bot_bottle.gateway.egress_addon_core import load_config + from bot_bottle.gateway.egress.addon_core import load_config b = _bottle([{"host": "registry-1.docker.io", "preserve_auth": True}]) routes = egress_routes_for_bottle(b) rendered = egress_render_routes(routes) @@ -416,7 +416,7 @@ class TestRenderRoutes(unittest.TestCase): b = _bottle([{"host": "x.example"}]) rendered = egress_render_routes(egress_routes_for_bottle(b)) self.assertNotIn("preserve_auth", rendered) - from bot_bottle.gateway.egress_addon_core import load_config + from bot_bottle.gateway.egress.addon_core import load_config self.assertFalse(load_config(rendered).routes[0].preserve_auth) def test_log_zero_omitted_from_render(self): @@ -434,7 +434,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertTrue(rendered.startswith(f"log: {level}\n")) def test_log_level_round_trips_to_addon_core(self): - from bot_bottle.gateway.egress_addon_core import load_config, LOG_FULL + from bot_bottle.gateway.egress.addon_core import load_config, LOG_FULL b = _bottle([{"host": "x.example"}]) routes = egress_routes_for_bottle(b) rendered = egress_render_routes(routes, log=LOG_FULL) @@ -444,7 +444,7 @@ class TestRenderRoutes(unittest.TestCase): def test_log_via_manifest_flows_to_render(self): from bot_bottle.manifest import ManifestIndex - from bot_bottle.gateway.egress_addon_core import load_config, LOG_BLOCKS + from bot_bottle.gateway.egress.addon_core import load_config, LOG_BLOCKS m = ManifestIndex.from_json_obj({ "bottles": {"dev": {"egress": { "log": 1, @@ -512,7 +512,7 @@ class TestRenderRoutesEscaping(unittest.TestCase): self.assertEqual('Bear"er', parsed[0]["inspect"]["auth_scheme"]) def test_path_value_with_double_quote_round_trips(self): - from bot_bottle.gateway.egress_addon_core import PathMatch, MatchEntry + from bot_bottle.gateway.egress.addon_core import PathMatch, MatchEntry routes = (EgressRoute( host="api.example", matches=(MatchEntry(paths=(PathMatch(type="prefix", value='/v1/"quoted"/'),)),), @@ -521,7 +521,7 @@ class TestRenderRoutesEscaping(unittest.TestCase): self.assertEqual('/v1/"quoted"/', parsed[0]["inspect"]["matches"][0]["paths"][0]["value"]) def test_header_value_with_double_quote_round_trips(self): - from bot_bottle.gateway.egress_addon_core import HeaderMatch, MatchEntry + from bot_bottle.gateway.egress.addon_core import HeaderMatch, MatchEntry routes = (EgressRoute( host="api.example", matches=(MatchEntry(headers=(HeaderMatch(name="x-h", value='val"ue'),)),), @@ -598,7 +598,7 @@ class TestCanaryGeneration(unittest.TestCase): self.assertNotEqual(plan_a.canary, plan_b.canary) def test_canary_detected_by_scan_known_secrets(self): - from bot_bottle.gateway.dlp_detectors import scan_known_secrets + from bot_bottle.gateway.egress.dlp_detectors import scan_known_secrets plan = self._make_plan() env = {plan.canary_env: plan.canary} diff --git a/tests/unit/test_egress_addon_core.py b/tests/unit/test_egress_addon_core.py index 593d6ca..a99e783 100644 --- a/tests/unit/test_egress_addon_core.py +++ b/tests/unit/test_egress_addon_core.py @@ -12,7 +12,7 @@ import unittest from pathlib import Path from urllib.parse import urlsplit -from bot_bottle.gateway.egress_addon_core import ( +from bot_bottle.gateway.egress.addon_core import ( LOG_BLOCKS, LOG_FULL, LOG_OFF, @@ -1377,15 +1377,15 @@ class TestScanOutboundEnhanced(unittest.TestCase): class TestOutboundDetectorNames(unittest.TestCase): def test_entropy_in_outbound_detector_names(self): - from bot_bottle.gateway.egress_addon_core import OUTBOUND_DETECTOR_NAMES + from bot_bottle.gateway.egress.addon_core import OUTBOUND_DETECTOR_NAMES self.assertIn("entropy", OUTBOUND_DETECTOR_NAMES) def test_known_secrets_in_outbound_detector_names(self): - from bot_bottle.gateway.egress_addon_core import OUTBOUND_DETECTOR_NAMES + from bot_bottle.gateway.egress.addon_core import OUTBOUND_DETECTOR_NAMES self.assertIn("known_secrets", OUTBOUND_DETECTOR_NAMES) def test_token_patterns_in_outbound_detector_names(self): - from bot_bottle.gateway.egress_addon_core import OUTBOUND_DETECTOR_NAMES + from bot_bottle.gateway.egress.addon_core import OUTBOUND_DETECTOR_NAMES self.assertIn("token_patterns", OUTBOUND_DETECTOR_NAMES) diff --git a/tests/unit/test_egress_addon_log_redaction.py b/tests/unit/test_egress_addon_log_redaction.py index 44af815..c6196d4 100644 --- a/tests/unit/test_egress_addon_log_redaction.py +++ b/tests/unit/test_egress_addon_log_redaction.py @@ -36,7 +36,7 @@ def _ensure_shims() -> None: _ensure_shims() -from bot_bottle.gateway.egress_addon import EgressAddon # noqa: E402 (import after shims) +from bot_bottle.gateway.egress.addon import EgressAddon # noqa: E402 (import after shims) # --------------------------------------------------------------------------- diff --git a/tests/unit/test_egress_addon_request_flow.py b/tests/unit/test_egress_addon_request_flow.py index a8a06d2..415bb8d 100644 --- a/tests/unit/test_egress_addon_request_flow.py +++ b/tests/unit/test_egress_addon_request_flow.py @@ -194,15 +194,15 @@ def _ensure_shims() -> None: _ensure_shims() -import bot_bottle.gateway.egress_addon as _ea_mod # noqa: E402 (after shims) -from bot_bottle.gateway.egress_addon import EgressAddon # noqa: E402 (after shims) -from bot_bottle.gateway.egress_addon import ( # noqa: E402 +import bot_bottle.gateway.egress.addon as _ea_mod # noqa: E402 (after shims) +from bot_bottle.gateway.egress.addon import EgressAddon # noqa: E402 (after shims) +from bot_bottle.gateway.egress.addon import ( # noqa: E402 DEFAULT_INBOUND_SCAN_LIMIT_BYTES, DEFAULT_TOKEN_ALLOW_TIMEOUT_SECONDS, _inbound_scan_limit_from_env, _token_allow_timeout_from_env, ) -from bot_bottle.gateway.egress_addon_core import ( # noqa: E402 +from bot_bottle.gateway.egress.addon_core import ( # noqa: E402 Config, LOG_BLOCKS, LOG_FULL, diff --git a/tests/unit/test_egress_core_parsing.py b/tests/unit/test_egress_core_parsing.py index 6a56e90..161efb4 100644 --- a/tests/unit/test_egress_core_parsing.py +++ b/tests/unit/test_egress_core_parsing.py @@ -8,7 +8,7 @@ from __future__ import annotations import unittest -from bot_bottle.gateway.egress_addon_core import ( +from bot_bottle.gateway.egress.addon_core import ( HeaderMatch, MatchEntry, PathMatch, diff --git a/tests/unit/test_egress_multitenant.py b/tests/unit/test_egress_multitenant.py index adabb3e..2aa5c86 100644 --- a/tests/unit/test_egress_multitenant.py +++ b/tests/unit/test_egress_multitenant.py @@ -4,7 +4,7 @@ from __future__ import annotations import unittest -from bot_bottle.gateway.egress_addon_core import ( +from bot_bottle.gateway.egress.addon_core import ( DENY_RESOLVER_ERROR, DENY_UNATTRIBUTED, DENY_UNPARSEABLE, diff --git a/tests/unit/test_firecracker_infra_vm.py b/tests/unit/test_firecracker_infra_vm.py index e0ae9bb..7dcf4ea 100644 --- a/tests/unit/test_firecracker_infra_vm.py +++ b/tests/unit/test_firecracker_infra_vm.py @@ -72,7 +72,7 @@ class TestBuildInfraRootfs(unittest.TestCase): self.assertIn("bot_bottle.orchestrator", init) # Gateway launches via the installed package (there is no # /app/gateway_init.py file since the daemons moved into bot_bottle). - self.assertIn("bot_bottle.gateway.gateway_init", init) + self.assertIn("bot_bottle.gateway.bootstrap", init) self.assertIn("export PATH=", init) # Persistent registry volume mounted at the DB dir before the CP starts. self.assertIn("/dev/vdb", init) diff --git a/tests/unit/test_gateway_init.py b/tests/unit/test_gateway_init.py index bb987ba..d43cfe6 100644 --- a/tests/unit/test_gateway_init.py +++ b/tests/unit/test_gateway_init.py @@ -1,6 +1,6 @@ """Unit: gateway data-plane init supervisor (PRD 0070; PRD 0024 bundle shape). -Tests both the helper functions in `bot_bottle.gateway.gateway_init` +Tests both the helper functions in `bot_bottle.gateway.bootstrap` and the supervisor's end-to-end signal / exit-code behavior. The end-to-end tests use real subprocesses (`sleep`, `/bin/sh -c '...'`) — short-lived, no docker required — so they run under `tests/unit/` @@ -18,7 +18,7 @@ import warnings from pathlib import Path from unittest.mock import patch -from bot_bottle.gateway.gateway_init import ( +from bot_bottle.gateway.bootstrap import ( _DaemonSpec, _Supervisor, _argv_for_daemon, @@ -489,7 +489,7 @@ class TestSupervisor(unittest.TestCase): time.sleep(0.3) # let `trap` register sup.request_shutdown(reason="test") - with patch("bot_bottle.gateway.gateway_init._GRACE_SECONDS", 0.3): + with patch("bot_bottle.gateway.bootstrap._GRACE_SECONDS", 0.3): rc = self._drive(sup, max_wait_s=4.0) # Process was SIGKILL'd → returncode -9 on POSIX. @@ -531,7 +531,7 @@ class TestMainEndToEnd(unittest.TestCase): helper = ( "import os, runpy, sys\n" - "from bot_bottle.gateway import gateway_init as si\n" + "from bot_bottle.gateway import bootstrap as si\n" "si._DAEMONS = (\n" f" si._DaemonSpec('alpha', ({SLEEP!r},'30')),\n" f" si._DaemonSpec('beta', ({SLEEP!r},'30')),\n" diff --git a/tests/unit/test_git_gate_provision_render.py b/tests/unit/test_git_gate_provision_render.py index b9bce15..d2ba9a7 100644 --- a/tests/unit/test_git_gate_provision_render.py +++ b/tests/unit/test_git_gate_provision_render.py @@ -4,7 +4,7 @@ from __future__ import annotations import unittest -from bot_bottle.gateway.git_gate_render import ( +from bot_bottle.gateway.git_gate.render import ( GitGateUpstream, git_gate_render_entrypoint, git_gate_render_provision, diff --git a/tests/unit/test_git_http_backend.py b/tests/unit/test_git_http_backend.py index 6c58772..89f162c 100644 --- a/tests/unit/test_git_http_backend.py +++ b/tests/unit/test_git_http_backend.py @@ -10,7 +10,7 @@ from pathlib import Path from unittest import mock from bot_bottle.git_gate import GIT_GATE_TIMEOUT_SECS -from bot_bottle.gateway.git_http_backend import GitHttpHandler, MAX_BODY_BYTES +from bot_bottle.gateway.git_gate.http_backend import GitHttpHandler, MAX_BODY_BYTES # The git-http backend is resolver-only: every request is attributed to a @@ -199,7 +199,7 @@ class TestGitHttpBackend(unittest.TestCase): subprocess.CompletedProcess(["git"], 0, backend_response, b""), ] with mock.patch( - "bot_bottle.gateway.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_gate.http_backend.subprocess.run", side_effect=calls, ) as run: request = urllib.request.Request( @@ -265,7 +265,7 @@ class TestGitHttpBackend(unittest.TestCase): subprocess.CompletedProcess(["git"], 0, backend_response, b""), ] with mock.patch( - "bot_bottle.gateway.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_gate.http_backend.subprocess.run", side_effect=calls, ) as run: req = urllib.request.Request( @@ -309,7 +309,7 @@ class TestGitHttpBackend(unittest.TestCase): denial = b"git-gate: upstream fetch failed; refusing to serve stale data\n" with mock.patch( - "bot_bottle.gateway.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_gate.http_backend.subprocess.run", return_value=subprocess.CompletedProcess( ["hook"], 1, b"", denial, ), @@ -355,7 +355,7 @@ class TestGitHttpBackend(unittest.TestCase): self.addCleanup(server.server_close) with mock.patch( - "bot_bottle.gateway.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_gate.http_backend.subprocess.run", return_value=subprocess.CompletedProcess( ["hook"], 2, b"", b"", ), @@ -402,7 +402,7 @@ class TestGitHttpBackend(unittest.TestCase): self.addCleanup(server.server_close) with mock.patch( - "bot_bottle.gateway.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_gate.http_backend.subprocess.run", side_effect=PermissionError(13, "Permission denied"), ): buf = io.StringIO() @@ -461,7 +461,7 @@ class TestMalformedStatusHeader(unittest.TestCase): def _get_with_backend_response(self, cgi_response: bytes) -> int: with mock.patch( - "bot_bottle.gateway.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_gate.http_backend.subprocess.run", return_value=mock.Mock(returncode=0, stdout=cgi_response), ): req = urllib.request.Request( @@ -545,7 +545,7 @@ class TestContentLengthBounds(unittest.TestCase): # With a valid Content-Length the handler proceeds into # git http-backend; that will fail (no real git repo) but the # status won't be 400 or 413. - with mock.patch("bot_bottle.gateway.git_http_backend.subprocess.run") as run: + with mock.patch("bot_bottle.gateway.git_gate.http_backend.subprocess.run") as run: run.return_value = mock.Mock( returncode=0, stdout=( diff --git a/tests/unit/test_git_http_multitenant.py b/tests/unit/test_git_http_multitenant.py index 16acff1..229e7df 100644 --- a/tests/unit/test_git_http_multitenant.py +++ b/tests/unit/test_git_http_multitenant.py @@ -10,7 +10,7 @@ from __future__ import annotations import unittest from pathlib import Path -from bot_bottle.gateway.git_http_backend import resolve_sandbox_root +from bot_bottle.gateway.git_gate.http_backend import resolve_sandbox_root from bot_bottle.gateway.policy_resolver import PolicyResolveError _BASE = Path("/git") diff --git a/tests/unit/test_macos_infra.py b/tests/unit/test_macos_infra.py index 7d80840..914ed1c 100644 --- a/tests/unit/test_macos_infra.py +++ b/tests/unit/test_macos_infra.py @@ -48,7 +48,7 @@ class TestInfraRun(unittest.TestCase): self.assertIn("bot_bottle.orchestrator", script) # Gateway launches via the installed package (there is no # /app/gateway_init.py file since the daemons moved into bot_bottle). - self.assertIn("bot_bottle.gateway.gateway_init", script) + self.assertIn("bot_bottle.gateway.bootstrap", script) self.assertIn("127.0.0.1", script) # they reach each other on loopback def test_db_is_a_container_only_volume(self) -> None: diff --git a/tests/unit/test_orchestrator_registration.py b/tests/unit/test_orchestrator_registration.py index ffb3752..738ecf8 100644 --- a/tests/unit/test_orchestrator_registration.py +++ b/tests/unit/test_orchestrator_registration.py @@ -7,7 +7,7 @@ import unittest from pathlib import Path from bot_bottle.egress import EgressPlan, EgressRoute -from bot_bottle.gateway.egress_addon_core import LOG_BLOCKS, load_config +from bot_bottle.gateway.egress.addon_core import LOG_BLOCKS, load_config from bot_bottle.orchestrator.registration import ( RegistrationInputs, egress_policy, diff --git a/tests/unit/test_supervise_server.py b/tests/unit/test_supervise_server.py index 0ae9ce5..52c575c 100644 --- a/tests/unit/test_supervise_server.py +++ b/tests/unit/test_supervise_server.py @@ -22,8 +22,8 @@ from bot_bottle.orchestrator import supervisor as _sv from bot_bottle.orchestrator.store import queue_store as _qs from bot_bottle.store import audit_store as _as -from bot_bottle.gateway import supervise_server # noqa: E402 -from bot_bottle.gateway.supervise_server import ( +from bot_bottle.gateway.supervisor import server as supervise_server # noqa: E402 +from bot_bottle.gateway.supervisor.server import ( ERR_INTERNAL, ERR_INVALID_PARAMS, ERR_INVALID_REQUEST, -- 2.52.0 From 74dc984cf8cb773a5a8a01f737597a965165d199 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 17:08:34 -0400 Subject: [PATCH 24/39] refactor(gateway): rename bootstrap's _Supervisor to _DaemonManager The PID-1 process manager in gateway/bootstrap.py was named _Supervisor, which collided conceptually with the supervise permissions service (Supervisor / gateway.supervisor). It manages the _DaemonSpec child set, so _DaemonManager names what it does without the "supervise" echo. Co-Authored-By: Claude Opus 4.8 --- bot_bottle/gateway/bootstrap.py | 4 ++-- tests/unit/test_gateway_init.py | 40 ++++++++++++++++----------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/bot_bottle/gateway/bootstrap.py b/bot_bottle/gateway/bootstrap.py index cd36179..060a980 100644 --- a/bot_bottle/gateway/bootstrap.py +++ b/bot_bottle/gateway/bootstrap.py @@ -173,7 +173,7 @@ def _spawn(spec: _DaemonSpec) -> subprocess.Popen[bytes]: return proc -class _Supervisor: +class _DaemonManager: """Holds the running children + shutdown state. Pulled out so the test suite can drive it with fake commands.""" @@ -387,7 +387,7 @@ def main(argv: Sequence[str] | None = None) -> int: _log("no daemons selected; nothing to do") return 0 - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() signal.signal(signal.SIGTERM, lambda *_: sup.request_shutdown("SIGTERM")) # type: ignore diff --git a/tests/unit/test_gateway_init.py b/tests/unit/test_gateway_init.py index d43cfe6..7f5272c 100644 --- a/tests/unit/test_gateway_init.py +++ b/tests/unit/test_gateway_init.py @@ -20,7 +20,7 @@ from unittest.mock import patch from bot_bottle.gateway.bootstrap import ( _DaemonSpec, - _Supervisor, + _DaemonManager, _argv_for_daemon, _env_for_daemon, _selected_daemons, @@ -169,7 +169,7 @@ class TestDaemonArgv(unittest.TestCase): class TestSupervisor(unittest.TestCase): - """End-to-end: drive `_Supervisor` directly with fake commands. + """End-to-end: drive `_DaemonManager` directly with fake commands. We don't go through `main()` because main installs signal handlers process-wide, which collides with the test runner.""" @@ -177,7 +177,7 @@ class TestSupervisor(unittest.TestCase): warnings.simplefilter("error", ResourceWarning) self.addCleanup(warnings.resetwarnings) - def _drive(self, sup: _Supervisor, max_wait_s: float = 6.0) -> int: + def _drive(self, sup: _DaemonManager, max_wait_s: float = 6.0) -> int: deadline = time.monotonic() + max_wait_s while not sup.tick(): if time.monotonic() > deadline: @@ -193,7 +193,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("a", ("/bin/sh", "-c", ":")), _DaemonSpec("b", ("/bin/sh", "-c", ":")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.1) sup.request_shutdown(reason="test") @@ -209,7 +209,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("crasher", ("/bin/sh", "-c", "exit 1")), _DaemonSpec("longrun", (SLEEP, "30")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() original_pid = sup.procs[0][1].pid @@ -235,7 +235,7 @@ class TestSupervisor(unittest.TestCase): def test_single_daemon_crash_is_restarted_before_tick_completes(self): specs = [_DaemonSpec("crasher", ("/bin/sh", "-c", "exit 1"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() original_pid = sup.procs[0][1].pid time.sleep(0.1) @@ -259,7 +259,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("crasher", ("/bin/sh", "-c", "exit 1")), _DaemonSpec("longrun", (SLEEP, "30")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.3) # let crasher die sup.request_shutdown(reason="test") @@ -271,7 +271,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("a", ("/bin/sh", "-c", "exit 0")), _DaemonSpec("b", ("/bin/sh", "-c", "exit 2")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() original_pids = [p.pid for _, p in sup.procs] time.sleep(0.1) @@ -309,7 +309,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("egress", sighup_marker), _DaemonSpec("other", (SLEEP, "30")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.3) # let Python install the handler @@ -331,7 +331,7 @@ class TestSupervisor(unittest.TestCase): def test_forward_signal_unknown_daemon_no_op(self): specs = [_DaemonSpec("a", (SLEEP, "30"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() delivered = sup.forward_signal(signal.SIGHUP, "ghost") self.assertFalse(delivered) @@ -345,7 +345,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("git-gate", (SLEEP, "30")), _DaemonSpec("supervise", (SLEEP, "30")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.1) old_git_gate_pid = sup.procs[0][1].pid @@ -370,7 +370,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("git-gate", (SLEEP, "30")), _DaemonSpec("supervise", (SLEEP, "30")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.1) old_git_gate_pid = sup.procs[0][1].pid @@ -392,7 +392,7 @@ class TestSupervisor(unittest.TestCase): def test_repeated_restart_requests_coalesce(self): specs = [_DaemonSpec("git-gate", (SLEEP, "30"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.1) @@ -415,7 +415,7 @@ class TestSupervisor(unittest.TestCase): def test_request_restart_unknown_daemon_no_op(self): specs = [_DaemonSpec("a", (SLEEP, "30"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() ok = sup.request_restart("ghost") self.assertFalse(ok) @@ -425,7 +425,7 @@ class TestSupervisor(unittest.TestCase): def test_restart_unknown_daemon_no_op(self): specs = [_DaemonSpec("a", (SLEEP, "30"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() ok = sup.restart_daemon("ghost") self.assertFalse(ok) @@ -434,7 +434,7 @@ class TestSupervisor(unittest.TestCase): def test_restart_during_shutdown_is_no_op(self): specs = [_DaemonSpec("git-gate", (SLEEP, "30"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() sup.request_shutdown(reason="test") ok = sup.restart_daemon("git-gate") @@ -444,7 +444,7 @@ class TestSupervisor(unittest.TestCase): def test_pending_restart_dropped_during_shutdown(self): specs = [_DaemonSpec("git-gate", (SLEEP, "30"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.1) old_pid = sup.procs[0][1].pid @@ -464,7 +464,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("a", (SLEEP, "60")), _DaemonSpec("b", (SLEEP, "60")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.2) # let them actually start sup.request_shutdown(reason="test") @@ -484,7 +484,7 @@ class TestSupervisor(unittest.TestCase): "trap '' TERM; sleep 30", ) specs = [_DaemonSpec("stubborn", ignore_term)] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.3) # let `trap` register sup.request_shutdown(reason="test") @@ -498,7 +498,7 @@ class TestSupervisor(unittest.TestCase): def test_idempotent_shutdown_requests(self): specs = [_DaemonSpec("a", (SLEEP, "60"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.1) first_at = None -- 2.52.0 From 82d02d2c4b24714d338621f7d9e8c9632b634358 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 17:14:26 -0400 Subject: [PATCH 25/39] refactor(gateway): move egress entrypoint into gateway/egress egress_entrypoint.sh is the mitmdump launcher for the egress data plane, so it belongs with the gateway egress service, not at the package root. Moved to bot_bottle/gateway/egress/entrypoint.sh (prefix stripped now that the package namespaces it). Only the Dockerfile.gateway COPY source changes; the runtime target (/app/egress-entrypoint.sh, referenced by bootstrap's egress DaemonSpec) is unchanged, and the infra images inherit it via FROM gateway. Updated the test that resolves the script path and the infra-artifact docstring example. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- Dockerfile.gateway | 2 +- bot_bottle/backend/firecracker/infra_artifact.py | 2 +- .../{egress_entrypoint.sh => gateway/egress/entrypoint.sh} | 0 tests/unit/test_egress_entrypoint.py | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename bot_bottle/{egress_entrypoint.sh => gateway/egress/entrypoint.sh} (100%) diff --git a/Dockerfile.gateway b/Dockerfile.gateway index 47ff584..ad7001c 100644 --- a/Dockerfile.gateway +++ b/Dockerfile.gateway @@ -103,7 +103,7 @@ RUN pip install --no-cache-dir /src/ # (nothing created /app before this point). WORKDIR /app RUN printf 'from bot_bottle.gateway.egress.addon import addons\n' > /app/egress_addon.py -COPY bot_bottle/egress_entrypoint.sh /app/egress-entrypoint.sh +COPY bot_bottle/gateway/egress/entrypoint.sh /app/egress-entrypoint.sh RUN chmod +x /app/egress-entrypoint.sh # Pre-create runtime directories the compose renderer + start diff --git a/bot_bottle/backend/firecracker/infra_artifact.py b/bot_bottle/backend/firecracker/infra_artifact.py index 5c8ecb2..eb8396d 100644 --- a/bot_bottle/backend/firecracker/infra_artifact.py +++ b/bot_bottle/backend/firecracker/infra_artifact.py @@ -66,7 +66,7 @@ def infra_artifact_version(init_script: str, *, repo_root: Path = _REPO_ROOT) -> The package is `COPY bot_bottle /app/bot_bottle`'d wholesale into the image, so hash *every* regular file under it — not just `*.py`. Non-Python inputs - (e.g. `egress_entrypoint.sh`, `netpool.defaults.env`) are baked in too, and + (e.g. `gateway/egress/entrypoint.sh`, `netpool.defaults.env`) are baked in too, and a change to one must bump the version or a launch host could boot a stale rootfs whose code differs from its checkout. `__pycache__`/`.pyc` are the only exclusions — build artifacts, never copied.""" diff --git a/bot_bottle/egress_entrypoint.sh b/bot_bottle/gateway/egress/entrypoint.sh similarity index 100% rename from bot_bottle/egress_entrypoint.sh rename to bot_bottle/gateway/egress/entrypoint.sh diff --git a/tests/unit/test_egress_entrypoint.py b/tests/unit/test_egress_entrypoint.py index fc0bb12..0c846ac 100644 --- a/tests/unit/test_egress_entrypoint.py +++ b/tests/unit/test_egress_entrypoint.py @@ -21,7 +21,7 @@ from pathlib import Path _SCRIPT = ( Path(__file__).resolve().parent.parent.parent - / "bot_bottle" / "egress_entrypoint.sh" + / "bot_bottle" / "gateway" / "egress" / "entrypoint.sh" ) -- 2.52.0 From 50a67c04bd3fa3018413af72f31c458a59475f8b Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 17:25:25 -0400 Subject: [PATCH 26/39] refactor(cli): group subcommand handlers under cli/commands/ Move the eleven per-command modules (backend, cleanup, commit, edit, info, init, list, login, resume, start, supervise) into a new bot_bottle/cli/commands/ package, leaving the dispatcher (__init__), entrypoint (__main__), and shared helpers (_common, tui) at the cli root. Makes the command surface obvious at a glance and separates handlers from the plumbing that registers them. Updated the dispatcher's COMMANDS imports to .commands.*, bumped the moved files' relative-import depths (.._common, .. import tui, sibling .start unchanged), and repointed test references to bot_bottle.cli.commands.*. Full unit suite green (2243); CLI dispatch verified via `python -m bot_bottle.cli --help`. Co-Authored-By: Claude Opus 4.8 --- bot_bottle/cli/__init__.py | 22 +++++----- bot_bottle/cli/commands/__init__.py | 6 +++ bot_bottle/cli/{ => commands}/backend.py | 4 +- bot_bottle/cli/{ => commands}/cleanup.py | 6 +-- bot_bottle/cli/{ => commands}/commit.py | 12 ++--- bot_bottle/cli/{ => commands}/edit.py | 4 +- bot_bottle/cli/{ => commands}/info.py | 6 +-- bot_bottle/cli/{ => commands}/init.py | 4 +- bot_bottle/cli/{ => commands}/list.py | 6 +-- bot_bottle/cli/{ => commands}/login.py | 2 +- bot_bottle/cli/{ => commands}/resume.py | 10 ++--- bot_bottle/cli/{ => commands}/start.py | 26 +++++------ bot_bottle/cli/{ => commands}/supervise.py | 12 ++--- tests/unit/test_cli_backend.py | 2 +- tests/unit/test_cli_cleanup_cross_backend.py | 2 +- tests/unit/test_cli_commit.py | 20 ++++----- tests/unit/test_cli_login.py | 44 +++++++++---------- tests/unit/test_cli_start_headless.py | 24 +++++----- tests/unit/test_cli_start_selector.py | 26 +++++------ tests/unit/test_cli_start_settle.py | 2 +- tests/unit/test_cli_start_stale.py | 10 ++--- tests/unit/test_supervise_cli.py | 2 +- .../unit/test_supervise_cli_crash_logging.py | 2 +- 23 files changed, 130 insertions(+), 124 deletions(-) create mode 100644 bot_bottle/cli/commands/__init__.py rename bot_bottle/cli/{ => commands}/backend.py (94%) rename bot_bottle/cli/{ => commands}/cleanup.py (93%) rename bot_bottle/cli/{ => commands}/commit.py (86%) rename bot_bottle/cli/{ => commands}/edit.py (95%) rename bot_bottle/cli/{ => commands}/info.py (93%) rename bot_bottle/cli/{ => commands}/init.py (98%) rename bot_bottle/cli/{ => commands}/list.py (93%) rename bot_bottle/cli/{ => commands}/login.py (99%) rename bot_bottle/cli/{ => commands}/resume.py (90%) rename bot_bottle/cli/{ => commands}/start.py (97%) rename bot_bottle/cli/{ => commands}/supervise.py (98%) diff --git a/bot_bottle/cli/__init__.py b/bot_bottle/cli/__init__.py index d1d550a..41ffa7d 100644 --- a/bot_bottle/cli/__init__.py +++ b/bot_bottle/cli/__init__.py @@ -12,17 +12,17 @@ from ..log import Die, die, error from ..manifest import ManifestError from ..orchestrator.store.store_manager import StoreManager from ._common import PROG -from . import list as _list_mod -from .backend import cmd_backend -from .cleanup import cmd_cleanup -from .commit import cmd_commit -from .edit import cmd_edit -from .info import cmd_info -from .init import cmd_init -from .login import cmd_login -from .resume import cmd_resume -from .start import cmd_start -from .supervise import cmd_supervise +from .commands import list as _list_mod +from .commands.backend import cmd_backend +from .commands.cleanup import cmd_cleanup +from .commands.commit import cmd_commit +from .commands.edit import cmd_edit +from .commands.info import cmd_info +from .commands.init import cmd_init +from .commands.login import cmd_login +from .commands.resume import cmd_resume +from .commands.start import cmd_start +from .commands.supervise import cmd_supervise cmd_list = _list_mod.cmd_list diff --git a/bot_bottle/cli/commands/__init__.py b/bot_bottle/cli/commands/__init__.py new file mode 100644 index 0000000..0daac76 --- /dev/null +++ b/bot_bottle/cli/commands/__init__.py @@ -0,0 +1,6 @@ +"""CLI subcommand handlers — one module per `bot-bottle `. + +Each module exposes a `cmd_(argv)` handler that the dispatcher +(`bot_bottle.cli`) registers in its COMMANDS table. Shared CLI helpers +(`_common`, `tui`) stay one level up in the `cli` package. +""" diff --git a/bot_bottle/cli/backend.py b/bot_bottle/cli/commands/backend.py similarity index 94% rename from bot_bottle/cli/backend.py rename to bot_bottle/cli/commands/backend.py index ed6808f..2997db3 100644 --- a/bot_bottle/cli/backend.py +++ b/bot_bottle/cli/commands/backend.py @@ -15,8 +15,8 @@ from __future__ import annotations import argparse -from ..backend import get_bottle_backend, known_backend_names -from ._common import PROG +from ...backend import get_bottle_backend, known_backend_names +from .._common import PROG def cmd_backend(args: list[str]) -> int: diff --git a/bot_bottle/cli/cleanup.py b/bot_bottle/cli/commands/cleanup.py similarity index 93% rename from bot_bottle/cli/cleanup.py rename to bot_bottle/cli/commands/cleanup.py index 34d6de9..685f6bc 100644 --- a/bot_bottle/cli/cleanup.py +++ b/bot_bottle/cli/commands/cleanup.py @@ -21,9 +21,9 @@ from __future__ import annotations import sys -from ..backend import get_bottle_backend, has_backend, known_backend_names -from ..log import info -from ._common import read_tty_line +from ...backend import get_bottle_backend, has_backend, known_backend_names +from ...log import info +from .._common import read_tty_line def cmd_cleanup(_argv: list[str]) -> int: diff --git a/bot_bottle/cli/commit.py b/bot_bottle/cli/commands/commit.py similarity index 86% rename from bot_bottle/cli/commit.py rename to bot_bottle/cli/commands/commit.py index 51d94d9..502ae69 100644 --- a/bot_bottle/cli/commit.py +++ b/bot_bottle/cli/commands/commit.py @@ -12,12 +12,12 @@ from __future__ import annotations import argparse -from ..backend import enumerate_active_agents -from ..backend.freeze import CommitCancelled, get_freezer -from ..bottle_state import read_metadata -from ..log import die -from ._common import PROG -from . import tui +from ...backend import enumerate_active_agents +from ...backend.freeze import CommitCancelled, get_freezer +from ...bottle_state import read_metadata +from ...log import die +from .._common import PROG +from .. import tui def cmd_commit(argv: list[str]) -> int: diff --git a/bot_bottle/cli/edit.py b/bot_bottle/cli/commands/edit.py similarity index 95% rename from bot_bottle/cli/edit.py rename to bot_bottle/cli/commands/edit.py index 8c412d9..82ca91b 100644 --- a/bot_bottle/cli/edit.py +++ b/bot_bottle/cli/commands/edit.py @@ -7,8 +7,8 @@ import json import os from pathlib import Path -from ..log import die -from ._common import PROG, USER_CWD +from ...log import die +from .._common import PROG, USER_CWD def cmd_edit(argv: list[str]) -> int: diff --git a/bot_bottle/cli/info.py b/bot_bottle/cli/commands/info.py similarity index 93% rename from bot_bottle/cli/info.py rename to bot_bottle/cli/commands/info.py index 61b8945..d15b561 100644 --- a/bot_bottle/cli/info.py +++ b/bot_bottle/cli/commands/info.py @@ -4,9 +4,9 @@ from __future__ import annotations import argparse -from ..log import info -from ..manifest import ManifestIndex -from ._common import PROG, USER_CWD +from ...log import info +from ...manifest import ManifestIndex +from .._common import PROG, USER_CWD def cmd_info(argv: list[str]) -> int: diff --git a/bot_bottle/cli/init.py b/bot_bottle/cli/commands/init.py similarity index 98% rename from bot_bottle/cli/init.py rename to bot_bottle/cli/commands/init.py index 21b62a0..ca169f1 100644 --- a/bot_bottle/cli/init.py +++ b/bot_bottle/cli/commands/init.py @@ -10,8 +10,8 @@ import sys from pathlib import Path from typing import Any -from ..log import die, info, warn -from ._common import PROG, USER_CWD, read_tty_line +from ...log import die, info, warn +from .._common import PROG, USER_CWD, read_tty_line def cmd_init(argv: list[str]) -> int: diff --git a/bot_bottle/cli/list.py b/bot_bottle/cli/commands/list.py similarity index 93% rename from bot_bottle/cli/list.py rename to bot_bottle/cli/commands/list.py index c727428..5c0caf3 100644 --- a/bot_bottle/cli/list.py +++ b/bot_bottle/cli/commands/list.py @@ -6,9 +6,9 @@ import argparse import os import sys -from ..backend import enumerate_active_agents -from ..manifest import ManifestIndex -from ._common import PROG, USER_CWD +from ...backend import enumerate_active_agents +from ...manifest import ManifestIndex +from .._common import PROG, USER_CWD _ANSI_COLOR_CODES: dict[str, str] = { "red": "\033[91m", diff --git a/bot_bottle/cli/login.py b/bot_bottle/cli/commands/login.py similarity index 99% rename from bot_bottle/cli/login.py rename to bot_bottle/cli/commands/login.py index 3d00f17..55e04bf 100644 --- a/bot_bottle/cli/login.py +++ b/bot_bottle/cli/commands/login.py @@ -25,7 +25,7 @@ import urllib.request from pathlib import Path from typing import Any -from ..paths import bot_bottle_root +from ...paths import bot_bottle_root _CONSOLE_URL_ENV = "BB_CONSOLE_URL" _POLL_SLEEP = 2 # seconds between polls; matches console's poll_interval default diff --git a/bot_bottle/cli/resume.py b/bot_bottle/cli/commands/resume.py similarity index 90% rename from bot_bottle/cli/resume.py rename to bot_bottle/cli/commands/resume.py index 5dd421b..6d8224a 100644 --- a/bot_bottle/cli/resume.py +++ b/bot_bottle/cli/commands/resume.py @@ -16,11 +16,11 @@ from __future__ import annotations import argparse -from ..backend import BottleSpec -from ..bottle_state import read_metadata -from ..log import die -from ..manifest import ManifestIndex -from ._common import PROG, USER_CWD +from ...backend import BottleSpec +from ...bottle_state import read_metadata +from ...log import die +from ...manifest import ManifestIndex +from .._common import PROG, USER_CWD from .start import _launch_bottle diff --git a/bot_bottle/cli/start.py b/bot_bottle/cli/commands/start.py similarity index 97% rename from bot_bottle/cli/start.py rename to bot_bottle/cli/commands/start.py index b587216..24c2e9c 100644 --- a/bot_bottle/cli/start.py +++ b/bot_bottle/cli/commands/start.py @@ -22,25 +22,25 @@ import tempfile from pathlib import Path from typing import Callable -from ..agent_provider import get_provider, runtime_for -from ..backend import ( +from ...agent_provider import get_provider, runtime_for +from ...backend import ( Bottle, BottleSpec, enumerate_active_agents, get_bottle_backend, ) -from ..backend.docker import util as docker_mod -from ..backend.docker.bottle_plan import DockerBottlePlan -from ..bottle_state import ( +from ...backend.docker import util as docker_mod +from ...backend.docker.bottle_plan import DockerBottlePlan +from ...bottle_state import ( cleanup_state, is_preserved, mark_preserved, ) -from ..image_cache import StaleImageError -from ..log import info, die -from ..manifest import Manifest, ManifestIndex -from ._common import PROG, USER_CWD, read_tty_line -from . import tui +from ...image_cache import StaleImageError +from ...log import info, die +from ...manifest import Manifest, ManifestIndex +from .._common import PROG, USER_CWD, read_tty_line +from .. import tui def cmd_start(argv: list[str]) -> int: @@ -380,8 +380,8 @@ def _peek_agent_bottle(manifest: ManifestIndex, agent_name: str) -> str: return manifest.agents[agent_name].bottle return "" - from ..manifest.loader import scan_agent_names - from ..yaml_subset import YamlSubsetError, parse_frontmatter + from ...manifest.loader import scan_agent_names + from ...yaml_subset import YamlSubsetError, parse_frontmatter home_agents = scan_agent_names(manifest.home_md / "agents") cwd_agents: dict[str, Path] = {} @@ -449,7 +449,7 @@ def _bottle_lineage(manifest: ManifestIndex) -> dict[str, str]: if not bottles_dir.is_dir(): return {} - from ..yaml_subset import YamlSubsetError, parse_frontmatter + from ...yaml_subset import YamlSubsetError, parse_frontmatter extends_of: dict[str, str] = {} for path in bottles_dir.glob("*.md"): diff --git a/bot_bottle/cli/supervise.py b/bot_bottle/cli/commands/supervise.py similarity index 98% rename from bot_bottle/cli/supervise.py rename to bot_bottle/cli/commands/supervise.py index abf7d70..c2beb03 100644 --- a/bot_bottle/cli/supervise.py +++ b/bot_bottle/cli/commands/supervise.py @@ -19,22 +19,22 @@ from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path -from ..paths import bot_bottle_root -from ..log import Die, error, info -from ..orchestrator.client import ( +from ...paths import bot_bottle_root +from ...log import Die, error, info +from ...orchestrator.client import ( OrchestratorClient, OrchestratorClientError, discover_orchestrator_url, ) -from ..supervisor.types import ( +from ...supervisor.types import ( Proposal, TOOL_EGRESS_ALLOW, TOOL_EGRESS_BLOCK, TOOL_GITLEAKS_ALLOW, TOOL_EGRESS_TOKEN_ALLOW, ) -from ._common import PROG +from .._common import PROG _REFRESH_INTERVAL_MS = 1000 @@ -81,7 +81,7 @@ def _resolve_orchestrator_url() -> str: try: return discover_orchestrator_url() except OrchestratorClientError: - from ..backend import get_bottle_backend + from ...backend import get_bottle_backend backend = get_bottle_backend() info(f"no orchestrator control plane running; starting one ({backend.name})…") return backend.ensure_orchestrator() diff --git a/tests/unit/test_cli_backend.py b/tests/unit/test_cli_backend.py index 16eea2c..40f2890 100644 --- a/tests/unit/test_cli_backend.py +++ b/tests/unit/test_cli_backend.py @@ -12,7 +12,7 @@ import subprocess import unittest from unittest.mock import MagicMock, patch -from bot_bottle.cli import backend as cmd +from bot_bottle.cli.commands import backend as cmd from bot_bottle.backend.docker import setup as docker_setup diff --git a/tests/unit/test_cli_cleanup_cross_backend.py b/tests/unit/test_cli_cleanup_cross_backend.py index e1f4be8..47ab133 100644 --- a/tests/unit/test_cli_cleanup_cross_backend.py +++ b/tests/unit/test_cli_cleanup_cross_backend.py @@ -11,7 +11,7 @@ from __future__ import annotations import unittest from unittest.mock import patch, MagicMock -from bot_bottle.cli import cleanup as cmd +from bot_bottle.cli.commands import cleanup as cmd def _make_backend(empty: bool = True): diff --git a/tests/unit/test_cli_commit.py b/tests/unit/test_cli_commit.py index 6a7e7ff..9eee542 100644 --- a/tests/unit/test_cli_commit.py +++ b/tests/unit/test_cli_commit.py @@ -7,7 +7,7 @@ import unittest from pathlib import Path from unittest.mock import MagicMock, patch -from bot_bottle.cli.commit import cmd_commit +from bot_bottle.cli.commands.commit import cmd_commit from tests.unit import use_bottle_root from bot_bottle import bottle_state from bot_bottle.backend.freeze import CommitCancelled @@ -42,7 +42,7 @@ class TestCmdCommitSlugArg(_FakeHomeMixin, unittest.TestCase): slug = "dev-abc12" self._write_meta(slug, "docker") - with patch("bot_bottle.cli.commit.get_freezer") as mock_gf: + with patch("bot_bottle.cli.commands.commit.get_freezer") as mock_gf: mock_freezer = MagicMock() mock_gf.return_value = mock_freezer rc = cmd_commit([slug]) @@ -56,7 +56,7 @@ class TestCmdCommitSlugArg(_FakeHomeMixin, unittest.TestCase): slug = "dev-abc12" self._write_meta(slug, "") - with patch("bot_bottle.cli.commit.get_freezer") as mock_gf: + with patch("bot_bottle.cli.commands.commit.get_freezer") as mock_gf: mock_freezer = MagicMock() mock_gf.return_value = mock_freezer rc = cmd_commit([slug]) @@ -68,7 +68,7 @@ class TestCmdCommitSlugArg(_FakeHomeMixin, unittest.TestCase): slug = "dev-abc12" self._write_meta(slug, "macos-container") - with patch("bot_bottle.cli.commit.get_freezer") as mock_gf: + with patch("bot_bottle.cli.commands.commit.get_freezer") as mock_gf: mock_freezer = MagicMock() mock_gf.return_value = mock_freezer rc = cmd_commit([slug]) @@ -81,7 +81,7 @@ class TestCmdCommitSlugArg(_FakeHomeMixin, unittest.TestCase): slug = "dev-abc12" self._write_meta(slug, "firecracker") - with patch("bot_bottle.cli.commit.get_freezer") as mock_gf: + with patch("bot_bottle.cli.commands.commit.get_freezer") as mock_gf: mock_freezer = MagicMock() mock_gf.return_value = mock_freezer rc = cmd_commit([slug]) @@ -93,7 +93,7 @@ class TestCmdCommitSlugArg(_FakeHomeMixin, unittest.TestCase): slug = "dev-abc12" self._write_meta(slug, "macos-container") - with patch("bot_bottle.cli.commit.get_freezer") as mock_gf: + with patch("bot_bottle.cli.commands.commit.get_freezer") as mock_gf: mock_freezer = MagicMock() mock_freezer.commit_slug.side_effect = CommitCancelled mock_gf.return_value = mock_freezer @@ -111,9 +111,9 @@ class TestCmdCommitNoActiveBottles(_FakeHomeMixin, unittest.TestCase): def test_dies_when_no_active_bottles_and_no_slug(self): with patch( - "bot_bottle.cli.commit.enumerate_active_agents", return_value=[], + "bot_bottle.cli.commands.commit.enumerate_active_agents", return_value=[], ), patch( - "bot_bottle.cli.commit.die", side_effect=SystemExit("die"), + "bot_bottle.cli.commands.commit.die", side_effect=SystemExit("die"), ) as mock_die: with self.assertRaises(SystemExit): cmd_commit([]) @@ -124,9 +124,9 @@ class TestCmdCommitNoActiveBottles(_FakeHomeMixin, unittest.TestCase): active = MagicMock() active.slug = "dev-abc12" with patch( - "bot_bottle.cli.commit.enumerate_active_agents", return_value=[active], + "bot_bottle.cli.commands.commit.enumerate_active_agents", return_value=[active], ), patch( - "bot_bottle.cli.commit.tui.filter_select", return_value=None, + "bot_bottle.cli.commands.commit.tui.filter_select", return_value=None, ): rc = cmd_commit([]) diff --git a/tests/unit/test_cli_login.py b/tests/unit/test_cli_login.py index 0d4c13c..67c00e0 100644 --- a/tests/unit/test_cli_login.py +++ b/tests/unit/test_cli_login.py @@ -14,27 +14,27 @@ from unittest.mock import MagicMock, patch class TestFlagParsing(unittest.TestCase): def test_console_url_flag(self) -> None: - from bot_bottle.cli.login import _flag + from bot_bottle.cli.commands.login import _flag self.assertEqual(_flag(["--console-url", "http://x"], "--console-url"), "http://x") def test_console_url_equals_form(self) -> None: - from bot_bottle.cli.login import _flag + from bot_bottle.cli.commands.login import _flag self.assertEqual( _flag(["--console-url=http://x"], "--console-url"), "http://x" ) def test_label_flag(self) -> None: - from bot_bottle.cli.login import _flag + from bot_bottle.cli.commands.login import _flag self.assertEqual(_flag(["--label", "my-mac"], "--label"), "my-mac") def test_missing_flag_returns_none(self) -> None: - from bot_bottle.cli.login import _flag + from bot_bottle.cli.commands.login import _flag self.assertIsNone(_flag([], "--console-url")) class TestHttpHelpers(unittest.TestCase): def test_post_sends_json_and_decodes_response(self) -> None: - from bot_bottle.cli.login import _post + from bot_bottle.cli.commands.login import _post response = MagicMock() response.__enter__.return_value.read.return_value = b'{"ok": true}' @@ -48,7 +48,7 @@ class TestHttpHelpers(unittest.TestCase): self.assertEqual(request.get_header("Content-type"), "application/json") def test_get_decodes_success_response(self) -> None: - from bot_bottle.cli.login import _get + from bot_bottle.cli.commands.login import _get response = MagicMock() response.__enter__.return_value.status = 200 @@ -59,7 +59,7 @@ class TestHttpHelpers(unittest.TestCase): ) def test_get_returns_http_error_status(self) -> None: - from bot_bottle.cli.login import _get + from bot_bottle.cli.commands.login import _get error = urllib.error.HTTPError( "http://console/status", 410, "gone", Message(), None @@ -70,7 +70,7 @@ class TestHttpHelpers(unittest.TestCase): class TestSaveCredentials(unittest.TestCase): def test_writes_json_and_sets_perms(self) -> None: - from bot_bottle.cli.login import _save_credentials + from bot_bottle.cli.commands.login import _save_credentials with tempfile.TemporaryDirectory() as tmp: with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}): @@ -85,7 +85,7 @@ class TestSaveCredentials(unittest.TestCase): def test_temp_file_is_private_before_replace(self) -> None: """Temp file must be 0600 at the moment os.replace is called.""" - from bot_bottle.cli.login import _save_credentials + from bot_bottle.cli.commands.login import _save_credentials from pathlib import Path as _Path tmp_perms_at_replace: list[int] = [] @@ -107,7 +107,7 @@ class TestSaveCredentials(unittest.TestCase): def test_cleanup_on_write_failure(self) -> None: """Temp file is removed and no credentials remain if replace fails.""" - from bot_bottle.cli.login import _save_credentials + from bot_bottle.cli.commands.login import _save_credentials with tempfile.TemporaryDirectory() as tmp: with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}): @@ -120,12 +120,12 @@ class TestSaveCredentials(unittest.TestCase): class TestCmdLoginMissingUrl(unittest.TestCase): def test_help_returns_0(self) -> None: - from bot_bottle.cli.login import cmd_login + from bot_bottle.cli.commands.login import cmd_login self.assertEqual(cmd_login(["--help"]), 0) def test_returns_1_without_url(self) -> None: - from bot_bottle.cli.login import cmd_login + from bot_bottle.cli.commands.login import cmd_login with patch.dict(os.environ, {}, clear=True): os.environ.pop("BB_CONSOLE_URL", None) result = cmd_login([]) @@ -133,7 +133,7 @@ class TestCmdLoginMissingUrl(unittest.TestCase): def test_reads_env_var(self) -> None: """Exits 1 (network error) not because of missing URL when env var is set.""" - from bot_bottle.cli.login import cmd_login + from bot_bottle.cli.commands.login import cmd_login def _fail_post(_url: str, _payload: dict[str, Any]) -> dict[str, Any]: raise OSError("connection refused") @@ -146,7 +146,7 @@ class TestCmdLoginMissingUrl(unittest.TestCase): "BOT_BOTTLE_ROOT": tmp, }, ): - with patch("bot_bottle.cli.login._post", side_effect=_fail_post): + with patch("bot_bottle.cli.commands.login._post", side_effect=_fail_post): result = cmd_login([]) self.assertEqual(result, 1) @@ -155,7 +155,7 @@ class TestCmdLoginFlow(unittest.TestCase): def _run_with_mocks( self, poll_responses: list[dict[str, Any]], tmp: str ) -> int: - from bot_bottle.cli.login import cmd_login + from bot_bottle.cli.commands.login import cmd_login start_resp = { "device_code": "dc123", @@ -175,8 +175,8 @@ class TestCmdLoginFlow(unittest.TestCase): return 200, next(poll_iter, {"status": "pending"}) with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}): - with patch("bot_bottle.cli.login._post", side_effect=_fake_post): - with patch("bot_bottle.cli.login._get", side_effect=_fake_get): + with patch("bot_bottle.cli.commands.login._post", side_effect=_fake_post): + with patch("bot_bottle.cli.commands.login._get", side_effect=_fake_get): with patch("time.sleep"): return cmd_login(["--console-url", "http://console"]) @@ -202,7 +202,7 @@ class TestCmdLoginFlow(unittest.TestCase): self.assertEqual(result, 1) def test_timeout_returns_1(self) -> None: - from bot_bottle.cli.login import cmd_login + from bot_bottle.cli.commands.login import cmd_login start_resp = { "device_code": "dc", @@ -213,13 +213,13 @@ class TestCmdLoginFlow(unittest.TestCase): with tempfile.TemporaryDirectory() as tmp: with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}): - with patch("bot_bottle.cli.login._post", return_value=start_resp): + with patch("bot_bottle.cli.commands.login._post", return_value=start_resp): result = cmd_login(["--console-url", "http://console"]) self.assertEqual(result, 1) def test_poll_interval_from_server_is_used(self) -> None: """time.sleep must be called with the server-provided poll_interval.""" - from bot_bottle.cli.login import cmd_login + from bot_bottle.cli.commands.login import cmd_login server_interval = 7 start_resp = { @@ -241,9 +241,9 @@ class TestCmdLoginFlow(unittest.TestCase): with tempfile.TemporaryDirectory() as tmp: with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}): - with patch("bot_bottle.cli.login._post", return_value=start_resp): + with patch("bot_bottle.cli.commands.login._post", return_value=start_resp): with patch( - "bot_bottle.cli.login._get", side_effect=_fake_get + "bot_bottle.cli.commands.login._get", side_effect=_fake_get ): with patch("time.sleep") as mock_sleep: result = cmd_login(["--console-url", "http://console"]) diff --git a/tests/unit/test_cli_start_headless.py b/tests/unit/test_cli_start_headless.py index 092b26c..e259acb 100644 --- a/tests/unit/test_cli_start_headless.py +++ b/tests/unit/test_cli_start_headless.py @@ -14,7 +14,7 @@ import os import unittest from unittest.mock import MagicMock, patch -import bot_bottle.cli.start as start_mod +import bot_bottle.cli.commands.start as start_mod import bot_bottle.cli.tui as tui_mod from bot_bottle.backend import ActiveAgent from bot_bottle.log import Die @@ -53,15 +53,15 @@ class TestCmdStartHeadless(unittest.TestCase): ["researcher", "implementer"], ["claude", "dev"], agent_bottle="claude" ) patch( - "bot_bottle.cli.start.ManifestIndex.resolve", + "bot_bottle.cli.commands.start.ManifestIndex.resolve", return_value=self._manifest, ).start() self._launch_mock = patch( - "bot_bottle.cli.start._launch_bottle", return_value=0 + "bot_bottle.cli.commands.start._launch_bottle", return_value=0 ).start() # No bottles running by default → no label collision. patch( - "bot_bottle.cli.start.enumerate_active_agents", return_value=[] + "bot_bottle.cli.commands.start.enumerate_active_agents", return_value=[] ).start() # If any TUI picker fires in headless mode, that's a bug. self._agent_picker = patch.object(tui_mod, "filter_select").start() @@ -71,7 +71,7 @@ class TestCmdStartHeadless(unittest.TestCase): os.environ.pop("BOT_BOTTLE_BACKEND", None) # PTY check uses os.isatty(sys.stdin.fileno()); stub both so # headless unit tests aren't blocked on a real TTY. - patch("bot_bottle.cli.start.os.isatty", return_value=True).start() + patch("bot_bottle.cli.commands.start.os.isatty", return_value=True).start() self.addCleanup(patch.stopall) def _spec(self): @@ -128,7 +128,7 @@ class TestCmdStartHeadless(unittest.TestCase): def test_no_bottle_and_no_default_dies(self): manifest = _make_manifest(["researcher"], ["claude"], agent_bottle="") with patch( - "bot_bottle.cli.start.ManifestIndex.resolve", return_value=manifest + "bot_bottle.cli.commands.start.ManifestIndex.resolve", return_value=manifest ): with self.assertRaises(Die): start_mod.cmd_start( @@ -170,7 +170,7 @@ class TestCmdStartHeadless(unittest.TestCase): def test_label_collision_uniquifies(self): with patch( - "bot_bottle.cli.start.enumerate_active_agents", + "bot_bottle.cli.commands.start.enumerate_active_agents", return_value=[_active_agent("researcher")], ): start_mod.cmd_start( @@ -203,9 +203,9 @@ class TestPrepareWithPreflight(unittest.TestCase): mock_backend.name = "test-backend" render = MagicMock() - with patch("bot_bottle.cli.start.get_bottle_backend", return_value=mock_backend), \ - patch("bot_bottle.cli.start._identity_from_plan", return_value="id"), \ - patch("bot_bottle.cli.start.info"): + with patch("bot_bottle.cli.commands.start.get_bottle_backend", return_value=mock_backend), \ + patch("bot_bottle.cli.commands.start._identity_from_plan", return_value="id"), \ + patch("bot_bottle.cli.commands.start.info"): start_mod.prepare_with_preflight( MagicMock(), stage_dir=Path("/tmp"), @@ -223,7 +223,7 @@ class TestNoAgentsDefined(unittest.TestCase): def test_no_agents_defined_returns_1(self): manifest = _make_manifest([], []) with patch( - "bot_bottle.cli.start.ManifestIndex.resolve", return_value=manifest + "bot_bottle.cli.commands.start.ManifestIndex.resolve", return_value=manifest ), patch("sys.stderr", io.StringIO()) as err: rc = start_mod.cmd_start([]) self.assertEqual(1, rc) @@ -236,7 +236,7 @@ class TestTextRenderPreflight(unittest.TestCase): def test_backend_name_in_output(self): render = start_mod._text_render_preflight() plan = MagicMock() - with patch("bot_bottle.cli.start._manifest_to_yaml", return_value=""), \ + with patch("bot_bottle.cli.commands.start._manifest_to_yaml", return_value=""), \ patch("sys.stderr", io.StringIO()) as err: render(plan, "my-backend") self.assertIn("my-backend", err.getvalue()) diff --git a/tests/unit/test_cli_start_selector.py b/tests/unit/test_cli_start_selector.py index 4f41f07..5819dac 100644 --- a/tests/unit/test_cli_start_selector.py +++ b/tests/unit/test_cli_start_selector.py @@ -14,7 +14,7 @@ import unittest from collections.abc import Mapping, Sequence from unittest.mock import MagicMock, patch -import bot_bottle.cli.start as start_mod +import bot_bottle.cli.commands.start as start_mod import bot_bottle.cli.tui as tui_mod from bot_bottle.backend import ActiveAgent @@ -38,13 +38,13 @@ class TestCmdStartSelector(unittest.TestCase): def setUp(self): self._manifest = _make_manifest(["researcher", "implementer"], ["claude", "dev"]) self._resolve_patch = patch( - "bot_bottle.cli.start.ManifestIndex.resolve", + "bot_bottle.cli.commands.start.ManifestIndex.resolve", return_value=self._manifest, ) self._resolve_patch.start() self._launch_patch = patch( - "bot_bottle.cli.start._launch_bottle", + "bot_bottle.cli.commands.start._launch_bottle", return_value=0, ) self._launch_mock = self._launch_patch.start() @@ -66,7 +66,7 @@ class TestCmdStartSelector(unittest.TestCase): self._modal_patch.start() self._image_policy_patch = patch( - "bot_bottle.cli.start._select_image_policy", + "bot_bottle.cli.commands.start._select_image_policy", return_value="fresh", ) self._image_policy_patch.start() @@ -141,14 +141,14 @@ class TestCmdStartSelector(unittest.TestCase): self.assertEqual(("claude", "dev"), spec.bottle_names) def test_image_policy_forwarded_to_spec(self): - with patch("bot_bottle.cli.start._select_image_policy", return_value="cached"): + with patch("bot_bottle.cli.commands.start._select_image_policy", return_value="cached"): start_mod.cmd_start(["researcher"]) self._launch_mock.assert_called_once() spec = self._launch_mock.call_args[0][0] self.assertEqual("cached", spec.image_policy) def test_image_policy_cancel_returns_0(self): - with patch("bot_bottle.cli.start._select_image_policy", return_value=None): + with patch("bot_bottle.cli.commands.start._select_image_policy", return_value=None): rc = start_mod.cmd_start(["researcher"]) self.assertEqual(0, rc) self._launch_mock.assert_not_called() @@ -169,7 +169,7 @@ class TestCmdStartSelector(unittest.TestCase): ["implementer"], ["claude", "dev"], agent_bottle="claude" ) with patch( - "bot_bottle.cli.start.ManifestIndex.resolve", return_value=manifest + "bot_bottle.cli.commands.start.ManifestIndex.resolve", return_value=manifest ): start_mod.cmd_start(["implementer"]) call_kwargs = self._bottle_picker_mock.call_args @@ -178,7 +178,7 @@ class TestCmdStartSelector(unittest.TestCase): def test_no_agent_bottle_empty_initial(self): manifest = _make_manifest(["researcher"], ["claude", "dev"], agent_bottle="") with patch( - "bot_bottle.cli.start.ManifestIndex.resolve", return_value=manifest + "bot_bottle.cli.commands.start.ManifestIndex.resolve", return_value=manifest ): start_mod.cmd_start(["researcher"]) call_kwargs = self._bottle_picker_mock.call_args @@ -228,19 +228,19 @@ class TestCmdStartLabelCollision(unittest.TestCase): def setUp(self): self._manifest = _make_manifest(["researcher"], ["claude"]) - patch("bot_bottle.cli.start.ManifestIndex.resolve", return_value=self._manifest).start() + patch("bot_bottle.cli.commands.start.ManifestIndex.resolve", return_value=self._manifest).start() self._launch_mock = patch( - "bot_bottle.cli.start._launch_bottle", return_value=0, + "bot_bottle.cli.commands.start._launch_bottle", return_value=0, ).start() # Stub the bottle picker to always return a selection. patch.object(tui_mod, "filter_multiselect", return_value=["claude"]).start() - patch("bot_bottle.cli.start._select_image_policy", return_value="fresh").start() + patch("bot_bottle.cli.commands.start._select_image_policy", return_value="fresh").start() self.addCleanup(patch.stopall) def test_no_collision_proceeds_without_reprompt(self): with ( patch.object(tui_mod, "name_color_modal", return_value=("researcher", "")) as modal, - patch("bot_bottle.cli.start.enumerate_active_agents", return_value=[]), + patch("bot_bottle.cli.commands.start.enumerate_active_agents", return_value=[]), ): rc = start_mod.cmd_start(["researcher"]) self.assertEqual(0, rc) @@ -261,7 +261,7 @@ class TestCmdStartLabelCollision(unittest.TestCase): with ( patch.object(tui_mod, "name_color_modal", side_effect=_modal) as modal, patch( - "bot_bottle.cli.start.enumerate_active_agents", + "bot_bottle.cli.commands.start.enumerate_active_agents", side_effect=[[collision_agent], []], ), ): diff --git a/tests/unit/test_cli_start_settle.py b/tests/unit/test_cli_start_settle.py index 4b1c405..55666b8 100644 --- a/tests/unit/test_cli_start_settle.py +++ b/tests/unit/test_cli_start_settle.py @@ -10,7 +10,7 @@ from pathlib import Path from tests.unit import use_bottle_root from bot_bottle import bottle_state -from bot_bottle.cli import start as start_mod +from bot_bottle.cli.commands import start as start_mod class _FakeHomeMixin: diff --git a/tests/unit/test_cli_start_stale.py b/tests/unit/test_cli_start_stale.py index 87625ae..97651a1 100644 --- a/tests/unit/test_cli_start_stale.py +++ b/tests/unit/test_cli_start_stale.py @@ -56,7 +56,7 @@ class TestLaunchBottleStaleHandling(unittest.TestCase): ) def _run_launch(self, **patch_kwargs: Any) -> int: - import bot_bottle.cli.start as start_mod + import bot_bottle.cli.commands.start as start_mod spec = self._spec() with patch.object(start_mod, "prepare_with_preflight", return_value=(_fake_plan(), "dev-abc")), \ @@ -71,7 +71,7 @@ class TestLaunchBottleStaleHandling(unittest.TestCase): def test_headless_stale_calls_die(self) -> None: """In headless mode (assume_yes=True), a StaleImageError from prelaunch_checks must call die().""" - import bot_bottle.cli.start as start_mod + import bot_bottle.cli.commands.start as start_mod backend_mock = MagicMock() backend_mock.prelaunch_checks.side_effect = StaleImageError("image is 5 day(s) old") @@ -85,7 +85,7 @@ class TestLaunchBottleStaleHandling(unittest.TestCase): def test_interactive_user_declines_stops_before_launch(self) -> None: """Interactive user answering 'n' → launch is never called.""" - import bot_bottle.cli.start as start_mod + import bot_bottle.cli.commands.start as start_mod backend_mock = MagicMock() backend_mock.prelaunch_checks.side_effect = StaleImageError("image is 5 day(s) old") @@ -100,7 +100,7 @@ class TestLaunchBottleStaleHandling(unittest.TestCase): def test_interactive_user_confirms_launches_once(self) -> None: """Interactive user answering 'y' → prelaunch stale error is bypassed; launch called once.""" - import bot_bottle.cli.start as start_mod + import bot_bottle.cli.commands.start as start_mod bottle_mock = MagicMock() bottle_mock.name = "dev-abc" @@ -122,7 +122,7 @@ class TestLaunchBottleStaleHandling(unittest.TestCase): def test_interactive_yes_uppercase_also_accepted(self) -> None: """'Y' or 'YES' should also be accepted as confirmation.""" - import bot_bottle.cli.start as start_mod + import bot_bottle.cli.commands.start as start_mod bottle_mock = MagicMock() bottle_mock.name = "dev-abc" diff --git a/tests/unit/test_supervise_cli.py b/tests/unit/test_supervise_cli.py index f3f4477..1d6679b 100644 --- a/tests/unit/test_supervise_cli.py +++ b/tests/unit/test_supervise_cli.py @@ -12,7 +12,7 @@ import unittest from datetime import datetime, timezone from unittest.mock import MagicMock, patch -from bot_bottle.cli import supervise as supervise_cli +from bot_bottle.cli.commands import supervise as supervise_cli from bot_bottle.orchestrator.supervisor import ( Proposal, TOOL_EGRESS_ALLOW, diff --git a/tests/unit/test_supervise_cli_crash_logging.py b/tests/unit/test_supervise_cli_crash_logging.py index 0f07f08..e36c0c3 100644 --- a/tests/unit/test_supervise_cli_crash_logging.py +++ b/tests/unit/test_supervise_cli_crash_logging.py @@ -18,7 +18,7 @@ from pathlib import Path from unittest import mock from tests.unit import use_bottle_root -from bot_bottle.cli import supervise as supervise_cli +from bot_bottle.cli.commands import supervise as supervise_cli from bot_bottle.log import Die, die -- 2.52.0 From a21f2358c625afaa6ba9aea6bab527fa2bbf76c3 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 17:31:14 -0400 Subject: [PATCH 27/39] refactor(cli): trim _common to a PROG constants leaf; inline os.getcwd() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _common had rotted into a junk drawer. Trim it to what actually justifies a shared leaf module and rename for legibility: * REPO_DIR: deleted (dead — nothing imported it). * read_tty_line: dropped the pointless re-export; cleanup/start/init now import it straight from bot_bottle.util. * USER_CWD: deleted. It captured os.getcwd() at import time, but nothing chdirs and no test patched it, so it was equivalent to a live os.getcwd() — inlined at the six call sites. * PROG: kept, now the sole member. It's still a leaf (both the dispatcher and the commands it imports need it) so it can't move into __init__ without a circular import. _common.py -> constants.py. Full unit suite green (2243); CLI dispatch verified. Co-Authored-By: Claude Opus 4.8 --- bot_bottle/cli/__init__.py | 2 +- bot_bottle/cli/_common.py | 12 ------------ bot_bottle/cli/commands/__init__.py | 2 +- bot_bottle/cli/commands/backend.py | 2 +- bot_bottle/cli/commands/cleanup.py | 2 +- bot_bottle/cli/commands/commit.py | 2 +- bot_bottle/cli/commands/edit.py | 4 ++-- bot_bottle/cli/commands/info.py | 5 +++-- bot_bottle/cli/commands/init.py | 5 +++-- bot_bottle/cli/commands/list.py | 4 ++-- bot_bottle/cli/commands/resume.py | 7 ++++--- bot_bottle/cli/commands/start.py | 9 +++++---- bot_bottle/cli/commands/supervise.py | 2 +- bot_bottle/cli/constants.py | 8 ++++++++ 14 files changed, 33 insertions(+), 33 deletions(-) delete mode 100644 bot_bottle/cli/_common.py create mode 100644 bot_bottle/cli/constants.py diff --git a/bot_bottle/cli/__init__.py b/bot_bottle/cli/__init__.py index 41ffa7d..b26642a 100644 --- a/bot_bottle/cli/__init__.py +++ b/bot_bottle/cli/__init__.py @@ -11,7 +11,7 @@ from ..errors import MissingEnvVarError from ..log import Die, die, error from ..manifest import ManifestError from ..orchestrator.store.store_manager import StoreManager -from ._common import PROG +from .constants import PROG from .commands import list as _list_mod from .commands.backend import cmd_backend from .commands.cleanup import cmd_cleanup diff --git a/bot_bottle/cli/_common.py b/bot_bottle/cli/_common.py deleted file mode 100644 index 5fc6c33..0000000 --- a/bot_bottle/cli/_common.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Shared constants and tty helper for cli subcommands.""" - -from __future__ import annotations - -import os -from pathlib import Path - -from ..util import read_tty_line as read_tty_line - -PROG = "cli.py" -USER_CWD = os.getcwd() -REPO_DIR = str(Path(__file__).resolve().parent.parent.parent) diff --git a/bot_bottle/cli/commands/__init__.py b/bot_bottle/cli/commands/__init__.py index 0daac76..7f34d76 100644 --- a/bot_bottle/cli/commands/__init__.py +++ b/bot_bottle/cli/commands/__init__.py @@ -2,5 +2,5 @@ Each module exposes a `cmd_(argv)` handler that the dispatcher (`bot_bottle.cli`) registers in its COMMANDS table. Shared CLI helpers -(`_common`, `tui`) stay one level up in the `cli` package. +(`constants`, `tui`) stay one level up in the `cli` package. """ diff --git a/bot_bottle/cli/commands/backend.py b/bot_bottle/cli/commands/backend.py index 2997db3..a06e44f 100644 --- a/bot_bottle/cli/commands/backend.py +++ b/bot_bottle/cli/commands/backend.py @@ -16,7 +16,7 @@ from __future__ import annotations import argparse from ...backend import get_bottle_backend, known_backend_names -from .._common import PROG +from ..constants import PROG def cmd_backend(args: list[str]) -> int: diff --git a/bot_bottle/cli/commands/cleanup.py b/bot_bottle/cli/commands/cleanup.py index 685f6bc..d83ef14 100644 --- a/bot_bottle/cli/commands/cleanup.py +++ b/bot_bottle/cli/commands/cleanup.py @@ -23,7 +23,7 @@ import sys from ...backend import get_bottle_backend, has_backend, known_backend_names from ...log import info -from .._common import read_tty_line +from ...util import read_tty_line def cmd_cleanup(_argv: list[str]) -> int: diff --git a/bot_bottle/cli/commands/commit.py b/bot_bottle/cli/commands/commit.py index 502ae69..fcf454d 100644 --- a/bot_bottle/cli/commands/commit.py +++ b/bot_bottle/cli/commands/commit.py @@ -16,7 +16,7 @@ from ...backend import enumerate_active_agents from ...backend.freeze import CommitCancelled, get_freezer from ...bottle_state import read_metadata from ...log import die -from .._common import PROG +from ..constants import PROG from .. import tui diff --git a/bot_bottle/cli/commands/edit.py b/bot_bottle/cli/commands/edit.py index 82ca91b..8607517 100644 --- a/bot_bottle/cli/commands/edit.py +++ b/bot_bottle/cli/commands/edit.py @@ -8,7 +8,7 @@ import os from pathlib import Path from ...log import die -from .._common import PROG, USER_CWD +from ..constants import PROG def cmd_edit(argv: list[str]) -> int: @@ -20,7 +20,7 @@ def cmd_edit(argv: list[str]) -> int: if args.scope == "user": target_file = Path(os.environ["HOME"]) / "bot-bottle.json" else: - target_file = Path(USER_CWD) / "bot-bottle.json" + target_file = Path(os.getcwd()) / "bot-bottle.json" if not target_file.is_file(): die(f"{target_file} does not exist") diff --git a/bot_bottle/cli/commands/info.py b/bot_bottle/cli/commands/info.py index d15b561..7850a2d 100644 --- a/bot_bottle/cli/commands/info.py +++ b/bot_bottle/cli/commands/info.py @@ -3,10 +3,11 @@ from __future__ import annotations import argparse +import os from ...log import info from ...manifest import ManifestIndex -from .._common import PROG, USER_CWD +from ..constants import PROG def cmd_info(argv: list[str]) -> int: @@ -14,7 +15,7 @@ def cmd_info(argv: list[str]) -> int: parser.add_argument("name", help="agent name defined in bot-bottle.json") args = parser.parse_args(argv) - names = ManifestIndex.resolve(USER_CWD) + names = ManifestIndex.resolve(os.getcwd()) names.require_agent(args.name) manifest = names.load_for_agent(args.name) diff --git a/bot_bottle/cli/commands/init.py b/bot_bottle/cli/commands/init.py index ca169f1..3444177 100644 --- a/bot_bottle/cli/commands/init.py +++ b/bot_bottle/cli/commands/init.py @@ -11,7 +11,8 @@ from pathlib import Path from typing import Any from ...log import die, info, warn -from .._common import PROG, USER_CWD, read_tty_line +from ..constants import PROG +from ...util import read_tty_line def cmd_init(argv: list[str]) -> int: @@ -22,7 +23,7 @@ def cmd_init(argv: list[str]) -> int: if args.scope == "user": target_file = Path(os.environ["HOME"]) / "bot-bottle.json" else: - target_file = Path(USER_CWD) / "bot-bottle.json" + target_file = Path(os.getcwd()) / "bot-bottle.json" print(file=sys.stderr) info(f"bot-bottle init — adding a new agent to {target_file}") diff --git a/bot_bottle/cli/commands/list.py b/bot_bottle/cli/commands/list.py index 5c0caf3..6b83c4e 100644 --- a/bot_bottle/cli/commands/list.py +++ b/bot_bottle/cli/commands/list.py @@ -8,7 +8,7 @@ import sys from ...backend import enumerate_active_agents from ...manifest import ManifestIndex -from .._common import PROG, USER_CWD +from ..constants import PROG _ANSI_COLOR_CODES: dict[str, str] = { "red": "\033[91m", @@ -40,7 +40,7 @@ def cmd_list(argv: list[str]) -> int: args = parser.parse_args(argv) if args.scope == "available": - manifest = ManifestIndex.resolve(USER_CWD) + manifest = ManifestIndex.resolve(os.getcwd()) for name in manifest.all_agent_names: print(name) return 0 diff --git a/bot_bottle/cli/commands/resume.py b/bot_bottle/cli/commands/resume.py index 6d8224a..e3b6bc2 100644 --- a/bot_bottle/cli/commands/resume.py +++ b/bot_bottle/cli/commands/resume.py @@ -15,12 +15,13 @@ to bring up the replacement from the recorded state. from __future__ import annotations import argparse +import os from ...backend import BottleSpec from ...bottle_state import read_metadata from ...log import die from ...manifest import ManifestIndex -from .._common import PROG, USER_CWD +from ..constants import PROG from .start import _launch_bottle @@ -40,14 +41,14 @@ def cmd_resume(argv: list[str]) -> int: f"check ~/.bot-bottle/state/ or run `cli.py start` to create a new bottle" ) - manifest = ManifestIndex.resolve(USER_CWD) + manifest = ManifestIndex.resolve(os.getcwd()) manifest.require_agent(metadata.agent_name) spec = BottleSpec( manifest=manifest, agent_name=metadata.agent_name, copy_cwd=metadata.copy_cwd, - user_cwd=metadata.cwd or USER_CWD, + user_cwd=metadata.cwd or os.getcwd(), identity=metadata.identity, bottle_names=tuple(metadata.bottle_names), ) diff --git a/bot_bottle/cli/commands/start.py b/bot_bottle/cli/commands/start.py index 24c2e9c..2082f92 100644 --- a/bot_bottle/cli/commands/start.py +++ b/bot_bottle/cli/commands/start.py @@ -39,7 +39,8 @@ from ...bottle_state import ( from ...image_cache import StaleImageError from ...log import info, die from ...manifest import Manifest, ManifestIndex -from .._common import PROG, USER_CWD, read_tty_line +from ..constants import PROG +from ...util import read_tty_line from .. import tui @@ -116,7 +117,7 @@ def cmd_start(argv: list[str]) -> int: # threading a no_cache field through every backend's plan dataclass. os.environ["BOT_BOTTLE_NO_CACHE"] = "1" - manifest = ManifestIndex.resolve(USER_CWD) + manifest = ManifestIndex.resolve(os.getcwd()) if args.headless: return _start_headless( @@ -167,7 +168,7 @@ def cmd_start(argv: list[str]) -> int: manifest=manifest, agent_name=agent_name, copy_cwd=args.cwd, - user_cwd=USER_CWD, + user_cwd=os.getcwd(), label=label, color=color, bottle_names=bottle_names, @@ -235,7 +236,7 @@ def _start_headless( manifest=manifest, agent_name=agent_name, copy_cwd=args.cwd, - user_cwd=USER_CWD, + user_cwd=os.getcwd(), label=label, color=args.color or "", bottle_names=bottle_names, diff --git a/bot_bottle/cli/commands/supervise.py b/bot_bottle/cli/commands/supervise.py index c2beb03..cbb349f 100644 --- a/bot_bottle/cli/commands/supervise.py +++ b/bot_bottle/cli/commands/supervise.py @@ -34,7 +34,7 @@ from ...supervisor.types import ( TOOL_GITLEAKS_ALLOW, TOOL_EGRESS_TOKEN_ALLOW, ) -from .._common import PROG +from ..constants import PROG _REFRESH_INTERVAL_MS = 1000 diff --git a/bot_bottle/cli/constants.py b/bot_bottle/cli/constants.py new file mode 100644 index 0000000..4df182e --- /dev/null +++ b/bot_bottle/cli/constants.py @@ -0,0 +1,8 @@ +"""Shared CLI constants. + +Kept as a leaf module (imports nothing from the `cli` package) so both the +dispatcher (`cli/__init__.py`) and the command modules it imports can share +`PROG` without a circular import. +""" + +PROG = "cli.py" -- 2.52.0 From 3ccd308613da31a192884362ba787aa0baffe95f Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 17:40:05 -0400 Subject: [PATCH 28/39] refactor(cli): make help a first-class command; drop usage() from dispatcher Move the top-level usage/command-list text out of the dispatcher into commands/help.py as cmd_help, registered in COMMANDS so `bb help` now works as a real command. The dispatcher's -h/--help, no-args, and unknown-command fallbacks call cmd_help() for the text while keeping their exit codes (0 for help/-h, 2 for bare no-args, Die for unknown). Added `help` to NO_MIGRATION_COMMANDS so it prints without a migrated DB, and listed it in the command summary. Full unit suite green (2243); dispatch + migration-gate tests pass, `bb help` verified. Co-Authored-By: Claude Opus 4.8 --- bot_bottle/cli/__init__.py | 39 +++++++-------------------------- bot_bottle/cli/commands/help.py | 37 +++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 31 deletions(-) create mode 100644 bot_bottle/cli/commands/help.py diff --git a/bot_bottle/cli/__init__.py b/bot_bottle/cli/__init__.py index b26642a..6a1ea3b 100644 --- a/bot_bottle/cli/__init__.py +++ b/bot_bottle/cli/__init__.py @@ -1,6 +1,7 @@ """Main CLI dispatcher. -Commands: backend, cleanup, commit, edit, info, init, list, resume, start, supervise +Commands: backend, cleanup, commit, edit, help, info, init, list, login, +resume, start, supervise """ from __future__ import annotations @@ -17,6 +18,7 @@ from .commands.backend import cmd_backend from .commands.cleanup import cmd_cleanup from .commands.commit import cmd_commit from .commands.edit import cmd_edit +from .commands.help import cmd_help from .commands.info import cmd_info from .commands.init import cmd_init from .commands.login import cmd_login @@ -31,6 +33,7 @@ COMMANDS = { "cleanup": cmd_cleanup, "commit": cmd_commit, "edit": cmd_edit, + "help": cmd_help, "info": cmd_info, "init": cmd_init, "list": cmd_list, @@ -45,49 +48,23 @@ COMMANDS = { # the host (TAP pool, /dev/kvm, firecracker) and never opens the store, so # gating it on the schema breaks preflight on a fresh CI runner where stdin # isn't a TTY and the migration prompt can't be answered. -NO_MIGRATION_COMMANDS = frozenset({"backend", "login"}) - - -def usage() -> None: - sys.stderr.write(f"usage: {PROG} [args...]\n\n") - sys.stderr.write("Commands:\n") - sys.stderr.write(" backend set up / check / undo a backend's host prerequisites (setup|status|teardown)\n") - sys.stderr.write(" cleanup stop and remove all active bot-bottle containers\n") - sys.stderr.write(" commit snapshot a running bottle's container state to a Docker image\n") - sys.stderr.write(" edit open an agent in vim for editing\n") - sys.stderr.write(" info print env, skills, and prompt details for a named agent\n") - sys.stderr.write(" init interactively create a new agent and add it to bot-bottle.json\n") - sys.stderr.write(" list list available agents or active containers\n") - sys.stderr.write(" login register this host with a bot-bottle console\n") - sys.stderr.write( - " resume re-launch a bottle by its identity " - "(continues state from PRD 0016)\n" - ) - sys.stderr.write( - " start boot a container for a named agent and " - "attach an interactive session\n" - ) - sys.stderr.write( - " supervise view + approve/modify/reject pending supervise " - "proposals (PRD 0013)\n\n" - ) - sys.stderr.write(f"Run '{PROG} --help' for command-specific usage.\n") +NO_MIGRATION_COMMANDS = frozenset({"backend", "help", "login"}) def main(argv: list[str] | None = None) -> int: if argv is None: argv = sys.argv[1:] if not argv: - usage() + cmd_help() return 2 command = argv[0] rest = argv[1:] if command in ("-h", "--help"): - usage() + cmd_help() return 0 handler = COMMANDS.get(command) if handler is None: - usage() + cmd_help() die(f"unknown command: {command}") mgr = StoreManager.instance() if command not in NO_MIGRATION_COMMANDS and not mgr.is_migrated(): diff --git a/bot_bottle/cli/commands/help.py b/bot_bottle/cli/commands/help.py new file mode 100644 index 0000000..d7ee633 --- /dev/null +++ b/bot_bottle/cli/commands/help.py @@ -0,0 +1,37 @@ +"""help: print the top-level command list and usage. + +Rendered by the dispatcher for the `help` command and for its +`-h`/`--help`, no-args, and unknown-command fallbacks. The per-command +summaries live here; keep them in sync with the COMMANDS table in +`bot_bottle.cli`. +""" + +from __future__ import annotations + +import sys + +from ..constants import PROG + + +def cmd_help(argv: list[str] | None = None) -> int: + """Write the top-level usage + command list to stderr. Returns 0; + the dispatcher chooses the process exit code per entry path (0 for an + explicit `help`/`-h`, 2 for the bare no-args usage error).""" + del argv # help takes no arguments + w = sys.stderr.write + w(f"usage: {PROG} [args...]\n\n") + w("Commands:\n") + w(" backend set up / check / undo a backend's host prerequisites (setup|status|teardown)\n") + w(" cleanup stop and remove all active bot-bottle containers\n") + w(" commit snapshot a running bottle's container state to a Docker image\n") + w(" edit open an agent in vim for editing\n") + w(" help show this command list\n") + w(" info print env, skills, and prompt details for a named agent\n") + w(" init interactively create a new agent and add it to bot-bottle.json\n") + w(" list list available agents or active containers\n") + w(" login register this host with a bot-bottle console\n") + w(" resume re-launch a bottle by its identity (continues state from PRD 0016)\n") + w(" start boot a container for a named agent and attach an interactive session\n") + w(" supervise view + approve/modify/reject pending supervise proposals (PRD 0013)\n\n") + w(f"Run '{PROG} --help' for command-specific usage.\n") + return 0 -- 2.52.0 From b276227cbbf54e067e6100fd2a181d730a73ac03 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 17:52:52 -0400 Subject: [PATCH 29/39] refactor(cli): registry in commands/__init__, dispatcher main() in __main__ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split the cli package's two responsibilities out of __init__: * commands/__init__.py now assembles the COMMANDS registry and NO_MIGRATION_COMMANDS from the per-command modules — the command surface lives entirely under commands/. * __main__.py now owns main() (dispatch + migration gate + exit-code mapping) alongside the runnable entry guard. cli/__init__.py shrinks to a shim that re-exports main / COMMANDS / NO_MIGRATION_COMMANDS, so bot_bottle.cli.main (repo-root cli.py entry) and the tests' bot_bottle.cli.COMMANDS keep working unchanged. Verified `python -m bot_bottle.cli` (runpy) and `cli.py` both dispatch; full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/cli/__init__.py | 96 +++-------------------------- bot_bottle/cli/__main__.py | 60 ++++++++++++++++-- bot_bottle/cli/commands/__init__.py | 47 +++++++++++++- 3 files changed, 108 insertions(+), 95 deletions(-) diff --git a/bot_bottle/cli/__init__.py b/bot_bottle/cli/__init__.py index 6a1ea3b..56a73d0 100644 --- a/bot_bottle/cli/__init__.py +++ b/bot_bottle/cli/__init__.py @@ -1,93 +1,15 @@ -"""Main CLI dispatcher. +"""bot-bottle CLI package. -Commands: backend, cleanup, commit, edit, help, info, init, list, login, -resume, start, supervise +The subcommand handlers live in `commands/` and are assembled into the +COMMANDS registry by `commands/__init__.py`; the dispatcher `main()` lives +in `__main__.py`. They are re-exported here so `bot_bottle.cli.main`, +`bot_bottle.cli.COMMANDS`, and `bot_bottle.cli.NO_MIGRATION_COMMANDS` stay +importable (the repo-root `cli.py` entry point and the tests use them). """ from __future__ import annotations -import sys +from .__main__ import main +from .commands import COMMANDS, NO_MIGRATION_COMMANDS -from ..errors import MissingEnvVarError -from ..log import Die, die, error -from ..manifest import ManifestError -from ..orchestrator.store.store_manager import StoreManager -from .constants import PROG -from .commands import list as _list_mod -from .commands.backend import cmd_backend -from .commands.cleanup import cmd_cleanup -from .commands.commit import cmd_commit -from .commands.edit import cmd_edit -from .commands.help import cmd_help -from .commands.info import cmd_info -from .commands.init import cmd_init -from .commands.login import cmd_login -from .commands.resume import cmd_resume -from .commands.start import cmd_start -from .commands.supervise import cmd_supervise - -cmd_list = _list_mod.cmd_list - -COMMANDS = { - "backend": cmd_backend, - "cleanup": cmd_cleanup, - "commit": cmd_commit, - "edit": cmd_edit, - "help": cmd_help, - "info": cmd_info, - "init": cmd_init, - "list": cmd_list, - "login": cmd_login, - "resume": cmd_resume, - "start": cmd_start, - "supervise": cmd_supervise, -} - -# Commands that manage host prerequisites (or are otherwise store-free) and -# must run before — or without — a migrated DB. `backend` provisions/probes -# the host (TAP pool, /dev/kvm, firecracker) and never opens the store, so -# gating it on the schema breaks preflight on a fresh CI runner where stdin -# isn't a TTY and the migration prompt can't be answered. -NO_MIGRATION_COMMANDS = frozenset({"backend", "help", "login"}) - - -def main(argv: list[str] | None = None) -> int: - if argv is None: - argv = sys.argv[1:] - if not argv: - cmd_help() - return 2 - command = argv[0] - rest = argv[1:] - if command in ("-h", "--help"): - cmd_help() - return 0 - handler = COMMANDS.get(command) - if handler is None: - cmd_help() - die(f"unknown command: {command}") - mgr = StoreManager.instance() - if command not in NO_MIGRATION_COMMANDS and not mgr.is_migrated(): - sys.stderr.write("bot-bottle: database schema is out of date\n") - sys.stderr.write("Migrate now? [y/N] ") - sys.stderr.flush() - try: - answer = sys.stdin.readline().strip().lower() - except EOFError: - answer = "" - if answer != "y": - error("migration required — re-run and confirm to migrate") - return 1 - mgr.migrate() - try: - return handler(rest) or 0 - except MissingEnvVarError as e: - error(str(e)) - return 1 - except ManifestError as e: - error(str(e)) - return 1 - except Die as e: - return e.code if isinstance(e.code, int) else 1 - except KeyboardInterrupt: - return 130 +__all__ = ["main", "COMMANDS", "NO_MIGRATION_COMMANDS"] diff --git a/bot_bottle/cli/__main__.py b/bot_bottle/cli/__main__.py index 3cf5f19..91a7e5a 100644 --- a/bot_bottle/cli/__main__.py +++ b/bot_bottle/cli/__main__.py @@ -1,15 +1,65 @@ -"""Entry point for `python -m bot_bottle.cli`. +"""Entry point + dispatcher for `python -m bot_bottle.cli`. -`cli.py` at the repo root is the usual way in; this makes the package -runnable too, so the CLI works from an installed copy where there is no -`cli.py` on disk to point at. +Maps `bot-bottle ` to its handler in the COMMANDS registry +(`bot_bottle.cli.commands`), enforces the schema-migration gate, and +translates handler exceptions into process exit codes. The repo-root +`cli.py` is the usual way in; this makes the package runnable too, so the +CLI works from an installed copy where there is no `cli.py` on disk. """ from __future__ import annotations import sys -from . import main +from ..errors import MissingEnvVarError +from ..log import Die, die, error +from ..manifest import ManifestError +from ..orchestrator.store.store_manager import StoreManager +from .commands import COMMANDS, NO_MIGRATION_COMMANDS +from .commands.help import cmd_help + + +def main(argv: list[str] | None = None) -> int: + if argv is None: + argv = sys.argv[1:] + if not argv: + cmd_help() + return 2 + command = argv[0] + rest = argv[1:] + if command in ("-h", "--help"): + cmd_help() + return 0 + handler = COMMANDS.get(command) + if handler is None: + cmd_help() + die(f"unknown command: {command}") + mgr = StoreManager.instance() + if command not in NO_MIGRATION_COMMANDS and not mgr.is_migrated(): + sys.stderr.write("bot-bottle: database schema is out of date\n") + sys.stderr.write("Migrate now? [y/N] ") + sys.stderr.flush() + try: + answer = sys.stdin.readline().strip().lower() + except EOFError: + answer = "" + if answer != "y": + error("migration required — re-run and confirm to migrate") + return 1 + mgr.migrate() + try: + return handler(rest) or 0 + except MissingEnvVarError as e: + error(str(e)) + return 1 + except ManifestError as e: + error(str(e)) + return 1 + except Die as e: + return e.code if isinstance(e.code, int) else 1 + except KeyboardInterrupt: + return 130 + if __name__ == "__main__": sys.exit(main()) diff --git a/bot_bottle/cli/commands/__init__.py b/bot_bottle/cli/commands/__init__.py index 7f34d76..cda230d 100644 --- a/bot_bottle/cli/commands/__init__.py +++ b/bot_bottle/cli/commands/__init__.py @@ -1,6 +1,47 @@ -"""CLI subcommand handlers — one module per `bot-bottle `. +"""CLI subcommand registry. -Each module exposes a `cmd_(argv)` handler that the dispatcher -(`bot_bottle.cli`) registers in its COMMANDS table. Shared CLI helpers +One module per `bot-bottle `, each exposing a `cmd_(argv)` +handler. This package `__init__` assembles them into the COMMANDS table +the dispatcher (`bot_bottle.cli.__main__`) reads. Shared CLI helpers (`constants`, `tui`) stay one level up in the `cli` package. """ + +from __future__ import annotations + +from .backend import cmd_backend +from .cleanup import cmd_cleanup +from .commit import cmd_commit +from .edit import cmd_edit +from .help import cmd_help +from .info import cmd_info +from .init import cmd_init +from .list import cmd_list +from .login import cmd_login +from .resume import cmd_resume +from .start import cmd_start +from .supervise import cmd_supervise + +COMMANDS = { + "backend": cmd_backend, + "cleanup": cmd_cleanup, + "commit": cmd_commit, + "edit": cmd_edit, + "help": cmd_help, + "info": cmd_info, + "init": cmd_init, + "list": cmd_list, + "login": cmd_login, + "resume": cmd_resume, + "start": cmd_start, + "supervise": cmd_supervise, +} + +# Commands that manage host prerequisites (or are otherwise store-free) and +# must run before — or without — a migrated DB. `backend` provisions/probes +# the host (TAP pool, /dev/kvm, firecracker) and never opens the store, so +# gating it on the schema breaks preflight on a fresh CI runner where stdin +# isn't a TTY and the migration prompt can't be answered. `help` and `login` +# likewise never touch the store. +NO_MIGRATION_COMMANDS = frozenset({"backend", "help", "login"}) + +__all__ = ["COMMANDS", "NO_MIGRATION_COMMANDS"] -- 2.52.0 From 4166057abc2bcf6ba53f1c286784b4c752121e0a Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 17:57:49 -0400 Subject: [PATCH 30/39] refactor(orchestrator): rename control_plane module to server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit orchestrator/control_plane.py -> orchestrator/server.py. Within the orchestrator package the "control_plane" filename stutters (the orchestrator *is* the control plane), and `orchestrator.server` reads as "the orchestrator's HTTP server", pairing with service.py (domain logic) and matching gateway/supervisor/server.py. Scope is the module name only. The class ControlPlaneServer, the CONTROL_AUTH_HEADER constant, and the "control plane" architectural term (CONTROL_PLANE_PORT, host_control_plane_token, control_plane_url, …) are deliberately unchanged — that's the load-bearing control-plane/data-plane distinction. Test file renamed to match; full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/control_auth.py | 4 ++-- bot_bottle/orchestrator/__init__.py | 2 +- bot_bottle/orchestrator/__main__.py | 2 +- bot_bottle/orchestrator/client.py | 2 +- bot_bottle/orchestrator/{control_plane.py => server.py} | 0 bot_bottle/paths.py | 2 +- ...hestrator_control_plane.py => test_orchestrator_server.py} | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) rename bot_bottle/orchestrator/{control_plane.py => server.py} (100%) rename tests/unit/{test_orchestrator_control_plane.py => test_orchestrator_server.py} (99%) diff --git a/bot_bottle/control_auth.py b/bot_bottle/control_auth.py index 23cf351..54f4762 100644 --- a/bot_bottle/control_auth.py +++ b/bot_bottle/control_auth.py @@ -15,8 +15,8 @@ Only the orchestrator (and the host CLI, which shares the host trust domain) holds the signing key; the gateway is handed a pre-minted ``gateway`` token it cannot rewrite into a ``cli`` token. So a compromised data-plane process can no longer approve its own supervise proposals or drive operator routes — it can -only present the ``gateway`` role it was issued (control_plane rejects it on -operator routes with 403). +only present the ``gateway`` role it was issued (the control plane rejects it +on operator routes with 403). Stdlib-only (HMAC-SHA256 over a JSON payload); no JWT dependency — the project carries no runtime pip deps. Tokens are **signed, not encrypted** (the role is diff --git a/bot_bottle/orchestrator/__init__.py b/bot_bottle/orchestrator/__init__.py index 5346c00..8a0a9da 100644 --- a/bot_bottle/orchestrator/__init__.py +++ b/bot_bottle/orchestrator/__init__.py @@ -40,7 +40,7 @@ from .broker import ( from .docker_broker import DockerBroker, DockerBrokerError from ..gateway import Gateway, GatewayError from .service import Orchestrator -from .control_plane import ControlPlaneServer, dispatch, make_server +from .server import ControlPlaneServer, dispatch, make_server __all__ = [ "BottleRecord", diff --git a/bot_bottle/orchestrator/__main__.py b/bot_bottle/orchestrator/__main__.py index cd532f9..0370c83 100644 --- a/bot_bottle/orchestrator/__main__.py +++ b/bot_bottle/orchestrator/__main__.py @@ -18,7 +18,7 @@ from pathlib import Path from .. import log from .store.store_manager import StoreManager from .broker import LaunchBroker, StubBroker -from .control_plane import make_server +from .server import make_server from .docker_broker import DockerBroker from .registry import RegistryStore, default_db_path from .service import Orchestrator diff --git a/bot_bottle/orchestrator/client.py b/bot_bottle/orchestrator/client.py index a349c64..fb80c5e 100644 --- a/bot_bottle/orchestrator/client.py +++ b/bot_bottle/orchestrator/client.py @@ -20,7 +20,7 @@ from dataclasses import dataclass from ..control_auth import ROLE_CLI, mint from ..paths import host_control_plane_token -from .control_plane import CONTROL_AUTH_HEADER +from .server import CONTROL_AUTH_HEADER DEFAULT_TIMEOUT_SECONDS = 5.0 diff --git a/bot_bottle/orchestrator/control_plane.py b/bot_bottle/orchestrator/server.py similarity index 100% rename from bot_bottle/orchestrator/control_plane.py rename to bot_bottle/orchestrator/server.py diff --git a/bot_bottle/paths.py b/bot_bottle/paths.py index 629edb5..e1723c4 100644 --- a/bot_bottle/paths.py +++ b/bot_bottle/paths.py @@ -33,7 +33,7 @@ HOST_DB_FILENAME = "bot-bottle.db" # The per-host control-plane secret file, and the env var the launchers inject # its value into. The control plane requires this secret on every mutating / -# reading route (see orchestrator/control_plane.py); it is held only by the +# reading route (see orchestrator/server.py); it is held only by the # trusted callers (control plane, gateway, host CLI) and never handed to an # agent, so an agent that can reach the control-plane port still can't drive it. CONTROL_PLANE_TOKEN_FILENAME = "control-plane-token" diff --git a/tests/unit/test_orchestrator_control_plane.py b/tests/unit/test_orchestrator_server.py similarity index 99% rename from tests/unit/test_orchestrator_control_plane.py rename to tests/unit/test_orchestrator_server.py index 8210df5..2d5c736 100644 --- a/tests/unit/test_orchestrator_control_plane.py +++ b/tests/unit/test_orchestrator_server.py @@ -21,7 +21,7 @@ from unittest.mock import patch from bot_bottle.control_auth import ROLE_CLI, ROLE_GATEWAY, mint from bot_bottle.orchestrator.broker import StubBroker -from bot_bottle.orchestrator.control_plane import dispatch, make_server +from bot_bottle.orchestrator.server import dispatch, make_server from bot_bottle.orchestrator.registry import BottleRecord, RegistryStore from bot_bottle.orchestrator.service import Orchestrator from bot_bottle.orchestrator.store.store_manager import StoreManager -- 2.52.0 From ca1d341d4ffded224c5579268b1d19b51e8b60ea Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 18:17:48 -0400 Subject: [PATCH 31/39] =?UTF-8?q?refactor:=20unify=20component=20naming=20?= =?UTF-8?q?=E2=80=94=20control=5Fplane/control=5Fauth=20->=20orchestrator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The codebase used "control plane" both as an architectural role term AND as an identifier alias for the orchestrator component, producing duplicate names for one thing (control_plane_url vs orchestrator_url, CONTROL_PLANE_PORT, host_control_plane_token, …). Going forward the concrete component is always named for what it is — Gateway or Orchestrator — and the plane vocabulary is reserved for prose (module descriptions, the security argument). Renamed (identifiers + the in-repo env/wire/file string values, all setters/getters are in this repo so the change is atomic): ControlPlaneServer -> OrchestratorServer control_plane_url -> orchestrator_url probe_control_plane_url -> probe_orchestrator_url host_control_plane_token -> host_orchestrator_token CONTROL_PLANE_PORT -> ORCHESTRATOR_PORT CONTROL_PLANE_TOKEN_ENV/FILE -> ORCHESTRATOR_TOKEN_ENV/FILENAME BOT_BOTTLE_CONTROL_PLANE_TOKEN-> BOT_BOTTLE_ORCHESTRATOR_TOKEN control-plane-token (file) -> orchestrator-token control_auth (module) -> orchestrator_auth (stays top-level; the gateway imports it and must not import the orchestrator/ package) CONTROL_AUTH_HEADER -> ORCHESTRATOR_AUTH_HEADER x-bot-bottle-control-auth -> x-bot-bottle-orchestrator-auth CONTROL_AUTH_JWT_ENV -> ORCHESTRATOR_AUTH_JWT_ENV BOT_BOTTLE_CONTROL_AUTH_JWT -> BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT _control_auth_headers -> _orchestrator_auth_headers Prose plane-terms ("control plane", "data plane") are preserved, including the test name test_data_plane_daemons_get_jwt_not_key (it names the security invariant). Gateway and orchestrator verified to agree on the renamed wire header; full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/backend/docker/gateway.py | 10 +++--- bot_bottle/backend/docker/infra.py | 18 +++++------ bot_bottle/backend/firecracker/backend.py | 2 +- .../firecracker/consolidated_launch.py | 2 +- bot_bottle/backend/firecracker/infra_vm.py | 32 +++++++++---------- bot_bottle/backend/macos_container/backend.py | 2 +- .../macos_container/consolidated_launch.py | 2 +- bot_bottle/backend/macos_container/infra.py | 26 +++++++-------- bot_bottle/gateway/bootstrap.py | 8 ++--- bot_bottle/gateway/policy_resolver.py | 12 +++---- bot_bottle/orchestrator/__init__.py | 6 ++-- bot_bottle/orchestrator/client.py | 18 +++++------ bot_bottle/orchestrator/server.py | 32 +++++++++---------- .../{control_auth.py => orchestrator_auth.py} | 0 bot_bottle/paths.py | 23 +++++++------ ...th.py => test_orchestrator_docker_auth.py} | 12 +++---- tests/unit/test_backend_selection.py | 4 +-- tests/unit/test_firecracker_infra_vm.py | 16 +++++----- tests/unit/test_gateway_init.py | 14 ++++---- tests/unit/test_macos_consolidated_launch.py | 4 +-- tests/unit/test_macos_infra.py | 10 +++--- ...trol_auth.py => test_orchestrator_auth.py} | 8 ++--- tests/unit/test_orchestrator_client.py | 6 ++-- tests/unit/test_orchestrator_server.py | 10 +++--- tests/unit/test_policy_resolver.py | 16 +++++----- 25 files changed, 146 insertions(+), 147 deletions(-) rename bot_bottle/{control_auth.py => orchestrator_auth.py} (100%) rename tests/integration/{test_orchestrator_docker_control_plane_auth.py => test_orchestrator_docker_auth.py} (95%) rename tests/unit/{test_control_auth.py => test_orchestrator_auth.py} (91%) diff --git a/bot_bottle/backend/docker/gateway.py b/bot_bottle/backend/docker/gateway.py index c45ff8e..221de21 100644 --- a/bot_bottle/backend/docker/gateway.py +++ b/bot_bottle/backend/docker/gateway.py @@ -4,11 +4,11 @@ import os import time from pathlib import Path -from ...control_auth import ROLE_GATEWAY, mint +from ...orchestrator_auth import ROLE_GATEWAY, mint from .util import run_docker from ...paths import ( - CONTROL_AUTH_JWT_ENV, - host_control_plane_token, + ORCHESTRATOR_AUTH_JWT_ENV, + host_orchestrator_token, host_gateway_ca_dir, ) from ...gateway import ( @@ -158,8 +158,8 @@ class DockerGateway(Gateway): # a `cli` token, so a compromised data-plane process can't drive the # operator routes (issue #469 review). Bare `--env NAME` keeps the value # off argv / `docker inspect`; only the gateway (not the agent) is given it. - argv += ["--env", CONTROL_AUTH_JWT_ENV] - run_env[CONTROL_AUTH_JWT_ENV] = mint(ROLE_GATEWAY, host_control_plane_token()) + argv += ["--env", ORCHESTRATOR_AUTH_JWT_ENV] + run_env[ORCHESTRATOR_AUTH_JWT_ENV] = mint(ROLE_GATEWAY, host_orchestrator_token()) argv.append(self.image_ref) proc = run_docker(argv, env=run_env) if proc.returncode != 0: diff --git a/bot_bottle/backend/docker/infra.py b/bot_bottle/backend/docker/infra.py index 5a1709e..d9d119d 100644 --- a/bot_bottle/backend/docker/infra.py +++ b/bot_bottle/backend/docker/infra.py @@ -25,13 +25,13 @@ import urllib.request from pathlib import Path from ... import log -from ...control_auth import ROLE_GATEWAY, mint +from ...orchestrator_auth import ROLE_GATEWAY, mint from .util import run_docker from ...paths import ( - CONTROL_AUTH_JWT_ENV, - CONTROL_PLANE_TOKEN_ENV, + ORCHESTRATOR_AUTH_JWT_ENV, + ORCHESTRATOR_TOKEN_ENV, bot_bottle_root, - host_control_plane_token, + host_orchestrator_token, host_gateway_ca_dir, ) from ...gateway import ( @@ -176,7 +176,7 @@ class DockerInfraService: so a later `ensure_running` can detect a real code change.""" self._ensure_network() run_docker(["docker", "rm", "--force", self._infra_name]) - _signing_key = host_control_plane_token() + _signing_key = host_orchestrator_token() proc = run_docker([ "docker", "run", "--detach", "--name", self._infra_name, @@ -204,8 +204,8 @@ class DockerInfraService: # pre-minted `gateway` JWT (data-plane daemons: present it). gateway_init # scopes each to its process, so a compromised data-plane daemon never # sees the key and can't mint a `cli` token (issue #469 review). - "--env", CONTROL_PLANE_TOKEN_ENV, - "--env", CONTROL_AUTH_JWT_ENV, + "--env", ORCHESTRATOR_TOKEN_ENV, + "--env", ORCHESTRATOR_AUTH_JWT_ENV, # Gateway daemons reach the orchestrator over loopback at its # fixed internal port (DEFAULT_PORT), independent of self.port. "--env", f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{DEFAULT_PORT}", @@ -214,8 +214,8 @@ class DockerInfraService: self.image, ], env={ **os.environ, - CONTROL_PLANE_TOKEN_ENV: _signing_key, - CONTROL_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key), + ORCHESTRATOR_TOKEN_ENV: _signing_key, + ORCHESTRATOR_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key), }) if proc.returncode != 0: raise OrchestratorStartError( diff --git a/bot_bottle/backend/firecracker/backend.py b/bot_bottle/backend/firecracker/backend.py index a528182..4119383 100644 --- a/bot_bottle/backend/firecracker/backend.py +++ b/bot_bottle/backend/firecracker/backend.py @@ -120,4 +120,4 @@ class FirecrackerBottleBackend( def ensure_orchestrator(self) -> str: from . import infra_vm - return infra_vm.ensure_running().control_plane_url + return infra_vm.ensure_running().orchestrator_url diff --git a/bot_bottle/backend/firecracker/consolidated_launch.py b/bot_bottle/backend/firecracker/consolidated_launch.py index 503d0af..24e8232 100644 --- a/bot_bottle/backend/firecracker/consolidated_launch.py +++ b/bot_bottle/backend/firecracker/consolidated_launch.py @@ -124,7 +124,7 @@ def launch_consolidated( provision its git-gate state into the gateway VM. Returns the context the agent-VM launch needs. Raises on failure — the caller tears down.""" infra = infra_vm.ensure_running() - url = infra.control_plane_url + url = infra.orchestrator_url client = OrchestratorClient(url) _reprovision_running_bottles(client) diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py index 1542a60..cf644ab 100644 --- a/bot_bottle/backend/firecracker/infra_vm.py +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -33,7 +33,7 @@ from pathlib import Path from typing import Generator from ...log import die, info -from ...paths import CONTROL_PLANE_TOKEN_FILENAME, bot_bottle_root +from ...paths import ORCHESTRATOR_TOKEN_FILENAME, bot_bottle_root from .. import util as backend_util from ..docker import util as docker_mod from ..docker.gateway_provision import GatewayProvisionError @@ -42,7 +42,7 @@ from . import firecracker_vm, infra_artifact, netpool, util # Where the infra VM keeps its control-plane signing key (generated on the # persistent /dev/vdb volume mounted at BOT_BOTTLE_ROOT). The host mirrors it # back so the CLI signs `cli` tokens the VM verifies (issue #469 review). -_GUEST_SIGNING_KEY_PATH = "/var/lib/bot-bottle/control-plane-token" +_GUEST_SIGNING_KEY_PATH = "/var/lib/bot-bottle/orchestrator-token" # The single infra-VM image: gateway data plane + baked control-plane source # (Dockerfile.infra FROM the gateway image). Built from source by default; @@ -52,7 +52,7 @@ _GATEWAY_IMAGE = "bot-bottle-gateway:latest" _ORCHESTRATOR_IMAGE = "bot-bottle-orchestrator:latest" _REPO_ROOT = Path(__file__).resolve().parents[3] -CONTROL_PLANE_PORT = 8099 +ORCHESTRATOR_PORT = 8099 # Gateway data-plane ports (agent-facing): egress proxy, supervise MCP, # git-http. Reached by agent VMs over VM-to-VM routing (added next). EGRESS_PORT = 9099 @@ -84,8 +84,8 @@ class InfraVm: vm: firecracker_vm.VmHandle | None = None @property - def control_plane_url(self) -> str: - return f"http://{self.guest_ip}:{CONTROL_PLANE_PORT}" + def orchestrator_url(self) -> str: + return f"http://{self.guest_ip}:{ORCHESTRATOR_PORT}" def terminate(self) -> None: """Stop the infra VM — via the live handle if we booted it, else the @@ -168,7 +168,7 @@ def ensure_running() -> InfraVm: flock, so two simultaneous first launches don't both boot on the same rootfs/PID. The healthy fast-path takes no lock.""" slot = netpool.orch_slot() - url = f"http://{slot.guest_ip}:{CONTROL_PLANE_PORT}" + url = f"http://{slot.guest_ip}:{ORCHESTRATOR_PORT}" key = _infra_dir() / "id_ed25519" want = _expected_version() if _adoptable(key, url, want): @@ -192,7 +192,7 @@ def ensure_running() -> InfraVm: def _with_signing_key(infra: InfraVm) -> InfraVm: """Mirror the infra VM's control-plane signing key (generated on its - persistent volume) into the host's control-plane-token file, so the host CLI + persistent volume) into the host's orchestrator-token file, so the host CLI signs `cli` tokens the VM verifies (issue #469 review). Best-effort: an unreadable key is logged, not fatal — the VM still enforces auth, but the CLI may then be rejected until the key is readable. Returns `infra` for chaining.""" @@ -205,7 +205,7 @@ def _with_signing_key(infra: InfraVm) -> InfraVm: if proc.returncode != 0 or not signing_key: info("infra signing key not yet readable; control-plane auth may fail") return infra - path = bot_bottle_root() / CONTROL_PLANE_TOKEN_FILENAME + path = bot_bottle_root() / ORCHESTRATOR_TOKEN_FILENAME path.parent.mkdir(parents=True, exist_ok=True) fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) with os.fdopen(fd, "w") as f: @@ -458,7 +458,7 @@ def wait_for_health( ) -> None: """Poll the control plane's /health until it answers 200 or the deadline passes. Dies (with the console tail) if the VMM exits early.""" - url = f"{infra.control_plane_url}/health" + url = f"{infra.orchestrator_url}/health" deadline = time.monotonic() + timeout while time.monotonic() < deadline: if infra.vm is not None and not infra.vm.is_alive(): @@ -467,7 +467,7 @@ def wait_for_health( try: with urllib.request.urlopen(url, timeout=1.0) as resp: if resp.status == 200: - info(f"infra control plane healthy at {infra.control_plane_url}") + info(f"infra control plane healthy at {infra.orchestrator_url}") return except (urllib.error.URLError, TimeoutError, OSError): pass @@ -525,11 +525,11 @@ cd /app # same VM — reaching the orchestrator over 127.0.0.1, past the nft boundary that # only fences off the separate agent VM — could drive the operator routes # (approve its own supervise proposals, rewrite policy, read injected tokens). -CP_KEY=$(BOT_BOTTLE_ROOT=/var/lib/bot-bottle python3 -c 'from bot_bottle.paths import host_control_plane_token as t; print(t())') -GW_JWT=$(BB_SIGNING_KEY="$CP_KEY" python3 -c 'import os; from bot_bottle.control_auth import mint, ROLE_GATEWAY; print(mint(ROLE_GATEWAY, os.environ["BB_SIGNING_KEY"]))') +CP_KEY=$(BOT_BOTTLE_ROOT=/var/lib/bot-bottle python3 -c 'from bot_bottle.paths import host_orchestrator_token as t; print(t())') +GW_JWT=$(BB_SIGNING_KEY="$CP_KEY" python3 -c 'import os; from bot_bottle.orchestrator_auth import mint, ROLE_GATEWAY; print(mint(ROLE_GATEWAY, os.environ["BB_SIGNING_KEY"]))') -BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_CONTROL_PLANE_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator \\ - --host 0.0.0.0 --port {CONTROL_PLANE_PORT} --broker stub & +BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_ORCHESTRATOR_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator \\ + --host 0.0.0.0 --port {ORCHESTRATOR_PORT} --broker stub & # Gateway data plane, multi-tenant: each request resolves source-IP -> # policy against the local control plane. The VM backend reaches git over @@ -540,8 +540,8 @@ BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_CONTROL_PLANE_TOKEN="$CP_KEY" pyt # the pre-minted `gateway` JWT; gateway_init keeps the signing key out of the # data-plane daemons' env. BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise \\ -BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{CONTROL_PLANE_PORT} \\ -BOT_BOTTLE_CONTROL_AUTH_JWT="$GW_JWT" \\ +BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{ORCHESTRATOR_PORT} \\ +BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT="$GW_JWT" \\ python3 -m bot_bottle.gateway.bootstrap & # Reap as PID 1; children are backgrounded, so `wait` blocks. diff --git a/bot_bottle/backend/macos_container/backend.py b/bot_bottle/backend/macos_container/backend.py index 71b4d5e..e9225ed 100644 --- a/bot_bottle/backend/macos_container/backend.py +++ b/bot_bottle/backend/macos_container/backend.py @@ -102,7 +102,7 @@ class MacosContainerBottleBackend( (`supervise`) call when no control plane is running yet. Mirrors firecracker's infra-VM bring-up.""" from .infra import MacosInfraService - return MacosInfraService().ensure_running().control_plane_url + return MacosInfraService().ensure_running().orchestrator_url def prepare_cleanup(self) -> MacosContainerBottleCleanupPlan: return _cleanup.prepare_cleanup() diff --git a/bot_bottle/backend/macos_container/consolidated_launch.py b/bot_bottle/backend/macos_container/consolidated_launch.py index 94976fb..a641a9b 100644 --- a/bot_bottle/backend/macos_container/consolidated_launch.py +++ b/bot_bottle/backend/macos_container/consolidated_launch.py @@ -90,7 +90,7 @@ def ensure_gateway( service = service or MacosInfraService() infra = service.ensure_running() endpoint = GatewayEndpoint( - orchestrator_url=infra.control_plane_url, + orchestrator_url=infra.orchestrator_url, gateway_ip=infra.gateway_ip, gateway_ca_pem=service.ca_cert_pem(), network=service.network, diff --git a/bot_bottle/backend/macos_container/infra.py b/bot_bottle/backend/macos_container/infra.py index 5bb7f75..c2f14cc 100644 --- a/bot_bottle/backend/macos_container/infra.py +++ b/bot_bottle/backend/macos_container/infra.py @@ -48,12 +48,12 @@ from ...orchestrator.lifecycle import ( OrchestratorStartError, source_hash, ) -from ...control_auth import ROLE_GATEWAY, mint +from ...orchestrator_auth import ROLE_GATEWAY, mint from ...paths import ( - CONTROL_AUTH_JWT_ENV, - CONTROL_PLANE_TOKEN_ENV, + ORCHESTRATOR_AUTH_JWT_ENV, + ORCHESTRATOR_TOKEN_ENV, HOST_DB_FILENAME, - host_control_plane_token, + host_orchestrator_token, host_gateway_ca_dir, ) from .. import util as backend_util @@ -117,7 +117,7 @@ class InfraEndpoint: """How to reach the running infra container. The control plane and the gateway are the same container, so one address serves both.""" - control_plane_url: str # http://:8099 — host CLI + registration + orchestrator_url: str # http://:8099 — host CLI + registration gateway_ip: str # same container; agents' proxy / git-http / MCP target @@ -181,7 +181,7 @@ class MacosInfraService: return None url = self._resolve_url() if url and self.is_healthy(url): - return InfraEndpoint(control_plane_url=url, gateway_ip=_ip_of(url)) + return InfraEndpoint(orchestrator_url=url, gateway_ip=_ip_of(url)) return None def ensure_built(self) -> None: @@ -247,17 +247,17 @@ class MacosInfraService: # run process below, so neither lands on argv or in `container # inspect`'s command line. The agent runs in a SEPARATE container that # is never given these vars, which is the whole point. - "--env", CONTROL_PLANE_TOKEN_ENV, - "--env", CONTROL_AUTH_JWT_ENV, + "--env", ORCHESTRATOR_TOKEN_ENV, + "--env", ORCHESTRATOR_AUTH_JWT_ENV, "--entrypoint", "sh", self.image, "-c", _init_script(self.port), ] - _signing_key = host_control_plane_token() + _signing_key = host_orchestrator_token() run_env = { **os.environ, - CONTROL_PLANE_TOKEN_ENV: _signing_key, - CONTROL_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key), + ORCHESTRATOR_TOKEN_ENV: _signing_key, + ORCHESTRATOR_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key), } result = container_mod.run_container_argv(argv, env=run_env) if result.returncode != 0: @@ -272,7 +272,7 @@ class MacosInfraService: url = self._resolve_url() if url and self.is_healthy(url): log.info("infra container healthy", context={"url": url}) - return InfraEndpoint(control_plane_url=url, gateway_ip=_ip_of(url)) + return InfraEndpoint(orchestrator_url=url, gateway_ip=_ip_of(url)) if time.monotonic() >= deadline: raise OrchestratorStartError( f"infra container did not become healthy within " @@ -306,7 +306,7 @@ def _ip_of(url: str) -> str: return url.split("://", 1)[-1].rsplit(":", 1)[0] -def probe_control_plane_url(port: int = DEFAULT_PORT) -> str: +def probe_orchestrator_url(port: int = DEFAULT_PORT) -> str: """The running infra container's control-plane URL, or "" if it isn't up. Used by host-side control-plane discovery (`discover_orchestrator_url`); safe to call on any host — returns "" when the container or the `container` diff --git a/bot_bottle/gateway/bootstrap.py b/bot_bottle/gateway/bootstrap.py index 060a980..4e1c6d1 100644 --- a/bot_bottle/gateway/bootstrap.py +++ b/bot_bottle/gateway/bootstrap.py @@ -58,10 +58,10 @@ _READY_GATED_DAEMONS: tuple[str, ...] = ("git-gate", "git-http") # The data-plane daemons instead hold the pre-minted `gateway` JWT they present. # Scoping each to its process (even in the combined infra container) keeps a # compromised data-plane daemon from reading the key and minting a `cli` token -# (issue #469 review). Values match paths.CONTROL_PLANE_TOKEN_ENV / -# CONTROL_AUTH_JWT_ENV; hardcoded here so this supervisor stays import-light. -_SIGNING_KEY_ENV = "BOT_BOTTLE_CONTROL_PLANE_TOKEN" -_GATEWAY_JWT_ENV = "BOT_BOTTLE_CONTROL_AUTH_JWT" +# (issue #469 review). Values match paths.ORCHESTRATOR_TOKEN_ENV / +# ORCHESTRATOR_AUTH_JWT_ENV; hardcoded here so this supervisor stays import-light. +_SIGNING_KEY_ENV = "BOT_BOTTLE_ORCHESTRATOR_TOKEN" +_GATEWAY_JWT_ENV = "BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT" # Daemons that must be requested explicitly via BOT_BOTTLE_GATEWAY_DAEMONS # and are NOT started in the default (env-var-unset) case. The orchestrator diff --git a/bot_bottle/gateway/policy_resolver.py b/bot_bottle/gateway/policy_resolver.py index 9298a12..a61f3da 100644 --- a/bot_bottle/gateway/policy_resolver.py +++ b/bot_bottle/gateway/policy_resolver.py @@ -41,16 +41,16 @@ DEFAULT_TIMEOUT_SECONDS = 2.0 # rather than imported because this module is COPYed flat into the gateway image, # free of bot-bottle imports — same rationale as IDENTITY_HEADER in egress_addon # / git_http_backend. -CONTROL_AUTH_HEADER = "x-bot-bottle-control-auth" -CONTROL_AUTH_JWT_ENV = "BOT_BOTTLE_CONTROL_AUTH_JWT" +ORCHESTRATOR_AUTH_HEADER = "x-bot-bottle-orchestrator-auth" +ORCHESTRATOR_AUTH_JWT_ENV = "BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT" -def _control_auth_headers() -> dict[str, str]: +def _orchestrator_auth_headers() -> dict[str, str]: """The auth header to send, or {} when no token is configured (an open control plane, e.g. Firecracker behind its nft boundary — sending nothing is correct there and harmlessly ignored).""" - token = os.environ.get(CONTROL_AUTH_JWT_ENV, "").strip() - return {CONTROL_AUTH_HEADER: token} if token else {} + token = os.environ.get(ORCHESTRATOR_AUTH_JWT_ENV, "").strip() + return {ORCHESTRATOR_AUTH_HEADER: token} if token else {} class PolicyResolveError(RuntimeError): @@ -74,7 +74,7 @@ class PolicyResolver: body = json.dumps(payload).encode() req = urllib.request.Request( f"{self._base}{path}", data=body, method="POST", - headers={"Content-Type": "application/json", **_control_auth_headers()}, + headers={"Content-Type": "application/json", **_orchestrator_auth_headers()}, ) try: with urllib.request.urlopen(req, timeout=self._timeout) as resp: diff --git a/bot_bottle/orchestrator/__init__.py b/bot_bottle/orchestrator/__init__.py index 8a0a9da..06e3d18 100644 --- a/bot_bottle/orchestrator/__init__.py +++ b/bot_bottle/orchestrator/__init__.py @@ -17,7 +17,7 @@ backend-neutral "consolidation core" that needs no VM packaging: * `service` — the `Orchestrator`: owns the registry, brokers the launch lifecycle (launch/teardown), manages the shared gateway, attributes. - * `control_plane` — the HTTP control-plane RPC (launch / teardown / + * `server` — the HTTP control-plane RPC (launch / teardown / list / attribute / gateway / health). The actual backend-native launch (a real docker/firecracker broker) and @@ -40,7 +40,7 @@ from .broker import ( from .docker_broker import DockerBroker, DockerBrokerError from ..gateway import Gateway, GatewayError from .service import Orchestrator -from .server import ControlPlaneServer, dispatch, make_server +from .server import OrchestratorServer, dispatch, make_server __all__ = [ "BottleRecord", @@ -57,7 +57,7 @@ __all__ = [ "sign_request", "verify_request", "Orchestrator", - "ControlPlaneServer", + "OrchestratorServer", "dispatch", "make_server", ] diff --git a/bot_bottle/orchestrator/client.py b/bot_bottle/orchestrator/client.py index fb80c5e..b6c9549 100644 --- a/bot_bottle/orchestrator/client.py +++ b/bot_bottle/orchestrator/client.py @@ -18,9 +18,9 @@ import urllib.request from collections.abc import Iterable from dataclasses import dataclass -from ..control_auth import ROLE_CLI, mint -from ..paths import host_control_plane_token -from .server import CONTROL_AUTH_HEADER +from ..orchestrator_auth import ROLE_CLI, mint +from ..paths import host_orchestrator_token +from .server import ORCHESTRATOR_AUTH_HEADER DEFAULT_TIMEOUT_SECONDS = 5.0 @@ -32,7 +32,7 @@ def _host_auth_token() -> str: "" means 'send no auth header' — correct against an open (unconfigured) control plane, and harmlessly rejected by a secured one.""" try: - return mint(ROLE_CLI, host_control_plane_token()) + return mint(ROLE_CLI, host_orchestrator_token()) except (OSError, ValueError): return "" @@ -83,7 +83,7 @@ class OrchestratorClient: data = json.dumps(body).encode() if body is not None else None headers = {"Content-Type": "application/json"} if data is not None else {} if self._auth_token: - headers[CONTROL_AUTH_HEADER] = self._auth_token + headers[ORCHESTRATOR_AUTH_HEADER] = self._auth_token req = urllib.request.Request( f"{self._base}{path}", data=data, method=method, headers=headers, ) @@ -252,14 +252,14 @@ def discover_orchestrator_url(*, timeout: float = 2.0) -> str: candidates.append("http://127.0.0.1:8099") try: # firecracker: infra VM control plane on the orchestrator TAP from ..backend.firecracker import netpool - from ..backend.firecracker.infra_vm import CONTROL_PLANE_PORT + from ..backend.firecracker.infra_vm import ORCHESTRATOR_PORT candidates.append( - f"http://{netpool.orch_slot().guest_ip}:{CONTROL_PLANE_PORT}") + f"http://{netpool.orch_slot().guest_ip}:{ORCHESTRATOR_PORT}") except Exception: # noqa: BLE001 — backend optional / not firecracker pass try: # macOS: infra container control plane on its host-only address - from ..backend.macos_container.infra import probe_control_plane_url - url = probe_control_plane_url() + from ..backend.macos_container.infra import probe_orchestrator_url + url = probe_orchestrator_url() if url: candidates.append(url) except Exception: # noqa: BLE001 — backend optional / not macOS diff --git a/bot_bottle/orchestrator/server.py b/bot_bottle/orchestrator/server.py index 4daed70..a7ec7a4 100644 --- a/bot_bottle/orchestrator/server.py +++ b/bot_bottle/orchestrator/server.py @@ -48,7 +48,7 @@ via the orchestrator. Register/deregister without a launch are internal to `Orchestrator`, not exposed here. Routing/handling is the pure function `dispatch()` so it is unit-testable -without a socket; `Handler` / `ControlPlaneServer` / `make_server` are a +without a socket; `Handler` / `OrchestratorServer` / `make_server` are a thin stdlib adapter around it. Listing redacts identity tokens — they are returned only once, to the caller that launches the bottle. """ @@ -63,8 +63,8 @@ import sys import typing from urllib.parse import urlsplit -from ..control_auth import ROLE_CLI, ROLES, verify -from ..paths import CONTROL_PLANE_TOKEN_ENV +from ..orchestrator_auth import ROLE_CLI, ROLES, verify +from ..paths import ORCHESTRATOR_TOKEN_ENV from ..supervisor.types import TOOLS from .service import Orchestrator @@ -72,13 +72,13 @@ from .service import Orchestrator Json = dict[str, object] # The request header carrying the caller's role-scoped control-plane token (a -# signed JWT naming the caller's role — see control_auth). The role gates which +# signed JWT naming the caller's role — see orchestrator_auth). The role gates which # routes the caller may reach: the data plane holds a `gateway` token good only # for the agent-facing lookups; the host CLI holds a `cli` token for the # operator/mutating routes. An agent that can merely *reach* the port holds no # token at all, and a compromised gateway holds only `gateway` — neither can # drive the operator routes (approve proposals, rewrite policy, read tokens). -CONTROL_AUTH_HEADER = "x-bot-bottle-control-auth" +ORCHESTRATOR_AUTH_HEADER = "x-bot-bottle-orchestrator-auth" # The routes the data plane (role `gateway`) is allowed to reach — exactly the # per-request lookups PolicyResolver makes. Every other authenticated route is @@ -116,7 +116,7 @@ def dispatch( # pylint: disable=too-many-return-statements,too-many-branches `role` is the caller's verified control-plane role (`gateway` or `cli`), or None for an unauthenticated request; an open-mode server (no signing key - configured — see `ControlPlaneServer`) passes `cli`. Every route except + configured — see `OrchestratorServer`) passes `cli`. Every route except `GET /health` requires a role: a missing role is 401, and a role that doesn't cover the route is 403 — so a `gateway` data-plane token can reach `/resolve` + `/supervise/{propose,poll}` but not the operator routes @@ -365,10 +365,10 @@ class Handler(http.server.BaseHTTPRequestHandler): crashing the connection, so one bad request can't take the control plane down for the caller.""" server = self.server - assert isinstance(server, ControlPlaneServer) + assert isinstance(server, OrchestratorServer) length = int(self.headers.get("Content-Length") or 0) body = self.rfile.read(length) if length > 0 else b"" - role = server.role_for(self.headers.get(CONTROL_AUTH_HEADER, "")) + role = server.role_for(self.headers.get(ORCHESTRATOR_AUTH_HEADER, "")) try: status, payload = dispatch( server.orchestrator, method, self.path, body, role=role) @@ -396,11 +396,11 @@ class Handler(http.server.BaseHTTPRequestHandler): self._serve("DELETE") -class ControlPlaneServer(socketserver.ThreadingMixIn, http.server.HTTPServer): +class OrchestratorServer(socketserver.ThreadingMixIn, http.server.HTTPServer): """Threading HTTP server that carries the orchestrator for its handlers. Holds the per-host control-plane *signing key* (from - `$BOT_BOTTLE_CONTROL_PLANE_TOKEN`, injected by the launcher into the + `$BOT_BOTTLE_ORCHESTRATOR_TOKEN`, injected by the launcher into the orchestrator process only) and verifies each request's role-scoped token against it. When a key is set, every route but `/health` requires a valid token whose role covers the route; when it is unset the server runs **open** @@ -413,11 +413,11 @@ class ControlPlaneServer(socketserver.ThreadingMixIn, http.server.HTTPServer): def __init__(self, address: tuple[str, int], orchestrator: Orchestrator) -> None: self.orchestrator = orchestrator - self._signing_key = os.environ.get(CONTROL_PLANE_TOKEN_ENV, "").strip() + self._signing_key = os.environ.get(ORCHESTRATOR_TOKEN_ENV, "").strip() if not self._signing_key: sys.stderr.write( "orchestrator: WARNING — no control-plane signing key " - f"(${CONTROL_PLANE_TOKEN_ENV}); running WITHOUT caller " + f"(${ORCHESTRATOR_TOKEN_ENV}); running WITHOUT caller " "authentication. Any client that can reach this port can drive " "it. Backends that put the control plane on an agent-reachable " "network MUST set this.\n" @@ -438,13 +438,13 @@ class ControlPlaneServer(socketserver.ThreadingMixIn, http.server.HTTPServer): def make_server( orchestrator: Orchestrator, host: str = "127.0.0.1", port: int = 0 -) -> ControlPlaneServer: +) -> OrchestratorServer: """Build (but do not start) a control-plane server. `port=0` binds an ephemeral port — read `server.server_address` for the actual one.""" - return ControlPlaneServer((host, port), orchestrator) + return OrchestratorServer((host, port), orchestrator) __all__ = [ - "dispatch", "Handler", "ControlPlaneServer", "make_server", "Json", - "CONTROL_AUTH_HEADER", + "dispatch", "Handler", "OrchestratorServer", "make_server", "Json", + "ORCHESTRATOR_AUTH_HEADER", ] diff --git a/bot_bottle/control_auth.py b/bot_bottle/orchestrator_auth.py similarity index 100% rename from bot_bottle/control_auth.py rename to bot_bottle/orchestrator_auth.py diff --git a/bot_bottle/paths.py b/bot_bottle/paths.py index e1723c4..7b991cb 100644 --- a/bot_bottle/paths.py +++ b/bot_bottle/paths.py @@ -36,17 +36,16 @@ HOST_DB_FILENAME = "bot-bottle.db" # reading route (see orchestrator/server.py); it is held only by the # trusted callers (control plane, gateway, host CLI) and never handed to an # agent, so an agent that can reach the control-plane port still can't drive it. -CONTROL_PLANE_TOKEN_FILENAME = "control-plane-token" -# The env var carrying the control-plane *signing key* — held only by the +ORCHESTRATOR_TOKEN_FILENAME = "orchestrator-token" +# The env var carrying the orchestrator's *signing key* — held only by the # orchestrator (to verify tokens) and the host CLI (to mint its own), never by -# the data plane. Same value as the host token file; the name is unchanged for -# backward compatibility with existing launchers. -CONTROL_PLANE_TOKEN_ENV = "BOT_BOTTLE_CONTROL_PLANE_TOKEN" +# the data plane. Same value as the host token file. +ORCHESTRATOR_TOKEN_ENV = "BOT_BOTTLE_ORCHESTRATOR_TOKEN" # The env var carrying the data plane's pre-minted `gateway`-role token (a # signed JWT the launcher mints from the signing key). The gateway presents this # on /resolve + /supervise/{propose,poll}; it never holds the signing key, so it # cannot forge a higher-privilege `cli` token (issue #469 review). -CONTROL_AUTH_JWT_ENV = "BOT_BOTTLE_CONTROL_AUTH_JWT" +ORCHESTRATOR_AUTH_JWT_ENV = "BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT" # The host directory holding the gateway's persistent mitmproxy CA. Bind-mounted # into the infra/gateway container at mitmproxy's confdir so the self-generated @@ -98,7 +97,7 @@ def host_gateway_ca_dir() -> Path: return ca_dir -def host_control_plane_token() -> str: +def host_orchestrator_token() -> str: """The per-host control-plane secret, minted (256-bit, url-safe) and persisted 0600 on first use, then reused. @@ -107,7 +106,7 @@ def host_control_plane_token() -> str: *host* artifact — the file lives under the root the agent never mounts, and the env var is set only on the trusted containers — so reading it here is safe on the host launch path but the value never reaches a bottle.""" - path = bot_bottle_root() / CONTROL_PLANE_TOKEN_FILENAME + path = bot_bottle_root() / ORCHESTRATOR_TOKEN_FILENAME try: existing = path.read_text().strip() if existing: @@ -131,13 +130,13 @@ def host_control_plane_token() -> str: __all__ = [ "HOST_DB_FILENAME", - "CONTROL_PLANE_TOKEN_FILENAME", - "CONTROL_PLANE_TOKEN_ENV", - "CONTROL_AUTH_JWT_ENV", + "ORCHESTRATOR_TOKEN_FILENAME", + "ORCHESTRATOR_TOKEN_ENV", + "ORCHESTRATOR_AUTH_JWT_ENV", "GATEWAY_CA_DIRNAME", "bot_bottle_root", "host_db_path", "host_db_dir", "host_gateway_ca_dir", - "host_control_plane_token", + "host_orchestrator_token", ] diff --git a/tests/integration/test_orchestrator_docker_control_plane_auth.py b/tests/integration/test_orchestrator_docker_auth.py similarity index 95% rename from tests/integration/test_orchestrator_docker_control_plane_auth.py rename to tests/integration/test_orchestrator_docker_auth.py index 08a6c62..50ecaa5 100644 --- a/tests/integration/test_orchestrator_docker_control_plane_auth.py +++ b/tests/integration/test_orchestrator_docker_auth.py @@ -25,10 +25,10 @@ import tempfile import unittest from pathlib import Path -from bot_bottle.control_auth import ROLE_CLI, ROLE_GATEWAY, mint +from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY, mint from bot_bottle.orchestrator.client import OrchestratorClient from bot_bottle.backend.docker.infra import DockerInfraService -from bot_bottle.paths import host_control_plane_token +from bot_bottle.paths import host_orchestrator_token from tests._docker import skip_unless_docker # Fixed (not per-run-suffixed) so repeated runs reuse the same layer-cached @@ -46,20 +46,20 @@ _TEST_INFRA_IMAGE = "bot-bottle-infra:itest" "runner's /workspace — same host-bind-mount constraint as the other " "bottle-bringup integration tests", ) -class TestDockerControlPlaneAuthIntegration(unittest.TestCase): +class TestDockerOrchestratorAuthIntegration(unittest.TestCase): @classmethod def setUpClass(cls) -> None: suffix = secrets.token_hex(4) cls._tmp = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with cls.addClassCleanup(cls._tmp.cleanup) - # host_control_plane_token() — both the token read below and the one + # host_orchestrator_token() — both the token read below and the one # DockerInfraService injects into the container's env — resolves its # path via the *ambient* BOT_BOTTLE_ROOT env var, not the host_root # kwarg passed to the constructor (that kwarg only controls the DB # bind-mount destination). Without pointing the env var at the same # throwaway dir, this "isolated" test would read/write the developer's - # real ~/.bot-bottle/control-plane-token. + # real ~/.bot-bottle/orchestrator-token. previous_root = os.environ.get("BOT_BOTTLE_ROOT") def _restore_root() -> None: @@ -88,7 +88,7 @@ class TestDockerControlPlaneAuthIntegration(unittest.TestCase): cls.svc.ensure_running() # The control plane now verifies role-scoped signed tokens, not the raw # key. Mint one of each role from the host signing key (issue #469 review). - signing_key = host_control_plane_token() + signing_key = host_orchestrator_token() cls.cli_token = mint(ROLE_CLI, signing_key) cls.gateway_token = mint(ROLE_GATEWAY, signing_key) diff --git a/tests/unit/test_backend_selection.py b/tests/unit/test_backend_selection.py index 1a2304d..25b221c 100644 --- a/tests/unit/test_backend_selection.py +++ b/tests/unit/test_backend_selection.py @@ -408,7 +408,7 @@ class TestEnsureOrchestrator(unittest.TestCase): with patch( "bot_bottle.backend.firecracker.infra_vm.ensure_running" ) as ensure_running: - ensure_running.return_value.control_plane_url = ( + ensure_running.return_value.orchestrator_url = ( "http://10.243.255.1:8099" ) url = b.ensure_orchestrator() @@ -419,7 +419,7 @@ class TestEnsureOrchestrator(unittest.TestCase): with patch( "bot_bottle.backend.macos_container.infra.MacosInfraService" ) as service_cls: - service_cls.return_value.ensure_running.return_value.control_plane_url = ( + service_cls.return_value.ensure_running.return_value.orchestrator_url = ( "http://192.168.128.2:8099" ) url = b.ensure_orchestrator() diff --git a/tests/unit/test_firecracker_infra_vm.py b/tests/unit/test_firecracker_infra_vm.py index 7dcf4ea..157a5c5 100644 --- a/tests/unit/test_firecracker_infra_vm.py +++ b/tests/unit/test_firecracker_infra_vm.py @@ -31,7 +31,7 @@ class TestSigningKeySync(unittest.TestCase): patch.object(infra_vm, "bot_bottle_root", return_value=root): out = infra_vm._with_signing_key(self._infra()) self.assertIsInstance(out, infra_vm.InfraVm) # returned for chaining - token = root / infra_vm.CONTROL_PLANE_TOKEN_FILENAME + token = root / infra_vm.ORCHESTRATOR_TOKEN_FILENAME self.assertEqual("the-signing-key", token.read_text()) self.assertEqual(0o600, token.stat().st_mode & 0o777) # It cat'd the guest volume path over SSH. @@ -44,16 +44,16 @@ class TestSigningKeySync(unittest.TestCase): with patch.object(infra_vm.subprocess, "run", return_value=proc), \ patch.object(infra_vm, "bot_bottle_root", return_value=root): infra_vm._with_signing_key(self._infra()) # no raise - self.assertFalse((root / infra_vm.CONTROL_PLANE_TOKEN_FILENAME).exists()) + self.assertFalse((root / infra_vm.ORCHESTRATOR_TOKEN_FILENAME).exists()) -class TestControlPlaneUrl(unittest.TestCase): +class TestOrchestratorUrl(unittest.TestCase): def test_url_uses_guest_ip_and_port(self): infra = infra_vm.InfraVm( vm=MagicMock(), guest_ip="10.243.255.1", private_key=Path("/k")) self.assertEqual( - f"http://10.243.255.1:{infra_vm.CONTROL_PLANE_PORT}", - infra.control_plane_url, + f"http://10.243.255.1:{infra_vm.ORCHESTRATOR_PORT}", + infra.orchestrator_url, ) @@ -81,11 +81,11 @@ class TestBuildInfraRootfs(unittest.TestCase): # Role-scoped control-plane auth (issue #469 review): the orchestrator # gets the signing key, the gateway daemons get a pre-minted `gateway` # JWT — never open mode in the infra VM. - self.assertIn("host_control_plane_token", init) # key generated on the volume + self.assertIn("host_orchestrator_token", init) # key generated on the volume self.assertIn("mint, ROLE_GATEWAY", init) # gateway JWT minted from it - self.assertIn('BOT_BOTTLE_CONTROL_PLANE_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator', + self.assertIn('BOT_BOTTLE_ORCHESTRATOR_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator', init) # key -> orchestrator only - self.assertIn('BOT_BOTTLE_CONTROL_AUTH_JWT="$GW_JWT"', init) # JWT -> gateway daemons + self.assertIn('BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT="$GW_JWT"', init) # JWT -> gateway daemons class TestSshGatewayTransport(unittest.TestCase): diff --git a/tests/unit/test_gateway_init.py b/tests/unit/test_gateway_init.py index 7f5272c..1fb471f 100644 --- a/tests/unit/test_gateway_init.py +++ b/tests/unit/test_gateway_init.py @@ -64,7 +64,7 @@ class TestEnvForDaemon(unittest.TestCase): self.assertNotIn("X", self._BASE) -class TestControlPlaneEnvScoping(unittest.TestCase): +class TestOrchestratorEnvScoping(unittest.TestCase): """The control-plane signing key stays with the orchestrator; the pre-minted `gateway` JWT goes to the data-plane daemons (issue #469 review). Scoping them per-process keeps a compromised data-plane daemon from reading the key @@ -72,20 +72,20 @@ class TestControlPlaneEnvScoping(unittest.TestCase): _BASE = { "PATH": "/usr/bin", - "BOT_BOTTLE_CONTROL_PLANE_TOKEN": "sk-x", - "BOT_BOTTLE_CONTROL_AUTH_JWT": "gw-jwt", + "BOT_BOTTLE_ORCHESTRATOR_TOKEN": "sk-x", + "BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT": "gw-jwt", } def test_orchestrator_gets_key_not_jwt(self): env = _env_for_daemon("orchestrator", self._BASE) - self.assertEqual("sk-x", env["BOT_BOTTLE_CONTROL_PLANE_TOKEN"]) - self.assertNotIn("BOT_BOTTLE_CONTROL_AUTH_JWT", env) + self.assertEqual("sk-x", env["BOT_BOTTLE_ORCHESTRATOR_TOKEN"]) + self.assertNotIn("BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT", env) def test_data_plane_daemons_get_jwt_not_key(self): for name in ("egress", "git-gate", "git-http", "supervise"): env = _env_for_daemon(name, self._BASE) - self.assertNotIn("BOT_BOTTLE_CONTROL_PLANE_TOKEN", env, name) - self.assertEqual("gw-jwt", env["BOT_BOTTLE_CONTROL_AUTH_JWT"], name) + self.assertNotIn("BOT_BOTTLE_ORCHESTRATOR_TOKEN", env, name) + self.assertEqual("gw-jwt", env["BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT"], name) class TestSelectedDaemons(unittest.TestCase): diff --git a/tests/unit/test_macos_consolidated_launch.py b/tests/unit/test_macos_consolidated_launch.py index b0f0b84..812e14e 100644 --- a/tests/unit/test_macos_consolidated_launch.py +++ b/tests/unit/test_macos_consolidated_launch.py @@ -58,7 +58,7 @@ class TestEnsureGateway(unittest.TestCase): from bot_bottle.backend.macos_container.infra import InfraEndpoint service = MagicMock() service.ensure_running.return_value = InfraEndpoint( - control_plane_url="http://192.168.128.2:8099", + orchestrator_url="http://192.168.128.2:8099", gateway_ip="192.168.128.2", ) service.network = "bot-bottle-mac-gateway" @@ -72,7 +72,7 @@ class TestEnsureGateway(unittest.TestCase): self.assertEqual("PEM", endpoint.gateway_ca_pem) self.assertEqual("bot-bottle-mac-gateway", endpoint.network) - def test_control_plane_and_gateway_share_one_address(self) -> None: + def test_orchestrator_and_gateway_share_one_address(self) -> None: """One infra container hosts both, so the gateway IP and the control-plane host are the same.""" endpoint = self._run(self._service()) diff --git a/tests/unit/test_macos_infra.py b/tests/unit/test_macos_infra.py index 914ed1c..f1341ec 100644 --- a/tests/unit/test_macos_infra.py +++ b/tests/unit/test_macos_infra.py @@ -10,7 +10,7 @@ from bot_bottle.backend.macos_container.infra import ( INFRA_DB_VOLUME, MacosInfraService, OrchestratorStartError, - probe_control_plane_url, + probe_orchestrator_url, ) _INFRA = "bot_bottle.backend.macos_container.infra" @@ -110,7 +110,7 @@ class TestInfraEnsureRunning(unittest.TestCase): mod.try_container_ipv4_on_network.return_value = "192.168.128.2" endpoint = svc.ensure_running() run.assert_not_called() - self.assertEqual("http://192.168.128.2:8099", endpoint.control_plane_url) + self.assertEqual("http://192.168.128.2:8099", endpoint.orchestrator_url) self.assertEqual("192.168.128.2", endpoint.gateway_ip) def test_changed_source_recreates(self) -> None: @@ -176,16 +176,16 @@ class TestCaCertPem(unittest.TestCase): svc.ca_cert_pem(timeout=0) -class TestProbeControlPlane(unittest.TestCase): +class TestProbeOrchestrator(unittest.TestCase): def test_returns_url_when_running(self) -> None: with patch(f"{_INFRA}.container_mod") as mod: mod.try_container_ipv4_on_network.return_value = "192.168.128.2" - self.assertEqual("http://192.168.128.2:8099", probe_control_plane_url()) + self.assertEqual("http://192.168.128.2:8099", probe_orchestrator_url()) def test_empty_when_absent(self) -> None: with patch(f"{_INFRA}.container_mod") as mod: mod.try_container_ipv4_on_network.return_value = "" - self.assertEqual("", probe_control_plane_url()) + self.assertEqual("", probe_orchestrator_url()) if __name__ == "__main__": diff --git a/tests/unit/test_control_auth.py b/tests/unit/test_orchestrator_auth.py similarity index 91% rename from tests/unit/test_control_auth.py rename to tests/unit/test_orchestrator_auth.py index bfcb7cc..78e2559 100644 --- a/tests/unit/test_control_auth.py +++ b/tests/unit/test_orchestrator_auth.py @@ -6,7 +6,7 @@ import base64 import json import unittest -from bot_bottle.control_auth import ROLE_CLI, ROLE_GATEWAY, ROLES, mint, verify +from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY, ROLES, mint, verify _KEY = "test-key" @@ -45,14 +45,14 @@ class TestMintVerify(unittest.TestCase): def test_validly_signed_but_wrong_alg_rejected(self) -> None: # Alg-confusion: even a *correctly signed* token whose header claims a # non-HS256 alg must be rejected. - from bot_bottle.control_auth import _sign # noqa: PLC0415 + from bot_bottle.orchestrator_auth import _sign # noqa: PLC0415 signing_input = f"{_b64({'alg': 'none', 'typ': 'JWT'})}.{_b64({'role': 'cli'})}" self.assertIsNone(verify(f"{signing_input}.{_sign(_KEY, signing_input)}", _KEY)) def test_validly_signed_but_undecodable_rejected(self) -> None: # A correct signature over a header that isn't valid base64/JSON still # fails closed rather than raising. - from bot_bottle.control_auth import _sign # noqa: PLC0415 + from bot_bottle.orchestrator_auth import _sign # noqa: PLC0415 signing_input = f"!!!not-base64!!!.{_b64({'role': 'cli'})}" self.assertIsNone(verify(f"{signing_input}.{_sign(_KEY, signing_input)}", _KEY)) @@ -67,7 +67,7 @@ class TestMintVerify(unittest.TestCase): header, _p, _s = mint(ROLE_CLI, _KEY).split(".") # Re-sign a token carrying an unknown role — a valid signature but a # role the control plane doesn't recognise must still be rejected. - from bot_bottle.control_auth import _sign # noqa: PLC0415 + from bot_bottle.orchestrator_auth import _sign # noqa: PLC0415 payload = _b64({"role": "root"}) signing_input = f"{header}.{payload}" forged = f"{signing_input}.{_sign(_KEY, signing_input)}" diff --git a/tests/unit/test_orchestrator_client.py b/tests/unit/test_orchestrator_client.py index 4086ade..a9f77da 100644 --- a/tests/unit/test_orchestrator_client.py +++ b/tests/unit/test_orchestrator_client.py @@ -7,7 +7,7 @@ import unittest import urllib.error from unittest.mock import MagicMock, patch -from bot_bottle.control_auth import ROLE_CLI, verify +from bot_bottle.orchestrator_auth import ROLE_CLI, verify from bot_bottle.orchestrator.client import ( OrchestratorClient, OrchestratorClientError, @@ -20,13 +20,13 @@ _URLOPEN = "bot_bottle.orchestrator.client.urllib.request.urlopen" class TestHostAuthToken(unittest.TestCase): def test_mints_a_cli_token_from_the_host_key(self) -> None: - with patch("bot_bottle.orchestrator.client.host_control_plane_token", + with patch("bot_bottle.orchestrator.client.host_orchestrator_token", return_value="signing-key"): tok = _host_auth_token() self.assertEqual(ROLE_CLI, verify(tok, "signing-key")) def test_returns_empty_when_key_unreadable(self) -> None: - with patch("bot_bottle.orchestrator.client.host_control_plane_token", + with patch("bot_bottle.orchestrator.client.host_orchestrator_token", side_effect=OSError("no host root")): self.assertEqual("", _host_auth_token()) diff --git a/tests/unit/test_orchestrator_server.py b/tests/unit/test_orchestrator_server.py index 2d5c736..d55908b 100644 --- a/tests/unit/test_orchestrator_server.py +++ b/tests/unit/test_orchestrator_server.py @@ -19,7 +19,7 @@ from contextlib import closing from pathlib import Path from unittest.mock import patch -from bot_bottle.control_auth import ROLE_CLI, ROLE_GATEWAY, mint +from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY, mint from bot_bottle.orchestrator.broker import StubBroker from bot_bottle.orchestrator.server import dispatch, make_server from bot_bottle.orchestrator.registry import BottleRecord, RegistryStore @@ -284,7 +284,7 @@ class TestServerRoundTrip(unittest.TestCase): self.assertEqual(reg["bottle_id"], attr["bottle_id"]) -class TestControlPlaneAuth(unittest.TestCase): +class TestOrchestratorAuth(unittest.TestCase): """Role-scoped control-plane tokens (issue #400 / #469 review): every route but /health needs a valid token, and the token's role gates which routes it reaches — a `gateway` data-plane token can't drive the operator routes.""" @@ -349,7 +349,7 @@ class TestControlPlaneAuth(unittest.TestCase): self.assertIsNotNone(self.orch.registry.get(rec.bottle_id)) def _server_with_key(self, signing_key: str): - with patch.dict("os.environ", {"BOT_BOTTLE_CONTROL_PLANE_TOKEN": signing_key}): + with patch.dict("os.environ", {"BOT_BOTTLE_ORCHESTRATOR_TOKEN": signing_key}): server = make_server(self.orch, "127.0.0.1", 0) self.addCleanup(server.server_close) threading.Thread(target=server.serve_forever, daemon=True).start() @@ -360,7 +360,7 @@ class TestControlPlaneAuth(unittest.TestCase): def _status(self, url: str, *, header: str | None = None) -> int: req = urllib.request.Request(url) if header is not None: - req.add_header("x-bot-bottle-control-auth", header) + req.add_header("x-bot-bottle-orchestrator-auth", header) try: return urllib.request.urlopen(req, timeout=5).status except urllib.error.HTTPError as e: @@ -384,7 +384,7 @@ class TestControlPlaneAuth(unittest.TestCase): grants full cli access, so existing round-trip behavior is unchanged.""" with patch.dict("os.environ", {}, clear=False): import os - os.environ.pop("BOT_BOTTLE_CONTROL_PLANE_TOKEN", None) + os.environ.pop("BOT_BOTTLE_ORCHESTRATOR_TOKEN", None) server = make_server(self.orch, "127.0.0.1", 0) self.addCleanup(server.server_close) self.assertEqual(ROLE_CLI, server.role_for("")) diff --git a/tests/unit/test_policy_resolver.py b/tests/unit/test_policy_resolver.py index 44178a3..e034d23 100644 --- a/tests/unit/test_policy_resolver.py +++ b/tests/unit/test_policy_resolver.py @@ -8,11 +8,11 @@ import urllib.error from unittest.mock import MagicMock, patch from bot_bottle.gateway.policy_resolver import ( - CONTROL_AUTH_HEADER, - CONTROL_AUTH_JWT_ENV, + ORCHESTRATOR_AUTH_HEADER, + ORCHESTRATOR_AUTH_JWT_ENV, PolicyResolveError, PolicyResolver, - _control_auth_headers, + _orchestrator_auth_headers, ) _URLOPEN = "bot_bottle.gateway.policy_resolver.urllib.request.urlopen" @@ -29,16 +29,16 @@ def _http_error(code: int) -> urllib.error.HTTPError: return urllib.error.HTTPError("http://x/resolve", code, "err", {}, None) # type: ignore[arg-type] -class TestControlAuthHeaders(unittest.TestCase): +class TestOrchestratorAuthHeaders(unittest.TestCase): def test_sends_the_gateway_jwt_when_configured(self) -> None: - with patch.dict("os.environ", {CONTROL_AUTH_JWT_ENV: "gateway.jwt.tok"}): - self.assertEqual({CONTROL_AUTH_HEADER: "gateway.jwt.tok"}, _control_auth_headers()) + with patch.dict("os.environ", {ORCHESTRATOR_AUTH_JWT_ENV: "gateway.jwt.tok"}): + self.assertEqual({ORCHESTRATOR_AUTH_HEADER: "gateway.jwt.tok"}, _orchestrator_auth_headers()) def test_sends_nothing_when_unset(self) -> None: import os with patch.dict("os.environ", {}, clear=False): - os.environ.pop(CONTROL_AUTH_JWT_ENV, None) - self.assertEqual({}, _control_auth_headers()) + os.environ.pop(ORCHESTRATOR_AUTH_JWT_ENV, None) + self.assertEqual({}, _orchestrator_auth_headers()) class TestPolicyResolver(unittest.TestCase): -- 2.52.0 From cb3a74edb00db83d20dcde527d8a9531098f56fa Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 18:22:03 -0400 Subject: [PATCH 32/39] refactor(orchestrator): collapse supervisor package into supervisor.py orchestrator/supervisor/ held only __init__.py, so the package no longer earns its own directory. Moved it to orchestrator/supervisor.py and removed the dir. External imports are unchanged (orchestrator.supervisor resolves identically as a module); only the file's own relative-import depths shift up one level. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- .../{supervisor/__init__.py => supervisor.py} | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) rename bot_bottle/orchestrator/{supervisor/__init__.py => supervisor.py} (91%) diff --git a/bot_bottle/orchestrator/supervisor/__init__.py b/bot_bottle/orchestrator/supervisor.py similarity index 91% rename from bot_bottle/orchestrator/supervisor/__init__.py rename to bot_bottle/orchestrator/supervisor.py index de4cf24..7db7839 100644 --- a/bot_bottle/orchestrator/supervisor/__init__.py +++ b/bot_bottle/orchestrator/supervisor.py @@ -19,12 +19,12 @@ from __future__ import annotations from pathlib import Path -from ...supervisor.types import * # noqa: F401,F403 — re-export the neutral vocabulary -from ...supervisor.plan import SupervisePlan -from ..store.store_manager import StoreManager -from ..store.queue_store import QueueStore -from ...store.audit_store import AuditStore -from ...supervisor.types import AuditEntry, Proposal, Response +from ..supervisor.types import * # noqa: F401,F403 — re-export the neutral vocabulary +from ..supervisor.plan import SupervisePlan +from .store.store_manager import StoreManager +from .store.queue_store import QueueStore +from ..store.audit_store import AuditStore +from ..supervisor.types import AuditEntry, Proposal, Response class Supervisor: -- 2.52.0 From 3efb014aceeed4ecc4228fffc4459db41f67c4f6 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 18:24:20 -0400 Subject: [PATCH 33/39] refactor(orchestrator): move registry into store/registry_store registry.py is a SQLite-backed store (its class is already RegistryStore), so it belongs with the other orchestrator-owned stores rather than at the package root. Moved orchestrator/registry.py -> orchestrator/store/ registry_store.py, joining queue_store / secret_store / config_store. Repointed the in-package importers (.registry -> .store.registry_store) and the tests, bumped the moved file's relative-import depths, renamed the test file to match, and listed it in the store package docstring. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/orchestrator/__init__.py | 2 +- bot_bottle/orchestrator/__main__.py | 2 +- bot_bottle/orchestrator/service.py | 2 +- bot_bottle/orchestrator/store/__init__.py | 6 +++--- .../orchestrator/{registry.py => store/registry_store.py} | 6 +++--- ...ator_registry.py => test_orchestrator_registry_store.py} | 2 +- tests/unit/test_orchestrator_server.py | 2 +- tests/unit/test_orchestrator_service.py | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) rename bot_bottle/orchestrator/{registry.py => store/registry_store.py} (99%) rename tests/unit/{test_orchestrator_registry.py => test_orchestrator_registry_store.py} (99%) diff --git a/bot_bottle/orchestrator/__init__.py b/bot_bottle/orchestrator/__init__.py index 06e3d18..0ae241d 100644 --- a/bot_bottle/orchestrator/__init__.py +++ b/bot_bottle/orchestrator/__init__.py @@ -28,7 +28,7 @@ orchestrator -> firecracker). from __future__ import annotations -from .registry import BottleRecord, RegistryStore, new_identity_token +from .store.registry_store import BottleRecord, RegistryStore, new_identity_token from .broker import ( BrokerAuthError, LaunchBroker, diff --git a/bot_bottle/orchestrator/__main__.py b/bot_bottle/orchestrator/__main__.py index 0370c83..6fad836 100644 --- a/bot_bottle/orchestrator/__main__.py +++ b/bot_bottle/orchestrator/__main__.py @@ -20,7 +20,7 @@ from .store.store_manager import StoreManager from .broker import LaunchBroker, StubBroker from .server import make_server from .docker_broker import DockerBroker -from .registry import RegistryStore, default_db_path +from .store.registry_store import RegistryStore, default_db_path from .service import Orchestrator diff --git a/bot_bottle/orchestrator/service.py b/bot_bottle/orchestrator/service.py index 740a6fa..5b7e109 100644 --- a/bot_bottle/orchestrator/service.py +++ b/bot_bottle/orchestrator/service.py @@ -26,7 +26,7 @@ from collections.abc import Iterable from datetime import datetime, timezone from .broker import LaunchBroker, LaunchRequest, sign_request -from .registry import DEFAULT_REAP_GRACE_SECONDS, BottleRecord, RegistryStore +from .store.registry_store import DEFAULT_REAP_GRACE_SECONDS, BottleRecord, RegistryStore from .supervisor import ( AuditEntry, COMPONENT_FOR_TOOL, diff --git a/bot_bottle/orchestrator/store/__init__.py b/bot_bottle/orchestrator/store/__init__.py index 1c4a81d..4b2b70f 100644 --- a/bot_bottle/orchestrator/store/__init__.py +++ b/bot_bottle/orchestrator/store/__init__.py @@ -1,9 +1,9 @@ """Orchestrator-owned SQLite stores. The stores whose tables only the orchestrator (control plane) opens: the -supervise proposal/response `queue_store`, the agent-secret `secret_store`, and -the orchestrator `config_store`. They build on the shared `DbStore` / -`migrations` base in `bot_bottle.store`. +per-host bottle `registry_store`, the supervise proposal/response `queue_store`, +the agent-secret `secret_store`, and the orchestrator `config_store`. They build +on the shared `DbStore` / `migrations` base in `bot_bottle.store`. Callers import the concrete module directly, e.g. `from bot_bottle.orchestrator.store.queue_store import QueueStore`. diff --git a/bot_bottle/orchestrator/registry.py b/bot_bottle/orchestrator/store/registry_store.py similarity index 99% rename from bot_bottle/orchestrator/registry.py rename to bot_bottle/orchestrator/store/registry_store.py index a5afe4e..05a2eb6 100644 --- a/bot_bottle/orchestrator/registry.py +++ b/bot_bottle/orchestrator/store/registry_store.py @@ -36,9 +36,9 @@ from collections.abc import Iterable from dataclasses import dataclass from pathlib import Path -from ..store.db_store import DbStore -from ..store.migrations import TableMigrations -from ..paths import host_db_path +from ...store.db_store import DbStore +from ...store.migrations import TableMigrations +from ...paths import host_db_path # 256 bits of urandom, URL-safe — unguessable per-bottle identity token. IDENTITY_TOKEN_BYTES = 32 diff --git a/tests/unit/test_orchestrator_registry.py b/tests/unit/test_orchestrator_registry_store.py similarity index 99% rename from tests/unit/test_orchestrator_registry.py rename to tests/unit/test_orchestrator_registry_store.py index 90d0ff4..8b46397 100644 --- a/tests/unit/test_orchestrator_registry.py +++ b/tests/unit/test_orchestrator_registry_store.py @@ -9,7 +9,7 @@ import unittest from contextlib import closing from pathlib import Path -from bot_bottle.orchestrator.registry import ( +from bot_bottle.orchestrator.store.registry_store import ( BottleRecord, RegistryStore, new_identity_token, diff --git a/tests/unit/test_orchestrator_server.py b/tests/unit/test_orchestrator_server.py index d55908b..e3dfb1b 100644 --- a/tests/unit/test_orchestrator_server.py +++ b/tests/unit/test_orchestrator_server.py @@ -22,7 +22,7 @@ from unittest.mock import patch from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY, mint from bot_bottle.orchestrator.broker import StubBroker from bot_bottle.orchestrator.server import dispatch, make_server -from bot_bottle.orchestrator.registry import BottleRecord, RegistryStore +from bot_bottle.orchestrator.store.registry_store import BottleRecord, RegistryStore from bot_bottle.orchestrator.service import Orchestrator from bot_bottle.orchestrator.store.store_manager import StoreManager from bot_bottle.orchestrator.supervisor import ( diff --git a/tests/unit/test_orchestrator_service.py b/tests/unit/test_orchestrator_service.py index a203af3..8ba2589 100644 --- a/tests/unit/test_orchestrator_service.py +++ b/tests/unit/test_orchestrator_service.py @@ -12,7 +12,7 @@ from pathlib import Path from unittest.mock import patch from bot_bottle.orchestrator.broker import LaunchBroker, LaunchRequest, StubBroker -from bot_bottle.orchestrator.registry import RegistryStore +from bot_bottle.orchestrator.store.registry_store import RegistryStore from bot_bottle.orchestrator.service import Orchestrator from bot_bottle.orchestrator.store.secret_store import new_env_var_secret from bot_bottle.orchestrator.store.store_manager import StoreManager -- 2.52.0 From e2740842a039cbe50c6eaed189fbf4f348daaa7f Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 18:54:41 -0400 Subject: [PATCH 34/39] refactor: make the two remaining heavy __init__ modules lazy Audit of package __init__ import cost turned up two eager ones: orchestrator/__init__.py (27 -> 2): was pure `from .X import Y` re-exports (registry_store, broker, docker_broker, gateway, service, server). Because importing any submodule runs the parent __init__, the CLI migration gate (orchestrator.store.store_manager, hit on every command) transitively dragged backend.docker, docker_broker, server, and http.server. Converted to the same lazy __getattr__ + _LAZY facade used by manifest/backend/egress/git_gate; the re-export API is unchanged and the docker/http drag is gone. cli/commands/__init__.py (78 -> 23): the registry imported all twelve handlers to build COMMANDS, so importing one command pulled all of them plus their deps. COMMANDS now maps each name to a thin lazy wrapper that imports its handler's module on first dispatch. Values stay callable, so the dispatcher and the patch.dict dispatch tests are unchanged; a CLI run now loads only the one command it dispatches. The remaining 23 is the dispatcher's own baseline (store_manager migration gate + help + log). Full unit suite green (2243); `bb help` + dispatch/migration-gate tests verified. Co-Authored-By: Claude Opus 4.8 --- bot_bottle/cli/commands/__init__.py | 67 +++++++++++++--------- bot_bottle/orchestrator/__init__.py | 88 ++++++++++++++++++----------- 2 files changed, 94 insertions(+), 61 deletions(-) diff --git a/bot_bottle/cli/commands/__init__.py b/bot_bottle/cli/commands/__init__.py index cda230d..813524f 100644 --- a/bot_bottle/cli/commands/__init__.py +++ b/bot_bottle/cli/commands/__init__.py @@ -1,41 +1,52 @@ """CLI subcommand registry. One module per `bot-bottle `, each exposing a `cmd_(argv)` -handler. This package `__init__` assembles them into the COMMANDS table -the dispatcher (`bot_bottle.cli.__main__`) reads. Shared CLI helpers -(`constants`, `tui`) stay one level up in the `cli` package. +handler. This package `__init__` maps command names to their handlers +**lazily**: a short-lived CLI run dispatches exactly one command, so +importing all twelve handlers (and their transitive deps — backend, +manifest, orchestrator, …) up front is wasted work. Each COMMANDS value is +a thin wrapper that imports its handler's module on first call. Shared CLI +helpers (`constants`, `tui`) stay one level up in the `cli` package. """ from __future__ import annotations -from .backend import cmd_backend -from .cleanup import cmd_cleanup -from .commit import cmd_commit -from .edit import cmd_edit -from .help import cmd_help -from .info import cmd_info -from .init import cmd_init -from .list import cmd_list -from .login import cmd_login -from .resume import cmd_resume -from .start import cmd_start -from .supervise import cmd_supervise +from importlib import import_module +from typing import Callable -COMMANDS = { - "backend": cmd_backend, - "cleanup": cmd_cleanup, - "commit": cmd_commit, - "edit": cmd_edit, - "help": cmd_help, - "info": cmd_info, - "init": cmd_init, - "list": cmd_list, - "login": cmd_login, - "resume": cmd_resume, - "start": cmd_start, - "supervise": cmd_supervise, +# command name -> ":". Kept as strings so building +# the registry imports nothing; the module loads only when dispatched. +_HANDLERS: dict[str, str] = { + "backend": "backend:cmd_backend", + "cleanup": "cleanup:cmd_cleanup", + "commit": "commit:cmd_commit", + "edit": "edit:cmd_edit", + "help": "help:cmd_help", + "info": "info:cmd_info", + "init": "init:cmd_init", + "list": "list:cmd_list", + "login": "login:cmd_login", + "resume": "resume:cmd_resume", + "start": "start:cmd_start", + "supervise": "supervise:cmd_supervise", } + +def _lazy(spec: str) -> Callable[[list[str]], "int | None"]: + """Wrap a `:` handler so its module is imported only when + the command is actually dispatched, not when the registry is built.""" + module, attr = spec.split(":") + + def run(argv: list[str]) -> "int | None": + handler = getattr(import_module(f".{module}", __name__), attr) + return handler(argv) + + run.__name__ = attr + return run + + +COMMANDS = {name: _lazy(spec) for name, spec in _HANDLERS.items()} + # Commands that manage host prerequisites (or are otherwise store-free) and # must run before — or without — a migrated DB. `backend` provisions/probes # the host (TAP pool, /dev/kvm, firecracker) and never opens the store, so diff --git a/bot_bottle/orchestrator/__init__.py b/bot_bottle/orchestrator/__init__.py index 0ae241d..0858e0e 100644 --- a/bot_bottle/orchestrator/__init__.py +++ b/bot_bottle/orchestrator/__init__.py @@ -5,7 +5,7 @@ A single persistent per-host service that will run the gateway functions agent launches. This package is being built bottom-up, starting with the backend-neutral "consolidation core" that needs no VM packaging: - * `registry` — the SQLite runtime-state store + fail-closed + * `store.registry_store` — the SQLite runtime-state store + fail-closed attribution (source IP + per-bottle identity token). * `broker` — the signed, structured launch-request contract + a `LaunchBroker` (stub for the harness) that verifies @@ -28,36 +28,58 @@ orchestrator -> firecracker). from __future__ import annotations -from .store.registry_store import BottleRecord, RegistryStore, new_identity_token -from .broker import ( - BrokerAuthError, - LaunchBroker, - LaunchRequest, - StubBroker, - sign_request, - verify_request, -) -from .docker_broker import DockerBroker, DockerBrokerError -from ..gateway import Gateway, GatewayError -from .service import Orchestrator -from .server import OrchestratorServer, dispatch, make_server +from typing import TYPE_CHECKING, Any -__all__ = [ - "BottleRecord", - "RegistryStore", - "new_identity_token", - "BrokerAuthError", - "LaunchBroker", - "LaunchRequest", - "StubBroker", - "DockerBroker", - "DockerBrokerError", - "Gateway", - "GatewayError", - "sign_request", - "verify_request", - "Orchestrator", - "OrchestratorServer", - "dispatch", - "make_server", -] +if TYPE_CHECKING: + from .store.registry_store import BottleRecord, RegistryStore, new_identity_token + from .broker import ( + BrokerAuthError, + LaunchBroker, + LaunchRequest, + StubBroker, + sign_request, + verify_request, + ) + from .docker_broker import DockerBroker, DockerBrokerError + from ..gateway import Gateway, GatewayError + from .service import Orchestrator + from .server import OrchestratorServer, dispatch, make_server + + +# Facade name -> submodule that defines it. Lazy so importing a leaf (or the +# parent package, e.g. via `orchestrator.store.store_manager`) doesn't drag the +# whole orchestrator — server, docker_broker + the docker backend, http — while +# `from bot_bottle.orchestrator import RegistryStore` keeps working. +_LAZY: dict[str, str] = { + "BottleRecord": ".store.registry_store", + "RegistryStore": ".store.registry_store", + "new_identity_token": ".store.registry_store", + "BrokerAuthError": ".broker", + "LaunchBroker": ".broker", + "LaunchRequest": ".broker", + "StubBroker": ".broker", + "sign_request": ".broker", + "verify_request": ".broker", + "DockerBroker": ".docker_broker", + "DockerBrokerError": ".docker_broker", + "Gateway": "..gateway", + "GatewayError": "..gateway", + "Orchestrator": ".service", + "OrchestratorServer": ".server", + "dispatch": ".server", + "make_server": ".server", +} + + +def __getattr__(name: str) -> Any: + src = _LAZY.get(name) + if src is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module + + value = getattr(import_module(src, __name__), name) + globals()[name] = value + return value + + +__all__ = list(_LAZY) -- 2.52.0 From bef45348f57155d9f08726362dfe819d09045655 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 19:11:37 -0400 Subject: [PATCH 35/39] fix(types): satisfy pyright strict on the refactored facades MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two facade issues from the reorg: * orchestrator/__init__.py used `__all__ = list(_LAZY)`, which pyright strict can't analyze (reportUnsupportedDunderAll) — and because it isn't a static literal, the TYPE_CHECKING re-export imports read as unused. Replaced with the explicit literal list the other lazy facades already use. * manifest/index.py imported six piece types (ManifestAgentProvider, EGRESS_AUTH_SCHEMES, ManifestEgressConfig/Route, ManifestGitEntry, ManifestKeyConfig) it never referenced — the package facade re-exports those straight from their submodules, so index.py needn't import them. Dropped the dead imports. pyright: 0 errors (was 24). Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/manifest/index.py | 9 ++------- bot_bottle/orchestrator/__init__.py | 20 +++++++++++++++++++- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/bot_bottle/manifest/index.py b/bot_bottle/manifest/index.py index 35966b4..8996560 100644 --- a/bot_bottle/manifest/index.py +++ b/bot_bottle/manifest/index.py @@ -16,15 +16,10 @@ from typing import Mapping from ..log import warn from .util import ManifestError, as_json_object -from .agent import ManifestAgent, ManifestAgentProvider +from .agent import ManifestAgent from .bottle import ManifestBottle -from .egress import ( - EGRESS_AUTH_SCHEMES, - ManifestEgressConfig, - ManifestEgressRoute, -) from .extends import merge_bottles_runtime, resolve_bottles -from .git import ManifestGitEntry, ManifestGitUser, ManifestKeyConfig +from .git import ManifestGitUser from .loader import ( check_stale_json, load_bottle_chain_from_dir, diff --git a/bot_bottle/orchestrator/__init__.py b/bot_bottle/orchestrator/__init__.py index 0858e0e..7597a69 100644 --- a/bot_bottle/orchestrator/__init__.py +++ b/bot_bottle/orchestrator/__init__.py @@ -82,4 +82,22 @@ def __getattr__(name: str) -> Any: return value -__all__ = list(_LAZY) +__all__ = [ + "BottleRecord", + "RegistryStore", + "new_identity_token", + "BrokerAuthError", + "LaunchBroker", + "LaunchRequest", + "StubBroker", + "sign_request", + "verify_request", + "DockerBroker", + "DockerBrokerError", + "Gateway", + "GatewayError", + "Orchestrator", + "OrchestratorServer", + "dispatch", + "make_server", +] -- 2.52.0 From e7fe00e2f5d992dcf8a3aa04be5611ba234398aa Mon Sep 17 00:00:00 2001 From: claude Date: Fri, 24 Jul 2026 16:38:03 +0000 Subject: [PATCH 36/39] fix(firecracker): keep the host key canonical instead of clobbering the host token file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reimplements the dropped firecracker fix on the refactored code, addressing didericis-codex's blocking review on #471. The previous firecracker path let the VM generate its own signing key on the guest volume and had `_with_signing_key()` unconditionally overwrite the single host-wide orchestrator-token with it. That breaks a co-running Docker/macOS control plane: their orchestrators still verify with the old key while new host clients start signing `cli` tokens with the guest key, so supervise/teardown/policy calls against those backends begin returning 401 (cross-backend operation is supported). Keep the host token file the single source of truth. The launcher now pushes the host-canonical key into the freshly booted infra VM over SSH (atomic write, mirroring persist_env_var_secret) via `_push_signing_key()`; the VM's init waits for it and refuses to start the control plane — rather than run OPEN — if it never arrives. Nothing writes back to the host file, so other backends are untouched, and the best-effort silent path is gone (both the push and the guest fail loudly). Ported onto the reorg'd names (host_orchestrator_token, BOT_BOTTLE_ORCHESTRATOR_ TOKEN, orchestrator-token, bot_bottle.orchestrator_auth, bot_bottle.gateway. bootstrap). pyright: 0 errors. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/backend/firecracker/infra_vm.py | 138 ++++++++++++--------- tests/unit/test_firecracker_infra_vm.py | 59 ++++----- 2 files changed, 112 insertions(+), 85 deletions(-) diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py index cf644ab..b63f9a5 100644 --- a/bot_bottle/backend/firecracker/infra_vm.py +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -33,15 +33,17 @@ from pathlib import Path from typing import Generator from ...log import die, info -from ...paths import ORCHESTRATOR_TOKEN_FILENAME, bot_bottle_root +from ...paths import host_orchestrator_token from .. import util as backend_util from ..docker import util as docker_mod from ..docker.gateway_provision import GatewayProvisionError from . import firecracker_vm, infra_artifact, netpool, util -# Where the infra VM keeps its control-plane signing key (generated on the -# persistent /dev/vdb volume mounted at BOT_BOTTLE_ROOT). The host mirrors it -# back so the CLI signs `cli` tokens the VM verifies (issue #469 review). +# The infra VM's control-plane signing-key path on its persistent /dev/vdb +# volume (mounted at BOT_BOTTLE_ROOT=/var/lib/bot-bottle). The launcher seeds +# this file with the host-canonical key BEFORE boot, so the VM verifies tokens +# with the same key the host CLI signs with — the host token file stays the +# single source of truth, never clobbered per-backend (issue #469 review). _GUEST_SIGNING_KEY_PATH = "/var/lib/bot-bottle/orchestrator-token" # The single infra-VM image: gateway data plane + baked control-plane source @@ -70,6 +72,10 @@ _INFRA_RESOLVER = "1.1.1.1" _HEALTH_TIMEOUT_SECONDS = 45.0 _HEALTH_POLL_SECONDS = 0.5 _CA_TIMEOUT_SECONDS = 30.0 +# How long the launcher retries pushing the signing key while the guest's SSH +# comes up. Below the init's own wait window, so a failed push dies here first. +_SIGNING_KEY_PUSH_TIMEOUT_SECONDS = 30.0 +_SIGNING_KEY_PUSH_POLL_SECONDS = 0.5 @dataclass @@ -173,45 +179,21 @@ def ensure_running() -> InfraVm: want = _expected_version() if _adoptable(key, url, want): info(f"adopting running infra VM at {url}") - return _with_signing_key(InfraVm(guest_ip=slot.guest_ip, private_key=key)) + return InfraVm(guest_ip=slot.guest_ip, private_key=key) with _singleton_lock(): # Re-check under the lock: another launcher may have booted it while # we waited for the lock (double-checked, so we adopt not re-boot). if _adoptable(key, url, want): info(f"adopting running infra VM at {url}") - return _with_signing_key(InfraVm(guest_ip=slot.guest_ip, private_key=key)) + return InfraVm(guest_ip=slot.guest_ip, private_key=key) # Clear a stale/hung/OUTDATED VM holding the link before booting fresh. stop() ensure_built() infra = boot() wait_for_health(infra) _record_booted_version(want) - return _with_signing_key(infra) - - -def _with_signing_key(infra: InfraVm) -> InfraVm: - """Mirror the infra VM's control-plane signing key (generated on its - persistent volume) into the host's orchestrator-token file, so the host CLI - signs `cli` tokens the VM verifies (issue #469 review). Best-effort: an - unreadable key is logged, not fatal — the VM still enforces auth, but the CLI - may then be rejected until the key is readable. Returns `infra` for chaining.""" - proc = subprocess.run( - util.ssh_base_argv(infra.private_key, infra.guest_ip) - + [f"cat {_GUEST_SIGNING_KEY_PATH}"], - capture_output=True, text=True, check=False, - ) - signing_key = proc.stdout.strip() - if proc.returncode != 0 or not signing_key: - info("infra signing key not yet readable; control-plane auth may fail") return infra - path = bot_bottle_root() / ORCHESTRATOR_TOKEN_FILENAME - path.parent.mkdir(parents=True, exist_ok=True) - fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) - with os.fdopen(fd, "w") as f: - f.write(signing_key) - os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) - return infra @contextmanager @@ -267,7 +249,13 @@ def boot() -> InfraVm: data_drive=_ensure_registry_volume(), ) _pid_file().write_text(str(vm.process.pid)) - return InfraVm(guest_ip=slot.guest_ip, private_key=private_key, vm=vm) + infra = InfraVm(guest_ip=slot.guest_ip, private_key=private_key, vm=vm) + # Push the host-canonical control-plane signing key into the guest over SSH + # (the init waits for it before starting the control plane). The host token + # file stays the single source of truth, so a co-running Docker/macOS control + # plane keeps working (issue #469 review). + _push_signing_key(infra) + return infra def _infra_dir() -> Path: @@ -339,6 +327,33 @@ def _ensure_registry_volume() -> Path: return vol +def _push_signing_key(infra: InfraVm) -> None: + """Push the host-canonical control-plane signing key into the freshly booted + infra VM over SSH (atomic write), so its orchestrator verifies tokens with + the same key the host CLI signs from. Mirrors the existing + `persist_env_var_secret` transport. Retries until the guest's SSH is up (it + comes up before the init waits for this file); dies if it never lands, since + the VM then refuses to start its control plane rather than run OPEN.""" + key = host_orchestrator_token() + push = ( + f"umask 077; cat > {_GUEST_SIGNING_KEY_PATH}.tmp " + f"&& mv {_GUEST_SIGNING_KEY_PATH}.tmp {_GUEST_SIGNING_KEY_PATH}" + ) + deadline = time.monotonic() + _SIGNING_KEY_PUSH_TIMEOUT_SECONDS + last = "" + while time.monotonic() < deadline: + proc = subprocess.run( + util.ssh_base_argv(infra.private_key, infra.guest_ip) + [push], + input=key, capture_output=True, text=True, check=False, + ) + if proc.returncode == 0: + return + last = proc.stderr.strip() + time.sleep(_SIGNING_KEY_PUSH_POLL_SECONDS) + die("could not push the control-plane signing key to the infra VM " + f"(its control plane will not start): {last or ''}") + + def _stable_keypair() -> tuple[Path, str]: """The infra VM's SSH keypair — generated once and reused, so any later launcher can SSH in (fetch CA / provision) even though a different process @@ -517,32 +532,43 @@ mount -t ext4 /dev/vdb /var/lib/bot-bottle 2>/dev/null || true # Control plane. Source is baked at /app; the package is stdlib-only. cd /app -# Control-plane signing key + a pre-minted `gateway` JWT, scoped per-process -# (issue #469 review). The key is generated once on the persistent volume and -# handed ONLY to the orchestrator (to verify tokens); the data-plane daemons get -# the `gateway` JWT they present, never the key. Without this the control plane -# would run OPEN and a compromised egress / supervise / git-http daemon in this -# same VM — reaching the orchestrator over 127.0.0.1, past the nft boundary that -# only fences off the separate agent VM — could drive the operator routes -# (approve its own supervise proposals, rewrite policy, read injected tokens). -CP_KEY=$(BOT_BOTTLE_ROOT=/var/lib/bot-bottle python3 -c 'from bot_bottle.paths import host_orchestrator_token as t; print(t())') -GW_JWT=$(BB_SIGNING_KEY="$CP_KEY" python3 -c 'import os; from bot_bottle.orchestrator_auth import mint, ROLE_GATEWAY; print(mint(ROLE_GATEWAY, os.environ["BB_SIGNING_KEY"]))') +# Wait for the launcher to push the host-canonical control-plane signing key +# over SSH, then hand it ONLY to the orchestrator (to verify tokens); the +# data-plane daemons get a pre-minted `gateway` JWT they present, never the key. +# If it never arrives, REFUSE to start the control plane rather than run OPEN — +# open mode would grant every unauthenticated caller the `cli` role, and a +# compromised egress / supervise / git-http daemon in this same VM (reaching the +# orchestrator over 127.0.0.1, past the nft boundary that only fences off the +# separate agent VM) could then drive the operator routes (issue #469). +CP_KEY="" +i=0 +while [ "$i" -lt 600 ]; do + CP_KEY=$(cat /var/lib/bot-bottle/orchestrator-token 2>/dev/null) + [ -n "$CP_KEY" ] && break + i=$((i + 1)) + sleep 0.1 +done +if [ -z "$CP_KEY" ]; then + echo "infra: control-plane signing key never arrived; refusing to start the control plane (would run OPEN)" >&2 +else + chmod 600 /var/lib/bot-bottle/orchestrator-token 2>/dev/null || true + GW_JWT=$(BB_SIGNING_KEY="$CP_KEY" python3 -c 'import os; from bot_bottle.orchestrator_auth import mint, ROLE_GATEWAY; print(mint(ROLE_GATEWAY, os.environ["BB_SIGNING_KEY"]))') + BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_ORCHESTRATOR_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator \\ + --host 0.0.0.0 --port {ORCHESTRATOR_PORT} --broker stub & -BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_ORCHESTRATOR_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator \\ - --host 0.0.0.0 --port {ORCHESTRATOR_PORT} --broker stub & - -# Gateway data plane, multi-tenant: each request resolves source-IP -> -# policy against the local control plane. The VM backend reaches git over -# git-http (9420), so the git:// daemon (git-gate, needs a per-bottle -# entrypoint the consolidated model doesn't use) is left out. No -# SUPERVISE_DB_PATH: the data plane reaches the supervise queue over the -# control-plane RPC and never opens bot-bottle.db (PRD 0070 / #469). It presents -# the pre-minted `gateway` JWT; gateway_init keeps the signing key out of the -# data-plane daemons' env. -BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise \\ -BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{ORCHESTRATOR_PORT} \\ -BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT="$GW_JWT" \\ - python3 -m bot_bottle.gateway.bootstrap & + # Gateway data plane, multi-tenant: each request resolves source-IP -> + # policy against the local control plane. The VM backend reaches git over + # git-http (9420), so the git:// daemon (git-gate, needs a per-bottle + # entrypoint the consolidated model doesn't use) is left out. No + # SUPERVISE_DB_PATH: the data plane reaches the supervise queue over the + # control-plane RPC and never opens bot-bottle.db (PRD 0070 / #469). It + # presents the pre-minted `gateway` JWT; gateway_init keeps the signing key + # out of the data-plane daemons' env. + BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise \\ + BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{ORCHESTRATOR_PORT} \\ + BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT="$GW_JWT" \\ + python3 -m bot_bottle.gateway.bootstrap & +fi # Reap as PID 1; children are backgrounded, so `wait` blocks. while : ; do wait ; done diff --git a/tests/unit/test_firecracker_infra_vm.py b/tests/unit/test_firecracker_infra_vm.py index 157a5c5..44d51a1 100644 --- a/tests/unit/test_firecracker_infra_vm.py +++ b/tests/unit/test_firecracker_infra_vm.py @@ -8,7 +8,6 @@ decisions that must hold without a VM. from __future__ import annotations import os -import tempfile import unittest from pathlib import Path from unittest.mock import MagicMock, patch @@ -16,35 +15,36 @@ from unittest.mock import MagicMock, patch from bot_bottle.backend.firecracker import infra_vm -class TestSigningKeySync(unittest.TestCase): - """`_with_signing_key` mirrors the VM's control-plane signing key into the - host token file so the CLI signs `cli` tokens the VM verifies (issue #469).""" +class TestPushSigningKey(unittest.TestCase): + """`_push_signing_key` pushes the host-canonical key into the guest over SSH + (the init waits for it). The host token file stays the single source of + truth, never clobbered per-backend (issue #469).""" def _infra(self) -> infra_vm.InfraVm: return infra_vm.InfraVm(guest_ip="10.0.0.1", private_key=Path("/k")) - def test_mirrors_guest_key_to_host_file(self): - with tempfile.TemporaryDirectory() as d: - root = Path(d) - proc = MagicMock(returncode=0, stdout="the-signing-key\n") - with patch.object(infra_vm.subprocess, "run", return_value=proc) as run, \ - patch.object(infra_vm, "bot_bottle_root", return_value=root): - out = infra_vm._with_signing_key(self._infra()) - self.assertIsInstance(out, infra_vm.InfraVm) # returned for chaining - token = root / infra_vm.ORCHESTRATOR_TOKEN_FILENAME - self.assertEqual("the-signing-key", token.read_text()) - self.assertEqual(0o600, token.stat().st_mode & 0o777) - # It cat'd the guest volume path over SSH. - self.assertIn(infra_vm._GUEST_SIGNING_KEY_PATH, run.call_args.args[0][-1]) + def test_writes_host_key_over_ssh_atomically(self): + proc = MagicMock(returncode=0, stderr="") + with patch.object(infra_vm, "host_orchestrator_token", return_value="host-key"), \ + patch.object(infra_vm.subprocess, "run", return_value=proc) as run: + infra_vm._push_signing_key(self._infra()) + # Piped the key to an atomic write of the guest token path. + self.assertEqual("host-key", run.call_args.kwargs["input"]) + remote_cmd = run.call_args.args[0][-1] + self.assertIn(f"cat > {infra_vm._GUEST_SIGNING_KEY_PATH}.tmp", remote_cmd) + self.assertIn(f"mv {infra_vm._GUEST_SIGNING_KEY_PATH}.tmp " + f"{infra_vm._GUEST_SIGNING_KEY_PATH}", remote_cmd) - def test_unreadable_key_is_not_fatal(self): - with tempfile.TemporaryDirectory() as d: - root = Path(d) - proc = MagicMock(returncode=255, stdout="") - with patch.object(infra_vm.subprocess, "run", return_value=proc), \ - patch.object(infra_vm, "bot_bottle_root", return_value=root): - infra_vm._with_signing_key(self._infra()) # no raise - self.assertFalse((root / infra_vm.ORCHESTRATOR_TOKEN_FILENAME).exists()) + def test_dies_when_push_never_succeeds(self): + proc = MagicMock(returncode=255, stderr="ssh: connect refused") + with patch.object(infra_vm, "host_orchestrator_token", return_value="host-key"), \ + patch.object(infra_vm, "_SIGNING_KEY_PUSH_TIMEOUT_SECONDS", 0.05), \ + patch.object(infra_vm, "_SIGNING_KEY_PUSH_POLL_SECONDS", 0.0), \ + patch.object(infra_vm.subprocess, "run", return_value=proc), \ + patch.object(infra_vm, "die", side_effect=SystemExit) as die: + with self.assertRaises(SystemExit): + infra_vm._push_signing_key(self._infra()) + die.assert_called_once() class TestOrchestratorUrl(unittest.TestCase): @@ -79,10 +79,11 @@ class TestBuildInfraRootfs(unittest.TestCase): # VM backend uses git-http (9420); the git:// daemon is left out. self.assertIn("BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise", init) # Role-scoped control-plane auth (issue #469 review): the orchestrator - # gets the signing key, the gateway daemons get a pre-minted `gateway` - # JWT — never open mode in the infra VM. - self.assertIn("host_orchestrator_token", init) # key generated on the volume - self.assertIn("mint, ROLE_GATEWAY", init) # gateway JWT minted from it + # gets the host-seeded signing key, the gateway daemons get a pre-minted + # `gateway` JWT, and the VM refuses to run OPEN if the key is missing. + self.assertIn("cat /var/lib/bot-bottle/orchestrator-token", init) # host-seeded key + self.assertIn("refusing to start the control plane", init) # no open mode + self.assertIn("mint, ROLE_GATEWAY", init) # gateway JWT self.assertIn('BOT_BOTTLE_ORCHESTRATOR_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator', init) # key -> orchestrator only self.assertIn('BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT="$GW_JWT"', init) # JWT -> gateway daemons -- 2.52.0 From ee26e9044f7afa2733458de15884149e9c07cdeb Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 20:41:34 -0400 Subject: [PATCH 37/39] docs(prd-0070): fold in the orchestrator/gateway plane-separation design MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that #469 got the DB off the data plane (removing the cross-kernel SQLite-coherence reason the planes were consolidated), separating them into distinct runtimes is possible again. Adds a Design subsection to 0070 covering: * Access — the only hop that changes is gateway->orchestrator (loopback -> network). A dedicated `bot-bottle-orchestrator` --internal control network with a dual-homed gateway gives Docker/macOS the L3 agent isolation that only Firecracker's nft currently has (both are token-only today); Firecracker gets it nearly free (agents already can't reach non-DNAT'd ports). * Memory — firecracker's mem_size_mib is a fixed ceiling (no reclaim without a balloon); the 4 GB combined VM is really buildah headroom. Right-sized: ~512 MiB orchestrator + ~1 GB gateway < today's 4 GB, if agent-image builds move off the gateway. * iroh belongs in the orchestrator (control-plane remote entry point); keep the data-plane gateway slim and P2P-free. Plus open questions: where agent-image builds run post-split, and whether agent-PTY-to-mobile relays through the orchestrator's iroh endpoint. Co-Authored-By: Claude Opus 4.8 --- docs/prds/0070-per-host-orchestrator.md | 119 ++++++++++++++++++++++++ 1 file changed, 119 insertions(+) diff --git a/docs/prds/0070-per-host-orchestrator.md b/docs/prds/0070-per-host-orchestrator.md index 0b7c579..af61119 100644 --- a/docs/prds/0070-per-host-orchestrator.md +++ b/docs/prds/0070-per-host-orchestrator.md @@ -353,6 +353,116 @@ in memory). Rotation does not auto-re-provision the new CA into running bottles — those re-attach to install the new anchor — so it is an operator action with a brief egress interruption, never an implicit one. +### Separating the planes: distinct orchestrator and gateway runtimes + +The combined per-host infra unit above (one VM on Firecracker/macOS, one +container on Docker running both planes) was forced by **DB coherence**, not +preference: two guest kernels sharing a virtiofs-mounted `bot-bottle.db` don't +get coherent `fcntl` locks, so both planes had to live in the one guest that +owned the file. **Issue #469 removed that constraint** — the data plane no +longer opens the DB at all; it reaches the queue/registry over the +control-plane RPC (see §State). With the coupling gone, the orchestrator +(control plane) and the gateway (data plane) can run as **separate runtimes**: +separate VMs on Firecracker, separate containers on macOS and Docker. (This +only re-splits the *orchestrator↔gateway* boundary; agents stay one +unit-each, unchanged.) + +**Why do it:** less surface to compromise (a breached gateway shares no +runtime with the control plane), independent lifecycle/placement (scale or +relocate the gateway; put the orchestrator elsewhere), and separation of +concerns the topology *enforces* rather than the process boundary merely +implying it. **The cost is lifecycle** — two units to build/start/adopt/ +health-check/stop per backend — not access (below) and not, right-sized, +memory (below). + +#### Access: keep the agent's isolation, gain it where it's missing + +The only relationship that changes is **gateway → orchestrator**: loopback +(`127.0.0.1:8099`) inside one unit today, a network hop when split. Everything +else is unchanged — CLI/console reach the orchestrator as before, and the +agent reaches only the gateway's data ports and **never** needs the +orchestrator (it flows agent → gateway supervise MCP → orchestrator, never +agent → orchestrator directly). + +The invariant to preserve is **agent cannot reach the orchestrator** — and +today it holds *unevenly*: + +| Backend | agent → orchestrator today | enforced by | +|---|---|---| +| Docker | reachable at `gateway_ip:8099` | **token only** — shared plain bridge, orchestrator binds `0.0.0.0` | +| macOS | reachable at `infra_ip:8099` | **token only** — no `pfctl` rule | +| Firecracker | **unreachable** | nft: /31 TAP, forward chain drops all but the DNAT'd gateway ports | + +So on Docker/macOS the agent is one `curl` from the control plane, gated +*only* by the JWT (one open-mode bug — exactly the class of the #469 +Firecracker regression — or one token leak from a bypass). Firecracker already +blocks it at L3. **The split is the moment to fix that, not a risk to it:** + +- **Docker / macOS — a dedicated `bot-bottle-orchestrator` control network.** + A second `--internal` network that **only** the orchestrator and gateway + join. The gateway is **dual-homed** (data network `bot-bottle-gateway` + + control network `bot-bottle-orchestrator`); the orchestrator joins the + control network only (plus the host-loopback publish the CLI already uses). + The agent is never attached to the control network, so it has **no route** + to the orchestrator — the same L3 block Firecracker already has. The one + wiring change: gateway daemons point `BOT_BOTTLE_ORCHESTRATOR_URL` at + `http://bot-bottle-orchestrator:8099` (control-net address) instead of + loopback. `DockerGateway` already takes an `orchestrator_url` and owns its + network — it needs the second `--network` and the URL retargeted; the + orchestrator side is `Dockerfile.orchestrator` run as its own container on + the control net. macOS mirrors this with a second container network and the + orchestrator's pinned control-net IP (no Apple DNS — reuse the + `gateway_hosts` `/etc/hosts` mechanism). +- **Firecracker — nearly free.** The nft forward chain already drops + everything a VM sends except the DNAT'd gateway ports, so a *separate* + orchestrator VM is unreachable by agent VMs with **no new agent rules**. The + new piece is a gateway-VM ↔ orchestrator-VM link — the same shape as today's + orch `/31` link, but between the two infra VMs instead of loopback within + one — with a forward-accept rule for that link alone. Agent egress DNATs to + the **gateway** VM's data ports; nothing DNATs to the orchestrator VM. + +Role-scoped JWT auth (#469) stays as defense-in-depth on all three, +unchanged. A *compromised gateway* can already reach the orchestrator today +(loopback, with its `gateway` JWT) and role scoping is what contains that — so +the split adds **no new surface** for the gateway-compromise case; it only +closes the *agent* path on Docker/macOS. + +#### Memory: right-size, don't double + +Firecracker guest memory is a **fixed ceiling** set at boot (`mem_size_mib`); +it's demand-paged by the host (RSS grows with what the guest touches) but there +is **no reclaim without a balloon device**, and none is configured — so plan +for the ceiling. The combined infra VM is **4096 MiB**, but that headroom is +for **buildah** (in-guest agent-image builds), not the daemons, which are +light. + +Split naïvely that reads as 2×, but right-sized it is **less** than today: + +- **Orchestrator** — stdlib-only Python HTTP + SQLite, no builds → **256–512 + MiB**. +- **Gateway** — mitmproxy (TLS bump + DLP body buffering) + git-http + + supervise + gitleaks → **512 MiB–1 GB**, *if it does not build images*. + +The real lever is **where agent-image builds run** (buildah's 2–4 GB working +set). Move builds off the gateway — to a host-controller / dedicated build unit +(a #468-adjacent decision) — and a slim orchestrator (~512 MiB) + slim gateway +(~1 GB) ≈ **1.5 GB total vs. today's 4 GB**. A per-unit **balloon device** to +reclaim idle memory is a later option, not needed for v1. + +#### iroh belongs in the orchestrator + +The remote-access transport (iroh — `web console -(iroh)-> orchestrator`, #468) +terminates at the **orchestrator**: everything the console/phone drives — +launch, teardown, list, supervise approvals, policy — is a control-plane +operation, so the trusted control plane is the single remote entry point. Keep +it **out of the gateway**: the data plane is the more-exposed, agent-facing +unit, and a goal of the split is to shrink its surface and keep its image slim +(no iroh/Rust dependency). Agent-PTY-to-mobile streaming is a *data-plane* +stream and a separate concern; if it must reach the phone over the same +endpoint, **relay it through the orchestrator's iroh door** rather than opening +a second door in the gateway — one authenticated remote entry point, plane +split intact. + ## Sequencing Jump straight to the **virtualized** end state (not a host-daemon stepping @@ -440,3 +550,12 @@ Keep the gateway **service one shared thing** throughout. plane (add/remove routes/keys/proposals without a restart). - **Identity-token delivery:** exactly how the per-bottle token is placed where the agent can present it but not swap in another bottle's. +- **Where agent-image builds run once the planes split:** buildah's 2–4 GB + working set is what keeps the combined infra VM at 4 GB. Keeping the gateway + slim (~1 GB) means builds move off it — to a host-controller / dedicated + build unit (#468-adjacent) — or the gateway keeps the build headroom and the + memory win is smaller. Decides the gateway VM/container size. +- **Agent-PTY-to-mobile streaming path:** relay it through the orchestrator's + iroh endpoint (default — one authenticated remote door, plane split intact), + or give the data-plane stream its own transport. Only matters once remote + terminal mirroring lands. -- 2.52.0 From 195e0f249dcc4887928246369b78a193af1114fd Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 20:59:30 -0400 Subject: [PATCH 38/39] =?UTF-8?q?docs(prd-0070):=20resolve=20build-placeme?= =?UTF-8?q?nt=20open=20question=20=E2=80=94=20builds=20stay=20in=20the=20o?= =?UTF-8?q?rchestrator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v1 decision: agent-image builds run in the orchestrator (the control-plane side already owns launches and hosts builds in today's combined unit; the data-plane gateway must not build). Consequence recorded in the memory section: the orchestrator keeps the ~4 GB buildah ceiling and the gateway is the slim unit (~1 GB), so the split is ~memory-neutral-to-+1 GB for now — the ~1.5 GB win waits on a later move to a dedicated slim build unit (#468-adjacent). Also flags the tension of co-locating buildah with the control plane. Removed the corresponding open question. Co-Authored-By: Claude Opus 4.8 --- docs/prds/0070-per-host-orchestrator.md | 38 ++++++++++++------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/prds/0070-per-host-orchestrator.md b/docs/prds/0070-per-host-orchestrator.md index af61119..618e618 100644 --- a/docs/prds/0070-per-host-orchestrator.md +++ b/docs/prds/0070-per-host-orchestrator.md @@ -427,27 +427,32 @@ unchanged. A *compromised gateway* can already reach the orchestrator today the split adds **no new surface** for the gateway-compromise case; it only closes the *agent* path on Docker/macOS. -#### Memory: right-size, don't double +#### Memory: fixed ceilings, and where builds run Firecracker guest memory is a **fixed ceiling** set at boot (`mem_size_mib`); it's demand-paged by the host (RSS grows with what the guest touches) but there is **no reclaim without a balloon device**, and none is configured — so plan for the ceiling. The combined infra VM is **4096 MiB**, but that headroom is for **buildah** (in-guest agent-image builds), not the daemons, which are -light. +light. Steady-state daemon footprints are small: the **orchestrator** +(stdlib-only Python HTTP + SQLite) is a few hundred MiB; the **gateway** +(mitmproxy TLS bump + DLP body buffering, git-http, supervise, gitleaks) is +**512 MiB–1 GB**. -Split naïvely that reads as 2×, but right-sized it is **less** than today: - -- **Orchestrator** — stdlib-only Python HTTP + SQLite, no builds → **256–512 - MiB**. -- **Gateway** — mitmproxy (TLS bump + DLP body buffering) + git-http + - supervise + gitleaks → **512 MiB–1 GB**, *if it does not build images*. - -The real lever is **where agent-image builds run** (buildah's 2–4 GB working -set). Move builds off the gateway — to a host-controller / dedicated build unit -(a #468-adjacent decision) — and a slim orchestrator (~512 MiB) + slim gateway -(~1 GB) ≈ **1.5 GB total vs. today's 4 GB**. A per-unit **balloon device** to -reclaim idle memory is a later option, not needed for v1. +The real ceiling driver is **where agent-image builds run** (buildah's 2–4 GB +working set). **v1 decision: builds stay in the orchestrator** — it is the +control-plane/management side that owns launches (and already hosts builds in +today's combined unit), and the data-plane gateway must not build. So the +**orchestrator keeps a ~4 GB ceiling** (mostly idle, spiking during a build) +and the **gateway is the slim unit** (~1 GB). Net for v1 the split is therefore +roughly **memory-neutral to ~+1 GB** — a slim gateway added beside a +still-build-capable orchestrator — *not* the ~1.5 GB total; that win is +deferred to a later move of builds onto a **dedicated slim build unit** +(#468-adjacent). Co-locating buildah with the control plane also keeps a +network-heavy, image-defined workload next to the signing key + registry — a +known tension the dedicated build unit resolves later. A per-unit **balloon +device** to reclaim the orchestrator's build-vs-idle swing is the other future +lever. The access and surface-separation gains land now regardless. #### iroh belongs in the orchestrator @@ -550,11 +555,6 @@ Keep the gateway **service one shared thing** throughout. plane (add/remove routes/keys/proposals without a restart). - **Identity-token delivery:** exactly how the per-bottle token is placed where the agent can present it but not swap in another bottle's. -- **Where agent-image builds run once the planes split:** buildah's 2–4 GB - working set is what keeps the combined infra VM at 4 GB. Keeping the gateway - slim (~1 GB) means builds move off it — to a host-controller / dedicated - build unit (#468-adjacent) — or the gateway keeps the build headroom and the - memory win is smaller. Decides the gateway VM/container size. - **Agent-PTY-to-mobile streaming path:** relay it through the orchestrator's iroh endpoint (default — one authenticated remote door, plane split intact), or give the data-plane stream its own transport. Only matters once remote -- 2.52.0 From 3d3d8fd7e8f85d918aa4222efe673f2443517ed4 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 21:14:20 -0400 Subject: [PATCH 39/39] docs(prd-0070): record remote-terminal resolution; track detail in #478 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remote terminal is an orchestrator operation, not a data-plane one: the agent PTY is exec_agent-sourced (ssh -t / container exec --tty / docker exec), which the orchestrator owns — the gateway never holds a PTY. A --remote flag publishes a running agent's session; local + remote clients share it via a session multiplexer (one PTY, many clients), operator-role-gated, so the split's "one authenticated remote door" holds for terminals too. Detailed design (multiplexer location, input arbitration, per-client rendering, always-on vs --remote) tracked in #478. Co-Authored-By: Claude Opus 4.8 --- docs/prds/0070-per-host-orchestrator.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/prds/0070-per-host-orchestrator.md b/docs/prds/0070-per-host-orchestrator.md index 618e618..86aed0c 100644 --- a/docs/prds/0070-per-host-orchestrator.md +++ b/docs/prds/0070-per-host-orchestrator.md @@ -462,11 +462,15 @@ launch, teardown, list, supervise approvals, policy — is a control-plane operation, so the trusted control plane is the single remote entry point. Keep it **out of the gateway**: the data plane is the more-exposed, agent-facing unit, and a goal of the split is to shrink its surface and keep its image slim -(no iroh/Rust dependency). Agent-PTY-to-mobile streaming is a *data-plane* -stream and a separate concern; if it must reach the phone over the same -endpoint, **relay it through the orchestrator's iroh door** rather than opening -a second door in the gateway — one authenticated remote entry point, plane -split intact. +(no iroh/Rust dependency). Remote **terminal** access rides the same door and is likewise an orchestrator +operation, not a data-plane one: the agent PTY is `exec_agent`-sourced (`ssh +-t` / `container exec --tty` / `docker exec`), which the orchestrator owns — the +gateway never holds a PTY. A `--remote` flag publishes a running agent's +session; local and remote clients share it through a **session multiplexer** +(one PTY, many attached clients), operator-role-gated, so the split's "one +authenticated remote door" holds for terminals too. Detailed design — +multiplexer location, input arbitration, per-client rendering, always-on vs. +`--remote` — is tracked in **#478**. ## Sequencing @@ -555,7 +559,5 @@ Keep the gateway **service one shared thing** throughout. plane (add/remove routes/keys/proposals without a restart). - **Identity-token delivery:** exactly how the per-bottle token is placed where the agent can present it but not swap in another bottle's. -- **Agent-PTY-to-mobile streaming path:** relay it through the orchestrator's - iroh endpoint (default — one authenticated remote door, plane split intact), - or give the data-plane stream its own transport. Only matters once remote - terminal mirroring lands. +- **Remote terminal design** (multiplexer location, input arbitration, + per-client rendering, always-on vs. `--remote`) is tracked in **#478**. -- 2.52.0