From dfce3d9505bbb438ceca8d6512759f86b129e8d0 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 12:39:58 -0400 Subject: [PATCH 01/30] refactor(gateway): move DockerGateway to the backend layer, drop the dead standalone-gateway path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `DockerGateway` container-lifecycle impl now lives in `backend/docker/gateway.py` (the shape backend gateway classes will share); `orchestrator/gateway.py` keeps only the backend-neutral pieces (the `Gateway` ABC, constants, `rotate_gateway_ca`). Delete the standalone-gateway path it was the only consumer of. `--gateway` on `python -m bot_bottle.orchestrator` was invoked nowhere — the production docker flow runs the gateway data plane inside the combined `bot-bottle-infra` container via `OrchestratorService`, never this class. Removing it takes with it `Orchestrator.ensure_gateway()` and the `gateway` ctor arg; `gateway_status()` becomes a stub reporting `configured: false` so the documented `GET /gateway` control-plane route keeps its contract. Fix a latent bug the move surfaced: `launch.py` read the shared gateway CA via `docker exec bot-bottle-orch-gateway`, a container the consolidated flow never creates — so the agent CA install was reaching a nonexistent name. Read it from `bot-bottle-infra` (INFRA_NAME) instead. Repoint the affected tests to the new module; drop the unit test + fake that covered the deleted standalone wiring. Co-Authored-By: Claude Opus 4.8 --- bot_bottle/backend/docker/gateway.py | 189 +++++++++++++++++ bot_bottle/backend/docker/launch.py | 10 +- bot_bottle/orchestrator/__init__.py | 11 +- bot_bottle/orchestrator/__main__.py | 20 +- bot_bottle/orchestrator/gateway.py | 192 +----------------- bot_bottle/orchestrator/service.py | 26 +-- .../test_orchestrator_docker_gateway.py | 2 +- .../test_orchestrator_docker_gateway_build.py | 2 +- tests/unit/test_orchestrator_gateway.py | 6 +- tests/unit/test_orchestrator_service.py | 45 +--- 10 files changed, 226 insertions(+), 277 deletions(-) create mode 100644 bot_bottle/backend/docker/gateway.py diff --git a/bot_bottle/backend/docker/gateway.py b/bot_bottle/backend/docker/gateway.py new file mode 100644 index 00000000..9bc9f93d --- /dev/null +++ b/bot_bottle/backend/docker/gateway.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +import os +import time +from pathlib import Path + +from ...control_auth import ROLE_GATEWAY, mint +from ...docker_cmd import run_docker +from ...paths import ( + CONTROL_AUTH_JWT_ENV, + host_control_plane_token, + host_gateway_ca_dir, +) +from ...orchestrator.gateway import ( + Gateway, GATEWAY_IMAGE, GATEWAY_NAME, GATEWAY_NETWORK, GATEWAY_DOCKERFILE, + REPO_ROOT, GATEWAY_LABEL, MITMPROXY_HOME, DEFAULT_CA_TIMEOUT_SECONDS, + CA_POLL_SECONDS, GATEWAY_CA_CERT, GatewayError +) + +class DockerGateway(Gateway): + """The consolidated gateway as a single, fixed-name Docker container. + + `image_ref` defaults to the gateway data-plane image; `ensure_built` + builds it from `Dockerfile.gateway` when it's missing. (Note: slice 5 + builds + launches the bundle container; wiring its per-bottle, + source-IP-keyed config is a later slice — see PRD 0070.)""" + + def __init__( + self, + image_ref: str = GATEWAY_IMAGE, + *, + name: str = GATEWAY_NAME, + network: str = GATEWAY_NETWORK, + orchestrator_url: str = "", + build_context: Path | None = None, + dockerfile: str | None = GATEWAY_DOCKERFILE, + host_port_bindings: tuple[int, ...] = (), + ) -> None: + self.image_ref = image_ref + self.name = name + self.network = network + # The control-plane URL the gateway's data plane resolves per bottle + # against — reached by container name over docker DNS on the shared + # network (container↔container, no host firewall). Mandatory to *run* + # the gateway (see `ensure_running`); empty is tolerated only for the + # construct-then-read-CA path (`ca_cert_pem` on an already-running + # container), which never launches a container. + self._orchestrator_url = orchestrator_url + self._build_context = build_context or REPO_ROOT + self._dockerfile = dockerfile + # Ports published on the host (0.0.0.0). Used by the Firecracker + # backend's dev-harness gateway so VMs can reach it via their TAP link; + # Docker's DNAT + the nft `ct status dnat accept` rule handle the rest. + self._host_port_bindings = host_port_bindings + + def image_exists(self) -> bool: + return run_docker(["docker", "image", "inspect", self.image_ref]).returncode == 0 + + def ensure_built(self) -> None: + """Build the bundle image from its Dockerfile, **cache-aware** — cheap + (a cache check) when nothing changed, a real rebuild when the flat + sources (egress addon / git-http / policy_resolver / supervise) moved. + + This deliberately builds every time rather than build-if-missing: the + per-bottle model kept the image fresh via compose's `build:` on up, and + a stale image silently runs the OLD single-tenant daemons. No-op only + when no dockerfile is configured (a pre-pulled image). BOT_BOTTLE_NO_CACHE + forces a full rebuild (parity with `start --no-cache`).""" + if self._dockerfile is None: + return + argv = ["docker", "build", "-t", self.image_ref, + "-f", str(self._build_context / self._dockerfile), + str(self._build_context)] + if os.environ.get("BOT_BOTTLE_NO_CACHE"): + argv.insert(2, "--no-cache") + proc = run_docker(argv) + if proc.returncode != 0: + raise GatewayError(f"gateway image build failed: {proc.stderr.strip()}") + + def is_running(self) -> bool: + proc = run_docker([ + "docker", "ps", + "--filter", f"name=^{self.name}$", + "--filter", "status=running", + "--format", "{{.Names}}", + ]) + return self.name in proc.stdout.split() + + def _running_image_is_current(self) -> bool: + """True iff the running gateway was created from the *current* + `image_ref`. When `ensure_built` rebuilds the image (a source change), + the running container is still the OLD image running the OLD flat + daemons — so this is how a rebuild actually takes effect: a mismatch + means recreate.""" + running = run_docker(["docker", "inspect", "--format", "{{.Image}}", self.name]) + current = run_docker(["docker", "image", "inspect", "--format", "{{.Id}}", self.image_ref]) + if running.returncode != 0 or current.returncode != 0: + return True # can't compare → don't churn a working container + return running.stdout.strip() == current.stdout.strip() + + def _ensure_network(self) -> None: + """Create the shared gateway network if it doesn't exist. Idempotent — + a concurrent create loses harmlessly (the loser sees 'already exists'). + Docker picks the subnet; the launcher reads it back to allocate IPs.""" + if run_docker(["docker", "network", "inspect", self.network]).returncode == 0: + return + proc = run_docker(["docker", "network", "create", self.network]) + if proc.returncode != 0 and "already exists" not in proc.stderr: + raise GatewayError( + f"gateway network {self.network} failed to create: {proc.stderr.strip()}" + ) + + def ensure_running(self) -> None: + # Fail closed on a missing policy source. The data-plane daemons are + # resolver-only now (PRD 0070) — without an orchestrator URL egress + # raises, git-http exits 1, and supervise exits 2 — so launching a + # gateway without one would only crash-loop its daemons. Refuse here so + # the misconfiguration surfaces as a clear error, not a broken container. + if not self._orchestrator_url: + raise GatewayError( + "gateway requires an orchestrator URL to run " + "(resolver-only data plane; no single-tenant fallback)" + ) + # Recreate when the running container's image is stale (a rebuild), + # so source changes to the gateway's flat daemons take effect — not + # just when the container is absent. + if self.is_running() and self._running_image_is_current(): + return + self._ensure_network() + # Clear any stale (stopped OR outdated-image) container holding the + # fixed name, then start fresh. `rm --force` on an absent name is a + # tolerated no-op. + run_docker(["docker", "rm", "--force", self.name]) + argv = [ + "docker", "run", "--detach", + "--name", self.name, + "--label", GATEWAY_LABEL, + "--network", self.network, + # Persist the self-generated CA on the host so it survives both + # container recreation AND docker volume pruning (agents trust it) + # — see host_gateway_ca_dir / issue #450. + "--volume", f"{host_gateway_ca_dir()}:{MITMPROXY_HOME}", + # No DB mount: the data plane (egress / supervise / git-gate) reaches + # the supervise queue over the control-plane RPC and never opens + # bot-bottle.db, so the gateway container gets no file handle on it + # (PRD 0070 / issue #469). + ] + for port in self._host_port_bindings: + argv += ["--publish", f"0.0.0.0:{port}:{port}"] + run_env = dict(os.environ) + # The gateway's egress / git / supervise daemons resolve source-IP -> + # policy against the control plane per request (guaranteed non-empty by + # the check above). + argv += ["--env", f"BOT_BOTTLE_ORCHESTRATOR_URL={self._orchestrator_url}"] + # ...presenting a role-scoped `gateway` token (a signed JWT minted from + # the host signing key) on those calls. The gateway never receives the + # signing key — only this pre-minted token, which it cannot rewrite into + # a `cli` token, so a compromised data-plane process can't drive the + # operator routes (issue #469 review). Bare `--env NAME` keeps the value + # off argv / `docker inspect`; only the gateway (not the agent) is given it. + argv += ["--env", CONTROL_AUTH_JWT_ENV] + run_env[CONTROL_AUTH_JWT_ENV] = mint(ROLE_GATEWAY, host_control_plane_token()) + argv.append(self.image_ref) + proc = run_docker(argv, env=run_env) + if proc.returncode != 0: + raise GatewayError(f"gateway failed to start: {proc.stderr.strip()}") + + def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str: + """The gateway's CA certificate (PEM) that agents install to trust its + TLS interception. mitmproxy generates it a moment after the container + starts, so this **polls** for it (up to `timeout`) rather than assuming + it's already there on a fresh gateway — raising only if it never + appears.""" + deadline = time.monotonic() + timeout + while True: + proc = run_docker(["docker", "exec", self.name, "cat", GATEWAY_CA_CERT]) + if proc.returncode == 0 and proc.stdout.strip(): + return proc.stdout + if time.monotonic() >= deadline: + raise GatewayError( + f"gateway CA cert not available after {timeout:g}s: " + f"{proc.stderr.strip() or 'empty'}" + ) + time.sleep(CA_POLL_SECONDS) + + def stop(self) -> None: + proc = run_docker(["docker", "rm", "--force", self.name]) + if proc.returncode != 0 and "No such container" not in proc.stderr: + raise GatewayError(f"gateway failed to stop: {proc.stderr.strip()}") \ No newline at end of file diff --git a/bot_bottle/backend/docker/launch.py b/bot_bottle/backend/docker/launch.py index e767d922..47e2f642 100644 --- a/bot_bottle/backend/docker/launch.py +++ b/bot_bottle/backend/docker/launch.py @@ -66,7 +66,8 @@ from .compose import ( from .consolidated_compose import consolidated_agent_compose from ...orchestrator.config_store import resolve_teardown_timeout from .consolidated_launch import launch_consolidated, teardown_consolidated -from ...orchestrator.gateway import DockerGateway +from ...orchestrator.lifecycle import INFRA_NAME +from .gateway import DockerGateway # Where the repo root lives, for `docker build` context. Computed once. @@ -159,11 +160,14 @@ def launch( ) # Step 4: install the SHARED gateway CA into the agent (replaces the - # per-bottle CA) — read it out of the running gateway. + # per-bottle CA) — read it out of the running gateway. The consolidated + # flow runs the gateway data plane inside the per-host infra container + # (INFRA_NAME), so read the CA from there, not the legacy standalone + # gateway container name. ca_dir = egress_state_dir(plan.slug) / "gateway-ca" ca_dir.mkdir(parents=True, exist_ok=True) ca_file = ca_dir / "gateway-ca.pem" - ca_file.write_text(DockerGateway(network=ctx.network).ca_cert_pem()) + ca_file.write_text(DockerGateway(name=INFRA_NAME).ca_cert_pem()) egress_plan = dataclasses.replace( plan.egress_plan, mitmproxy_ca_host_path=ca_file, diff --git a/bot_bottle/orchestrator/__init__.py b/bot_bottle/orchestrator/__init__.py index 47ff2826..3f03a7e5 100644 --- a/bot_bottle/orchestrator/__init__.py +++ b/bot_bottle/orchestrator/__init__.py @@ -10,10 +10,10 @@ backend-neutral "consolidation core" that needs no VM packaging: * `broker` — the signed, structured launch-request contract + a `LaunchBroker` (stub for the harness) that verifies provenance before acting. - * `gateway` — the consolidated per-host gateway: a `Gateway` - lifecycle contract (idempotent singleton) + a - `DockerGateway` impl. One gateway shared by all - bottles instead of one per bottle. + * `gateway` — the consolidated per-host gateway: the `Gateway` + lifecycle contract (idempotent singleton). One + gateway shared by all bottles instead of one per + bottle; backend impls live under `backend/*/gateway`. * `service` — the `Orchestrator`: owns the registry, brokers the launch lifecycle (launch/teardown), manages the shared gateway, attributes. @@ -38,7 +38,7 @@ from .broker import ( verify_request, ) from .docker_broker import DockerBroker, DockerBrokerError -from .gateway import DockerGateway, Gateway, GatewayError +from .gateway import Gateway, GatewayError from .service import Orchestrator from .control_plane import ControlPlaneServer, dispatch, make_server @@ -53,7 +53,6 @@ __all__ = [ "DockerBroker", "DockerBrokerError", "Gateway", - "DockerGateway", "GatewayError", "sign_request", "verify_request", diff --git a/bot_bottle/orchestrator/__main__.py b/bot_bottle/orchestrator/__main__.py index 1606e12e..0869c907 100644 --- a/bot_bottle/orchestrator/__main__.py +++ b/bot_bottle/orchestrator/__main__.py @@ -22,7 +22,6 @@ from .control_plane import make_server from .docker_broker import DockerBroker from .registry import RegistryStore, default_db_path from .service import Orchestrator -from .gateway import DockerGateway, Gateway def main(argv: list[str] | None = None) -> int: @@ -38,10 +37,6 @@ def main(argv: list[str] | None = None) -> int: "--broker", choices=("stub", "docker"), default="stub", help="launch broker: 'stub' records requests; 'docker' runs containers", ) - parser.add_argument( - "--gateway", action="store_true", - help="run one consolidated per-host gateway (build-if-missing)", - ) args = parser.parse_args(argv) registry = RegistryStore(args.db) @@ -57,20 +52,7 @@ def main(argv: list[str] | None = None) -> int: # anything; 'docker' runs real containers (firecracker drops in later). secret = secrets.token_bytes(32) broker: LaunchBroker = DockerBroker(secret) if args.broker == "docker" else StubBroker(secret) - # Standalone `--gateway` (not the consolidated flow, where the host - # lifecycle runs the gateway). The gateway resolves against this same - # process; the URL is only reachable when they share a docker network. - gateway: Gateway | None = ( - DockerGateway(orchestrator_url=f"http://{args.host}:{args.port}") - if args.gateway else None - ) - orchestrator = Orchestrator(registry, broker, secret, gateway) - - # One persistent per-host gateway, shared by every bottle: build the - # bundle image if missing, then bring the singleton up (idempotent). - if gateway is not None: - orchestrator.ensure_gateway() - log.info("consolidated gateway ensured", context={"name": gateway.name}) + orchestrator = Orchestrator(registry, broker, secret) server = make_server(orchestrator, host=args.host, port=args.port) bound_host, bound_port = server.server_address[0], server.server_address[1] diff --git a/bot_bottle/orchestrator/gateway.py b/bot_bottle/orchestrator/gateway.py index cc12330e..a1ad85ab 100644 --- a/bot_bottle/orchestrator/gateway.py +++ b/bot_bottle/orchestrator/gateway.py @@ -7,8 +7,9 @@ because the attribution invariant (source IP + identity token, see so per-bottle policy lives in one long-lived process keyed on who's calling. `Gateway` is the backend-neutral lifecycle contract (mirrors `LaunchBroker`): -ensure the single instance is up, report it, tear it down. `DockerGateway` -is the docker implementation; a firecracker gateway VM slots in later. +ensure the single instance is up, report it, tear it down. The docker +implementation (`DockerGateway`) lives in `backend/docker/gateway.py`; a +firecracker gateway VM slots in later. The defining behaviour is **idempotent singleton**: `ensure_running` starts the instance if absent and is a no-op if it's already up, so N bottle @@ -19,20 +20,13 @@ from __future__ import annotations import abc import os -import time from pathlib import Path -from ..control_auth import ROLE_GATEWAY, mint -from ..docker_cmd import run_docker -from ..paths import ( - CONTROL_AUTH_JWT_ENV, - host_control_plane_token, - host_gateway_ca_dir, -) +from ..paths import host_gateway_ca_dir # The gateway's mitmproxy writes its CA a beat after the container starts, so # reads poll for it rather than assuming it's there on a fresh launch. -_CA_POLL_SECONDS = 0.5 +CA_POLL_SECONDS = 0.5 DEFAULT_CA_TIMEOUT_SECONDS = 30.0 GATEWAY_NAME = "bot-bottle-orch-gateway" @@ -66,7 +60,7 @@ GATEWAY_CA_GLOB = "mitmproxy-ca*" # that lands. Env override matches the backend's BOT_BOTTLE_GATEWAY_IMAGE. GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest") GATEWAY_DOCKERFILE = "Dockerfile.gateway" -_REPO_ROOT = Path(__file__).resolve().parents[2] +REPO_ROOT = Path(__file__).resolve().parents[2] def rotate_gateway_ca(ca_dir: Path | None = None) -> list[Path]: @@ -118,180 +112,8 @@ class Gateway(abc.ABC): """Remove the gateway. Idempotent — absent is success.""" -class DockerGateway(Gateway): - """The consolidated gateway as a single, fixed-name Docker container. - - `image_ref` defaults to the gateway data-plane image; `ensure_built` - builds it from `Dockerfile.gateway` when it's missing. (Note: slice 5 - builds + launches the bundle container; wiring its per-bottle, - source-IP-keyed config is a later slice — see PRD 0070.)""" - - def __init__( - self, - image_ref: str = GATEWAY_IMAGE, - *, - name: str = GATEWAY_NAME, - network: str = GATEWAY_NETWORK, - orchestrator_url: str = "", - build_context: Path | None = None, - dockerfile: str | None = GATEWAY_DOCKERFILE, - host_port_bindings: tuple[int, ...] = (), - ) -> None: - self.image_ref = image_ref - self.name = name - self.network = network - # The control-plane URL the gateway's data plane resolves per bottle - # against — reached by container name over docker DNS on the shared - # network (container↔container, no host firewall). Mandatory to *run* - # the gateway (see `ensure_running`); empty is tolerated only for the - # construct-then-read-CA path (`ca_cert_pem` on an already-running - # container), which never launches a container. - self._orchestrator_url = orchestrator_url - self._build_context = build_context or _REPO_ROOT - self._dockerfile = dockerfile - # Ports published on the host (0.0.0.0). Used by the Firecracker - # backend's dev-harness gateway so VMs can reach it via their TAP link; - # Docker's DNAT + the nft `ct status dnat accept` rule handle the rest. - self._host_port_bindings = host_port_bindings - - def image_exists(self) -> bool: - return run_docker(["docker", "image", "inspect", self.image_ref]).returncode == 0 - - def ensure_built(self) -> None: - """Build the bundle image from its Dockerfile, **cache-aware** — cheap - (a cache check) when nothing changed, a real rebuild when the flat - sources (egress addon / git-http / policy_resolver / supervise) moved. - - This deliberately builds every time rather than build-if-missing: the - per-bottle model kept the image fresh via compose's `build:` on up, and - a stale image silently runs the OLD single-tenant daemons. No-op only - when no dockerfile is configured (a pre-pulled image). BOT_BOTTLE_NO_CACHE - forces a full rebuild (parity with `start --no-cache`).""" - if self._dockerfile is None: - return - argv = ["docker", "build", "-t", self.image_ref, - "-f", str(self._build_context / self._dockerfile), - str(self._build_context)] - if os.environ.get("BOT_BOTTLE_NO_CACHE"): - argv.insert(2, "--no-cache") - proc = run_docker(argv) - if proc.returncode != 0: - raise GatewayError(f"gateway image build failed: {proc.stderr.strip()}") - - def is_running(self) -> bool: - proc = run_docker([ - "docker", "ps", - "--filter", f"name=^{self.name}$", - "--filter", "status=running", - "--format", "{{.Names}}", - ]) - return self.name in proc.stdout.split() - - def _running_image_is_current(self) -> bool: - """True iff the running gateway was created from the *current* - `image_ref`. When `ensure_built` rebuilds the image (a source change), - the running container is still the OLD image running the OLD flat - daemons — so this is how a rebuild actually takes effect: a mismatch - means recreate.""" - running = run_docker(["docker", "inspect", "--format", "{{.Image}}", self.name]) - current = run_docker(["docker", "image", "inspect", "--format", "{{.Id}}", self.image_ref]) - if running.returncode != 0 or current.returncode != 0: - return True # can't compare → don't churn a working container - return running.stdout.strip() == current.stdout.strip() - - def _ensure_network(self) -> None: - """Create the shared gateway network if it doesn't exist. Idempotent — - a concurrent create loses harmlessly (the loser sees 'already exists'). - Docker picks the subnet; the launcher reads it back to allocate IPs.""" - if run_docker(["docker", "network", "inspect", self.network]).returncode == 0: - return - proc = run_docker(["docker", "network", "create", self.network]) - if proc.returncode != 0 and "already exists" not in proc.stderr: - raise GatewayError( - f"gateway network {self.network} failed to create: {proc.stderr.strip()}" - ) - - def ensure_running(self) -> None: - # Fail closed on a missing policy source. The data-plane daemons are - # resolver-only now (PRD 0070) — without an orchestrator URL egress - # raises, git-http exits 1, and supervise exits 2 — so launching a - # gateway without one would only crash-loop its daemons. Refuse here so - # the misconfiguration surfaces as a clear error, not a broken container. - if not self._orchestrator_url: - raise GatewayError( - "gateway requires an orchestrator URL to run " - "(resolver-only data plane; no single-tenant fallback)" - ) - # Recreate when the running container's image is stale (a rebuild), - # so source changes to the gateway's flat daemons take effect — not - # just when the container is absent. - if self.is_running() and self._running_image_is_current(): - return - self._ensure_network() - # Clear any stale (stopped OR outdated-image) container holding the - # fixed name, then start fresh. `rm --force` on an absent name is a - # tolerated no-op. - run_docker(["docker", "rm", "--force", self.name]) - argv = [ - "docker", "run", "--detach", - "--name", self.name, - "--label", GATEWAY_LABEL, - "--network", self.network, - # Persist the self-generated CA on the host so it survives both - # container recreation AND docker volume pruning (agents trust it) - # — see host_gateway_ca_dir / issue #450. - "--volume", f"{host_gateway_ca_dir()}:{MITMPROXY_HOME}", - # No DB mount: the data plane (egress / supervise / git-gate) reaches - # the supervise queue over the control-plane RPC and never opens - # bot-bottle.db, so the gateway container gets no file handle on it - # (PRD 0070 / issue #469). - ] - for port in self._host_port_bindings: - argv += ["--publish", f"0.0.0.0:{port}:{port}"] - run_env = dict(os.environ) - # The gateway's egress / git / supervise daemons resolve source-IP -> - # policy against the control plane per request (guaranteed non-empty by - # the check above). - argv += ["--env", f"BOT_BOTTLE_ORCHESTRATOR_URL={self._orchestrator_url}"] - # ...presenting a role-scoped `gateway` token (a signed JWT minted from - # the host signing key) on those calls. The gateway never receives the - # signing key — only this pre-minted token, which it cannot rewrite into - # a `cli` token, so a compromised data-plane process can't drive the - # operator routes (issue #469 review). Bare `--env NAME` keeps the value - # off argv / `docker inspect`; only the gateway (not the agent) is given it. - argv += ["--env", CONTROL_AUTH_JWT_ENV] - run_env[CONTROL_AUTH_JWT_ENV] = mint(ROLE_GATEWAY, host_control_plane_token()) - argv.append(self.image_ref) - proc = run_docker(argv, env=run_env) - if proc.returncode != 0: - raise GatewayError(f"gateway failed to start: {proc.stderr.strip()}") - - def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str: - """The gateway's CA certificate (PEM) that agents install to trust its - TLS interception. mitmproxy generates it a moment after the container - starts, so this **polls** for it (up to `timeout`) rather than assuming - it's already there on a fresh gateway — raising only if it never - appears.""" - deadline = time.monotonic() + timeout - while True: - proc = run_docker(["docker", "exec", self.name, "cat", GATEWAY_CA_CERT]) - if proc.returncode == 0 and proc.stdout.strip(): - return proc.stdout - if time.monotonic() >= deadline: - raise GatewayError( - f"gateway CA cert not available after {timeout:g}s: " - f"{proc.stderr.strip() or 'empty'}" - ) - time.sleep(_CA_POLL_SECONDS) - - def stop(self) -> None: - proc = run_docker(["docker", "rm", "--force", self.name]) - if proc.returncode != 0 and "No such container" not in proc.stderr: - raise GatewayError(f"gateway failed to stop: {proc.stderr.strip()}") - - __all__ = [ - "Gateway", "DockerGateway", "GatewayError", "rotate_gateway_ca", + "Gateway", "GatewayError", "rotate_gateway_ca", "GATEWAY_NAME", "GATEWAY_LABEL", "GATEWAY_IMAGE", "GATEWAY_NETWORK", "GATEWAY_CA_CERT", "GATEWAY_CA_GLOB", ] diff --git a/bot_bottle/orchestrator/service.py b/bot_bottle/orchestrator/service.py index 629ab16c..d521937c 100644 --- a/bot_bottle/orchestrator/service.py +++ b/bot_bottle/orchestrator/service.py @@ -27,7 +27,6 @@ from datetime import datetime, timezone from .broker import LaunchBroker, LaunchRequest, sign_request from .registry import DEFAULT_REAP_GRACE_SECONDS, BottleRecord, RegistryStore -from .gateway import Gateway from ..supervise import ( AuditEntry, COMPONENT_FOR_TOOL, @@ -72,12 +71,10 @@ class Orchestrator: registry: RegistryStore, broker: LaunchBroker, sign_secret: bytes, - gateway: Gateway | None = None, ) -> None: self.registry = registry self._broker = broker self._secret = sign_secret - self._gateway = gateway # Per-bottle egress auth tokens (env_name -> value), keyed by bottle_id. # Held **in memory only** — never written to the registry DB — so the # gateway can inject each bottle's upstream credential without secrets @@ -383,22 +380,15 @@ class Orchestrator: # --- consolidated gateway ---------------------------------------------- - def ensure_gateway(self) -> None: - """Ensure the single per-host gateway is built and up (idempotent). - No-op when no gateway is configured.""" - if self._gateway is not None: - self._gateway.ensure_built() - self._gateway.ensure_running() - def gateway_status(self) -> dict[str, object]: - """Report the shared gateway for the control plane / console.""" - if self._gateway is None: - return {"configured": False} - return { - "configured": True, - "name": self._gateway.name, - "running": self._gateway.is_running(), - } + """Report the shared gateway for the control plane / console. + + The orchestrator no longer owns a standalone gateway lifecycle — the + consolidated flow runs the gateway data plane inside the per-host infra + container/VM (see `backend/*/gateway`), so this reports `configured: + false`. Retained for the documented `GET /gateway` control-plane + contract.""" + return {"configured": False} __all__ = ["Orchestrator"] diff --git a/tests/integration/test_orchestrator_docker_gateway.py b/tests/integration/test_orchestrator_docker_gateway.py index 38e4b762..453dccd1 100644 --- a/tests/integration/test_orchestrator_docker_gateway.py +++ b/tests/integration/test_orchestrator_docker_gateway.py @@ -10,7 +10,7 @@ import secrets import subprocess import unittest -from bot_bottle.orchestrator.gateway import DockerGateway +from bot_bottle.backend.docker.gateway import DockerGateway from tests._docker import skip_unless_docker IMAGE = "busybox" diff --git a/tests/integration/test_orchestrator_docker_gateway_build.py b/tests/integration/test_orchestrator_docker_gateway_build.py index 5de444f0..4eb59cb0 100644 --- a/tests/integration/test_orchestrator_docker_gateway_build.py +++ b/tests/integration/test_orchestrator_docker_gateway_build.py @@ -11,7 +11,7 @@ from __future__ import annotations import subprocess import unittest -from bot_bottle.orchestrator.gateway import DockerGateway +from bot_bottle.backend.docker.gateway import DockerGateway from tests._docker import skip_unless_docker IMAGE = "busybox" diff --git a/tests/unit/test_orchestrator_gateway.py b/tests/unit/test_orchestrator_gateway.py index 40a14c84..f712fae4 100644 --- a/tests/unit/test_orchestrator_gateway.py +++ b/tests/unit/test_orchestrator_gateway.py @@ -7,10 +7,10 @@ import unittest from pathlib import Path from unittest.mock import Mock, patch +from bot_bottle.backend.docker.gateway import DockerGateway from bot_bottle.orchestrator.gateway import ( GATEWAY_CA_CERT, GATEWAY_NAME, - DockerGateway, GatewayError, rotate_gateway_ca, ) @@ -20,7 +20,7 @@ from tests.unit import use_bottle_root _CA_PEM = "-----BEGIN CERTIFICATE-----\nMII...\n-----END CERTIFICATE-----\n" -_RUN_DOCKER = "bot_bottle.orchestrator.gateway.run_docker" +_RUN_DOCKER = "bot_bottle.backend.docker.gateway.run_docker" def _proc(returncode: int = 0, stdout: str = "", stderr: str = "") -> Mock: @@ -162,7 +162,7 @@ class TestDockerGateway(unittest.TestCase): # First read: CA not there yet; second read: present. seq = [_proc(returncode=1, stderr="No such file"), _proc(stdout=_CA_PEM)] with patch(_RUN_DOCKER, side_effect=seq), \ - patch("bot_bottle.orchestrator.gateway.time.sleep"): + patch("bot_bottle.backend.docker.gateway.time.sleep"): self.assertEqual(_CA_PEM, self.sc.ca_cert_pem(timeout=5)) def test_ensure_running_reuses_existing_network(self) -> None: diff --git a/tests/unit/test_orchestrator_service.py b/tests/unit/test_orchestrator_service.py index 06cf6114..e76c1c88 100644 --- a/tests/unit/test_orchestrator_service.py +++ b/tests/unit/test_orchestrator_service.py @@ -14,7 +14,6 @@ from unittest.mock import patch from bot_bottle.orchestrator.broker import LaunchBroker, LaunchRequest, StubBroker from bot_bottle.orchestrator.registry import RegistryStore from bot_bottle.orchestrator.service import Orchestrator -from bot_bottle.orchestrator.gateway import Gateway from bot_bottle.orchestrator.secret_store import new_env_var_secret from bot_bottle.store_manager import StoreManager from bot_bottle.supervise import ( @@ -38,29 +37,6 @@ class _FailingBroker(LaunchBroker): pass -class _FakeGateway(Gateway): - """In-memory gateway for wiring tests.""" - - def __init__(self) -> None: - self.name = "fake-gateway" - self.ensured = 0 - self.built = 0 - self._running = False - - def ensure_built(self) -> None: - self.built += 1 - - def ensure_running(self) -> None: - self.ensured += 1 - self._running = True - - def is_running(self) -> bool: - return self._running - - def stop(self) -> None: - self._running = False - - class TestOrchestrator(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() @@ -168,24 +144,11 @@ class TestOrchestrator(unittest.TestCase): orch.launch_bottle("10.243.0.9") self.assertEqual([], self.store.all()) # no orphan - def test_gateway_unconfigured_by_default(self) -> None: + def test_gateway_status_reports_unconfigured(self) -> None: + # The orchestrator no longer owns a standalone gateway lifecycle; the + # consolidated flow runs the gateway data plane in the per-host infra + # container/VM. `GET /gateway` therefore always reports unconfigured. self.assertEqual({"configured": False}, self.orch.gateway_status()) - self.orch.ensure_gateway() # no-op, must not raise - - def test_gateway_wired_and_ensured(self) -> None: - sc = _FakeGateway() - orch = Orchestrator(self.store, self.broker, self.secret, gateway=sc) - self.assertEqual( - {"configured": True, "name": "fake-gateway", "running": False}, - orch.gateway_status(), - ) - orch.ensure_gateway() - self.assertEqual(1, sc.built) # ensure_gateway builds first, - self.assertEqual(1, sc.ensured) # then runs - self.assertEqual( - {"configured": True, "name": "fake-gateway", "running": True}, - orch.gateway_status(), - ) class TestOrchestratorSupervise(unittest.TestCase): -- 2.52.0 From d7a58e52fd2ba5a150b4c1b417e988b1d2fd5b66 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 12:53:52 -0400 Subject: [PATCH 02/30] refactor(store): consolidate the SQLite store family into bot_bottle.store Move the DbStore base (db_store), the shared schema-migration base (migrations / TableMigrations), the concrete stores (audit_store, config_store, queue_store), and StoreManager out of the package root into a bot_bottle/store/ package, so persistence lives in one place rather than scattered across the root. Callers use direct submodule imports (from bot_bottle.store.store_manager import StoreManager). Inside the moved modules the relative imports point back up to their non-store siblings (..paths, ..supervise_types) while co-moved siblings stay package-local; the flat-import fallbacks are untouched. The orchestrator's own stores (config_store, secret_store, registry) stay in orchestrator/ and repoint to the moved DbStore / TableMigrations. No behavior change; full unit suite green. Co-Authored-By: Claude Opus 4.8 --- bot_bottle/cli/__init__.py | 2 +- bot_bottle/image_cache.py | 2 +- bot_bottle/orchestrator/__main__.py | 2 +- bot_bottle/orchestrator/config_store.py | 4 ++-- bot_bottle/orchestrator/registry.py | 4 ++-- bot_bottle/store/__init__.py | 9 +++++++++ bot_bottle/{ => store}/audit_store.py | 4 ++-- bot_bottle/{ => store}/config_store.py | 2 +- bot_bottle/{ => store}/db_store.py | 0 bot_bottle/{ => store}/migrations.py | 0 bot_bottle/{ => store}/queue_store.py | 4 ++-- bot_bottle/{ => store}/store_manager.py | 2 +- bot_bottle/supervise.py | 6 +++--- tests/unit/test_cli_dispatch.py | 4 ++-- tests/unit/test_config_store.py | 4 ++-- tests/unit/test_db_store.py | 4 ++-- tests/unit/test_orchestrator_control_plane.py | 2 +- tests/unit/test_orchestrator_service.py | 2 +- tests/unit/test_supervise.py | 4 ++-- tests/unit/test_supervise_edge.py | 4 ++-- tests/unit/test_supervise_server.py | 4 ++-- 21 files changed, 39 insertions(+), 30 deletions(-) create mode 100644 bot_bottle/store/__init__.py rename bot_bottle/{ => store}/audit_store.py (97%) rename bot_bottle/{ => store}/config_store.py (98%) rename bot_bottle/{ => store}/db_store.py (100%) rename bot_bottle/{ => store}/migrations.py (100%) rename bot_bottle/{ => store}/queue_store.py (98%) rename bot_bottle/{ => store}/store_manager.py (97%) diff --git a/bot_bottle/cli/__init__.py b/bot_bottle/cli/__init__.py index cf62f68c..c1618b66 100644 --- a/bot_bottle/cli/__init__.py +++ b/bot_bottle/cli/__init__.py @@ -10,7 +10,7 @@ import sys from ..errors import MissingEnvVarError from ..log import Die, die, error from ..manifest import ManifestError -from ..store_manager import StoreManager +from ..store.store_manager import StoreManager from ._common import PROG from . import list as _list_mod from .backend import cmd_backend diff --git a/bot_bottle/image_cache.py b/bot_bottle/image_cache.py index b2f802a4..ff165445 100644 --- a/bot_bottle/image_cache.py +++ b/bot_bottle/image_cache.py @@ -6,7 +6,7 @@ from datetime import datetime, timezone from pathlib import Path try: - from .config_store import ConfigStore + from .store.config_store import ConfigStore except ImportError: from config_store import ConfigStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module diff --git a/bot_bottle/orchestrator/__main__.py b/bot_bottle/orchestrator/__main__.py index 0869c907..38d4e0c1 100644 --- a/bot_bottle/orchestrator/__main__.py +++ b/bot_bottle/orchestrator/__main__.py @@ -16,7 +16,7 @@ import secrets from pathlib import Path from .. import log -from ..store_manager import StoreManager +from ..store.store_manager import StoreManager from .broker import LaunchBroker, StubBroker from .control_plane import make_server from .docker_broker import DockerBroker diff --git a/bot_bottle/orchestrator/config_store.py b/bot_bottle/orchestrator/config_store.py index 2cd6f854..7779b68a 100644 --- a/bot_bottle/orchestrator/config_store.py +++ b/bot_bottle/orchestrator/config_store.py @@ -11,8 +11,8 @@ import os import sqlite3 from pathlib import Path -from ..db_store import DbStore -from ..migrations import TableMigrations +from ..store.db_store import DbStore +from ..store.migrations import TableMigrations from ..paths import host_db_path TEARDOWN_TIMEOUT_ENV = "BOT_BOTTLE_ORCHESTRATOR_TEARDOWN_TIMEOUT_SECONDS" diff --git a/bot_bottle/orchestrator/registry.py b/bot_bottle/orchestrator/registry.py index a7f34b19..a5afe4e3 100644 --- a/bot_bottle/orchestrator/registry.py +++ b/bot_bottle/orchestrator/registry.py @@ -36,8 +36,8 @@ from collections.abc import Iterable from dataclasses import dataclass from pathlib import Path -from ..db_store import DbStore -from ..migrations import TableMigrations +from ..store.db_store import DbStore +from ..store.migrations import TableMigrations from ..paths import host_db_path # 256 bits of urandom, URL-safe — unguessable per-bottle identity token. diff --git a/bot_bottle/store/__init__.py b/bot_bottle/store/__init__.py new file mode 100644 index 00000000..ec5e39b4 --- /dev/null +++ b/bot_bottle/store/__init__.py @@ -0,0 +1,9 @@ +"""SQLite-backed persistence for bot-bottle. + +The store family: a shared `DbStore` base (versioned `migrations`) and the +concrete stores built on it — `ConfigStore`, `AuditStore`, `QueueStore` — plus +`StoreManager`, which migrates them together against the one per-host DB. + +Callers import the concrete module they need directly, e.g. +`from bot_bottle.store.store_manager import StoreManager`. +""" diff --git a/bot_bottle/audit_store.py b/bot_bottle/store/audit_store.py similarity index 97% rename from bot_bottle/audit_store.py rename to bot_bottle/store/audit_store.py index 2f48f7c8..13de4707 100644 --- a/bot_bottle/audit_store.py +++ b/bot_bottle/store/audit_store.py @@ -6,8 +6,8 @@ import sqlite3 from pathlib import Path try: - from .supervise_types import AuditEntry - from .paths import host_db_path + from ..supervise_types import AuditEntry + from ..paths import host_db_path from .db_store import DbStore from .migrations import TableMigrations except ImportError: diff --git a/bot_bottle/config_store.py b/bot_bottle/store/config_store.py similarity index 98% rename from bot_bottle/config_store.py rename to bot_bottle/store/config_store.py index bc2f607b..d282d719 100644 --- a/bot_bottle/config_store.py +++ b/bot_bottle/store/config_store.py @@ -7,7 +7,7 @@ from pathlib import Path try: from .db_store import DbStore from .migrations import TableMigrations - from .paths import host_db_path + from ..paths import host_db_path except ImportError: from db_store import DbStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module diff --git a/bot_bottle/db_store.py b/bot_bottle/store/db_store.py similarity index 100% rename from bot_bottle/db_store.py rename to bot_bottle/store/db_store.py diff --git a/bot_bottle/migrations.py b/bot_bottle/store/migrations.py similarity index 100% rename from bot_bottle/migrations.py rename to bot_bottle/store/migrations.py diff --git a/bot_bottle/queue_store.py b/bot_bottle/store/queue_store.py similarity index 98% rename from bot_bottle/queue_store.py rename to bot_bottle/store/queue_store.py index 2dae5c9e..09ef1cf3 100644 --- a/bot_bottle/queue_store.py +++ b/bot_bottle/store/queue_store.py @@ -7,8 +7,8 @@ import sqlite3 from pathlib import Path try: - from .supervise_types import Proposal, Response - from .paths import host_db_path + from ..supervise_types import Proposal, Response + from ..paths import host_db_path from .db_store import DbStore from .migrations import TableMigrations except ImportError: diff --git a/bot_bottle/store_manager.py b/bot_bottle/store/store_manager.py similarity index 97% rename from bot_bottle/store_manager.py rename to bot_bottle/store/store_manager.py index f8807a40..e7729cc2 100644 --- a/bot_bottle/store_manager.py +++ b/bot_bottle/store/store_manager.py @@ -26,7 +26,7 @@ class StoreManager: def __init__(self, db_path: Path | None = None) -> None: if db_path is None: try: - from .paths import host_db_path + from ..paths import host_db_path except ImportError: from paths import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module db_path = host_db_path() diff --git a/bot_bottle/supervise.py b/bot_bottle/supervise.py index 184500f9..96d25a37 100644 --- a/bot_bottle/supervise.py +++ b/bot_bottle/supervise.py @@ -98,9 +98,9 @@ def audit_log_path(component: str, slug: str) -> Path: try: - from .queue_store import QueueStore - from .audit_store import AuditStore - from .store_manager import StoreManager + from .store.queue_store import QueueStore + from .store.audit_store import AuditStore + from .store.store_manager import StoreManager except ImportError: # Gateway: files are flat-copied under /app, not a package. from queue_store import QueueStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module diff --git a/tests/unit/test_cli_dispatch.py b/tests/unit/test_cli_dispatch.py index 2981f73f..e5e10d54 100644 --- a/tests/unit/test_cli_dispatch.py +++ b/tests/unit/test_cli_dispatch.py @@ -12,10 +12,10 @@ from unittest.mock import patch import bot_bottle.cli as climod from bot_bottle.cli import main -from bot_bottle.db_store import DbStore +from bot_bottle.store.db_store import DbStore from bot_bottle.log import Die from bot_bottle.manifest import ManifestError -from bot_bottle.store_manager import StoreManager +from bot_bottle.store.store_manager import StoreManager class TestMainDispatch(unittest.TestCase): diff --git a/tests/unit/test_config_store.py b/tests/unit/test_config_store.py index 5987ae05..0f40576c 100644 --- a/tests/unit/test_config_store.py +++ b/tests/unit/test_config_store.py @@ -7,11 +7,11 @@ import tempfile import unittest from pathlib import Path -from bot_bottle.config_store import ( +from bot_bottle.store.config_store import ( DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS, ConfigStore, ) -from bot_bottle.store_manager import StoreManager +from bot_bottle.store.store_manager import StoreManager class TestConfigStore(unittest.TestCase): diff --git a/tests/unit/test_db_store.py b/tests/unit/test_db_store.py index c447d1d3..04046eea 100644 --- a/tests/unit/test_db_store.py +++ b/tests/unit/test_db_store.py @@ -7,8 +7,8 @@ import tempfile import unittest from pathlib import Path -from bot_bottle.db_store import DbStore -from bot_bottle.migrations import TableMigrations +from bot_bottle.store.db_store import DbStore +from bot_bottle.store.migrations import TableMigrations def _store(tmp: Path) -> DbStore: diff --git a/tests/unit/test_orchestrator_control_plane.py b/tests/unit/test_orchestrator_control_plane.py index b400bc5c..1cd06b80 100644 --- a/tests/unit/test_orchestrator_control_plane.py +++ b/tests/unit/test_orchestrator_control_plane.py @@ -24,7 +24,7 @@ from bot_bottle.orchestrator.broker import StubBroker from bot_bottle.orchestrator.control_plane import dispatch, make_server from bot_bottle.orchestrator.registry import BottleRecord, RegistryStore from bot_bottle.orchestrator.service import Orchestrator -from bot_bottle.store_manager import StoreManager +from bot_bottle.store.store_manager import StoreManager from bot_bottle.supervise import ( Proposal, TOOL_EGRESS_ALLOW, diff --git a/tests/unit/test_orchestrator_service.py b/tests/unit/test_orchestrator_service.py index e76c1c88..9441654c 100644 --- a/tests/unit/test_orchestrator_service.py +++ b/tests/unit/test_orchestrator_service.py @@ -15,7 +15,7 @@ from bot_bottle.orchestrator.broker import LaunchBroker, LaunchRequest, StubBrok from bot_bottle.orchestrator.registry import RegistryStore from bot_bottle.orchestrator.service import Orchestrator from bot_bottle.orchestrator.secret_store import new_env_var_secret -from bot_bottle.store_manager import StoreManager +from bot_bottle.store.store_manager import StoreManager from bot_bottle.supervise import ( Proposal, STATUS_APPROVED, diff --git a/tests/unit/test_supervise.py b/tests/unit/test_supervise.py index 6381af09..e3360a77 100644 --- a/tests/unit/test_supervise.py +++ b/tests/unit/test_supervise.py @@ -10,8 +10,8 @@ from pathlib import Path from bot_bottle import supervise from bot_bottle.paths import host_db_path from tests.unit import use_bottle_root -from bot_bottle.audit_store import AuditStore -from bot_bottle.queue_store import QueueStore +from bot_bottle.store.audit_store import AuditStore +from bot_bottle.store.queue_store import QueueStore from bot_bottle.supervise import ( AuditEntry, Proposal, diff --git a/tests/unit/test_supervise_edge.py b/tests/unit/test_supervise_edge.py index 4cedbfa3..acbd4202 100644 --- a/tests/unit/test_supervise_edge.py +++ b/tests/unit/test_supervise_edge.py @@ -11,9 +11,9 @@ from pathlib import Path from unittest.mock import patch from bot_bottle import supervise -from bot_bottle.audit_store import AuditStore +from bot_bottle.store.audit_store import AuditStore from bot_bottle.paths import bot_bottle_root -from bot_bottle.queue_store import QueueStore +from bot_bottle.store.queue_store import QueueStore from bot_bottle.supervise import ( AuditEntry, Proposal, diff --git a/tests/unit/test_supervise_server.py b/tests/unit/test_supervise_server.py index c666c556..3f596b0a 100644 --- a/tests/unit/test_supervise_server.py +++ b/tests/unit/test_supervise_server.py @@ -19,8 +19,8 @@ from pathlib import Path from tests.unit import use_bottle_root from bot_bottle import supervise as _sv -from bot_bottle import queue_store as _qs -from bot_bottle import audit_store as _as +from bot_bottle.store import queue_store as _qs +from bot_bottle.store import audit_store as _as from bot_bottle import supervise_server # noqa: E402 from bot_bottle.supervise_server import ( -- 2.52.0 From a845cba925f055582ae40de686b1cb023fc20bc3 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 13:04:39 -0400 Subject: [PATCH 03/30] refactor(manifest): consolidate the manifest modules into a bot_bottle.manifest package Move the nine root-level manifest modules into a bot_bottle/manifest/ package, dropping the redundant manifest_ prefix: the facade (manifest.py) becomes the package __init__ and keeps re-exporting Manifest / ManifestIndex and the piece types, while manifest_.py become manifest/.py (agent, bottle, egress, extends, git, loader, schema, util). Because the facade stays the package __init__, the ~13 callers that import `from bot_bottle.manifest import ...` are unchanged; only direct-piece imports move to `bot_bottle.manifest.`. Inside the package, intra-manifest imports drop the prefix and the non-manifest siblings (log, yaml_subset, agent_provider) move to `..`. No behavior change; full unit suite green. Co-Authored-By: Claude Opus 4.8 --- bot_bottle/cli/start.py | 2 +- bot_bottle/deploy_key_provisioner.py | 2 +- .../{manifest.py => manifest/__init__.py} | 20 ++++++++-------- .../{manifest_agent.py => manifest/agent.py} | 8 +++---- .../bottle.py} | 24 +++++++++---------- .../egress.py} | 2 +- .../extends.py} | 8 +++---- .../{manifest_git.py => manifest/git.py} | 2 +- .../loader.py} | 12 +++++----- .../schema.py} | 2 +- .../{manifest_util.py => manifest/util.py} | 0 tests/unit/test_cli_start_selector.py | 2 +- tests/unit/test_git_gate_render_provision.py | 2 +- tests/unit/test_manifest_bottle_merge.py | 4 ++-- tests/unit/test_manifest_validation.py | 4 ++-- 15 files changed, 47 insertions(+), 47 deletions(-) rename bot_bottle/{manifest.py => manifest/__init__.py} (97%) rename bot_bottle/{manifest_agent.py => manifest/agent.py} (98%) rename bot_bottle/{manifest_bottle.py => manifest/bottle.py} (87%) rename bot_bottle/{manifest_egress.py => manifest/egress.py} (99%) rename bot_bottle/{manifest_extends.py => manifest/extends.py} (98%) rename bot_bottle/{manifest_git.py => manifest/git.py} (99%) rename bot_bottle/{manifest_loader.py => manifest/loader.py} (93%) rename bot_bottle/{manifest_schema.py => manifest/schema.py} (98%) rename bot_bottle/{manifest_util.py => manifest/util.py} (100%) diff --git a/bot_bottle/cli/start.py b/bot_bottle/cli/start.py index 27bcf3ac..b587216a 100644 --- a/bot_bottle/cli/start.py +++ b/bot_bottle/cli/start.py @@ -380,7 +380,7 @@ def _peek_agent_bottle(manifest: ManifestIndex, agent_name: str) -> str: return manifest.agents[agent_name].bottle return "" - from ..manifest_loader import scan_agent_names + from ..manifest.loader import scan_agent_names from ..yaml_subset import YamlSubsetError, parse_frontmatter home_agents = scan_agent_names(manifest.home_md / "agents") diff --git a/bot_bottle/deploy_key_provisioner.py b/bot_bottle/deploy_key_provisioner.py index e733bc10..a6d90d5f 100644 --- a/bot_bottle/deploy_key_provisioner.py +++ b/bot_bottle/deploy_key_provisioner.py @@ -49,7 +49,7 @@ def get_provisioner( GiteaDeployKeyProvisioner, ) return GiteaDeployKeyProvisioner(token=token, api_url=api_url) - from .manifest_util import ManifestError + from .manifest.util import ManifestError raise ManifestError( f"unknown provisioned_key provider: {provider!r}; " f"available: gitea" diff --git a/bot_bottle/manifest.py b/bot_bottle/manifest/__init__.py similarity index 97% rename from bot_bottle/manifest.py rename to bot_bottle/manifest/__init__.py index f8b63076..8289298f 100644 --- a/bot_bottle/manifest.py +++ b/bot_bottle/manifest/__init__.py @@ -63,25 +63,25 @@ from dataclasses import dataclass, field, replace from pathlib import Path from typing import Mapping -from .log import warn -from .manifest_util import ManifestError, as_json_object -from .manifest_agent import ManifestAgent, ManifestAgentProvider -from .manifest_bottle import ManifestBottle -from .manifest_egress import ( +from ..log import warn +from .util import ManifestError, as_json_object +from .agent import ManifestAgent, ManifestAgentProvider +from .bottle import ManifestBottle +from .egress import ( EGRESS_AUTH_SCHEMES, ManifestEgressConfig, ManifestEgressRoute, ) -from .manifest_extends import merge_bottles_runtime, resolve_bottles -from .manifest_git import ManifestGitEntry, ManifestGitUser, ManifestKeyConfig -from .manifest_loader import ( +from .extends import merge_bottles_runtime, resolve_bottles +from .git import ManifestGitEntry, ManifestGitUser, ManifestKeyConfig +from .loader import ( check_stale_json, load_bottle_chain_from_dir, scan_agent_names, scan_bottle_names, ) -from .manifest_schema import validate_agent_frontmatter_keys -from .yaml_subset import YamlSubsetError, parse_frontmatter +from .schema import validate_agent_frontmatter_keys +from ..yaml_subset import YamlSubsetError, parse_frontmatter # Re-export everything that callers currently import from this module. __all__ = [ diff --git a/bot_bottle/manifest_agent.py b/bot_bottle/manifest/agent.py similarity index 98% rename from bot_bottle/manifest_agent.py rename to bot_bottle/manifest/agent.py index c6071ad4..f5d76ceb 100644 --- a/bot_bottle/manifest_agent.py +++ b/bot_bottle/manifest/agent.py @@ -5,10 +5,10 @@ from __future__ import annotations from dataclasses import dataclass, field from typing import cast -from .agent_provider import PROVIDER_TEMPLATES -from .manifest_util import ManifestError, as_json_object -from .manifest_git import ManifestGitUser -from .manifest_schema import AGENT_MODEL_KEYS, is_valid_entity_name +from ..agent_provider import PROVIDER_TEMPLATES +from .util import ManifestError, as_json_object +from .git import ManifestGitUser +from .schema import AGENT_MODEL_KEYS, is_valid_entity_name @dataclass(frozen=True) diff --git a/bot_bottle/manifest_bottle.py b/bot_bottle/manifest/bottle.py similarity index 87% rename from bot_bottle/manifest_bottle.py rename to bot_bottle/manifest/bottle.py index b4b2015e..ae76d7e3 100644 --- a/bot_bottle/manifest_bottle.py +++ b/bot_bottle/manifest/bottle.py @@ -1,13 +1,13 @@ """The `ManifestBottle` value type. -Split out of `manifest.py` so the `extends:`/loader resolvers can import it -without a circular dependency: `manifest.py` imports those resolvers, while -they only need this value type. Everything here depends on leaf modules -(`manifest_util`, `manifest_agent`, `manifest_egress`, `manifest_git`, -`manifest_schema`), so this module sits at the bottom of the manifest layer. +Split out of the package facade (`manifest/__init__.py`) so the `extends:`/ +loader resolvers can import it without a circular dependency: the facade +imports those resolvers, while they only need this value type. Everything here +depends on leaf modules (`util`, `agent`, `egress`, `git`, `schema`), so this +module sits at the bottom of the manifest layer. -`manifest.py` re-exports `ManifestBottle`, so existing -`from .manifest import ManifestBottle` callers are unaffected. +The facade re-exports `ManifestBottle`, so existing +`from bot_bottle.manifest import ManifestBottle` callers are unaffected. """ from __future__ import annotations @@ -15,11 +15,11 @@ from __future__ import annotations from dataclasses import dataclass, field from typing import Mapping -from .manifest_util import ManifestError, as_json_object -from .manifest_agent import ManifestAgentProvider -from .manifest_egress import ManifestEgressConfig -from .manifest_git import ManifestGitEntry, ManifestGitUser, parse_git_gate_config -from .manifest_schema import BOTTLE_KEYS +from .util import ManifestError, as_json_object +from .agent import ManifestAgentProvider +from .egress import ManifestEgressConfig +from .git import ManifestGitEntry, ManifestGitUser, parse_git_gate_config +from .schema import BOTTLE_KEYS __all__ = ["ManifestBottle"] diff --git a/bot_bottle/manifest_egress.py b/bot_bottle/manifest/egress.py similarity index 99% rename from bot_bottle/manifest_egress.py rename to bot_bottle/manifest/egress.py index c373f324..a9f22cdb 100644 --- a/bot_bottle/manifest_egress.py +++ b/bot_bottle/manifest/egress.py @@ -6,7 +6,7 @@ import re from dataclasses import dataclass from typing import cast -from .manifest_util import ManifestError, as_json_object +from .util import ManifestError, as_json_object EGRESS_AUTH_SCHEMES = ("Bearer", "token") diff --git a/bot_bottle/manifest_extends.py b/bot_bottle/manifest/extends.py similarity index 98% rename from bot_bottle/manifest_extends.py rename to bot_bottle/manifest/extends.py index 698d23f6..c96b032c 100644 --- a/bot_bottle/manifest_extends.py +++ b/bot_bottle/manifest/extends.py @@ -2,10 +2,10 @@ from __future__ import annotations -from .manifest_bottle import ManifestBottle -from .manifest_egress import ManifestEgressConfig, validate_egress_routes -from .manifest_git import ManifestGitUser, parse_git_gate_config -from .manifest_util import ManifestError, as_json_object +from .bottle import ManifestBottle +from .egress import ManifestEgressConfig, validate_egress_routes +from .git import ManifestGitUser, parse_git_gate_config +from .util import ManifestError, as_json_object def _overlay_declared_bool( diff --git a/bot_bottle/manifest_git.py b/bot_bottle/manifest/git.py similarity index 99% rename from bot_bottle/manifest_git.py rename to bot_bottle/manifest/git.py index f620c153..44c87b68 100644 --- a/bot_bottle/manifest_git.py +++ b/bot_bottle/manifest/git.py @@ -5,7 +5,7 @@ from __future__ import annotations import re from dataclasses import dataclass -from .manifest_util import ManifestError, as_json_object +from .util import ManifestError, as_json_object # Shell-safe characters for git-gate repo names. Names are embedded in # the generated entrypoint shell script (shlex.quote is the primary diff --git a/bot_bottle/manifest_loader.py b/bot_bottle/manifest/loader.py similarity index 93% rename from bot_bottle/manifest_loader.py rename to bot_bottle/manifest/loader.py index c88af320..f19130a3 100644 --- a/bot_bottle/manifest_loader.py +++ b/bot_bottle/manifest/loader.py @@ -4,15 +4,15 @@ from __future__ import annotations from pathlib import Path -from .log import warn -from .manifest_bottle import ManifestBottle -from .manifest_extends import resolve_bottles -from .manifest_schema import ( +from ..log import warn +from .bottle import ManifestBottle +from .extends import resolve_bottles +from .schema import ( entity_name_from_path, validate_bottle_frontmatter_keys, ) -from .manifest_util import ManifestError -from .yaml_subset import YamlSubsetError, parse_frontmatter +from .util import ManifestError +from ..yaml_subset import YamlSubsetError, parse_frontmatter def check_stale_json(dir_path: Path, md_dir: Path, label: str) -> None: diff --git a/bot_bottle/manifest_schema.py b/bot_bottle/manifest/schema.py similarity index 98% rename from bot_bottle/manifest_schema.py rename to bot_bottle/manifest/schema.py index 165718a3..512231a6 100644 --- a/bot_bottle/manifest_schema.py +++ b/bot_bottle/manifest/schema.py @@ -68,7 +68,7 @@ def _validate_frontmatter_keys( keys: object, allowed_keys: frozenset[str], ) -> None: - from .manifest_util import ManifestError + from .util import ManifestError key_set = set(keys) # type: ignore unknown = key_set - allowed_keys # type: ignore diff --git a/bot_bottle/manifest_util.py b/bot_bottle/manifest/util.py similarity index 100% rename from bot_bottle/manifest_util.py rename to bot_bottle/manifest/util.py diff --git a/tests/unit/test_cli_start_selector.py b/tests/unit/test_cli_start_selector.py index 6af13b44..4f41f07a 100644 --- a/tests/unit/test_cli_start_selector.py +++ b/tests/unit/test_cli_start_selector.py @@ -336,7 +336,7 @@ class TestManifestToYaml(unittest.TestCase): agent_provider_template: str = "claude", ): from bot_bottle.manifest import Manifest, ManifestBottle - from bot_bottle.manifest_agent import ManifestAgent, ManifestAgentProvider + from bot_bottle.manifest.agent import ManifestAgent, ManifestAgentProvider agent = ManifestAgent(skills=tuple(skills)) bottle = ManifestBottle( diff --git a/tests/unit/test_git_gate_render_provision.py b/tests/unit/test_git_gate_render_provision.py index 481a63ca..81263e47 100644 --- a/tests/unit/test_git_gate_render_provision.py +++ b/tests/unit/test_git_gate_render_provision.py @@ -21,7 +21,7 @@ from bot_bottle.git_gate import ( git_gate_render_gitconfig, revoke_git_gate_provisioned_keys, ) -from bot_bottle.manifest_git import ManifestGitEntry, ManifestKeyConfig +from bot_bottle.manifest.git import ManifestGitEntry, ManifestKeyConfig def _entry(**kw: Any) -> ManifestGitEntry: diff --git a/tests/unit/test_manifest_bottle_merge.py b/tests/unit/test_manifest_bottle_merge.py index 03fc7dea..256680d1 100644 --- a/tests/unit/test_manifest_bottle_merge.py +++ b/tests/unit/test_manifest_bottle_merge.py @@ -14,7 +14,7 @@ import unittest from pathlib import Path from bot_bottle.manifest import ManifestBottle, ManifestError, ManifestIndex -from bot_bottle.manifest_extends import merge_bottles_runtime +from bot_bottle.manifest.extends import merge_bottles_runtime def _index(bottles: dict[str, object], agents: dict[str, object]) -> ManifestIndex: @@ -44,7 +44,7 @@ class TestMergeBottlesRuntime(unittest.TestCase): self.assertEqual("y", result.env["ONLY_OVERRIDE"]) def test_egress_routes_concatenated(self): - from bot_bottle.manifest_egress import ManifestEgressConfig, ManifestEgressRoute + from bot_bottle.manifest.egress import ManifestEgressConfig, ManifestEgressRoute r1 = ManifestEgressRoute(Host="api.a.com") r2 = ManifestEgressRoute(Host="api.b.com") base = ManifestBottle(egress=ManifestEgressConfig(routes=(r1,))) diff --git a/tests/unit/test_manifest_validation.py b/tests/unit/test_manifest_validation.py index 618717d4..b762c2a0 100644 --- a/tests/unit/test_manifest_validation.py +++ b/tests/unit/test_manifest_validation.py @@ -14,12 +14,12 @@ from unittest.mock import patch from bot_bottle.env import resolve_env from bot_bottle.errors import MissingEnvVarError from bot_bottle.manifest import Manifest, ManifestBottle, ManifestIndex -from bot_bottle.manifest_agent import ( +from bot_bottle.manifest.agent import ( ManifestAgent, ManifestAgentProvider, _parse_provider_settings, ) -from bot_bottle.manifest_util import ManifestError +from bot_bottle.manifest.util import ManifestError def _idx(obj: dict[str, object]) -> ManifestIndex: -- 2.52.0 From f77023db1dd872fe2019ddcaef7c8259cfcc8029 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 13:25:10 -0400 Subject: [PATCH 04/30] refactor(gateway): move the data-plane daemons into a bot_bottle.gateway package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Separate the gateway (data plane) from the orchestrator (control plane) at the module level. The gateway runtime files move out of the package root — and the backend-neutral Gateway lifecycle ABC + GATEWAY_* constants move out of orchestrator/ — into a new bot_bottle/gateway/ package: gateway/__init__.py (was orchestrator/gateway.py: Gateway ABC + consts + rotate_gateway_ca) gateway/gateway_init.py (the PID-1 daemon supervisor) gateway/egress_addon.py, egress_addon_core.py, egress_dlp_config.py, dlp_detectors.py (the egress mitmproxy daemon) gateway/git_http_backend.py (the git-http daemon) gateway/git_gate_render.py (the git-gate pre-receive rendering) gateway/supervise_server.py (the supervise MCP daemon) gateway/policy_resolver.py (the data-plane control-plane RPC client) orchestrator/ now holds only control-plane files. The shared plan/types/auth layer (egress.py=EgressPlan, git_gate.py=GitGatePlan, supervise.py, supervise_types.py, control_auth.py) and the launch-time git-gate provisioning helpers stay at root, so orchestrator/ and backend/ still own them. Because these daemons are invoked as `python3 -m bot_bottle.`, loaded flat by mitmproxy, and referenced in Dockerfile.gateway, the move updates more than Python imports: the `-m` invocations (firecracker/macOS infra scripts), the Dockerfile.gateway addon shim + ENTRYPOINT, gateway_init's _DAEMONS module paths, and the git-gate CGI heredocs all now point at bot_bottle.gateway.*. No behavior change; full unit suite green (2251). Co-Authored-By: Claude Opus 4.8 --- Dockerfile.gateway | 4 ++-- .../backend/docker/consolidated_launch.py | 2 +- bot_bottle/backend/docker/gateway.py | 2 +- bot_bottle/backend/egress_apply.py | 2 +- bot_bottle/backend/firecracker/infra_vm.py | 2 +- bot_bottle/backend/macos_container/gateway.py | 2 +- bot_bottle/backend/macos_container/infra.py | 4 ++-- bot_bottle/backend/macos_container/launch.py | 2 +- bot_bottle/egress.py | 2 +- .../gateway.py => gateway/__init__.py} | 0 bot_bottle/{ => gateway}/dlp_detectors.py | 0 bot_bottle/{ => gateway}/egress_addon.py | 6 ++--- bot_bottle/{ => gateway}/egress_addon_core.py | 2 +- bot_bottle/{ => gateway}/egress_dlp_config.py | 0 bot_bottle/{ => gateway}/gateway_init.py | 4 ++-- bot_bottle/{ => gateway}/git_gate_render.py | 8 +++---- bot_bottle/{ => gateway}/git_http_backend.py | 2 +- bot_bottle/{ => gateway}/policy_resolver.py | 0 bot_bottle/{ => gateway}/supervise_server.py | 4 ++-- bot_bottle/git_gate.py | 2 +- bot_bottle/git_gate_provision.py | 2 +- bot_bottle/orchestrator/__init__.py | 2 +- bot_bottle/orchestrator/lifecycle.py | 2 +- bot_bottle/orchestrator/rotate_ca.py | 2 +- bot_bottle/supervise.py | 4 ++-- tests/integration/test_gateway_image.py | 2 +- .../integration/test_multitenant_isolation.py | 2 +- tests/unit/test_dlp_detectors.py | 8 +++---- tests/unit/test_egress.py | 22 +++++++++---------- tests/unit/test_egress_addon_core.py | 8 +++---- tests/unit/test_egress_addon_log_redaction.py | 2 +- tests/unit/test_egress_addon_request_flow.py | 10 ++++----- tests/unit/test_egress_core_parsing.py | 2 +- tests/unit/test_egress_multitenant.py | 4 ++-- tests/unit/test_firecracker_infra_vm.py | 2 +- tests/unit/test_gateway_init.py | 8 +++---- tests/unit/test_git_gate.py | 2 +- tests/unit/test_git_gate_provision_render.py | 2 +- tests/unit/test_git_http_backend.py | 16 +++++++------- tests/unit/test_git_http_multitenant.py | 4 ++-- tests/unit/test_macos_infra.py | 2 +- tests/unit/test_orchestrator_gateway.py | 2 +- tests/unit/test_orchestrator_lifecycle.py | 2 +- tests/unit/test_orchestrator_registration.py | 2 +- tests/unit/test_orchestrator_rotate_ca.py | 2 +- tests/unit/test_policy_resolver.py | 4 ++-- tests/unit/test_supervise_server.py | 4 ++-- 47 files changed, 87 insertions(+), 87 deletions(-) rename bot_bottle/{orchestrator/gateway.py => gateway/__init__.py} (100%) rename bot_bottle/{ => gateway}/dlp_detectors.py (100%) rename bot_bottle/{ => gateway}/egress_addon.py (99%) rename bot_bottle/{ => gateway}/egress_addon_core.py (99%) rename bot_bottle/{ => gateway}/egress_dlp_config.py (100%) rename bot_bottle/{ => gateway}/gateway_init.py (98%) rename bot_bottle/{ => gateway}/git_gate_render.py (98%) rename bot_bottle/{ => gateway}/git_http_backend.py (99%) rename bot_bottle/{ => gateway}/policy_resolver.py (100%) rename bot_bottle/{ => gateway}/supervise_server.py (99%) diff --git a/Dockerfile.gateway b/Dockerfile.gateway index 8849d214..91d0c52e 100644 --- a/Dockerfile.gateway +++ b/Dockerfile.gateway @@ -102,7 +102,7 @@ RUN pip install --no-cache-dir /src/ # WORKDIR here also creates /app so the shim + COPYs below can write into it # (nothing created /app before this point). WORKDIR /app -RUN printf 'from bot_bottle.egress_addon import addons\n' > /app/egress_addon.py +RUN printf 'from bot_bottle.gateway.egress_addon import addons\n' > /app/egress_addon.py COPY bot_bottle/egress_entrypoint.sh /app/egress-entrypoint.sh RUN chmod +x /app/egress-entrypoint.sh @@ -123,4 +123,4 @@ EXPOSE 8888 9099 9418 9420 9100 # PID 1 is the supervisor. It owns signal handling and exit-code # propagation; no `exec` chain in the entrypoint itself. -ENTRYPOINT ["python3", "-m", "bot_bottle.gateway_init"] +ENTRYPOINT ["python3", "-m", "bot_bottle.gateway.gateway_init"] diff --git a/bot_bottle/backend/docker/consolidated_launch.py b/bot_bottle/backend/docker/consolidated_launch.py index 9f9768a3..482a30f8 100644 --- a/bot_bottle/backend/docker/consolidated_launch.py +++ b/bot_bottle/backend/docker/consolidated_launch.py @@ -20,7 +20,7 @@ from ...docker_cmd import run_docker from ...egress import EgressPlan from ...git_gate import GitGatePlan from ...orchestrator.client import OrchestratorClient -from ...orchestrator.gateway import GATEWAY_NETWORK +from ...gateway import GATEWAY_NETWORK from ...orchestrator.lifecycle import INFRA_NAME, OrchestratorService from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME from ...orchestrator.reprovision import reprovision_bottles diff --git a/bot_bottle/backend/docker/gateway.py b/bot_bottle/backend/docker/gateway.py index 9bc9f93d..edea8e7d 100644 --- a/bot_bottle/backend/docker/gateway.py +++ b/bot_bottle/backend/docker/gateway.py @@ -11,7 +11,7 @@ from ...paths import ( host_control_plane_token, host_gateway_ca_dir, ) -from ...orchestrator.gateway import ( +from ...gateway import ( Gateway, GATEWAY_IMAGE, GATEWAY_NAME, GATEWAY_NETWORK, GATEWAY_DOCKERFILE, REPO_ROOT, GATEWAY_LABEL, MITMPROXY_HOME, DEFAULT_CA_TIMEOUT_SECONDS, CA_POLL_SECONDS, GATEWAY_CA_CERT, GatewayError diff --git a/bot_bottle/backend/egress_apply.py b/bot_bottle/backend/egress_apply.py index ed54cdf1..a1981073 100644 --- a/bot_bottle/backend/egress_apply.py +++ b/bot_bottle/backend/egress_apply.py @@ -11,7 +11,7 @@ from pathlib import Path from ..bottle_state import egress_state_dir from ..egress import EGRESS_ROUTES_FILENAME -from ..egress_addon_core import LOG_OFF, load_config +from ..gateway.egress_addon_core import LOG_OFF, load_config class EgressApplyError(RuntimeError): diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py index cb64dd28..a7ab12b7 100644 --- a/bot_bottle/backend/firecracker/infra_vm.py +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -542,7 +542,7 @@ BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_CONTROL_PLANE_TOKEN="$CP_KEY" pyt BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise \\ BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{CONTROL_PLANE_PORT} \\ BOT_BOTTLE_CONTROL_AUTH_JWT="$GW_JWT" \\ - python3 -m bot_bottle.gateway_init & + python3 -m bot_bottle.gateway.gateway_init & # Reap as PID 1; children are backgrounded, so `wait` blocks. while : ; do wait ; done diff --git a/bot_bottle/backend/macos_container/gateway.py b/bot_bottle/backend/macos_container/gateway.py index 3d296ff7..fcac8558 100644 --- a/bot_bottle/backend/macos_container/gateway.py +++ b/bot_bottle/backend/macos_container/gateway.py @@ -12,7 +12,7 @@ from __future__ import annotations import os -from ...orchestrator.gateway import GatewayError +from ...gateway import GatewayError from . import util as container_mod # The shared host-only network the infra container and every agent bottle sit diff --git a/bot_bottle/backend/macos_container/infra.py b/bot_bottle/backend/macos_container/infra.py index 731a62a0..85bc3f06 100644 --- a/bot_bottle/backend/macos_container/infra.py +++ b/bot_bottle/backend/macos_container/infra.py @@ -41,7 +41,7 @@ from dataclasses import dataclass from pathlib import Path from ... import log -from ...orchestrator.gateway import GATEWAY_CA_CERT, MITMPROXY_HOME +from ...gateway import GATEWAY_CA_CERT, MITMPROXY_HOME from ...orchestrator.lifecycle import ( DEFAULT_PORT, DEFAULT_STARTUP_TIMEOUT_SECONDS, @@ -107,7 +107,7 @@ def _init_script(port: int) -> str: # control-plane RPC and never opens bot-bottle.db (PRD 0070 / #469). f"( cd /app && BOT_BOTTLE_GATEWAY_DAEMONS={_GATEWAY_DAEMONS} " f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{port} " - f"python3 -m bot_bottle.gateway_init ) &\n" + f"python3 -m bot_bottle.gateway.gateway_init ) &\n" "while : ; do wait ; done\n" ) diff --git a/bot_bottle/backend/macos_container/launch.py b/bot_bottle/backend/macos_container/launch.py index 89022668..d54a20ac 100644 --- a/bot_bottle/backend/macos_container/launch.py +++ b/bot_bottle/backend/macos_container/launch.py @@ -52,7 +52,7 @@ from ...git_gate import ( provision_git_gate_dynamic_keys, revoke_git_gate_provisioned_keys, ) -from ...git_http_backend import DEFAULT_PORT as _GIT_HTTP_PORT +from ...gateway.git_http_backend import DEFAULT_PORT as _GIT_HTTP_PORT from ...image_cache import check_stale from ...log import die, info, warn from .. import BottleImages diff --git a/bot_bottle/egress.py b/bot_bottle/egress.py index 1abaf6ae..b933bad6 100644 --- a/bot_bottle/egress.py +++ b/bot_bottle/egress.py @@ -16,7 +16,7 @@ from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING -from .egress_addon_core import ( +from .gateway.egress_addon_core import ( ON_MATCH_REDACT, HeaderMatch as CoreHeaderMatch, MatchEntry as CoreMatchEntry, diff --git a/bot_bottle/orchestrator/gateway.py b/bot_bottle/gateway/__init__.py similarity index 100% rename from bot_bottle/orchestrator/gateway.py rename to bot_bottle/gateway/__init__.py diff --git a/bot_bottle/dlp_detectors.py b/bot_bottle/gateway/dlp_detectors.py similarity index 100% rename from bot_bottle/dlp_detectors.py rename to bot_bottle/gateway/dlp_detectors.py diff --git a/bot_bottle/egress_addon.py b/bot_bottle/gateway/egress_addon.py similarity index 99% rename from bot_bottle/egress_addon.py rename to bot_bottle/gateway/egress_addon.py index 8766623f..112de8b8 100644 --- a/bot_bottle/egress_addon.py +++ b/bot_bottle/gateway/egress_addon.py @@ -16,8 +16,8 @@ import typing from mitmproxy import http # type: ignore[import-not-found] # pylint: disable=import-error from bot_bottle.constants import IDENTITY_HEADER -from bot_bottle.dlp_detectors import redact_tokens, strip_crlf -from bot_bottle.egress_addon_core import ( +from bot_bottle.gateway.dlp_detectors import redact_tokens, strip_crlf +from bot_bottle.gateway.egress_addon_core import ( LOG_BLOCKS, LOG_FULL, DEFAULT_OUTBOUND_ON_MATCH, @@ -40,7 +40,7 @@ from bot_bottle.egress_addon_core import ( scan_inbound, scan_outbound, ) -from bot_bottle.policy_resolver import PolicyResolveError, PolicyResolver +from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver from bot_bottle.supervise_types import ( STATUS_APPROVED, STATUS_MODIFIED, diff --git a/bot_bottle/egress_addon_core.py b/bot_bottle/gateway/egress_addon_core.py similarity index 99% rename from bot_bottle/egress_addon_core.py rename to bot_bottle/gateway/egress_addon_core.py index bf14fbb2..e4800d52 100644 --- a/bot_bottle/egress_addon_core.py +++ b/bot_bottle/gateway/egress_addon_core.py @@ -16,7 +16,7 @@ import re import typing from dataclasses import dataclass -from .yaml_subset import YamlSubsetError, parse_yaml_subset +from ..yaml_subset import YamlSubsetError, parse_yaml_subset # DLP detector-config parsing lives in a sibling module. Re-exported below # so existing `from egress_addon_core import ON_MATCH_*` callers keep working. diff --git a/bot_bottle/egress_dlp_config.py b/bot_bottle/gateway/egress_dlp_config.py similarity index 100% rename from bot_bottle/egress_dlp_config.py rename to bot_bottle/gateway/egress_dlp_config.py diff --git a/bot_bottle/gateway_init.py b/bot_bottle/gateway/gateway_init.py similarity index 98% rename from bot_bottle/gateway_init.py rename to bot_bottle/gateway/gateway_init.py index 7e9df7e2..1e23be76 100644 --- a/bot_bottle/gateway_init.py +++ b/bot_bottle/gateway/gateway_init.py @@ -100,8 +100,8 @@ _DAEMONS: tuple[_DaemonSpec, ...] = ( )), _DaemonSpec("egress", ("/bin/sh", "/app/egress-entrypoint.sh")), _DaemonSpec("git-gate", ("/bin/sh", "/git-gate-entrypoint.sh")), - _DaemonSpec("git-http", ("python3", "-m", "bot_bottle.git_http_backend")), - _DaemonSpec("supervise", ("python3", "-m", "bot_bottle.supervise_server")), + _DaemonSpec("git-http", ("python3", "-m", "bot_bottle.gateway.git_http_backend")), + _DaemonSpec("supervise", ("python3", "-m", "bot_bottle.gateway.supervise_server")), ) diff --git a/bot_bottle/git_gate_render.py b/bot_bottle/gateway/git_gate_render.py similarity index 98% rename from bot_bottle/git_gate_render.py rename to bot_bottle/gateway/git_gate_render.py index 04c038c1..294d6756 100644 --- a/bot_bottle/git_gate_render.py +++ b/bot_bottle/gateway/git_gate_render.py @@ -14,8 +14,8 @@ import shlex from dataclasses import dataclass from pathlib import Path -from .constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER -from .manifest import ManifestBottle, ManifestGitEntry +from ..constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER +from ..manifest import ManifestBottle, ManifestGitEntry # Short network alias for git-gate inside the gateway. The # agent's `.gitconfig` insteadOf rewrites resolve through this name. @@ -282,7 +282,7 @@ from pathlib import Path # identity_token), resolved server-side, so the proposal lands under the # calling bottle exactly as a direct write once did. try: - from bot_bottle.policy_resolver import PolicyResolver, PolicyResolveError + from bot_bottle.gateway.policy_resolver import PolicyResolver, PolicyResolveError from bot_bottle.supervise_types import TOOL_GITLEAKS_ALLOW except ImportError: from policy_resolver import PolicyResolver, PolicyResolveError @@ -374,7 +374,7 @@ import sys # Non-blocking poll over the control plane. A decided proposal is archived # server-side on read, so no separate archive step is needed here. try: - from bot_bottle.policy_resolver import PolicyResolver, PolicyResolveError + from bot_bottle.gateway.policy_resolver import PolicyResolver, PolicyResolveError except ImportError: from policy_resolver import PolicyResolver, PolicyResolveError diff --git a/bot_bottle/git_http_backend.py b/bot_bottle/gateway/git_http_backend.py similarity index 99% rename from bot_bottle/git_http_backend.py rename to bot_bottle/gateway/git_http_backend.py index 21f924fa..c4860e7e 100644 --- a/bot_bottle/git_http_backend.py +++ b/bot_bottle/gateway/git_http_backend.py @@ -27,7 +27,7 @@ from pathlib import Path from urllib.parse import urlsplit from bot_bottle.constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER -from bot_bottle.policy_resolver import PolicyResolveError, PolicyResolver +from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver DEFAULT_PORT = 9420 diff --git a/bot_bottle/policy_resolver.py b/bot_bottle/gateway/policy_resolver.py similarity index 100% rename from bot_bottle/policy_resolver.py rename to bot_bottle/gateway/policy_resolver.py diff --git a/bot_bottle/supervise_server.py b/bot_bottle/gateway/supervise_server.py similarity index 99% rename from bot_bottle/supervise_server.py rename to bot_bottle/gateway/supervise_server.py index aad06d1a..26495a3d 100644 --- a/bot_bottle/supervise_server.py +++ b/bot_bottle/gateway/supervise_server.py @@ -58,10 +58,10 @@ import typing from dataclasses import dataclass from bot_bottle.constants import IDENTITY_HEADER -from bot_bottle.egress_addon_core import ( +from bot_bottle.gateway.egress_addon_core import ( LOG_OFF, load_config, resolve_client_context, route_to_yaml_dict, ) -from bot_bottle.policy_resolver import PolicyResolveError, PolicyResolver +from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver from bot_bottle import supervise as _sv diff --git a/bot_bottle/git_gate.py b/bot_bottle/git_gate.py index d7c0f963..cfb8213a 100644 --- a/bot_bottle/git_gate.py +++ b/bot_bottle/git_gate.py @@ -39,7 +39,7 @@ from .manifest import ManifestBottle # Rendering and the deploy-key lifecycle live in sibling modules; the # names are re-exported here (see __all__) so existing # `from bot_bottle.git_gate import …` callers are unchanged. -from .git_gate_render import ( +from .gateway.git_gate_render import ( GIT_GATE_HOSTNAME, GIT_GATE_TIMEOUT_SECS, GitGateUpstream, diff --git a/bot_bottle/git_gate_provision.py b/bot_bottle/git_gate_provision.py index 00f42fc6..ac9eebdd 100644 --- a/bot_bottle/git_gate_provision.py +++ b/bot_bottle/git_gate_provision.py @@ -17,7 +17,7 @@ from .bottle_state import globalize_slug from .errors import MissingEnvVarError from .log import info from .manifest import ManifestBottle, ManifestGitEntry -from .git_gate_render import GitGateUpstream +from .gateway.git_gate_render import GitGateUpstream if TYPE_CHECKING: from .git_gate import GitGatePlan diff --git a/bot_bottle/orchestrator/__init__.py b/bot_bottle/orchestrator/__init__.py index 3f03a7e5..5346c004 100644 --- a/bot_bottle/orchestrator/__init__.py +++ b/bot_bottle/orchestrator/__init__.py @@ -38,7 +38,7 @@ from .broker import ( verify_request, ) from .docker_broker import DockerBroker, DockerBrokerError -from .gateway import Gateway, GatewayError +from ..gateway import Gateway, GatewayError from .service import Orchestrator from .control_plane import ControlPlaneServer, dispatch, make_server diff --git a/bot_bottle/orchestrator/lifecycle.py b/bot_bottle/orchestrator/lifecycle.py index b75e87d4..5c7b9c61 100644 --- a/bot_bottle/orchestrator/lifecycle.py +++ b/bot_bottle/orchestrator/lifecycle.py @@ -31,7 +31,7 @@ from ..paths import ( host_control_plane_token, host_gateway_ca_dir, ) -from .gateway import ( +from ..gateway import ( GATEWAY_DOCKERFILE, GATEWAY_IMAGE, GATEWAY_NETWORK, diff --git a/bot_bottle/orchestrator/rotate_ca.py b/bot_bottle/orchestrator/rotate_ca.py index 5495c779..5ae7d2d2 100644 --- a/bot_bottle/orchestrator/rotate_ca.py +++ b/bot_bottle/orchestrator/rotate_ca.py @@ -25,7 +25,7 @@ from pathlib import Path from ..docker_cmd import run_docker from ..paths import host_gateway_ca_dir -from .gateway import GATEWAY_NAME, rotate_gateway_ca +from ..gateway import GATEWAY_NAME, rotate_gateway_ca from .lifecycle import INFRA_NAME # The containers whose mitmproxy would still be serving the old CA from memory: diff --git a/bot_bottle/supervise.py b/bot_bottle/supervise.py index 96d25a37..d71253ce 100644 --- a/bot_bottle/supervise.py +++ b/bot_bottle/supervise.py @@ -1,7 +1,7 @@ """Per-bottle supervise plane (PRD 0013). The supervise plane is the per-bottle MCP daemon plus its host-side -queue/audit support. The daemon (bot_bottle.supervise_server) +queue/audit support. The daemon (bot_bottle.gateway.supervise_server) sits on the bottle's internal network and exposes MCP tools the agent calls when it needs an operator-reviewed egress change: @@ -18,7 +18,7 @@ the response and returns `{status, notes}` to the agent. This module defines the host-side library: dataclasses for the queue record shapes, queue read/write helpers, the audit log writer, and the diff renderer. The in-gateway daemon lives in -bot_bottle/supervise_server.py; the supervise daemon's container +bot_bottle/gateway/supervise_server.py; the supervise daemon's container lifecycle is owned by the gateway (PRD 0024). For 0013 the supervisor's approval handlers are deliberately no-ops: diff --git a/tests/integration/test_gateway_image.py b/tests/integration/test_gateway_image.py index 87e5d971..624bda9b 100644 --- a/tests/integration/test_gateway_image.py +++ b/tests/integration/test_gateway_image.py @@ -91,7 +91,7 @@ class TestGatewayImage(unittest.TestCase): # Probe that the package imports resolve inside the image. rc, out = self._run_in_image( "python3", "-c", - "from bot_bottle import supervise, supervise_server; print('ok')", + "from bot_bottle import supervise; from bot_bottle.gateway import supervise_server; print('ok')", ) self.assertEqual(0, rc, msg=out) self.assertIn("ok", out) diff --git a/tests/integration/test_multitenant_isolation.py b/tests/integration/test_multitenant_isolation.py index 7ed30314..fdd8ea51 100644 --- a/tests/integration/test_multitenant_isolation.py +++ b/tests/integration/test_multitenant_isolation.py @@ -29,7 +29,7 @@ from bot_bottle.backend.docker.consolidated_launch import ( from bot_bottle.backend.docker.egress import EGRESS_PORT from bot_bottle.backend.docker.gateway_net import next_free_ip from bot_bottle.orchestrator.client import OrchestratorClient -from bot_bottle.orchestrator.gateway import GATEWAY_IMAGE, GATEWAY_NAME, GATEWAY_NETWORK +from bot_bottle.gateway import GATEWAY_IMAGE, GATEWAY_NAME, GATEWAY_NETWORK from bot_bottle.orchestrator.lifecycle import OrchestratorService from tests._docker import skip_unless_docker diff --git a/tests/unit/test_dlp_detectors.py b/tests/unit/test_dlp_detectors.py index 0cbd54f9..d62fb4dc 100644 --- a/tests/unit/test_dlp_detectors.py +++ b/tests/unit/test_dlp_detectors.py @@ -7,7 +7,7 @@ import base64 import gzip import unittest -from bot_bottle.dlp_detectors import ( +from bot_bottle.gateway.dlp_detectors import ( ENTROPY_BLOCK_THRESHOLD, PARTIAL_MATCH_MIN_LEN, REDACT, @@ -465,17 +465,17 @@ class TestMatchedAndSafeTokens(unittest.TestCase): class TestStripCrlf(unittest.TestCase): def test_removes_url_encoded_crlf(self): - from bot_bottle.dlp_detectors import strip_crlf + from bot_bottle.gateway.dlp_detectors import strip_crlf out = strip_crlf("next=%0d%0aX-Injected: evil") self.assertNotRegex(out, r"%0[dD]%0[aA]") def test_removes_literal_header_injection(self): - from bot_bottle.dlp_detectors import strip_crlf + from bot_bottle.gateway.dlp_detectors import strip_crlf out = strip_crlf("value\r\nX-Injected: evil") self.assertIsNone(scan_crlf_injection(out)) def test_leaves_clean_text_unchanged(self): - from bot_bottle.dlp_detectors import strip_crlf + from bot_bottle.gateway.dlp_detectors import strip_crlf self.assertEqual("/api/v1/data?q=hello", strip_crlf("/api/v1/data?q=hello")) class TestAlnumProjection(unittest.TestCase): diff --git a/tests/unit/test_egress.py b/tests/unit/test_egress.py index 6d3e1346..027df828 100644 --- a/tests/unit/test_egress.py +++ b/tests/unit/test_egress.py @@ -344,7 +344,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertEqual([], parse_yaml_subset(rendered)["routes"]) def test_round_trip_through_addon_core(self): - from bot_bottle.egress_addon_core import load_config + from bot_bottle.gateway.egress_addon_core import load_config b = _bottle([ {"host": "api.github.com", "auth": {"scheme": "Bearer", "token_ref": "GH_PAT"}, @@ -363,7 +363,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertEqual("", addon_routes[2].auth_scheme) def test_dlp_round_trips(self): - from bot_bottle.egress_addon_core import load_config + from bot_bottle.gateway.egress_addon_core import load_config b = _bottle([{"host": "x.example", "dlp": { "outbound_detectors": ["token_patterns"], "inbound_detectors": False, @@ -375,7 +375,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertEqual((), addon_routes[0].inbound_detectors) def test_outbound_on_match_round_trips(self): - from bot_bottle.egress_addon_core import load_config + from bot_bottle.gateway.egress_addon_core import load_config b = _bottle([{"host": "logs.example", "dlp": { "outbound_on_match": "redact", }}]) @@ -392,7 +392,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertNotIn("outbound_on_match", rendered) def test_git_fetch_policy_round_trips(self): - from bot_bottle.egress_addon_core import load_config + from bot_bottle.gateway.egress_addon_core import load_config b = _bottle([{"host": "github.com", "git": {"fetch": True}}]) routes = egress_routes_for_bottle(b) rendered = egress_render_routes(routes) @@ -405,7 +405,7 @@ class TestRenderRoutes(unittest.TestCase): it, but the renderer in between dropped it — so the flag never reached the proxy and registry pulls kept failing with "unauthorized" while the config looked correct everywhere it was inspected.""" - from bot_bottle.egress_addon_core import load_config + from bot_bottle.gateway.egress_addon_core import load_config b = _bottle([{"host": "registry-1.docker.io", "preserve_auth": True}]) routes = egress_routes_for_bottle(b) rendered = egress_render_routes(routes) @@ -416,7 +416,7 @@ class TestRenderRoutes(unittest.TestCase): b = _bottle([{"host": "x.example"}]) rendered = egress_render_routes(egress_routes_for_bottle(b)) self.assertNotIn("preserve_auth", rendered) - from bot_bottle.egress_addon_core import load_config + from bot_bottle.gateway.egress_addon_core import load_config self.assertFalse(load_config(rendered).routes[0].preserve_auth) def test_log_zero_omitted_from_render(self): @@ -434,7 +434,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertTrue(rendered.startswith(f"log: {level}\n")) def test_log_level_round_trips_to_addon_core(self): - from bot_bottle.egress_addon_core import load_config, LOG_FULL + from bot_bottle.gateway.egress_addon_core import load_config, LOG_FULL b = _bottle([{"host": "x.example"}]) routes = egress_routes_for_bottle(b) rendered = egress_render_routes(routes, log=LOG_FULL) @@ -444,7 +444,7 @@ class TestRenderRoutes(unittest.TestCase): def test_log_via_manifest_flows_to_render(self): from bot_bottle.manifest import ManifestIndex - from bot_bottle.egress_addon_core import load_config, LOG_BLOCKS + from bot_bottle.gateway.egress_addon_core import load_config, LOG_BLOCKS m = ManifestIndex.from_json_obj({ "bottles": {"dev": {"egress": { "log": 1, @@ -512,7 +512,7 @@ class TestRenderRoutesEscaping(unittest.TestCase): self.assertEqual('Bear"er', parsed[0]["inspect"]["auth_scheme"]) def test_path_value_with_double_quote_round_trips(self): - from bot_bottle.egress_addon_core import PathMatch, MatchEntry + from bot_bottle.gateway.egress_addon_core import PathMatch, MatchEntry routes = (EgressRoute( host="api.example", matches=(MatchEntry(paths=(PathMatch(type="prefix", value='/v1/"quoted"/'),)),), @@ -521,7 +521,7 @@ class TestRenderRoutesEscaping(unittest.TestCase): self.assertEqual('/v1/"quoted"/', parsed[0]["inspect"]["matches"][0]["paths"][0]["value"]) def test_header_value_with_double_quote_round_trips(self): - from bot_bottle.egress_addon_core import HeaderMatch, MatchEntry + from bot_bottle.gateway.egress_addon_core import HeaderMatch, MatchEntry routes = (EgressRoute( host="api.example", matches=(MatchEntry(headers=(HeaderMatch(name="x-h", value='val"ue'),)),), @@ -598,7 +598,7 @@ class TestCanaryGeneration(unittest.TestCase): self.assertNotEqual(plan_a.canary, plan_b.canary) def test_canary_detected_by_scan_known_secrets(self): - from bot_bottle.dlp_detectors import scan_known_secrets + from bot_bottle.gateway.dlp_detectors import scan_known_secrets plan = self._make_plan() env = {plan.canary_env: plan.canary} diff --git a/tests/unit/test_egress_addon_core.py b/tests/unit/test_egress_addon_core.py index a88d1ba0..593d6ca9 100644 --- a/tests/unit/test_egress_addon_core.py +++ b/tests/unit/test_egress_addon_core.py @@ -12,7 +12,7 @@ import unittest from pathlib import Path from urllib.parse import urlsplit -from bot_bottle.egress_addon_core import ( +from bot_bottle.gateway.egress_addon_core import ( LOG_BLOCKS, LOG_FULL, LOG_OFF, @@ -1377,15 +1377,15 @@ class TestScanOutboundEnhanced(unittest.TestCase): class TestOutboundDetectorNames(unittest.TestCase): def test_entropy_in_outbound_detector_names(self): - from bot_bottle.egress_addon_core import OUTBOUND_DETECTOR_NAMES + from bot_bottle.gateway.egress_addon_core import OUTBOUND_DETECTOR_NAMES self.assertIn("entropy", OUTBOUND_DETECTOR_NAMES) def test_known_secrets_in_outbound_detector_names(self): - from bot_bottle.egress_addon_core import OUTBOUND_DETECTOR_NAMES + from bot_bottle.gateway.egress_addon_core import OUTBOUND_DETECTOR_NAMES self.assertIn("known_secrets", OUTBOUND_DETECTOR_NAMES) def test_token_patterns_in_outbound_detector_names(self): - from bot_bottle.egress_addon_core import OUTBOUND_DETECTOR_NAMES + from bot_bottle.gateway.egress_addon_core import OUTBOUND_DETECTOR_NAMES self.assertIn("token_patterns", OUTBOUND_DETECTOR_NAMES) diff --git a/tests/unit/test_egress_addon_log_redaction.py b/tests/unit/test_egress_addon_log_redaction.py index 8b64f677..44af8152 100644 --- a/tests/unit/test_egress_addon_log_redaction.py +++ b/tests/unit/test_egress_addon_log_redaction.py @@ -36,7 +36,7 @@ def _ensure_shims() -> None: _ensure_shims() -from bot_bottle.egress_addon import EgressAddon # noqa: E402 (import after shims) +from bot_bottle.gateway.egress_addon import EgressAddon # noqa: E402 (import after shims) # --------------------------------------------------------------------------- diff --git a/tests/unit/test_egress_addon_request_flow.py b/tests/unit/test_egress_addon_request_flow.py index 6896e9d0..a8a06d2a 100644 --- a/tests/unit/test_egress_addon_request_flow.py +++ b/tests/unit/test_egress_addon_request_flow.py @@ -194,22 +194,22 @@ def _ensure_shims() -> None: _ensure_shims() -import bot_bottle.egress_addon as _ea_mod # noqa: E402 (after shims) -from bot_bottle.egress_addon import EgressAddon # noqa: E402 (after shims) -from bot_bottle.egress_addon import ( # noqa: E402 +import bot_bottle.gateway.egress_addon as _ea_mod # noqa: E402 (after shims) +from bot_bottle.gateway.egress_addon import EgressAddon # noqa: E402 (after shims) +from bot_bottle.gateway.egress_addon import ( # noqa: E402 DEFAULT_INBOUND_SCAN_LIMIT_BYTES, DEFAULT_TOKEN_ALLOW_TIMEOUT_SECONDS, _inbound_scan_limit_from_env, _token_allow_timeout_from_env, ) -from bot_bottle.egress_addon_core import ( # noqa: E402 +from bot_bottle.gateway.egress_addon_core import ( # noqa: E402 Config, LOG_BLOCKS, LOG_FULL, Route, route_to_yaml_dict, ) -from bot_bottle.policy_resolver import PolicyResolveError # noqa: E402 +from bot_bottle.gateway.policy_resolver import PolicyResolveError # noqa: E402 # --------------------------------------------------------------------------- diff --git a/tests/unit/test_egress_core_parsing.py b/tests/unit/test_egress_core_parsing.py index 7762a93d..6a56e901 100644 --- a/tests/unit/test_egress_core_parsing.py +++ b/tests/unit/test_egress_core_parsing.py @@ -8,7 +8,7 @@ from __future__ import annotations import unittest -from bot_bottle.egress_addon_core import ( +from bot_bottle.gateway.egress_addon_core import ( HeaderMatch, MatchEntry, PathMatch, diff --git a/tests/unit/test_egress_multitenant.py b/tests/unit/test_egress_multitenant.py index 6d99e0d3..adabb3e7 100644 --- a/tests/unit/test_egress_multitenant.py +++ b/tests/unit/test_egress_multitenant.py @@ -4,7 +4,7 @@ from __future__ import annotations import unittest -from bot_bottle.egress_addon_core import ( +from bot_bottle.gateway.egress_addon_core import ( DENY_RESOLVER_ERROR, DENY_UNATTRIBUTED, DENY_UNPARSEABLE, @@ -12,7 +12,7 @@ from bot_bottle.egress_addon_core import ( resolve_client_config, resolve_client_context, ) -from bot_bottle.policy_resolver import PolicyResolveError +from bot_bottle.gateway.policy_resolver import PolicyResolveError class _FakeResolver: diff --git a/tests/unit/test_firecracker_infra_vm.py b/tests/unit/test_firecracker_infra_vm.py index a809344b..e0ae9bb8 100644 --- a/tests/unit/test_firecracker_infra_vm.py +++ b/tests/unit/test_firecracker_infra_vm.py @@ -72,7 +72,7 @@ class TestBuildInfraRootfs(unittest.TestCase): self.assertIn("bot_bottle.orchestrator", init) # Gateway launches via the installed package (there is no # /app/gateway_init.py file since the daemons moved into bot_bottle). - self.assertIn("bot_bottle.gateway_init", init) + self.assertIn("bot_bottle.gateway.gateway_init", init) self.assertIn("export PATH=", init) # Persistent registry volume mounted at the DB dir before the CP starts. self.assertIn("/dev/vdb", init) diff --git a/tests/unit/test_gateway_init.py b/tests/unit/test_gateway_init.py index 17b5213b..bb987bac 100644 --- a/tests/unit/test_gateway_init.py +++ b/tests/unit/test_gateway_init.py @@ -1,6 +1,6 @@ """Unit: gateway data-plane init supervisor (PRD 0070; PRD 0024 bundle shape). -Tests both the helper functions in `bot_bottle.gateway_init` +Tests both the helper functions in `bot_bottle.gateway.gateway_init` and the supervisor's end-to-end signal / exit-code behavior. The end-to-end tests use real subprocesses (`sleep`, `/bin/sh -c '...'`) — short-lived, no docker required — so they run under `tests/unit/` @@ -18,7 +18,7 @@ import warnings from pathlib import Path from unittest.mock import patch -from bot_bottle.gateway_init import ( +from bot_bottle.gateway.gateway_init import ( _DaemonSpec, _Supervisor, _argv_for_daemon, @@ -489,7 +489,7 @@ class TestSupervisor(unittest.TestCase): time.sleep(0.3) # let `trap` register sup.request_shutdown(reason="test") - with patch("bot_bottle.gateway_init._GRACE_SECONDS", 0.3): + with patch("bot_bottle.gateway.gateway_init._GRACE_SECONDS", 0.3): rc = self._drive(sup, max_wait_s=4.0) # Process was SIGKILL'd → returncode -9 on POSIX. @@ -531,7 +531,7 @@ class TestMainEndToEnd(unittest.TestCase): helper = ( "import os, runpy, sys\n" - "from bot_bottle import gateway_init as si\n" + "from bot_bottle.gateway import gateway_init as si\n" "si._DAEMONS = (\n" f" si._DaemonSpec('alpha', ({SLEEP!r},'30')),\n" f" si._DaemonSpec('beta', ({SLEEP!r},'30')),\n" diff --git a/tests/unit/test_git_gate.py b/tests/unit/test_git_gate.py index fd899f0a..4befb5de 100644 --- a/tests/unit/test_git_gate.py +++ b/tests/unit/test_git_gate.py @@ -230,7 +230,7 @@ class TestHookRender(unittest.TestCase): # execute from the bare repo directory, so the embedded Python must # include /app and support both import layouts. self.assertIn('PYTHONPATH="/app${PYTHONPATH:+:$PYTHONPATH}"', hook) - self.assertIn("from bot_bottle.policy_resolver import PolicyResolver", hook) + self.assertIn("from bot_bottle.gateway.policy_resolver import PolicyResolver", hook) self.assertIn("from policy_resolver import PolicyResolver", hook) def test_inline_gitleaks_allow_fails_closed_without_supervisor(self): diff --git a/tests/unit/test_git_gate_provision_render.py b/tests/unit/test_git_gate_provision_render.py index 3fc1a076..b9bce15c 100644 --- a/tests/unit/test_git_gate_provision_render.py +++ b/tests/unit/test_git_gate_provision_render.py @@ -4,7 +4,7 @@ from __future__ import annotations import unittest -from bot_bottle.git_gate_render import ( +from bot_bottle.gateway.git_gate_render import ( GitGateUpstream, git_gate_render_entrypoint, git_gate_render_provision, diff --git a/tests/unit/test_git_http_backend.py b/tests/unit/test_git_http_backend.py index 405f0b04..6c587726 100644 --- a/tests/unit/test_git_http_backend.py +++ b/tests/unit/test_git_http_backend.py @@ -10,7 +10,7 @@ from pathlib import Path from unittest import mock from bot_bottle.git_gate import GIT_GATE_TIMEOUT_SECS -from bot_bottle.git_http_backend import GitHttpHandler, MAX_BODY_BYTES +from bot_bottle.gateway.git_http_backend import GitHttpHandler, MAX_BODY_BYTES # The git-http backend is resolver-only: every request is attributed to a @@ -199,7 +199,7 @@ class TestGitHttpBackend(unittest.TestCase): subprocess.CompletedProcess(["git"], 0, backend_response, b""), ] with mock.patch( - "bot_bottle.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_http_backend.subprocess.run", side_effect=calls, ) as run: request = urllib.request.Request( @@ -265,7 +265,7 @@ class TestGitHttpBackend(unittest.TestCase): subprocess.CompletedProcess(["git"], 0, backend_response, b""), ] with mock.patch( - "bot_bottle.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_http_backend.subprocess.run", side_effect=calls, ) as run: req = urllib.request.Request( @@ -309,7 +309,7 @@ class TestGitHttpBackend(unittest.TestCase): denial = b"git-gate: upstream fetch failed; refusing to serve stale data\n" with mock.patch( - "bot_bottle.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_http_backend.subprocess.run", return_value=subprocess.CompletedProcess( ["hook"], 1, b"", denial, ), @@ -355,7 +355,7 @@ class TestGitHttpBackend(unittest.TestCase): self.addCleanup(server.server_close) with mock.patch( - "bot_bottle.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_http_backend.subprocess.run", return_value=subprocess.CompletedProcess( ["hook"], 2, b"", b"", ), @@ -402,7 +402,7 @@ class TestGitHttpBackend(unittest.TestCase): self.addCleanup(server.server_close) with mock.patch( - "bot_bottle.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_http_backend.subprocess.run", side_effect=PermissionError(13, "Permission denied"), ): buf = io.StringIO() @@ -461,7 +461,7 @@ class TestMalformedStatusHeader(unittest.TestCase): def _get_with_backend_response(self, cgi_response: bytes) -> int: with mock.patch( - "bot_bottle.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_http_backend.subprocess.run", return_value=mock.Mock(returncode=0, stdout=cgi_response), ): req = urllib.request.Request( @@ -545,7 +545,7 @@ class TestContentLengthBounds(unittest.TestCase): # With a valid Content-Length the handler proceeds into # git http-backend; that will fail (no real git repo) but the # status won't be 400 or 413. - with mock.patch("bot_bottle.git_http_backend.subprocess.run") as run: + with mock.patch("bot_bottle.gateway.git_http_backend.subprocess.run") as run: run.return_value = mock.Mock( returncode=0, stdout=( diff --git a/tests/unit/test_git_http_multitenant.py b/tests/unit/test_git_http_multitenant.py index b425f5f9..16acff14 100644 --- a/tests/unit/test_git_http_multitenant.py +++ b/tests/unit/test_git_http_multitenant.py @@ -10,8 +10,8 @@ from __future__ import annotations import unittest from pathlib import Path -from bot_bottle.git_http_backend import resolve_sandbox_root -from bot_bottle.policy_resolver import PolicyResolveError +from bot_bottle.gateway.git_http_backend import resolve_sandbox_root +from bot_bottle.gateway.policy_resolver import PolicyResolveError _BASE = Path("/git") diff --git a/tests/unit/test_macos_infra.py b/tests/unit/test_macos_infra.py index cf746dca..7d80840a 100644 --- a/tests/unit/test_macos_infra.py +++ b/tests/unit/test_macos_infra.py @@ -48,7 +48,7 @@ class TestInfraRun(unittest.TestCase): self.assertIn("bot_bottle.orchestrator", script) # Gateway launches via the installed package (there is no # /app/gateway_init.py file since the daemons moved into bot_bottle). - self.assertIn("bot_bottle.gateway_init", script) + self.assertIn("bot_bottle.gateway.gateway_init", script) self.assertIn("127.0.0.1", script) # they reach each other on loopback def test_db_is_a_container_only_volume(self) -> None: diff --git a/tests/unit/test_orchestrator_gateway.py b/tests/unit/test_orchestrator_gateway.py index f712fae4..b23162e6 100644 --- a/tests/unit/test_orchestrator_gateway.py +++ b/tests/unit/test_orchestrator_gateway.py @@ -8,7 +8,7 @@ from pathlib import Path from unittest.mock import Mock, patch from bot_bottle.backend.docker.gateway import DockerGateway -from bot_bottle.orchestrator.gateway import ( +from bot_bottle.gateway import ( GATEWAY_CA_CERT, GATEWAY_NAME, GatewayError, diff --git a/tests/unit/test_orchestrator_lifecycle.py b/tests/unit/test_orchestrator_lifecycle.py index 26728530..1a659084 100644 --- a/tests/unit/test_orchestrator_lifecycle.py +++ b/tests/unit/test_orchestrator_lifecycle.py @@ -8,7 +8,7 @@ import urllib.error from pathlib import Path from unittest.mock import MagicMock, Mock, patch -from bot_bottle.orchestrator.gateway import GatewayError +from bot_bottle.gateway import GatewayError from bot_bottle.orchestrator.lifecycle import ( INFRA_NAME, INFRA_SOURCE_HASH_LABEL, diff --git a/tests/unit/test_orchestrator_registration.py b/tests/unit/test_orchestrator_registration.py index 846da4ea..ffb37524 100644 --- a/tests/unit/test_orchestrator_registration.py +++ b/tests/unit/test_orchestrator_registration.py @@ -7,7 +7,7 @@ import unittest from pathlib import Path from bot_bottle.egress import EgressPlan, EgressRoute -from bot_bottle.egress_addon_core import LOG_BLOCKS, load_config +from bot_bottle.gateway.egress_addon_core import LOG_BLOCKS, load_config from bot_bottle.orchestrator.registration import ( RegistrationInputs, egress_policy, diff --git a/tests/unit/test_orchestrator_rotate_ca.py b/tests/unit/test_orchestrator_rotate_ca.py index 22199b8a..3c2db894 100644 --- a/tests/unit/test_orchestrator_rotate_ca.py +++ b/tests/unit/test_orchestrator_rotate_ca.py @@ -8,7 +8,7 @@ from pathlib import Path from unittest.mock import Mock, patch from bot_bottle.orchestrator import rotate_ca -from bot_bottle.orchestrator.gateway import GATEWAY_NAME +from bot_bottle.gateway import GATEWAY_NAME from bot_bottle.orchestrator.lifecycle import INFRA_NAME from bot_bottle.paths import host_gateway_ca_dir from tests.unit import use_bottle_root diff --git a/tests/unit/test_policy_resolver.py b/tests/unit/test_policy_resolver.py index 83544ff1..44178a3b 100644 --- a/tests/unit/test_policy_resolver.py +++ b/tests/unit/test_policy_resolver.py @@ -7,7 +7,7 @@ import unittest import urllib.error from unittest.mock import MagicMock, patch -from bot_bottle.policy_resolver import ( +from bot_bottle.gateway.policy_resolver import ( CONTROL_AUTH_HEADER, CONTROL_AUTH_JWT_ENV, PolicyResolveError, @@ -15,7 +15,7 @@ from bot_bottle.policy_resolver import ( _control_auth_headers, ) -_URLOPEN = "bot_bottle.policy_resolver.urllib.request.urlopen" +_URLOPEN = "bot_bottle.gateway.policy_resolver.urllib.request.urlopen" def _resp(payload: object) -> MagicMock: diff --git a/tests/unit/test_supervise_server.py b/tests/unit/test_supervise_server.py index 3f596b0a..7f0f21d7 100644 --- a/tests/unit/test_supervise_server.py +++ b/tests/unit/test_supervise_server.py @@ -22,8 +22,8 @@ from bot_bottle import supervise as _sv from bot_bottle.store import queue_store as _qs from bot_bottle.store import audit_store as _as -from bot_bottle import supervise_server # noqa: E402 -from bot_bottle.supervise_server import ( +from bot_bottle.gateway import supervise_server # noqa: E402 +from bot_bottle.gateway.supervise_server import ( ERR_INTERNAL, ERR_INVALID_PARAMS, ERR_INVALID_REQUEST, -- 2.52.0 From 44e2b5a897ef5c29f6905c1aaaa942f2c6313a67 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 14:15:04 -0400 Subject: [PATCH 05/30] refactor(supervise): split the supervise plane by tier; per-service store managers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Give each service its own store package + manager, and cut the supervise module along the control/data-plane boundary so nothing in the shared layer reaches up into the orchestrator. Stores, by owner: - bot_bottle/store/ keeps only the shared base (DbStore, migrations) and the concrete stores that aren't service-owned (audit_store, config_store). - bot_bottle/orchestrator/store/ now houses the orchestrator-owned stores — queue_store (supervise queue), secret_store, config_store — plus a new orchestrator store_manager that migrates them (composing audit/config downward from the base). The old shared store_manager is gone. Supervise plane, by tier: - bot_bottle/supervisor/ (NEUTRAL, importable by every tier including the gateway): types.py (the Proposal/Response/AuditEntry wire types + the tool/ status/poll constants + the shared daemon constants moved out of supervise.py) and plan.py (SupervisePlan, a pure DTO). - bot_bottle/orchestrator/supervisor/ (orchestrator-only): queue.py (the queue/audit I/O wrappers + render_diff + sha256_hex) and supervise.py (the Supervise lifecycle that stages the DB via the store manager). Its __init__ re-exports the neutral vocabulary so orchestrator-side callers import from one place. The gateway now imports only bot_bottle.supervisor.types (never bot_bottle.supervise), so the data plane holds no code dependency on the orchestrator — it reaches the queue over the control-plane RPC. This removes the circular import that moving queue_store under orchestrator introduced (supervise -> orchestrator -> service -> supervise). supervise_types.py -> supervisor/types.py; supervise.py deleted (split). Full unit suite green (2251). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/backend/__init__.py | 2 +- bot_bottle/backend/consolidated_util.py | 4 +- bot_bottle/backend/docker/backend.py | 4 +- .../backend/docker/consolidated_compose.py | 2 +- .../backend/docker/consolidated_launch.py | 2 +- bot_bottle/backend/docker/launch.py | 4 +- bot_bottle/backend/docker/resolve_plan.py | 2 +- bot_bottle/backend/firecracker/backend.py | 2 +- .../firecracker/consolidated_launch.py | 2 +- bot_bottle/backend/firecracker/launch.py | 6 +- .../backend/firecracker/resolve_plan.py | 2 +- bot_bottle/backend/macos_container/backend.py | 2 +- .../macos_container/consolidated_launch.py | 2 +- bot_bottle/backend/macos_container/launch.py | 6 +- .../backend/macos_container/resolve_plan.py | 2 +- bot_bottle/backend/resolve_common.py | 3 +- bot_bottle/cli/__init__.py | 2 +- bot_bottle/cli/supervise.py | 2 +- bot_bottle/gateway/egress_addon.py | 2 +- bot_bottle/gateway/git_gate_render.py | 4 +- bot_bottle/gateway/supervise_server.py | 2 +- bot_bottle/orchestrator/__main__.py | 2 +- bot_bottle/orchestrator/control_plane.py | 2 +- bot_bottle/orchestrator/service.py | 6 +- bot_bottle/orchestrator/store/__init__.py | 10 + .../orchestrator/{ => store}/config_store.py | 6 +- .../{ => orchestrator}/store/queue_store.py | 10 +- .../orchestrator/{ => store}/secret_store.py | 0 .../{ => orchestrator}/store/store_manager.py | 32 +- .../orchestrator/supervisor/__init__.py | 34 ++ bot_bottle/orchestrator/supervisor/queue.py | 141 +++++++++ .../orchestrator/supervisor/supervise.py | 36 +++ bot_bottle/store/__init__.py | 14 +- bot_bottle/store/audit_store.py | 4 +- bot_bottle/supervise.py | 298 ------------------ bot_bottle/supervisor/__init__.py | 16 + bot_bottle/supervisor/plan.py | 26 ++ .../types.py} | 25 ++ tests/integration/test_gateway_image.py | 2 +- tests/unit/_docker_bottle_plan.py | 2 +- tests/unit/test_cli_dispatch.py | 2 +- tests/unit/test_config_store.py | 2 +- tests/unit/test_contrib_claude_provider.py | 2 +- tests/unit/test_contrib_codex_provider.py | 2 +- tests/unit/test_orchestrator_config_store.py | 2 +- tests/unit/test_orchestrator_control_plane.py | 4 +- tests/unit/test_orchestrator_secret_store.py | 2 +- tests/unit/test_orchestrator_service.py | 6 +- tests/unit/test_supervise.py | 6 +- tests/unit/test_supervise_cli.py | 2 +- tests/unit/test_supervise_edge.py | 6 +- tests/unit/test_supervise_server.py | 4 +- 52 files changed, 380 insertions(+), 385 deletions(-) create mode 100644 bot_bottle/orchestrator/store/__init__.py rename bot_bottle/orchestrator/{ => store}/config_store.py (96%) rename bot_bottle/{ => orchestrator}/store/queue_store.py (96%) rename bot_bottle/orchestrator/{ => store}/secret_store.py (100%) rename bot_bottle/{ => orchestrator}/store/store_manager.py (54%) create mode 100644 bot_bottle/orchestrator/supervisor/__init__.py create mode 100644 bot_bottle/orchestrator/supervisor/queue.py create mode 100644 bot_bottle/orchestrator/supervisor/supervise.py delete mode 100644 bot_bottle/supervise.py create mode 100644 bot_bottle/supervisor/__init__.py create mode 100644 bot_bottle/supervisor/plan.py rename bot_bottle/{supervise_types.py => supervisor/types.py} (84%) diff --git a/bot_bottle/backend/__init__.py b/bot_bottle/backend/__init__.py index ff5223ef..79cd7af3 100644 --- a/bot_bottle/backend/__init__.py +++ b/bot_bottle/backend/__init__.py @@ -48,7 +48,7 @@ from ..git_gate import GitGatePlan from ..log import die, info, warn from ..util import read_tty_line from ..manifest import Manifest, ManifestIndex -from ..supervise import SupervisePlan +from ..supervisor.plan import SupervisePlan from ..util import expand_tilde from ..env import resolve_env, ResolvedEnv from ..workspace import WorkspacePlan, workspace_plan diff --git a/bot_bottle/backend/consolidated_util.py b/bot_bottle/backend/consolidated_util.py index 520ca0a0..3aa85905 100644 --- a/bot_bottle/backend/consolidated_util.py +++ b/bot_bottle/backend/consolidated_util.py @@ -13,7 +13,7 @@ from ..egress import EgressPlan from ..git_gate import GitGatePlan from ..orchestrator.client import OrchestratorClient, RegisteredBottle from ..orchestrator.registration import registration_inputs -from ..orchestrator.secret_store import new_env_var_secret +from ..orchestrator.store.secret_store import new_env_var_secret from .docker.gateway_provision import GatewayTransport, deprovision_git_gate, provision_git_gate @@ -58,7 +58,7 @@ def teardown_consolidated( ) -> None: """Deregister the bottle and remove its git-gate state. Both steps are idempotent so this is safe from a cleanup trap.""" - from ..orchestrator.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS + from ..orchestrator.store.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS OrchestratorClient( orchestrator_url, timeout=timeout if timeout is not None else DEFAULT_TEARDOWN_TIMEOUT_SECONDS, diff --git a/bot_bottle/backend/docker/backend.py b/bot_bottle/backend/docker/backend.py index b2ae45ba..8cab6d77 100644 --- a/bot_bottle/backend/docker/backend.py +++ b/bot_bottle/backend/docker/backend.py @@ -24,12 +24,12 @@ from contextlib import contextmanager from pathlib import Path from typing import Generator, Sequence -from ...supervise import SUPERVISE_HOSTNAME, SUPERVISE_PORT +from ...supervisor.types import SUPERVISE_HOSTNAME, SUPERVISE_PORT from ...agent_provider import AgentProvisionPlan from ...egress import EgressPlan from ...env import ResolvedEnv from ...git_gate import GitGatePlan -from ...supervise import SupervisePlan +from ...supervisor.plan import SupervisePlan from ...manifest import Manifest from .. import ActiveAgent, BottleBackend, BottleImages, BottleSpec from . import cleanup as _cleanup diff --git a/bot_bottle/backend/docker/consolidated_compose.py b/bot_bottle/backend/docker/consolidated_compose.py index 00652248..45296b5c 100644 --- a/bot_bottle/backend/docker/consolidated_compose.py +++ b/bot_bottle/backend/docker/consolidated_compose.py @@ -17,7 +17,7 @@ from __future__ import annotations from typing import Any from ...egress import egress_agent_env_entries -from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME +from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH from .bottle_plan import DockerBottlePlan from .egress import EGRESS_PORT diff --git a/bot_bottle/backend/docker/consolidated_launch.py b/bot_bottle/backend/docker/consolidated_launch.py index 482a30f8..2f57efb9 100644 --- a/bot_bottle/backend/docker/consolidated_launch.py +++ b/bot_bottle/backend/docker/consolidated_launch.py @@ -22,7 +22,7 @@ from ...git_gate import GitGatePlan from ...orchestrator.client import OrchestratorClient from ...gateway import GATEWAY_NETWORK from ...orchestrator.lifecycle import INFRA_NAME, OrchestratorService -from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME +from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME from ...orchestrator.reprovision import reprovision_bottles from ..consolidated_util import provision_bottle from ..consolidated_util import teardown_consolidated as _teardown_util diff --git a/bot_bottle/backend/docker/launch.py b/bot_bottle/backend/docker/launch.py index 47e2f642..78c87855 100644 --- a/bot_bottle/backend/docker/launch.py +++ b/bot_bottle/backend/docker/launch.py @@ -64,7 +64,7 @@ from .compose import ( write_compose_file, ) from .consolidated_compose import consolidated_agent_compose -from ...orchestrator.config_store import resolve_teardown_timeout +from ...orchestrator.store.config_store import resolve_teardown_timeout from .consolidated_launch import launch_consolidated, teardown_consolidated from ...orchestrator.lifecycle import INFRA_NAME from .gateway import DockerGateway @@ -207,7 +207,7 @@ def launch( # spec, value only in the subprocess env so it is never written to disk. compose_env: dict[str, str] = {**os.environ, **plan.forwarded_env} if plan.env_var_secret: - from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME + from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME compose_env[ENV_VAR_SECRET_NAME] = plan.env_var_secret info( f"docker compose up -d (project {project}, agent on shared " diff --git a/bot_bottle/backend/docker/resolve_plan.py b/bot_bottle/backend/docker/resolve_plan.py index c48e3acd..eac5d2ae 100644 --- a/bot_bottle/backend/docker/resolve_plan.py +++ b/bot_bottle/backend/docker/resolve_plan.py @@ -19,7 +19,7 @@ from ...env import ResolvedEnv from ...agent_provider import AgentProvisionPlan from ...egress import EgressPlan from ...manifest import Manifest -from ...supervise import SupervisePlan +from ...supervisor.plan import SupervisePlan from ...git_gate import GitGatePlan def preflight() -> None: diff --git a/bot_bottle/backend/firecracker/backend.py b/bot_bottle/backend/firecracker/backend.py index a8a9df93..a5281822 100644 --- a/bot_bottle/backend/firecracker/backend.py +++ b/bot_bottle/backend/firecracker/backend.py @@ -17,7 +17,7 @@ from ...egress import EgressPlan from ...env import ResolvedEnv from ...git_gate import GitGatePlan from ...manifest import Manifest -from ...supervise import SupervisePlan +from ...supervisor.plan import SupervisePlan from .. import ActiveAgent, BottleBackend, BottleImages, BottleSpec from . import cleanup as _cleanup from . import enumerate as _enumerate diff --git a/bot_bottle/backend/firecracker/consolidated_launch.py b/bot_bottle/backend/firecracker/consolidated_launch.py index cb726907..503d0af5 100644 --- a/bot_bottle/backend/firecracker/consolidated_launch.py +++ b/bot_bottle/backend/firecracker/consolidated_launch.py @@ -38,7 +38,7 @@ from ...orchestrator.lifecycle import ( OrchestratorStartError, # re-exported so callers can catch it ) from ...orchestrator.reprovision import reprovision_bottles -from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME +from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME from ..consolidated_util import ( provision_bottle, teardown_consolidated as _teardown_util, diff --git a/bot_bottle/backend/firecracker/launch.py b/bot_bottle/backend/firecracker/launch.py index d72f40c9..37bfb2d6 100644 --- a/bot_bottle/backend/firecracker/launch.py +++ b/bot_bottle/backend/firecracker/launch.py @@ -48,14 +48,14 @@ from ...git_gate import ( ) from ...image_cache import check_stale_path from ...log import die, info, warn -from ...supervise import SUPERVISE_PORT +from ...supervisor.types import SUPERVISE_PORT from ..docker.egress import EGRESS_PORT from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH from . import firecracker_vm, image_builder, isolation_probe, netpool, util from .bottle import FirecrackerBottle from .bottle_plan import FirecrackerBottlePlan -from ...orchestrator.config_store import resolve_teardown_timeout -from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME +from ...orchestrator.store.config_store import resolve_teardown_timeout +from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME from .consolidated_launch import ( launch_consolidated, persist_env_var_secret, diff --git a/bot_bottle/backend/firecracker/resolve_plan.py b/bot_bottle/backend/firecracker/resolve_plan.py index 62921756..f5b0dd60 100644 --- a/bot_bottle/backend/firecracker/resolve_plan.py +++ b/bot_bottle/backend/firecracker/resolve_plan.py @@ -9,7 +9,7 @@ from ...egress import EgressPlan from ...env import ResolvedEnv from ...git_gate import GitGatePlan from ...manifest import Manifest -from ...supervise import SupervisePlan +from ...supervisor.plan import SupervisePlan from .. import BottleSpec from . import util from .bottle_plan import FirecrackerBottlePlan diff --git a/bot_bottle/backend/macos_container/backend.py b/bot_bottle/backend/macos_container/backend.py index 2448c002..71b4d5e1 100644 --- a/bot_bottle/backend/macos_container/backend.py +++ b/bot_bottle/backend/macos_container/backend.py @@ -10,7 +10,7 @@ from ...agent_provider import AgentProvisionPlan from ...egress import EgressPlan from ...env import ResolvedEnv from ...git_gate import GitGatePlan -from ...supervise import SupervisePlan +from ...supervisor.plan import SupervisePlan from ...manifest import Manifest from .. import ActiveAgent, BottleBackend, BottleImages, BottleSpec from . import cleanup as _cleanup diff --git a/bot_bottle/backend/macos_container/consolidated_launch.py b/bot_bottle/backend/macos_container/consolidated_launch.py index 6c25f5e4..94976fb9 100644 --- a/bot_bottle/backend/macos_container/consolidated_launch.py +++ b/bot_bottle/backend/macos_container/consolidated_launch.py @@ -39,7 +39,7 @@ from ...git_gate import GitGatePlan from ...log import info from ...orchestrator.client import OrchestratorClient, OrchestratorClientError from ...orchestrator.reprovision import reprovision_bottles -from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME +from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME from ..consolidated_util import ( provision_bottle, teardown_consolidated as _teardown_util, diff --git a/bot_bottle/backend/macos_container/launch.py b/bot_bottle/backend/macos_container/launch.py index d54a20ac..95f5d611 100644 --- a/bot_bottle/backend/macos_container/launch.py +++ b/bot_bottle/backend/macos_container/launch.py @@ -56,7 +56,7 @@ from ...gateway.git_http_backend import DEFAULT_PORT as _GIT_HTTP_PORT from ...image_cache import check_stale from ...log import die, info, warn from .. import BottleImages -from ...supervise import SUPERVISE_PORT +from ...supervisor.types import SUPERVISE_PORT from ..docker.egress import EGRESS_PORT from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH from . import util as container_mod @@ -68,8 +68,8 @@ from .gateway_hosts import ( ) from . import nested_containers as nested_containers_mod from .bottle_plan import MacosContainerBottlePlan -from ...orchestrator.config_store import resolve_teardown_timeout -from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME, new_env_var_secret +from ...orchestrator.store.config_store import resolve_teardown_timeout +from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME, new_env_var_secret from .consolidated_launch import ( GatewayEndpoint, ensure_gateway, diff --git a/bot_bottle/backend/macos_container/resolve_plan.py b/bot_bottle/backend/macos_container/resolve_plan.py index 4c9883ef..f8fb9758 100644 --- a/bot_bottle/backend/macos_container/resolve_plan.py +++ b/bot_bottle/backend/macos_container/resolve_plan.py @@ -8,7 +8,7 @@ from ...agent_provider import AgentProvisionPlan from ...egress import EgressPlan from ...env import ResolvedEnv from ...git_gate import GitGatePlan -from ...supervise import SupervisePlan +from ...supervisor.plan import SupervisePlan from ...manifest import Manifest from .. import BottleSpec from . import util as container_mod diff --git a/bot_bottle/backend/resolve_common.py b/bot_bottle/backend/resolve_common.py index 88f94ff9..612291bb 100644 --- a/bot_bottle/backend/resolve_common.py +++ b/bot_bottle/backend/resolve_common.py @@ -28,7 +28,8 @@ from ..egress import Egress, EgressPlan from ..git_gate import GitGate, GitGatePlan from ..log import die from ..manifest import Manifest, ManifestBottle -from ..supervise import Supervise, SupervisePlan +from ..supervisor.plan import SupervisePlan +from ..orchestrator.supervisor.supervise import Supervise from . import BottleSpec diff --git a/bot_bottle/cli/__init__.py b/bot_bottle/cli/__init__.py index c1618b66..d1d550ac 100644 --- a/bot_bottle/cli/__init__.py +++ b/bot_bottle/cli/__init__.py @@ -10,7 +10,7 @@ import sys from ..errors import MissingEnvVarError from ..log import Die, die, error from ..manifest import ManifestError -from ..store.store_manager import StoreManager +from ..orchestrator.store.store_manager import StoreManager from ._common import PROG from . import list as _list_mod from .backend import cmd_backend diff --git a/bot_bottle/cli/supervise.py b/bot_bottle/cli/supervise.py index d87726c9..abf7d70e 100644 --- a/bot_bottle/cli/supervise.py +++ b/bot_bottle/cli/supervise.py @@ -27,7 +27,7 @@ from ..orchestrator.client import ( discover_orchestrator_url, ) -from ..supervise import ( +from ..supervisor.types import ( Proposal, TOOL_EGRESS_ALLOW, TOOL_EGRESS_BLOCK, diff --git a/bot_bottle/gateway/egress_addon.py b/bot_bottle/gateway/egress_addon.py index 112de8b8..eda9402e 100644 --- a/bot_bottle/gateway/egress_addon.py +++ b/bot_bottle/gateway/egress_addon.py @@ -41,7 +41,7 @@ from bot_bottle.gateway.egress_addon_core import ( scan_outbound, ) from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver -from bot_bottle.supervise_types import ( +from bot_bottle.supervisor.types import ( STATUS_APPROVED, STATUS_MODIFIED, STATUSES, diff --git a/bot_bottle/gateway/git_gate_render.py b/bot_bottle/gateway/git_gate_render.py index 294d6756..d6fe825c 100644 --- a/bot_bottle/gateway/git_gate_render.py +++ b/bot_bottle/gateway/git_gate_render.py @@ -283,10 +283,10 @@ from pathlib import Path # calling bottle exactly as a direct write once did. try: from bot_bottle.gateway.policy_resolver import PolicyResolver, PolicyResolveError - from bot_bottle.supervise_types import TOOL_GITLEAKS_ALLOW + from bot_bottle.supervisor.types import TOOL_GITLEAKS_ALLOW except ImportError: from policy_resolver import PolicyResolver, PolicyResolveError - from supervise_types import TOOL_GITLEAKS_ALLOW + from supervisor.types import TOOL_GITLEAKS_ALLOW report_path = Path(sys.argv[1]) source_ip = os.environ.get("SUPERVISE_SOURCE_IP", "") diff --git a/bot_bottle/gateway/supervise_server.py b/bot_bottle/gateway/supervise_server.py index 26495a3d..520f1dae 100644 --- a/bot_bottle/gateway/supervise_server.py +++ b/bot_bottle/gateway/supervise_server.py @@ -62,7 +62,7 @@ from bot_bottle.gateway.egress_addon_core import ( LOG_OFF, load_config, resolve_client_context, route_to_yaml_dict, ) from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver -from bot_bottle import supervise as _sv +from bot_bottle.supervisor import types as _sv # --- JSON-RPC / MCP plumbing ---------------------------------------------- diff --git a/bot_bottle/orchestrator/__main__.py b/bot_bottle/orchestrator/__main__.py index 38d4e0c1..cd532f9f 100644 --- a/bot_bottle/orchestrator/__main__.py +++ b/bot_bottle/orchestrator/__main__.py @@ -16,7 +16,7 @@ import secrets from pathlib import Path from .. import log -from ..store.store_manager import StoreManager +from .store.store_manager import StoreManager from .broker import LaunchBroker, StubBroker from .control_plane import make_server from .docker_broker import DockerBroker diff --git a/bot_bottle/orchestrator/control_plane.py b/bot_bottle/orchestrator/control_plane.py index 5760923d..4daed700 100644 --- a/bot_bottle/orchestrator/control_plane.py +++ b/bot_bottle/orchestrator/control_plane.py @@ -65,7 +65,7 @@ from urllib.parse import urlsplit from ..control_auth import ROLE_CLI, ROLES, verify from ..paths import CONTROL_PLANE_TOKEN_ENV -from ..supervise_types import TOOLS +from ..supervisor.types import TOOLS from .service import Orchestrator # JSON body payload type (parsed request / rendered response). diff --git a/bot_bottle/orchestrator/service.py b/bot_bottle/orchestrator/service.py index d521937c..bf6a14dd 100644 --- a/bot_bottle/orchestrator/service.py +++ b/bot_bottle/orchestrator/service.py @@ -27,7 +27,7 @@ from datetime import datetime, timezone from .broker import LaunchBroker, LaunchRequest, sign_request from .registry import DEFAULT_REAP_GRACE_SECONDS, BottleRecord, RegistryStore -from ..supervise import ( +from .supervisor import ( AuditEntry, COMPONENT_FOR_TOOL, POLL_STATUS_PENDING, @@ -104,7 +104,7 @@ class Orchestrator: if tokens: self._tokens[rec.bottle_id] = dict(tokens) if env_var_secret: - from .secret_store import encrypt_value + from .store.secret_store import encrypt_value encrypted = {k: encrypt_value(env_var_secret, v) for k, v in tokens.items()} self.registry.store_agent_secrets(rec.bottle_id, encrypted) req = LaunchRequest( @@ -367,7 +367,7 @@ class Orchestrator: value with *env_var_secret*, and restores ``_tokens[bottle_id]``. Returns True on success, False when no stored secrets exist for this bottle or decryption fails (wrong key / corrupt data).""" - from .secret_store import decrypt_value + from .store.secret_store import decrypt_value encrypted = self.registry.get_agent_secrets(bottle_id) if not encrypted: return False diff --git a/bot_bottle/orchestrator/store/__init__.py b/bot_bottle/orchestrator/store/__init__.py new file mode 100644 index 00000000..1c4a81d1 --- /dev/null +++ b/bot_bottle/orchestrator/store/__init__.py @@ -0,0 +1,10 @@ +"""Orchestrator-owned SQLite stores. + +The stores whose tables only the orchestrator (control plane) opens: the +supervise proposal/response `queue_store`, the agent-secret `secret_store`, and +the orchestrator `config_store`. They build on the shared `DbStore` / +`migrations` base in `bot_bottle.store`. + +Callers import the concrete module directly, e.g. +`from bot_bottle.orchestrator.store.queue_store import QueueStore`. +""" diff --git a/bot_bottle/orchestrator/config_store.py b/bot_bottle/orchestrator/store/config_store.py similarity index 96% rename from bot_bottle/orchestrator/config_store.py rename to bot_bottle/orchestrator/store/config_store.py index 7779b68a..251f0ef6 100644 --- a/bot_bottle/orchestrator/config_store.py +++ b/bot_bottle/orchestrator/store/config_store.py @@ -11,9 +11,9 @@ import os import sqlite3 from pathlib import Path -from ..store.db_store import DbStore -from ..store.migrations import TableMigrations -from ..paths import host_db_path +from ...store.db_store import DbStore +from ...store.migrations import TableMigrations +from ...paths import host_db_path TEARDOWN_TIMEOUT_ENV = "BOT_BOTTLE_ORCHESTRATOR_TEARDOWN_TIMEOUT_SECONDS" DEFAULT_TEARDOWN_TIMEOUT_SECONDS = 30.0 diff --git a/bot_bottle/store/queue_store.py b/bot_bottle/orchestrator/store/queue_store.py similarity index 96% rename from bot_bottle/store/queue_store.py rename to bot_bottle/orchestrator/store/queue_store.py index 09ef1cf3..16637f94 100644 --- a/bot_bottle/store/queue_store.py +++ b/bot_bottle/orchestrator/store/queue_store.py @@ -7,12 +7,12 @@ import sqlite3 from pathlib import Path try: - from ..supervise_types import Proposal, Response - from ..paths import host_db_path - from .db_store import DbStore - from .migrations import TableMigrations + from ...supervisor.types import Proposal, Response + from ...paths import host_db_path + from ...store.db_store import DbStore + from ...store.migrations import TableMigrations except ImportError: - from supervise_types import Proposal, Response # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module + from supervisor.types import Proposal, Response # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module from paths import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module from db_store import DbStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module diff --git a/bot_bottle/orchestrator/secret_store.py b/bot_bottle/orchestrator/store/secret_store.py similarity index 100% rename from bot_bottle/orchestrator/secret_store.py rename to bot_bottle/orchestrator/store/secret_store.py diff --git a/bot_bottle/store/store_manager.py b/bot_bottle/orchestrator/store/store_manager.py similarity index 54% rename from bot_bottle/store/store_manager.py rename to bot_bottle/orchestrator/store/store_manager.py index e7729cc2..2e40e0fe 100644 --- a/bot_bottle/store/store_manager.py +++ b/bot_bottle/orchestrator/store/store_manager.py @@ -1,34 +1,34 @@ -"""Singleton manager for all bot-bottle SQLite stores (PRD 0013).""" +"""The orchestrator's store manager (PRD 0013 / 0070). + +One store manager per service. Post-PRD-0070 the orchestrator is the sole +opener of `bot-bottle.db`, so it owns migrating every table in it: the supervise +`queue_store` (local to `orchestrator.store`) plus the `audit_store` and +`config_store` from the shared `bot_bottle.store` base. The data plane never +touches these — it reaches state over the control-plane RPC. +""" from __future__ import annotations from pathlib import Path -try: - from .audit_store import AuditStore - from .config_store import ConfigStore - from .queue_store import QueueStore -except ImportError: - from audit_store import AuditStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from config_store import ConfigStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from queue_store import QueueStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module +from .queue_store import QueueStore +from ...store.audit_store import AuditStore +from ...store.config_store import ConfigStore _instance: StoreManager | None = None class StoreManager: - """Owns db_path and delegates migrate/is_migrated across all stores. + """Owns db_path and delegates migrate/is_migrated across the orchestrator's + stores. - Use instance() for normal access. Call reset(db_path) in tests to swap - the singleton to a temp path, then reset() with no args to restore the + Use instance() for normal access. Call reset(db_path) in tests to swap the + singleton to a temp path, then reset() with no args to restore the default.""" def __init__(self, db_path: Path | None = None) -> None: if db_path is None: - try: - from ..paths import host_db_path - except ImportError: - from paths import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module + from ...paths import host_db_path db_path = host_db_path() self.db_path = db_path diff --git a/bot_bottle/orchestrator/supervisor/__init__.py b/bot_bottle/orchestrator/supervisor/__init__.py new file mode 100644 index 00000000..c6d903e6 --- /dev/null +++ b/bot_bottle/orchestrator/supervisor/__init__.py @@ -0,0 +1,34 @@ +"""The orchestrator's view of the supervise plane. + +Bundles the orchestrator-owned supervise surface: the queue/audit I/O and diff +rendering (`queue`) and the `Supervise` lifecycle (`supervise`). For +convenience it re-exports the neutral vocabulary (`bot_bottle.supervisor`'s +types + `SupervisePlan`) so orchestrator-side callers — service, CLI, tests — +import from one place. + +The data plane must NOT import this package; it imports `bot_bottle.supervisor` +(neutral) directly and reaches the queue over the control-plane RPC. +""" + +from __future__ import annotations + +from ...supervisor.types import * # noqa: F401,F403 — re-export the neutral vocabulary +from ...supervisor.plan import SupervisePlan +from .queue import ( + archive_all_proposals, + archive_proposal, + audit_dir, + audit_log_path, + list_all_pending_proposals, + list_pending_proposals, + read_audit_entries, + read_proposal, + read_response, + render_diff, + sha256_hex, + wait_for_response, + write_audit_entry, + write_proposal, + write_response, +) +from .supervise import Supervise diff --git a/bot_bottle/orchestrator/supervisor/queue.py b/bot_bottle/orchestrator/supervisor/queue.py new file mode 100644 index 00000000..fed41325 --- /dev/null +++ b/bot_bottle/orchestrator/supervisor/queue.py @@ -0,0 +1,141 @@ +"""Orchestrator-side supervise queue + audit I/O and diff rendering. + +The host-side library the orchestrator uses to drive the supervise flow: read / +write proposals and responses against the queue store, append audit entries, +and render operator diffs. Only the orchestrator opens `bot-bottle.db` (PRD +0070), so this lives under `orchestrator/` — the data plane reaches it over the +control-plane RPC, never by importing this module. +""" + +from __future__ import annotations + +import difflib +import hashlib +import time +from pathlib import Path + +from ...supervisor.types import ( + AuditEntry, + DEFAULT_POLL_INTERVAL_SEC, + Proposal, + Response, +) +from ...paths import bot_bottle_root +from ..store.queue_store import QueueStore +from ...store.audit_store import AuditStore + + +# --- Paths ----------------------------------------------------------------- + + +def audit_dir() -> Path: + return bot_bottle_root() / "audit" + + +def audit_log_path(component: str, slug: str) -> Path: + return audit_dir() / f"{component}-{slug}.log" + + +# --- Queue I/O ------------------------------------------------------------- + + +def write_proposal(proposal: Proposal) -> Path: + """Persist `proposal` in the queue database, mode 0o600. + Directory is created if missing.""" + return QueueStore(proposal.bottle_slug).write_proposal(proposal) + + +def read_proposal(bottle_slug: str, proposal_id: str) -> Proposal: + return QueueStore(bottle_slug).read_proposal(proposal_id) + + +def list_pending_proposals(bottle_slug: str) -> list[Proposal]: + """All proposals for `bottle_slug` that do not yet have a matching + response. Sorted by `arrival_timestamp` so the operator sees the queue + FIFO.""" + return QueueStore(bottle_slug).list_pending_proposals() + + +def list_all_pending_proposals() -> list[Proposal]: + """All pending proposals across bottles, sorted FIFO.""" + return QueueStore("").list_all_pending_proposals() + + +def write_response(bottle_slug: str, response: Response) -> Path: + return QueueStore(bottle_slug).write_response(response) + + +def read_response(bottle_slug: str, proposal_id: str) -> Response: + return QueueStore(bottle_slug).read_response(proposal_id) + + +def wait_for_response( + bottle_slug: str, + proposal_id: str, + *, + poll_interval: float = DEFAULT_POLL_INTERVAL_SEC, + deadline: float | None = None, +) -> Response: + """Block until a response file appears for `proposal_id`, then return it. + `deadline` is an absolute time.monotonic() value after which the wait raises + TimeoutError. None waits forever — the natural shape, since the operator's + response time is unbounded. + + Polls SQLite so the implementation stays portable and stdlib-only.""" + store = QueueStore(bottle_slug) + while True: + try: + return store.read_response(proposal_id) + except FileNotFoundError: + pass + if deadline is not None and time.monotonic() >= deadline: + raise TimeoutError(f"no response for proposal {proposal_id!r}") + time.sleep(poll_interval) + + +def archive_proposal(bottle_slug: str, proposal_id: str) -> None: + """Mark both proposal and response rows processed. + Idempotent — missing rows are silently skipped.""" + QueueStore(bottle_slug).archive_proposal(proposal_id) + + +def archive_all_proposals(bottle_slug: str) -> None: + """Archive every proposal + response for `bottle_slug` (bottle teardown / + reconcile cleanup). Idempotent.""" + QueueStore(bottle_slug).archive_all() + + +# --- Audit log ------------------------------------------------------------- + + +def write_audit_entry(entry: AuditEntry) -> Path: + """Append `entry` to the host supervise audit table.""" + return AuditStore().write_audit_entry(entry) + + +def read_audit_entries(component: str, slug: str) -> list[AuditEntry]: + """Load all audit entries for the given component+slug.""" + return AuditStore().read_audit_entries(component, slug) + + +# --- Diff rendering -------------------------------------------------------- + + +def render_diff(before: str, after: str, *, label: str = "config") -> str: + """Unified diff suitable for the audit log + TUI. Empty diff (no changes) + renders as the empty string.""" + diff = difflib.unified_diff( + before.splitlines(keepends=True), + after.splitlines(keepends=True), + fromfile=f"{label} (current)", + tofile=f"{label} (proposed)", + lineterm="", + ) + parts = list(diff) + if not parts: + return "" + return "".join(p if p.endswith("\n") else p + "\n" for p in parts).rstrip("\n") + + +def sha256_hex(content: str) -> str: + return hashlib.sha256(content.encode("utf-8")).hexdigest() diff --git a/bot_bottle/orchestrator/supervisor/supervise.py b/bot_bottle/orchestrator/supervisor/supervise.py new file mode 100644 index 00000000..f2e51fe9 --- /dev/null +++ b/bot_bottle/orchestrator/supervisor/supervise.py @@ -0,0 +1,36 @@ +"""The `Supervise` lifecycle — host-side database staging (PRD 0013 / 0070). + +`Supervise.prepare` migrates the orchestrator's `bot-bottle.db` and returns the +neutral `SupervisePlan`. It touches the orchestrator store manager, so it lives +here rather than in the neutral `bot_bottle.supervisor` package; the backend +(which drives launch) may import it — backend → orchestrator is an allowed +direction, unlike gateway → orchestrator. +""" + +from __future__ import annotations + +from abc import ABC +from pathlib import Path + +from ..store.store_manager import StoreManager +from ...supervisor.plan import SupervisePlan + + +class Supervise(ABC): + """Per-bottle supervise daemon. Encapsulates host-side database staging; + the gateway's start/stop lifecycle is backend-specific.""" + + def prepare( + self, + slug: str, + stage_dir: Path, + ) -> SupervisePlan: + """Stage the host database. Returns the plan; `internal_network` must be + set by the launch step before .start runs.""" + del stage_dir + mgr = StoreManager.instance() + mgr.migrate() + return SupervisePlan( + slug=slug, + db_path=mgr.db_path, + ) diff --git a/bot_bottle/store/__init__.py b/bot_bottle/store/__init__.py index ec5e39b4..0a7f1f32 100644 --- a/bot_bottle/store/__init__.py +++ b/bot_bottle/store/__init__.py @@ -1,9 +1,13 @@ -"""SQLite-backed persistence for bot-bottle. +"""SQLite-backed persistence base for bot-bottle. -The store family: a shared `DbStore` base (versioned `migrations`) and the -concrete stores built on it — `ConfigStore`, `AuditStore`, `QueueStore` — plus -`StoreManager`, which migrates them together against the one per-host DB. +The shared store base: the `DbStore` class (versioned `migrations`) and the +concrete stores built on it that aren't owned by a single service — +`ConfigStore` and `AuditStore`. + +Service-owned stores live under their service: the supervise `queue_store`, the +`secret_store`, the orchestrator `config_store`, and the orchestrator +`store_manager` are under `bot_bottle.orchestrator.store`. Callers import the concrete module they need directly, e.g. -`from bot_bottle.store.store_manager import StoreManager`. +`from bot_bottle.store.db_store import DbStore`. """ diff --git a/bot_bottle/store/audit_store.py b/bot_bottle/store/audit_store.py index 13de4707..3a6738b4 100644 --- a/bot_bottle/store/audit_store.py +++ b/bot_bottle/store/audit_store.py @@ -6,12 +6,12 @@ import sqlite3 from pathlib import Path try: - from ..supervise_types import AuditEntry + from ..supervisor.types import AuditEntry from ..paths import host_db_path from .db_store import DbStore from .migrations import TableMigrations except ImportError: - from supervise_types import AuditEntry # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module + from supervisor.types import AuditEntry # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module from paths import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module from db_store import DbStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module diff --git a/bot_bottle/supervise.py b/bot_bottle/supervise.py deleted file mode 100644 index d71253ce..00000000 --- a/bot_bottle/supervise.py +++ /dev/null @@ -1,298 +0,0 @@ -"""Per-bottle supervise plane (PRD 0013). - -The supervise plane is the per-bottle MCP daemon plus its host-side -queue/audit support. The daemon (bot_bottle.gateway.supervise_server) -sits on the bottle's internal network and exposes MCP tools the agent -calls when it needs an operator-reviewed egress change: - - * egress-block / allow — agent proposes a new routes.yaml - -Each tool call: the agent passes the full proposed file plus a -justification text. The gateway validates the proposal syntactically, -writes it to the host SQLite queue table, and holds the tool-call -connection open. The operator's supervise TUI -(bot_bottle.cli.supervise) sees the proposal, accepts -approve / modify / reject, and writes a response row. The gateway sees -the response and returns `{status, notes}` to the agent. - -This module defines the host-side library: dataclasses for the queue -record shapes, queue read/write helpers, the audit log writer, and the -diff renderer. The in-gateway daemon lives in -bot_bottle/gateway/supervise_server.py; the supervise daemon's container -lifecycle is owned by the gateway (PRD 0024). - -For 0013 the supervisor's approval handlers are deliberately no-ops: -on approval the audit log is written and the response file is -delivered to the agent, but no host-side config change happens. The -remediation engines that wire real config changes land in PRDs 0014, -0015, and 0016. -""" - -from __future__ import annotations - -import difflib -import hashlib -import time -from abc import ABC -from dataclasses import dataclass -from pathlib import Path - -from .supervise_types import ( - ACTION_OPERATOR_EDIT, - AuditEntry, - POLL_STATUS_PENDING, - POLL_STATUS_UNKNOWN, - Proposal, - Response, - STATUSES, - STATUS_APPROVED, - STATUS_MODIFIED, - STATUS_REJECTED, - TOOLS, - TOOL_CHECK_PROPOSAL, - TOOL_EGRESS_ALLOW, - TOOL_EGRESS_BLOCK, - TOOL_EGRESS_TOKEN_ALLOW, - TOOL_GITLEAKS_ALLOW, - TOOL_LIST_EGRESS_ROUTES, -) - - -try: - from .paths import bot_bottle_root -except ImportError: # flat imports inside the gateway - from paths import bot_bottle_root # type: ignore[import-not-found,no-redef] # pylint: disable=import-error,no-name-in-module - - -SUPERVISE_HOSTNAME = "supervise" -SUPERVISE_PORT = 9100 - -# The supervise daemon uses these to query egress's -# introspection endpoint for the `list-egress-routes` MCP -# tool. The hostname + port match egress's docker network -# listen port (see backend.docker.egress.EGRESS_PORT). The supervise -# daemon runs inside the gateway alongside egress, so loopback -# is the stable address across docker, firecracker, and Apple -# Container backends. -EGRESS_FORWARD_PROXY = "http://127.0.0.1:9099" -EGRESS_INTROSPECT_URL = "http://_egress.local/allowlist" - -COMPONENT_FOR_TOOL: dict[str, str] = { - TOOL_EGRESS_ALLOW: "egress", - TOOL_EGRESS_BLOCK: "egress", -} - -DB_PATH_IN_CONTAINER = "/run/supervise/bot-bottle.db" -DEFAULT_POLL_INTERVAL_SEC = 0.5 - - -# --- Paths ----------------------------------------------------------------- - - -def audit_dir() -> Path: - return bot_bottle_root() / "audit" - - -def audit_log_path(component: str, slug: str) -> Path: - return audit_dir() / f"{component}-{slug}.log" - - -try: - from .store.queue_store import QueueStore - from .store.audit_store import AuditStore - from .store.store_manager import StoreManager -except ImportError: - # Gateway: files are flat-copied under /app, not a package. - from queue_store import QueueStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from audit_store import AuditStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from store_manager import StoreManager # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - - -# --- Queue I/O ------------------------------------------------------------- - - -def write_proposal(proposal: Proposal) -> Path: - """Persist `proposal` in the queue database, mode 0o600. - Directory is created if missing.""" - return QueueStore(proposal.bottle_slug).write_proposal(proposal) - - -def read_proposal(bottle_slug: str, proposal_id: str) -> Proposal: - return QueueStore(bottle_slug).read_proposal(proposal_id) - - -def list_pending_proposals(bottle_slug: str) -> list[Proposal]: - """All proposals for `bottle_slug` that do not yet have a matching - response. Sorted by `arrival_timestamp` so the operator - sees the queue FIFO.""" - return QueueStore(bottle_slug).list_pending_proposals() - - -def list_all_pending_proposals() -> list[Proposal]: - """All pending proposals across bottles, sorted FIFO.""" - return QueueStore("").list_all_pending_proposals() - - -def write_response(bottle_slug: str, response: Response) -> Path: - return QueueStore(bottle_slug).write_response(response) - - -def read_response(bottle_slug: str, proposal_id: str) -> Response: - return QueueStore(bottle_slug).read_response(proposal_id) - - -def wait_for_response( - bottle_slug: str, - proposal_id: str, - *, - poll_interval: float = DEFAULT_POLL_INTERVAL_SEC, - deadline: float | None = None, -) -> Response: - """Block until a response file appears for `proposal_id`, then - return it. `deadline` is an absolute time.monotonic() value after - which the wait raises TimeoutError. None waits forever — the - natural shape, since the operator's response time is unbounded. - - Polls SQLite so the implementation stays portable and stdlib-only.""" - store = QueueStore(bottle_slug) - while True: - try: - return store.read_response(proposal_id) - except FileNotFoundError: - pass - if deadline is not None and time.monotonic() >= deadline: - raise TimeoutError(f"no response for proposal {proposal_id!r}") - time.sleep(poll_interval) - - -def archive_proposal(bottle_slug: str, proposal_id: str) -> None: - """Mark both proposal and response rows processed. - Idempotent — missing rows are silently skipped.""" - QueueStore(bottle_slug).archive_proposal(proposal_id) - - -def archive_all_proposals(bottle_slug: str) -> None: - """Archive every proposal + response for `bottle_slug` (bottle teardown / - reconcile cleanup). Idempotent.""" - QueueStore(bottle_slug).archive_all() - - -# --- Audit log ------------------------------------------------------------- - - -def write_audit_entry(entry: AuditEntry) -> Path: - """Append `entry` to the host supervise audit table.""" - return AuditStore().write_audit_entry(entry) - - -def read_audit_entries(component: str, slug: str) -> list[AuditEntry]: - """Load all audit entries for the given component+slug.""" - return AuditStore().read_audit_entries(component, slug) - - -# --- Diff rendering -------------------------------------------------------- - - -def render_diff(before: str, after: str, *, label: str = "config") -> str: - """Unified diff suitable for the audit log + TUI. Empty diff (no - changes) renders as the empty string.""" - diff = difflib.unified_diff( - before.splitlines(keepends=True), - after.splitlines(keepends=True), - fromfile=f"{label} (current)", - tofile=f"{label} (proposed)", - lineterm="", - ) - parts = list(diff) - if not parts: - return "" - return "".join(p if p.endswith("\n") else p + "\n" for p in parts).rstrip("\n") - - -def sha256_hex(content: str) -> str: - return hashlib.sha256(content.encode("utf-8")).hexdigest() - - -# --- Gateway plan + abstract lifecycle ------------------------------------- - - -@dataclass(frozen=True) -class SupervisePlan: - """Output of Supervise.prepare; consumed by .start. - - `db_path` is the host database bind-mounted into the gateway at - /run/supervise/bot-bottle.db. `internal_network` is empty at - prepare time; the backend's launch step fills it via - dataclasses.replace before calling .start.""" - - slug: str - db_path: Path - internal_network: str = "" - - -class Supervise(ABC): - """Per-bottle supervise daemon. Encapsulates host-side database - staging; the gateway's start/stop lifecycle is backend-specific.""" - - def prepare( - self, - slug: str, - stage_dir: Path, - ) -> SupervisePlan: - """Stage the host database. Returns the plan; `internal_network` - must be set by the launch step before .start runs.""" - del stage_dir - mgr = StoreManager.instance() - mgr.migrate() - return SupervisePlan( - slug=slug, - db_path=mgr.db_path, - ) - - -__all__ = [ - "ACTION_OPERATOR_EDIT", - "AuditEntry", - "AuditStore", - "COMPONENT_FOR_TOOL", - "DEFAULT_POLL_INTERVAL_SEC", - "DB_PATH_IN_CONTAINER", - "Proposal", - "QueueStore", - "Response", - "StoreManager", - "STATUSES", - "STATUS_APPROVED", - "STATUS_MODIFIED", - "STATUS_REJECTED", - "POLL_STATUS_PENDING", - "POLL_STATUS_UNKNOWN", - "SUPERVISE_HOSTNAME", - "SUPERVISE_PORT", - "Supervise", - "SupervisePlan", - "TOOLS", - "EGRESS_FORWARD_PROXY", - "EGRESS_INTROSPECT_URL", - "TOOL_CHECK_PROPOSAL", - "TOOL_EGRESS_ALLOW", - "TOOL_EGRESS_BLOCK", - "TOOL_GITLEAKS_ALLOW", - "TOOL_EGRESS_TOKEN_ALLOW", - "TOOL_LIST_EGRESS_ROUTES", - "archive_proposal", - "archive_all_proposals", - "audit_dir", - "audit_log_path", - "list_pending_proposals", - "list_all_pending_proposals", - "read_audit_entries", - "read_proposal", - "read_response", - "render_diff", - "sha256_hex", - "wait_for_response", - "write_audit_entry", - "write_proposal", - "write_response", -] diff --git a/bot_bottle/supervisor/__init__.py b/bot_bottle/supervisor/__init__.py new file mode 100644 index 00000000..d76c1e9f --- /dev/null +++ b/bot_bottle/supervisor/__init__.py @@ -0,0 +1,16 @@ +"""Neutral supervise vocabulary — importable by every tier. + +The supervise plane spans the control plane (orchestrator) and the data plane +(gateway `supervise_server`): the gateway proposes/polls over RPC, the +orchestrator queues and applies operator decisions. Both sides must agree on +the wire types and constants, so those live here — a neutral package neither +tier's private code, importable by `bot_bottle.gateway` **and** +`bot_bottle.orchestrator` without either depending on the other. + + * `types` — the `Proposal`/`Response`/`AuditEntry` dataclasses, the tool / + status / poll-status constants, and the shared daemon constants. + * `plan` — `SupervisePlan`, the launch-time staging DTO. + +The orchestrator-only half (queue I/O, diff rendering, the `Supervise` +lifecycle) lives under `bot_bottle.orchestrator.supervisor`. +""" diff --git a/bot_bottle/supervisor/plan.py b/bot_bottle/supervisor/plan.py new file mode 100644 index 00000000..2962d82f --- /dev/null +++ b/bot_bottle/supervisor/plan.py @@ -0,0 +1,26 @@ +"""`SupervisePlan` — the launch-time supervise staging DTO. + +A pure value type (no store access), so it lives in the neutral package: the +backend builds it at launch and the orchestrator's `Supervise.prepare` returns +it. The behaviour that fills it in — staging the host database — is the +orchestrator's, in `bot_bottle.orchestrator.supervisor.supervise`. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class SupervisePlan: + """Output of `Supervise.prepare`; consumed by `.start`. + + `db_path` is the host database bind-mounted into the gateway at + /run/supervise/bot-bottle.db. `internal_network` is empty at prepare time; + the backend's launch step fills it via `dataclasses.replace` before calling + `.start`.""" + + slug: str + db_path: Path + internal_network: str = "" diff --git a/bot_bottle/supervise_types.py b/bot_bottle/supervisor/types.py similarity index 84% rename from bot_bottle/supervise_types.py rename to bot_bottle/supervisor/types.py index 6bb65e5d..e09fbde2 100644 --- a/bot_bottle/supervise_types.py +++ b/bot_bottle/supervisor/types.py @@ -48,6 +48,24 @@ POLL_STATUS_UNKNOWN = "unknown" ACTION_OPERATOR_EDIT = "operator-edit" +# Shared supervise-daemon constants. The gateway `supervise_server` and the +# backend read these, so they live with the wire types in the neutral package +# (never in the orchestrator's private code the data plane can't import). +SUPERVISE_HOSTNAME = "supervise" +SUPERVISE_PORT = 9100 +# The supervise daemon queries egress's introspection endpoint for the +# `list-egress-routes` tool over loopback (egress runs alongside it in the +# gateway), stable across docker / firecracker / Apple Container backends. +EGRESS_FORWARD_PROXY = "http://127.0.0.1:9099" +EGRESS_INTROSPECT_URL = "http://_egress.local/allowlist" +DB_PATH_IN_CONTAINER = "/run/supervise/bot-bottle.db" +DEFAULT_POLL_INTERVAL_SEC = 0.5 + +COMPONENT_FOR_TOOL: dict[str, str] = { + TOOL_EGRESS_ALLOW: "egress", + TOOL_EGRESS_BLOCK: "egress", +} + def _require_str(raw: dict[str, object], key: str) -> str: value = raw.get(key) @@ -175,4 +193,11 @@ __all__ = [ "TOOL_EGRESS_TOKEN_ALLOW", "TOOL_GITLEAKS_ALLOW", "TOOL_LIST_EGRESS_ROUTES", + "SUPERVISE_HOSTNAME", + "SUPERVISE_PORT", + "EGRESS_FORWARD_PROXY", + "EGRESS_INTROSPECT_URL", + "DB_PATH_IN_CONTAINER", + "DEFAULT_POLL_INTERVAL_SEC", + "COMPONENT_FOR_TOOL", ] diff --git a/tests/integration/test_gateway_image.py b/tests/integration/test_gateway_image.py index 624bda9b..213c93b6 100644 --- a/tests/integration/test_gateway_image.py +++ b/tests/integration/test_gateway_image.py @@ -91,7 +91,7 @@ class TestGatewayImage(unittest.TestCase): # Probe that the package imports resolve inside the image. rc, out = self._run_in_image( "python3", "-c", - "from bot_bottle import supervise; from bot_bottle.gateway import supervise_server; print('ok')", + "from bot_bottle.supervisor import types; from bot_bottle.gateway import supervise_server; print('ok')", ) self.assertEqual(0, rc, msg=out) self.assertIn("ok", out) diff --git a/tests/unit/_docker_bottle_plan.py b/tests/unit/_docker_bottle_plan.py index c3a16c2f..78f4b19e 100644 --- a/tests/unit/_docker_bottle_plan.py +++ b/tests/unit/_docker_bottle_plan.py @@ -16,7 +16,7 @@ from bot_bottle.backend.docker.bottle_plan import DockerBottlePlan from bot_bottle.egress import EgressPlan, EgressRoute from bot_bottle.git_gate import GitGatePlan, GitGateUpstream from bot_bottle.manifest import ManifestIndex -from bot_bottle.supervise import SupervisePlan +from bot_bottle.supervisor.plan import SupervisePlan SLUG = "demo-abc12" diff --git a/tests/unit/test_cli_dispatch.py b/tests/unit/test_cli_dispatch.py index e5e10d54..bdc37bf9 100644 --- a/tests/unit/test_cli_dispatch.py +++ b/tests/unit/test_cli_dispatch.py @@ -15,7 +15,7 @@ from bot_bottle.cli import main from bot_bottle.store.db_store import DbStore from bot_bottle.log import Die from bot_bottle.manifest import ManifestError -from bot_bottle.store.store_manager import StoreManager +from bot_bottle.orchestrator.store.store_manager import StoreManager class TestMainDispatch(unittest.TestCase): diff --git a/tests/unit/test_config_store.py b/tests/unit/test_config_store.py index 0f40576c..b47cb6dc 100644 --- a/tests/unit/test_config_store.py +++ b/tests/unit/test_config_store.py @@ -11,7 +11,7 @@ from bot_bottle.store.config_store import ( DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS, ConfigStore, ) -from bot_bottle.store.store_manager import StoreManager +from bot_bottle.orchestrator.store.store_manager import StoreManager class TestConfigStore(unittest.TestCase): diff --git a/tests/unit/test_contrib_claude_provider.py b/tests/unit/test_contrib_claude_provider.py index 0e8589b2..1dc5db3c 100644 --- a/tests/unit/test_contrib_claude_provider.py +++ b/tests/unit/test_contrib_claude_provider.py @@ -25,7 +25,7 @@ from bot_bottle.contrib.claude.agent_provider import ClaudeAgentProvider from bot_bottle.egress import EgressPlan from bot_bottle.git_gate import GitGatePlan from bot_bottle.manifest import ManifestIndex -from bot_bottle.supervise import SupervisePlan +from bot_bottle.supervisor.plan import SupervisePlan _URL = "http://supervise:9100/" diff --git a/tests/unit/test_contrib_codex_provider.py b/tests/unit/test_contrib_codex_provider.py index c53d2fb3..4b2ca1f3 100644 --- a/tests/unit/test_contrib_codex_provider.py +++ b/tests/unit/test_contrib_codex_provider.py @@ -25,7 +25,7 @@ from bot_bottle.contrib.codex.agent_provider import CodexAgentProvider from bot_bottle.egress import EgressPlan from bot_bottle.git_gate import GitGatePlan from bot_bottle.manifest import ManifestIndex -from bot_bottle.supervise import SupervisePlan +from bot_bottle.supervisor.plan import SupervisePlan _URL = "http://supervise:9100/" diff --git a/tests/unit/test_orchestrator_config_store.py b/tests/unit/test_orchestrator_config_store.py index ff439877..538a475d 100644 --- a/tests/unit/test_orchestrator_config_store.py +++ b/tests/unit/test_orchestrator_config_store.py @@ -15,7 +15,7 @@ import unittest from pathlib import Path from types import ModuleType -from bot_bottle.orchestrator.config_store import ( +from bot_bottle.orchestrator.store.config_store import ( DEFAULT_TEARDOWN_TIMEOUT_SECONDS, TEARDOWN_TIMEOUT_ENV, OrchestratorConfigStore, diff --git a/tests/unit/test_orchestrator_control_plane.py b/tests/unit/test_orchestrator_control_plane.py index 1cd06b80..0bacd900 100644 --- a/tests/unit/test_orchestrator_control_plane.py +++ b/tests/unit/test_orchestrator_control_plane.py @@ -24,8 +24,8 @@ from bot_bottle.orchestrator.broker import StubBroker from bot_bottle.orchestrator.control_plane import dispatch, make_server from bot_bottle.orchestrator.registry import BottleRecord, RegistryStore from bot_bottle.orchestrator.service import Orchestrator -from bot_bottle.store.store_manager import StoreManager -from bot_bottle.supervise import ( +from bot_bottle.orchestrator.store.store_manager import StoreManager +from bot_bottle.orchestrator.supervisor import ( Proposal, TOOL_EGRESS_ALLOW, sha256_hex, diff --git a/tests/unit/test_orchestrator_secret_store.py b/tests/unit/test_orchestrator_secret_store.py index 64dd453d..2b0e0d51 100644 --- a/tests/unit/test_orchestrator_secret_store.py +++ b/tests/unit/test_orchestrator_secret_store.py @@ -4,7 +4,7 @@ from __future__ import annotations import unittest -from bot_bottle.orchestrator.secret_store import ( +from bot_bottle.orchestrator.store.secret_store import ( ENV_VAR_SECRET_NAME, decrypt_value, encrypt_value, diff --git a/tests/unit/test_orchestrator_service.py b/tests/unit/test_orchestrator_service.py index 9441654c..a83f8c17 100644 --- a/tests/unit/test_orchestrator_service.py +++ b/tests/unit/test_orchestrator_service.py @@ -14,9 +14,9 @@ from unittest.mock import patch from bot_bottle.orchestrator.broker import LaunchBroker, LaunchRequest, StubBroker from bot_bottle.orchestrator.registry import RegistryStore from bot_bottle.orchestrator.service import Orchestrator -from bot_bottle.orchestrator.secret_store import new_env_var_secret -from bot_bottle.store.store_manager import StoreManager -from bot_bottle.supervise import ( +from bot_bottle.orchestrator.store.secret_store import new_env_var_secret +from bot_bottle.orchestrator.store.store_manager import StoreManager +from bot_bottle.orchestrator.supervisor import ( Proposal, STATUS_APPROVED, TOOL_EGRESS_ALLOW, diff --git a/tests/unit/test_supervise.py b/tests/unit/test_supervise.py index e3360a77..f0f41be2 100644 --- a/tests/unit/test_supervise.py +++ b/tests/unit/test_supervise.py @@ -7,12 +7,12 @@ import unittest from datetime import datetime, timezone from pathlib import Path -from bot_bottle import supervise +from bot_bottle.orchestrator import supervisor as supervise from bot_bottle.paths import host_db_path from tests.unit import use_bottle_root from bot_bottle.store.audit_store import AuditStore -from bot_bottle.store.queue_store import QueueStore -from bot_bottle.supervise import ( +from bot_bottle.orchestrator.store.queue_store import QueueStore +from bot_bottle.orchestrator.supervisor import ( AuditEntry, Proposal, Response, diff --git a/tests/unit/test_supervise_cli.py b/tests/unit/test_supervise_cli.py index b785a43f..f35105b4 100644 --- a/tests/unit/test_supervise_cli.py +++ b/tests/unit/test_supervise_cli.py @@ -13,7 +13,7 @@ from datetime import datetime, timezone from unittest.mock import MagicMock, patch from bot_bottle.cli import supervise as supervise_cli -from bot_bottle.supervise import ( +from bot_bottle.orchestrator.supervisor import ( Proposal, TOOL_EGRESS_ALLOW, TOOL_EGRESS_BLOCK, diff --git a/tests/unit/test_supervise_edge.py b/tests/unit/test_supervise_edge.py index acbd4202..c56d9c55 100644 --- a/tests/unit/test_supervise_edge.py +++ b/tests/unit/test_supervise_edge.py @@ -10,11 +10,11 @@ import unittest from pathlib import Path from unittest.mock import patch -from bot_bottle import supervise +from bot_bottle.orchestrator import supervisor as supervise from bot_bottle.store.audit_store import AuditStore from bot_bottle.paths import bot_bottle_root -from bot_bottle.store.queue_store import QueueStore -from bot_bottle.supervise import ( +from bot_bottle.orchestrator.store.queue_store import QueueStore +from bot_bottle.orchestrator.supervisor import ( AuditEntry, Proposal, STATUS_APPROVED, diff --git a/tests/unit/test_supervise_server.py b/tests/unit/test_supervise_server.py index 7f0f21d7..a082998b 100644 --- a/tests/unit/test_supervise_server.py +++ b/tests/unit/test_supervise_server.py @@ -18,8 +18,8 @@ from pathlib import Path from tests.unit import use_bottle_root -from bot_bottle import supervise as _sv -from bot_bottle.store import queue_store as _qs +from bot_bottle.orchestrator import supervisor as _sv +from bot_bottle.orchestrator.store import queue_store as _qs from bot_bottle.store import audit_store as _as from bot_bottle.gateway import supervise_server # noqa: E402 -- 2.52.0 From 3e2cbcab881a8df0c5b5772d7a257dc6aaa0235c Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 14:24:06 -0400 Subject: [PATCH 06/30] refactor(supervise): fold Supervise into the facade as Supervisor; drop dead flat-fallbacks Move the `Supervise` lifecycle out of its own `orchestrator/supervisor/ supervise.py` and into the package `__init__`, renaming the class to `Supervisor`. Callers now import it from `bot_bottle.orchestrator.supervisor` alongside the queue surface it belongs with. Remove the dead `try/except ImportError` flat-import fallbacks from the package-only store modules (db_store, audit_store, config_store, queue_store) and image_cache. Those fallbacks existed for when the store files were flat-copied into the gateway; post-PRD-0070 the data plane never opens the DB, so these modules are only ever imported as part of the package. The two gateway data-plane files that may still be loaded flat (egress_addon_core, git_gate_render) keep their fallbacks. Full unit suite green (2251). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/backend/resolve_common.py | 4 +-- bot_bottle/image_cache.py | 5 +-- bot_bottle/orchestrator/store/queue_store.py | 14 +++----- .../orchestrator/supervisor/__init__.py | 32 ++++++++++++++--- .../orchestrator/supervisor/supervise.py | 36 ------------------- bot_bottle/store/audit_store.py | 14 +++----- bot_bottle/store/config_store.py | 11 ++---- bot_bottle/store/db_store.py | 5 +-- bot_bottle/supervisor/__init__.py | 2 +- bot_bottle/supervisor/plan.py | 6 ++-- tests/unit/test_supervise.py | 2 +- 11 files changed, 47 insertions(+), 84 deletions(-) delete mode 100644 bot_bottle/orchestrator/supervisor/supervise.py diff --git a/bot_bottle/backend/resolve_common.py b/bot_bottle/backend/resolve_common.py index 612291bb..7ea9c145 100644 --- a/bot_bottle/backend/resolve_common.py +++ b/bot_bottle/backend/resolve_common.py @@ -29,7 +29,7 @@ from ..git_gate import GitGate, GitGatePlan from ..log import die from ..manifest import Manifest, ManifestBottle from ..supervisor.plan import SupervisePlan -from ..orchestrator.supervisor.supervise import Supervise +from ..orchestrator.supervisor import Supervisor from . import BottleSpec @@ -102,7 +102,7 @@ def prepare_supervise(bottle: ManifestBottle, slug: str) -> SupervisePlan | None return None supervise_dir = supervise_state_dir(slug) supervise_dir.mkdir(parents=True, exist_ok=True) - return Supervise().prepare(slug, supervise_dir) + return Supervisor().prepare(slug, supervise_dir) def merge_provision_env_vars(provision: AgentProvisionPlan) -> AgentProvisionPlan: diff --git a/bot_bottle/image_cache.py b/bot_bottle/image_cache.py index ff165445..65538e39 100644 --- a/bot_bottle/image_cache.py +++ b/bot_bottle/image_cache.py @@ -5,10 +5,7 @@ from __future__ import annotations from datetime import datetime, timezone from pathlib import Path -try: - from .store.config_store import ConfigStore -except ImportError: - from config_store import ConfigStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module +from .store.config_store import ConfigStore class StaleImageError(Exception): diff --git a/bot_bottle/orchestrator/store/queue_store.py b/bot_bottle/orchestrator/store/queue_store.py index 16637f94..a8428e37 100644 --- a/bot_bottle/orchestrator/store/queue_store.py +++ b/bot_bottle/orchestrator/store/queue_store.py @@ -6,16 +6,10 @@ import os import sqlite3 from pathlib import Path -try: - from ...supervisor.types import Proposal, Response - from ...paths import host_db_path - from ...store.db_store import DbStore - from ...store.migrations import TableMigrations -except ImportError: - from supervisor.types import Proposal, Response # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from paths import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from db_store import DbStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module +from ...supervisor.types import Proposal, Response +from ...paths import host_db_path +from ...store.db_store import DbStore +from ...store.migrations import TableMigrations class QueueStore(DbStore): diff --git a/bot_bottle/orchestrator/supervisor/__init__.py b/bot_bottle/orchestrator/supervisor/__init__.py index c6d903e6..e222677f 100644 --- a/bot_bottle/orchestrator/supervisor/__init__.py +++ b/bot_bottle/orchestrator/supervisor/__init__.py @@ -1,10 +1,10 @@ """The orchestrator's view of the supervise plane. Bundles the orchestrator-owned supervise surface: the queue/audit I/O and diff -rendering (`queue`) and the `Supervise` lifecycle (`supervise`). For -convenience it re-exports the neutral vocabulary (`bot_bottle.supervisor`'s -types + `SupervisePlan`) so orchestrator-side callers — service, CLI, tests — -import from one place. +rendering (`queue`), and the `Supervisor` lifecycle (host-side database +staging). For convenience it re-exports the neutral vocabulary +(`bot_bottle.supervisor`'s types + `SupervisePlan`) so orchestrator-side +callers — service, CLI, tests — import from one place. The data plane must NOT import this package; it imports `bot_bottle.supervisor` (neutral) directly and reaches the queue over the control-plane RPC. @@ -12,8 +12,12 @@ The data plane must NOT import this package; it imports `bot_bottle.supervisor` from __future__ import annotations +from abc import ABC +from pathlib import Path + from ...supervisor.types import * # noqa: F401,F403 — re-export the neutral vocabulary from ...supervisor.plan import SupervisePlan +from ..store.store_manager import StoreManager from .queue import ( archive_all_proposals, archive_proposal, @@ -31,4 +35,22 @@ from .queue import ( write_proposal, write_response, ) -from .supervise import Supervise + + +class Supervisor(ABC): + """Per-bottle supervise lifecycle. Encapsulates host-side database staging; + the gateway's start/stop lifecycle is backend-specific. + + `prepare` migrates the orchestrator's `bot-bottle.db` and returns the + neutral `SupervisePlan`. It touches the orchestrator store manager, so it + lives here rather than in the neutral `bot_bottle.supervisor` package; the + backend (which drives launch) may import it — backend → orchestrator is an + allowed direction, unlike gateway → orchestrator.""" + + def prepare(self, slug: str, stage_dir: Path) -> SupervisePlan: + """Stage the host database. Returns the plan; `internal_network` must be + set by the launch step before .start runs.""" + del stage_dir + mgr = StoreManager.instance() + mgr.migrate() + return SupervisePlan(slug=slug, db_path=mgr.db_path) diff --git a/bot_bottle/orchestrator/supervisor/supervise.py b/bot_bottle/orchestrator/supervisor/supervise.py deleted file mode 100644 index f2e51fe9..00000000 --- a/bot_bottle/orchestrator/supervisor/supervise.py +++ /dev/null @@ -1,36 +0,0 @@ -"""The `Supervise` lifecycle — host-side database staging (PRD 0013 / 0070). - -`Supervise.prepare` migrates the orchestrator's `bot-bottle.db` and returns the -neutral `SupervisePlan`. It touches the orchestrator store manager, so it lives -here rather than in the neutral `bot_bottle.supervisor` package; the backend -(which drives launch) may import it — backend → orchestrator is an allowed -direction, unlike gateway → orchestrator. -""" - -from __future__ import annotations - -from abc import ABC -from pathlib import Path - -from ..store.store_manager import StoreManager -from ...supervisor.plan import SupervisePlan - - -class Supervise(ABC): - """Per-bottle supervise daemon. Encapsulates host-side database staging; - the gateway's start/stop lifecycle is backend-specific.""" - - def prepare( - self, - slug: str, - stage_dir: Path, - ) -> SupervisePlan: - """Stage the host database. Returns the plan; `internal_network` must be - set by the launch step before .start runs.""" - del stage_dir - mgr = StoreManager.instance() - mgr.migrate() - return SupervisePlan( - slug=slug, - db_path=mgr.db_path, - ) diff --git a/bot_bottle/store/audit_store.py b/bot_bottle/store/audit_store.py index 3a6738b4..d80451b1 100644 --- a/bot_bottle/store/audit_store.py +++ b/bot_bottle/store/audit_store.py @@ -5,16 +5,10 @@ from __future__ import annotations import sqlite3 from pathlib import Path -try: - from ..supervisor.types import AuditEntry - from ..paths import host_db_path - from .db_store import DbStore - from .migrations import TableMigrations -except ImportError: - from supervisor.types import AuditEntry # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from paths import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from db_store import DbStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module +from ..supervisor.types import AuditEntry +from ..paths import host_db_path +from .db_store import DbStore +from .migrations import TableMigrations class AuditStore(DbStore): diff --git a/bot_bottle/store/config_store.py b/bot_bottle/store/config_store.py index d282d719..8d7811e7 100644 --- a/bot_bottle/store/config_store.py +++ b/bot_bottle/store/config_store.py @@ -4,14 +4,9 @@ from __future__ import annotations from pathlib import Path -try: - from .db_store import DbStore - from .migrations import TableMigrations - from ..paths import host_db_path -except ImportError: - from db_store import DbStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from paths import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module +from .db_store import DbStore +from .migrations import TableMigrations +from ..paths import host_db_path DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS = 1 diff --git a/bot_bottle/store/db_store.py b/bot_bottle/store/db_store.py index 256f29e0..49bcced0 100644 --- a/bot_bottle/store/db_store.py +++ b/bot_bottle/store/db_store.py @@ -6,10 +6,7 @@ import sqlite3 from contextlib import contextmanager from pathlib import Path -try: - from .migrations import TableMigrations -except ImportError: - from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module +from .migrations import TableMigrations class DbVersionError(Exception): diff --git a/bot_bottle/supervisor/__init__.py b/bot_bottle/supervisor/__init__.py index d76c1e9f..7d002f9e 100644 --- a/bot_bottle/supervisor/__init__.py +++ b/bot_bottle/supervisor/__init__.py @@ -11,6 +11,6 @@ tier's private code, importable by `bot_bottle.gateway` **and** status / poll-status constants, and the shared daemon constants. * `plan` — `SupervisePlan`, the launch-time staging DTO. -The orchestrator-only half (queue I/O, diff rendering, the `Supervise` +The orchestrator-only half (queue I/O, diff rendering, the `Supervisor` lifecycle) lives under `bot_bottle.orchestrator.supervisor`. """ diff --git a/bot_bottle/supervisor/plan.py b/bot_bottle/supervisor/plan.py index 2962d82f..e61cf915 100644 --- a/bot_bottle/supervisor/plan.py +++ b/bot_bottle/supervisor/plan.py @@ -1,9 +1,9 @@ """`SupervisePlan` — the launch-time supervise staging DTO. A pure value type (no store access), so it lives in the neutral package: the -backend builds it at launch and the orchestrator's `Supervise.prepare` returns +backend builds it at launch and the orchestrator's `Supervisor.prepare` returns it. The behaviour that fills it in — staging the host database — is the -orchestrator's, in `bot_bottle.orchestrator.supervisor.supervise`. +orchestrator's, in `bot_bottle.orchestrator.supervisor.Supervisor`. """ from __future__ import annotations @@ -14,7 +14,7 @@ from pathlib import Path @dataclass(frozen=True) class SupervisePlan: - """Output of `Supervise.prepare`; consumed by `.start`. + """Output of `Supervisor.prepare`; consumed by `.start`. `db_path` is the host database bind-mounted into the gateway at /run/supervise/bot-bottle.db. `internal_network` is empty at prepare time; diff --git a/tests/unit/test_supervise.py b/tests/unit/test_supervise.py index f0f41be2..7dbdb68e 100644 --- a/tests/unit/test_supervise.py +++ b/tests/unit/test_supervise.py @@ -352,7 +352,7 @@ class TestToolConstants(unittest.TestCase): ) -class _StubSupervise(supervise.Supervise): +class _StubSupervise(supervise.Supervisor): """Concrete Supervise subclass for testing the prepare template.""" def start(self, plan): # type: ignore -- 2.52.0 From 27a122e24be1817b52dfbeaed2dd534d129c3c61 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 14:36:32 -0400 Subject: [PATCH 07/30] refactor(supervise): drop obsolete queue helpers (archive_proposal, wait_for_response, audit_dir/path) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the supervise queue helpers that no live path uses anymore — they're mechanisms the PRD-0070 / idempotent-poll migration superseded: - archive_proposal (+ QueueStore.archive_proposal): the poll stopped archiving on read (#469 review), so single-proposal archiving has no caller; decided proposals drop off the pending list via their response row and are reaped in bulk by archive_all_proposals on teardown/reconcile. - wait_for_response: the data plane polls the queue over the control-plane RPC now, so nothing block-waits on it. - audit_dir / audit_log_path: file-based audit paths, replaced by the SQLite AuditStore. Also drops the tests that covered those obsolete helpers. Kept the prod-unused read accessors list_pending_proposals and read_audit_entries: those aren't dead mechanisms — they're the read side of live write paths that many tests use to verify real queue/audit behavior. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/orchestrator/store/queue_store.py | 19 ------- .../orchestrator/supervisor/__init__.py | 4 -- bot_bottle/orchestrator/supervisor/queue.py | 50 +------------------ tests/unit/test_supervise.py | 46 ----------------- tests/unit/test_supervise_edge.py | 39 --------------- 5 files changed, 1 insertion(+), 157 deletions(-) diff --git a/bot_bottle/orchestrator/store/queue_store.py b/bot_bottle/orchestrator/store/queue_store.py index a8428e37..b9d8b1a8 100644 --- a/bot_bottle/orchestrator/store/queue_store.py +++ b/bot_bottle/orchestrator/store/queue_store.py @@ -167,25 +167,6 @@ class QueueStore(DbStore): raise FileNotFoundError(proposal_id) return self._row_to_response(row) - def archive_proposal(self, proposal_id: str) -> None: - if not self.db_path.is_file(): - return - with self._connection() as conn: - conn.execute( - """ - UPDATE supervise_proposals SET archived = 1 - WHERE queue_key = ? AND id = ? - """, - (self.queue_key, proposal_id), - ) - conn.execute( - """ - UPDATE supervise_responses SET archived = 1 - WHERE queue_key = ? AND proposal_id = ? - """, - (self.queue_key, proposal_id), - ) - def archive_all(self) -> None: """Archive every proposal + response for this queue key. Used to reap a bottle's supervise records when it is torn down or reconciled away — diff --git a/bot_bottle/orchestrator/supervisor/__init__.py b/bot_bottle/orchestrator/supervisor/__init__.py index e222677f..48b96b04 100644 --- a/bot_bottle/orchestrator/supervisor/__init__.py +++ b/bot_bottle/orchestrator/supervisor/__init__.py @@ -20,9 +20,6 @@ from ...supervisor.plan import SupervisePlan from ..store.store_manager import StoreManager from .queue import ( archive_all_proposals, - archive_proposal, - audit_dir, - audit_log_path, list_all_pending_proposals, list_pending_proposals, read_audit_entries, @@ -30,7 +27,6 @@ from .queue import ( read_response, render_diff, sha256_hex, - wait_for_response, write_audit_entry, write_proposal, write_response, diff --git a/bot_bottle/orchestrator/supervisor/queue.py b/bot_bottle/orchestrator/supervisor/queue.py index fed41325..17b0c7f9 100644 --- a/bot_bottle/orchestrator/supervisor/queue.py +++ b/bot_bottle/orchestrator/supervisor/queue.py @@ -11,31 +11,13 @@ from __future__ import annotations import difflib import hashlib -import time from pathlib import Path -from ...supervisor.types import ( - AuditEntry, - DEFAULT_POLL_INTERVAL_SEC, - Proposal, - Response, -) -from ...paths import bot_bottle_root +from ...supervisor.types import AuditEntry, Proposal, Response from ..store.queue_store import QueueStore from ...store.audit_store import AuditStore -# --- Paths ----------------------------------------------------------------- - - -def audit_dir() -> Path: - return bot_bottle_root() / "audit" - - -def audit_log_path(component: str, slug: str) -> Path: - return audit_dir() / f"{component}-{slug}.log" - - # --- Queue I/O ------------------------------------------------------------- @@ -69,36 +51,6 @@ def read_response(bottle_slug: str, proposal_id: str) -> Response: return QueueStore(bottle_slug).read_response(proposal_id) -def wait_for_response( - bottle_slug: str, - proposal_id: str, - *, - poll_interval: float = DEFAULT_POLL_INTERVAL_SEC, - deadline: float | None = None, -) -> Response: - """Block until a response file appears for `proposal_id`, then return it. - `deadline` is an absolute time.monotonic() value after which the wait raises - TimeoutError. None waits forever — the natural shape, since the operator's - response time is unbounded. - - Polls SQLite so the implementation stays portable and stdlib-only.""" - store = QueueStore(bottle_slug) - while True: - try: - return store.read_response(proposal_id) - except FileNotFoundError: - pass - if deadline is not None and time.monotonic() >= deadline: - raise TimeoutError(f"no response for proposal {proposal_id!r}") - time.sleep(poll_interval) - - -def archive_proposal(bottle_slug: str, proposal_id: str) -> None: - """Mark both proposal and response rows processed. - Idempotent — missing rows are silently skipped.""" - QueueStore(bottle_slug).archive_proposal(proposal_id) - - def archive_all_proposals(bottle_slug: str) -> None: """Archive every proposal + response for `bottle_slug` (bottle teardown / reconcile cleanup). Idempotent.""" diff --git a/tests/unit/test_supervise.py b/tests/unit/test_supervise.py index 7dbdb68e..da2b5f0d 100644 --- a/tests/unit/test_supervise.py +++ b/tests/unit/test_supervise.py @@ -1,8 +1,6 @@ """Unit: supervise queue + audit log + diff helpers (PRD 0013).""" import tempfile -import threading -import time import unittest from datetime import datetime, timezone from pathlib import Path @@ -21,14 +19,12 @@ from bot_bottle.orchestrator.supervisor import ( STATUS_REJECTED, TOOL_EGRESS_ALLOW, TOOL_GITLEAKS_ALLOW, - archive_proposal, list_pending_proposals, read_audit_entries, read_proposal, read_response, render_diff, sha256_hex, - wait_for_response, write_audit_entry, write_proposal, write_response, @@ -173,48 +169,6 @@ class TestQueueIO(unittest.TestCase): write_response(self.slug, r) self.assertEqual(r, read_response(self.slug, "xyz")) - def test_wait_for_response_returns_when_file_appears(self): - p = _proposal() - write_proposal(p) - - def write_after_delay(): - time.sleep(0.05) - write_response(self.slug, Response( - proposal_id=p.id, status=STATUS_APPROVED, notes="ok", - )) - - t = threading.Thread(target=write_after_delay) - t.start() - try: - r = wait_for_response(self.slug, p.id, poll_interval=0.01) - finally: - t.join() - self.assertEqual(STATUS_APPROVED, r.status) - self.assertEqual("ok", r.notes) - - def test_wait_for_response_times_out(self): - deadline = time.monotonic() + 0.05 - with self.assertRaises(TimeoutError): - wait_for_response( - self.slug, "never", - poll_interval=0.01, deadline=deadline, - ) - - def test_archive_proposal_hides_rows(self): - p = _proposal() - write_proposal(p) - write_response(self.slug, Response( - proposal_id=p.id, status=STATUS_APPROVED, notes="", - )) - archive_proposal(self.slug, p.id) - self.assertEqual([], list_pending_proposals(self.slug)) - with self.assertRaises(FileNotFoundError): - read_response(self.slug, p.id) - - def test_archive_is_idempotent_on_missing_files(self): - # Should not raise. - archive_proposal(self.slug, "nope") - class TestAuditLog(unittest.TestCase): def setUp(self): diff --git a/tests/unit/test_supervise_edge.py b/tests/unit/test_supervise_edge.py index c56d9c55..8548040f 100644 --- a/tests/unit/test_supervise_edge.py +++ b/tests/unit/test_supervise_edge.py @@ -5,7 +5,6 @@ fallback paths.""" from __future__ import annotations import tempfile -import time import unittest from pathlib import Path from unittest.mock import patch @@ -23,7 +22,6 @@ from bot_bottle.orchestrator.supervisor import ( read_audit_entries, read_proposal, read_response, - wait_for_response, write_audit_entry, ) @@ -81,22 +79,6 @@ class TestReadMalformed(unittest.TestCase): self.assertEqual([], list_pending_proposals("slug")) -class TestWaitForResponse(unittest.TestCase): - def test_missing_response_times_out(self) -> None: - with tempfile.TemporaryDirectory() as d: - with patch.dict("os.environ", {"HOME": d}): - QueueStore("slug").migrate() - with self.assertRaises(TimeoutError): - wait_for_response("slug", "p", deadline=time.monotonic()) - - def test_empty_db_response_does_not_count(self) -> None: - with tempfile.TemporaryDirectory() as d: - with patch.dict("os.environ", {"HOME": d}): - QueueStore("slug").migrate() - with self.assertRaises(TimeoutError): - wait_for_response("slug", "p", deadline=time.monotonic()) - - class TestReadAuditEntries(unittest.TestCase): def test_missing_log_returns_empty(self) -> None: with tempfile.TemporaryDirectory() as home, \ @@ -129,19 +111,6 @@ class TestReadAuditEntries(unittest.TestCase): self.assertEqual(1, len(entries)) self.assertEqual("approve", entries[0].operator_action) - def test_legacy_audit_log_file_does_not_count(self) -> None: - with tempfile.TemporaryDirectory() as home, \ - patch.dict("os.environ", {"HOME": home}): - path = supervise.audit_log_path("egress", "slug") - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text( - '{"timestamp": "t", "bottle_slug": "slug", "component": "egress",' - ' "operator_action": "approve", "operator_notes": "",' - ' "justification": "", "diff": ""}\n' - ) - entries = read_audit_entries("egress", "slug") - self.assertEqual([], entries) - class TestStoreGuardBranches(unittest.TestCase): """Direct QueueStore / AuditStore construction and early-return guard branches.""" @@ -170,14 +139,6 @@ class TestStoreGuardBranches(unittest.TestCase): db.unlink() self.assertEqual([], store.list_all_pending_proposals()) - def test_queue_store_missing_db_archive_is_noop(self): - with tempfile.TemporaryDirectory() as d: - db = Path(d) / "q.db" - store = QueueStore("key", db_path=db) - store.migrate() - db.unlink() - store.archive_proposal("anything") # must not raise - def test_queue_store_chmod_oserror_is_swallowed(self): with tempfile.TemporaryDirectory() as d: db = Path(d) / "q.db" -- 2.52.0 From 8abccf7ffecab3cbfae3d1026d5acc5d1505a310 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 14:52:17 -0400 Subject: [PATCH 08/30] refactor(supervise): make Supervisor a service class the orchestrator calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turn the loose queue/audit functions in orchestrator/supervisor/queue.py into methods on a concrete Supervisor service. The Orchestrator now owns an injectable `self._supervisor` and calls `self._supervisor.write_proposal(...)` etc., instead of module-level free functions — the supervise dependency is explicit and mockable, and a Supervisor can be scoped to a `db_path` (defaults to the host DB) so tests can point it at a temp database. - Supervisor (in the package __init__) drops the ABC and gains write_proposal, read_proposal, list_pending_proposals, list_all_pending_proposals, write_response, read_response, archive_all_proposals, write_audit_entry, read_audit_entries, plus the launch-time prepare(). - render_diff / sha256_hex are pure and stateless, so they move to orchestrator/supervisor/util.py (re-exported from the facade) rather than becoming methods. - queue.py is deleted; service.py + the supervise tests call through a Supervisor instance. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/orchestrator/service.py | 30 +++-- .../orchestrator/supervisor/__init__.py | 105 ++++++++++++------ bot_bottle/orchestrator/supervisor/queue.py | 93 ---------------- bot_bottle/orchestrator/supervisor/util.py | 31 ++++++ tests/unit/test_orchestrator_control_plane.py | 4 +- tests/unit/test_orchestrator_service.py | 9 +- tests/unit/test_supervise.py | 56 +++++----- tests/unit/test_supervise_edge.py | 30 +++-- tests/unit/test_supervise_server.py | 35 +++--- 9 files changed, 183 insertions(+), 210 deletions(-) delete mode 100644 bot_bottle/orchestrator/supervisor/queue.py create mode 100644 bot_bottle/orchestrator/supervisor/util.py diff --git a/bot_bottle/orchestrator/service.py b/bot_bottle/orchestrator/service.py index bf6a14dd..452af25a 100644 --- a/bot_bottle/orchestrator/service.py +++ b/bot_bottle/orchestrator/service.py @@ -39,15 +39,9 @@ from .supervisor import ( STATUS_REJECTED, TOOL_EGRESS_ALLOW, TOOL_EGRESS_BLOCK, - archive_all_proposals, - list_all_pending_proposals, - read_proposal, - read_response, + Supervisor, render_diff, sha256_hex, - write_audit_entry, - write_proposal, - write_response, ) @@ -71,10 +65,14 @@ class Orchestrator: registry: RegistryStore, broker: LaunchBroker, sign_secret: bytes, + supervisor: Supervisor | None = None, ) -> None: self.registry = registry self._broker = broker self._secret = sign_secret + # The supervise service (queue + audit I/O). Injectable so tests can + # point it at a temp DB; defaults to the host DB. + self._supervisor = supervisor or Supervisor() # Per-bottle egress auth tokens (env_name -> value), keyed by bottle_id. # Held **in memory only** — never written to the registry DB — so the # gateway can inject each bottle's upstream credential without secrets @@ -134,7 +132,7 @@ class Orchestrator: self.registry.deregister(bottle_id) self._tokens.pop(bottle_id, None) # Reap any supervise proposals the gone bottle never acknowledged. - archive_all_proposals(bottle_id) + self._supervisor.archive_all_proposals(bottle_id) return True def reconcile( @@ -160,7 +158,7 @@ class Orchestrator: for rec in reaped: self._tokens.pop(rec.bottle_id, None) # Reap any supervise proposals the gone bottle never acknowledged. - archive_all_proposals(rec.bottle_id) + self._supervisor.archive_all_proposals(rec.bottle_id) return [rec.bottle_id for rec in reaped] def tokens_for(self, bottle_id: str) -> dict[str, str]: @@ -220,7 +218,7 @@ class Orchestrator: justification=justification, current_file_hash=sha256_hex(proposed_file), ) - write_proposal(proposal) + self._supervisor.write_proposal(proposal) return proposal.id def supervise_poll_response(self, bottle_id: str, proposal_id: str) -> dict[str, object]: @@ -241,10 +239,10 @@ class Orchestrator: Reads are scoped to `bottle_id` (the queue key), so a caller can never read another bottle's proposal even with a guessed id.""" try: - response = read_response(bottle_id, proposal_id) + response = self._supervisor.read_response(bottle_id, proposal_id) except FileNotFoundError: try: - read_proposal(bottle_id, proposal_id) + self._supervisor.read_proposal(bottle_id, proposal_id) except FileNotFoundError: return {"status": POLL_STATUS_UNKNOWN} return {"status": POLL_STATUS_PENDING} @@ -263,7 +261,7 @@ class Orchestrator: orchestrator-assigned bottle_id, which is opaque to an operator). The CLI renders the label but still responds against `bottle_slug`.""" out: list[dict[str, object]] = [] - for p in list_all_pending_proposals(): + for p in self._supervisor.list_all_pending_proposals(): d = p.to_dict() d["bottle_label"] = self._label_for(p.bottle_slug) out.append(d) @@ -326,7 +324,7 @@ class Orchestrator: if status is None: return False, f"unknown decision {decision!r}" try: - proposal = read_proposal(bottle_slug, proposal_id) + proposal = self._supervisor.read_proposal(bottle_slug, proposal_id) except FileNotFoundError: return False, "no such proposal" @@ -342,12 +340,12 @@ class Orchestrator: diff_before, diff_after = rec.policy, new_policy self.set_policy(rec.bottle_id, new_policy) - write_response(bottle_slug, Response( + self._supervisor.write_response(bottle_slug, Response( proposal_id=proposal_id, status=status, notes=notes, final_file=final_file, )) component = COMPONENT_FOR_TOOL.get(proposal.tool) if component is not None: - write_audit_entry(AuditEntry( + self._supervisor.write_audit_entry(AuditEntry( timestamp=datetime.now(timezone.utc).isoformat(), bottle_slug=bottle_slug, component=component, diff --git a/bot_bottle/orchestrator/supervisor/__init__.py b/bot_bottle/orchestrator/supervisor/__init__.py index 48b96b04..8a2d9127 100644 --- a/bot_bottle/orchestrator/supervisor/__init__.py +++ b/bot_bottle/orchestrator/supervisor/__init__.py @@ -1,52 +1,91 @@ -"""The orchestrator's view of the supervise plane. +"""The orchestrator's supervise service. -Bundles the orchestrator-owned supervise surface: the queue/audit I/O and diff -rendering (`queue`), and the `Supervisor` lifecycle (host-side database -staging). For convenience it re-exports the neutral vocabulary -(`bot_bottle.supervisor`'s types + `SupervisePlan`) so orchestrator-side -callers — service, CLI, tests — import from one place. +`Supervisor` is the service object the `Orchestrator` owns and calls: it wraps +the supervise queue + audit stores (queue proposals, record operator responses, +write audit entries, stage the DB at launch). Binding those operations to one +object — optionally scoped to a `db_path` — makes the orchestrator's supervise +dependency explicit and injectable in tests. -The data plane must NOT import this package; it imports `bot_bottle.supervisor` -(neutral) directly and reaches the queue over the control-plane RPC. +For convenience this package also re-exports the neutral vocabulary +(`bot_bottle.supervisor`'s types + `SupervisePlan`) and the pure `render_diff` +/ `sha256_hex` helpers (`util`), so orchestrator-side callers import from one +place. The data plane must NOT import this package — it imports +`bot_bottle.supervisor` (neutral) directly and reaches the queue over the +control-plane RPC. """ from __future__ import annotations -from abc import ABC from pathlib import Path from ...supervisor.types import * # noqa: F401,F403 — re-export the neutral vocabulary from ...supervisor.plan import SupervisePlan from ..store.store_manager import StoreManager -from .queue import ( - archive_all_proposals, - list_all_pending_proposals, - list_pending_proposals, - read_audit_entries, - read_proposal, - read_response, - render_diff, - sha256_hex, - write_audit_entry, - write_proposal, - write_response, -) +from ..store.queue_store import QueueStore +from ...store.audit_store import AuditStore +from ...supervisor.types import AuditEntry, Proposal, Response +from .util import render_diff, sha256_hex # noqa: F401 — re-exported for callers -class Supervisor(ABC): - """Per-bottle supervise lifecycle. Encapsulates host-side database staging; - the gateway's start/stop lifecycle is backend-specific. +class Supervisor: + """Host-side supervise service: the queue + audit I/O and launch-time DB + staging the orchestrator drives. Stateless apart from an optional `db_path` + (None → the host DB, resolved per store as before), so a test can point a + `Supervisor` at a temp database.""" - `prepare` migrates the orchestrator's `bot-bottle.db` and returns the - neutral `SupervisePlan`. It touches the orchestrator store manager, so it - lives here rather than in the neutral `bot_bottle.supervisor` package; the - backend (which drives launch) may import it — backend → orchestrator is an - allowed direction, unlike gateway → orchestrator.""" + def __init__(self, db_path: Path | None = None) -> None: + self._db_path = db_path + + def _queue(self, queue_key: str) -> QueueStore: + return QueueStore(queue_key, self._db_path) + + def _audit(self) -> AuditStore: + return AuditStore(self._db_path) + + # --- Queue I/O --------------------------------------------------------- + + def write_proposal(self, proposal: Proposal) -> Path: + """Persist `proposal` in the queue database, mode 0o600.""" + return self._queue(proposal.bottle_slug).write_proposal(proposal) + + def read_proposal(self, bottle_slug: str, proposal_id: str) -> Proposal: + return self._queue(bottle_slug).read_proposal(proposal_id) + + def list_pending_proposals(self, bottle_slug: str) -> list[Proposal]: + """A single bottle's undecided proposals, FIFO by arrival.""" + return self._queue(bottle_slug).list_pending_proposals() + + def list_all_pending_proposals(self) -> list[Proposal]: + """All pending proposals across bottles, sorted FIFO.""" + return self._queue("").list_all_pending_proposals() + + def write_response(self, bottle_slug: str, response: Response) -> Path: + return self._queue(bottle_slug).write_response(response) + + def read_response(self, bottle_slug: str, proposal_id: str) -> Response: + return self._queue(bottle_slug).read_response(proposal_id) + + def archive_all_proposals(self, bottle_slug: str) -> None: + """Archive every proposal + response for `bottle_slug` (bottle teardown + / reconcile cleanup). Idempotent.""" + self._queue(bottle_slug).archive_all() + + # --- Audit log --------------------------------------------------------- + + def write_audit_entry(self, entry: AuditEntry) -> Path: + """Append `entry` to the host supervise audit table.""" + return self._audit().write_audit_entry(entry) + + def read_audit_entries(self, component: str, slug: str) -> list[AuditEntry]: + """Load all audit entries for the given component+slug.""" + return self._audit().read_audit_entries(component, slug) + + # --- Launch-time staging ---------------------------------------------- def prepare(self, slug: str, stage_dir: Path) -> SupervisePlan: - """Stage the host database. Returns the plan; `internal_network` must be - set by the launch step before .start runs.""" + """Stage the host database and return the plan. Called by the backend at + launch; `internal_network` is set by the launch step before `.start`.""" del stage_dir - mgr = StoreManager.instance() + mgr = StoreManager(self._db_path) if self._db_path else StoreManager.instance() mgr.migrate() return SupervisePlan(slug=slug, db_path=mgr.db_path) diff --git a/bot_bottle/orchestrator/supervisor/queue.py b/bot_bottle/orchestrator/supervisor/queue.py deleted file mode 100644 index 17b0c7f9..00000000 --- a/bot_bottle/orchestrator/supervisor/queue.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Orchestrator-side supervise queue + audit I/O and diff rendering. - -The host-side library the orchestrator uses to drive the supervise flow: read / -write proposals and responses against the queue store, append audit entries, -and render operator diffs. Only the orchestrator opens `bot-bottle.db` (PRD -0070), so this lives under `orchestrator/` — the data plane reaches it over the -control-plane RPC, never by importing this module. -""" - -from __future__ import annotations - -import difflib -import hashlib -from pathlib import Path - -from ...supervisor.types import AuditEntry, Proposal, Response -from ..store.queue_store import QueueStore -from ...store.audit_store import AuditStore - - -# --- Queue I/O ------------------------------------------------------------- - - -def write_proposal(proposal: Proposal) -> Path: - """Persist `proposal` in the queue database, mode 0o600. - Directory is created if missing.""" - return QueueStore(proposal.bottle_slug).write_proposal(proposal) - - -def read_proposal(bottle_slug: str, proposal_id: str) -> Proposal: - return QueueStore(bottle_slug).read_proposal(proposal_id) - - -def list_pending_proposals(bottle_slug: str) -> list[Proposal]: - """All proposals for `bottle_slug` that do not yet have a matching - response. Sorted by `arrival_timestamp` so the operator sees the queue - FIFO.""" - return QueueStore(bottle_slug).list_pending_proposals() - - -def list_all_pending_proposals() -> list[Proposal]: - """All pending proposals across bottles, sorted FIFO.""" - return QueueStore("").list_all_pending_proposals() - - -def write_response(bottle_slug: str, response: Response) -> Path: - return QueueStore(bottle_slug).write_response(response) - - -def read_response(bottle_slug: str, proposal_id: str) -> Response: - return QueueStore(bottle_slug).read_response(proposal_id) - - -def archive_all_proposals(bottle_slug: str) -> None: - """Archive every proposal + response for `bottle_slug` (bottle teardown / - reconcile cleanup). Idempotent.""" - QueueStore(bottle_slug).archive_all() - - -# --- Audit log ------------------------------------------------------------- - - -def write_audit_entry(entry: AuditEntry) -> Path: - """Append `entry` to the host supervise audit table.""" - return AuditStore().write_audit_entry(entry) - - -def read_audit_entries(component: str, slug: str) -> list[AuditEntry]: - """Load all audit entries for the given component+slug.""" - return AuditStore().read_audit_entries(component, slug) - - -# --- Diff rendering -------------------------------------------------------- - - -def render_diff(before: str, after: str, *, label: str = "config") -> str: - """Unified diff suitable for the audit log + TUI. Empty diff (no changes) - renders as the empty string.""" - diff = difflib.unified_diff( - before.splitlines(keepends=True), - after.splitlines(keepends=True), - fromfile=f"{label} (current)", - tofile=f"{label} (proposed)", - lineterm="", - ) - parts = list(diff) - if not parts: - return "" - return "".join(p if p.endswith("\n") else p + "\n" for p in parts).rstrip("\n") - - -def sha256_hex(content: str) -> str: - return hashlib.sha256(content.encode("utf-8")).hexdigest() diff --git a/bot_bottle/orchestrator/supervisor/util.py b/bot_bottle/orchestrator/supervisor/util.py new file mode 100644 index 00000000..3776b566 --- /dev/null +++ b/bot_bottle/orchestrator/supervisor/util.py @@ -0,0 +1,31 @@ +"""Pure supervise helpers — diff rendering and content hashing. + +Stateless utilities used alongside the `Supervisor` service. Kept as plain +functions (not methods) because they carry no state and are also handy to +callers that only need to hash or diff. +""" + +from __future__ import annotations + +import difflib +import hashlib + + +def render_diff(before: str, after: str, *, label: str = "config") -> str: + """Unified diff suitable for the audit log + TUI. Empty diff (no changes) + renders as the empty string.""" + diff = difflib.unified_diff( + before.splitlines(keepends=True), + after.splitlines(keepends=True), + fromfile=f"{label} (current)", + tofile=f"{label} (proposed)", + lineterm="", + ) + parts = list(diff) + if not parts: + return "" + return "".join(p if p.endswith("\n") else p + "\n" for p in parts).rstrip("\n") + + +def sha256_hex(content: str) -> str: + return hashlib.sha256(content.encode("utf-8")).hexdigest() diff --git a/tests/unit/test_orchestrator_control_plane.py b/tests/unit/test_orchestrator_control_plane.py index 0bacd900..66119c3e 100644 --- a/tests/unit/test_orchestrator_control_plane.py +++ b/tests/unit/test_orchestrator_control_plane.py @@ -29,7 +29,7 @@ from bot_bottle.orchestrator.supervisor import ( Proposal, TOOL_EGRESS_ALLOW, sha256_hex, - write_proposal, + Supervisor, ) @@ -421,7 +421,7 @@ class TestDispatchSupervise(unittest.TestCase): p = Proposal.new( bottle_slug=slug, tool=TOOL_EGRESS_ALLOW, proposed_file=proposed, justification="need it", current_file_hash=sha256_hex(proposed)) - write_proposal(p) + Supervisor().write_proposal(p) return p.id def test_list_pending(self) -> None: diff --git a/tests/unit/test_orchestrator_service.py b/tests/unit/test_orchestrator_service.py index a83f8c17..d090f3ab 100644 --- a/tests/unit/test_orchestrator_service.py +++ b/tests/unit/test_orchestrator_service.py @@ -20,9 +20,8 @@ from bot_bottle.orchestrator.supervisor import ( Proposal, STATUS_APPROVED, TOOL_EGRESS_ALLOW, - read_response, sha256_hex, - write_proposal, + Supervisor, ) @@ -185,7 +184,7 @@ class TestOrchestratorSupervise(unittest.TestCase): p = Proposal.new( bottle_slug=slug, tool=TOOL_EGRESS_ALLOW, proposed_file=proposed, justification="need it", current_file_hash=sha256_hex(proposed)) - write_proposal(p) + Supervisor().write_proposal(p) return p.id def test_pending_lists_queued_proposal(self) -> None: @@ -208,7 +207,7 @@ class TestOrchestratorSupervise(unittest.TestCase): assert rec is not None self.assertEqual(new_routes, rec.policy) # response written -> agent unblocks, proposal no longer pending - self.assertEqual(STATUS_APPROVED, read_response("demo", pid).status) + self.assertEqual(STATUS_APPROVED, Supervisor().read_response("demo", pid).status) self.assertEqual([], self.orch.supervise_pending()) def test_pending_carries_human_label(self) -> None: @@ -261,7 +260,7 @@ class TestOrchestratorSupervise(unittest.TestCase): rec = self.store.get(bottle_id) assert rec is not None self.assertEqual("routes:\n - host: existing.com\n", rec.policy) - self.assertEqual("rejected", read_response("demo", pid).status) + self.assertEqual("rejected", Supervisor().read_response("demo", pid).status) def test_unknown_proposal_is_error(self) -> None: ok, err = self.orch.supervise_respond( diff --git a/tests/unit/test_supervise.py b/tests/unit/test_supervise.py index da2b5f0d..c609dbe6 100644 --- a/tests/unit/test_supervise.py +++ b/tests/unit/test_supervise.py @@ -17,19 +17,15 @@ from bot_bottle.orchestrator.supervisor import ( STATUS_APPROVED, STATUS_MODIFIED, STATUS_REJECTED, + Supervisor, TOOL_EGRESS_ALLOW, TOOL_GITLEAKS_ALLOW, - list_pending_proposals, - read_audit_entries, - read_proposal, - read_response, render_diff, sha256_hex, - write_audit_entry, - write_proposal, - write_response, ) +_SV = Supervisor() + FIXED_TS = datetime(2026, 5, 25, 12, 0, 0, tzinfo=timezone.utc) @@ -123,26 +119,26 @@ class TestQueueIO(unittest.TestCase): def test_write_and_read_proposal(self): p = _proposal() - path = write_proposal(p) + path = _SV.write_proposal(p) self.assertTrue(path.exists()) self.assertEqual(host_db_path(), path) self.assertEqual(0o600, path.stat().st_mode & 0o777) - loaded = read_proposal(self.slug, p.id) + loaded = _SV.read_proposal(self.slug, p.id) self.assertEqual(p, loaded) def test_list_pending_excludes_responded(self): a = _proposal(justification="first") b = _proposal(justification="second") - write_proposal(a) - write_proposal(b) - write_response(self.slug, Response( + _SV.write_proposal(a) + _SV.write_proposal(b) + _SV.write_response(self.slug, Response( proposal_id=a.id, status=STATUS_APPROVED, notes="", )) - pending = list_pending_proposals(self.slug) + pending = _SV.list_pending_proposals(self.slug) self.assertEqual([b.id], [p.id for p in pending]) def test_list_pending_returns_empty_for_missing_slug(self): - self.assertEqual([], list_pending_proposals("nope")) + self.assertEqual([], _SV.list_pending_proposals("nope")) def test_list_pending_sorted_by_arrival(self): # Fabricate two with explicit timestamps. @@ -159,15 +155,15 @@ class TestQueueIO(unittest.TestCase): now=datetime(2026, 5, 25, 14, 0, 0, tzinfo=timezone.utc), ) # Write in reverse order. - write_proposal(b) - write_proposal(a) - ordered = list_pending_proposals(self.slug) + _SV.write_proposal(b) + _SV.write_proposal(a) + ordered = _SV.list_pending_proposals(self.slug) self.assertEqual([a.id, b.id], [p.id for p in ordered]) def test_write_and_read_response(self): r = Response(proposal_id="xyz", status=STATUS_REJECTED, notes="no") - write_response(self.slug, r) - self.assertEqual(r, read_response(self.slug, "xyz")) + _SV.write_response(self.slug, r) + self.assertEqual(r, _SV.read_response(self.slug, "xyz")) class TestAuditLog(unittest.TestCase): @@ -193,15 +189,15 @@ class TestAuditLog(unittest.TestCase): justification="agent needed gh-api token", diff="--- before\n+++ after\n", ) - path = write_audit_entry(e) + path = _SV.write_audit_entry(e) self.assertEqual(host_db_path(), path) self.assertEqual(0o600, path.stat().st_mode & 0o777) - loaded = read_audit_entries("cred-proxy", "dev") + loaded = _SV.read_audit_entries("cred-proxy", "dev") self.assertEqual([e], loaded) def test_appends_one_line_per_entry(self): for i in range(3): - write_audit_entry(AuditEntry( + _SV.write_audit_entry(AuditEntry( timestamp=f"2026-05-25T12:00:0{i}+00:00", bottle_slug="dev", component="egress", @@ -210,7 +206,7 @@ class TestAuditLog(unittest.TestCase): justification="", diff="", )) - entries = read_audit_entries("egress", "dev") + entries = _SV.read_audit_entries("egress", "dev") self.assertEqual(3, len(entries)) self.assertEqual( ["2026-05-25T12:00:00+00:00", "2026-05-25T12:00:01+00:00", @@ -219,7 +215,7 @@ class TestAuditLog(unittest.TestCase): ) def test_separate_logs_per_component_slug(self): - write_audit_entry(AuditEntry( + _SV.write_audit_entry(AuditEntry( timestamp="t", bottle_slug="dev", component="cred-proxy", @@ -228,7 +224,7 @@ class TestAuditLog(unittest.TestCase): justification="", diff="", )) - write_audit_entry(AuditEntry( + _SV.write_audit_entry(AuditEntry( timestamp="t", bottle_slug="dev", component="egress", @@ -237,7 +233,7 @@ class TestAuditLog(unittest.TestCase): justification="", diff="", )) - write_audit_entry(AuditEntry( + _SV.write_audit_entry(AuditEntry( timestamp="t", bottle_slug="other", component="cred-proxy", @@ -246,12 +242,12 @@ class TestAuditLog(unittest.TestCase): justification="", diff="", )) - self.assertEqual(1, len(read_audit_entries("cred-proxy", "dev"))) - self.assertEqual(1, len(read_audit_entries("egress", "dev"))) - self.assertEqual(1, len(read_audit_entries("cred-proxy", "other"))) + self.assertEqual(1, len(_SV.read_audit_entries("cred-proxy", "dev"))) + self.assertEqual(1, len(_SV.read_audit_entries("egress", "dev"))) + self.assertEqual(1, len(_SV.read_audit_entries("cred-proxy", "other"))) def test_read_audit_entries_missing_log_returns_empty(self): - self.assertEqual([], read_audit_entries("cred-proxy", "no-such-bottle")) + self.assertEqual([], _SV.read_audit_entries("cred-proxy", "no-such-bottle")) class TestDiffAndHash(unittest.TestCase): diff --git a/tests/unit/test_supervise_edge.py b/tests/unit/test_supervise_edge.py index 8548040f..d29135b7 100644 --- a/tests/unit/test_supervise_edge.py +++ b/tests/unit/test_supervise_edge.py @@ -17,14 +17,12 @@ from bot_bottle.orchestrator.supervisor import ( AuditEntry, Proposal, STATUS_APPROVED, + Supervisor, TOOL_EGRESS_ALLOW, - list_pending_proposals, - read_audit_entries, - read_proposal, - read_response, - write_audit_entry, ) +_SV = Supervisor() + def _proposal() -> Proposal: return Proposal.new( @@ -47,21 +45,21 @@ class TestReadMalformed(unittest.TestCase): with patch.dict("os.environ", {"HOME": d}): QueueStore("slug").migrate() with self.assertRaises(FileNotFoundError): - read_proposal("slug", "p") + _SV.read_proposal("slug", "p") def test_read_response_missing_row(self) -> None: with tempfile.TemporaryDirectory() as d: with patch.dict("os.environ", {"HOME": d}): QueueStore("slug").migrate() with self.assertRaises(FileNotFoundError): - read_response("slug", "p") + _SV.read_response("slug", "p") def test_list_pending_reads_db_only(self) -> None: with tempfile.TemporaryDirectory() as d: with patch.dict("os.environ", {"HOME": d}): QueueStore("slug").migrate() - supervise.write_proposal(_proposal()) - pending = list_pending_proposals("slug") + _SV.write_proposal(_proposal()) + pending = _SV.list_pending_proposals("slug") self.assertEqual(1, len(pending)) self.assertEqual("slug", pending[0].bottle_slug) @@ -70,26 +68,26 @@ class TestReadMalformed(unittest.TestCase): with patch.dict("os.environ", {"HOME": d}): QueueStore("slug").migrate() p = _proposal() - supervise.write_proposal(p) - supervise.write_response("slug", supervise.Response( + _SV.write_proposal(p) + _SV.write_response("slug", supervise.Response( proposal_id=p.id, status=STATUS_APPROVED, notes="", )) - self.assertEqual([], list_pending_proposals("slug")) + self.assertEqual([], _SV.list_pending_proposals("slug")) class TestReadAuditEntries(unittest.TestCase): def test_missing_log_returns_empty(self) -> None: with tempfile.TemporaryDirectory() as home, \ patch.dict("os.environ", {"HOME": home}): - self.assertEqual([], read_audit_entries("egress", "nope")) + self.assertEqual([], _SV.read_audit_entries("egress", "nope")) def test_reads_entries_from_db(self) -> None: with tempfile.TemporaryDirectory() as home, \ patch.dict("os.environ", {"HOME": home}): AuditStore().migrate() - write_audit_entry(AuditEntry( + _SV.write_audit_entry(AuditEntry( timestamp="t", bottle_slug="slug", component="egress", @@ -98,7 +96,7 @@ class TestReadAuditEntries(unittest.TestCase): justification="", diff="", )) - write_audit_entry(AuditEntry( + _SV.write_audit_entry(AuditEntry( timestamp="t", bottle_slug="other", component="egress", @@ -107,7 +105,7 @@ class TestReadAuditEntries(unittest.TestCase): justification="", diff="", )) - entries = read_audit_entries("egress", "slug") + entries = _SV.read_audit_entries("egress", "slug") self.assertEqual(1, len(entries)) self.assertEqual("approve", entries[0].operator_action) diff --git a/tests/unit/test_supervise_server.py b/tests/unit/test_supervise_server.py index a082998b..fdaff909 100644 --- a/tests/unit/test_supervise_server.py +++ b/tests/unit/test_supervise_server.py @@ -50,6 +50,11 @@ from bot_bottle.gateway.supervise_server import ( validate_proposed_file, ) +# The orchestrator's supervise service, backing the fake resolver's queue reads +# and writes (the fake stands in for the RPC; state still goes through the real +# queue store). +_SV = _sv.Supervisor() + # Fixed caller identity for the handler tests. The control plane attributes by # (source_ip, identity_token); the fake resolver ignores them and answers for a # fixed bottle, since attribution itself is covered by the orchestrator tests. @@ -89,7 +94,7 @@ class _FakeSuperviseResolver: bottle_slug=self.bottle_id, tool=tool, proposed_file=proposed_file, justification=justification, current_file_hash=_sv.sha256_hex(proposed_file), ) - _sv.write_proposal(proposal) + _SV.write_proposal(proposal) return proposal.id def poll_supervise( @@ -101,10 +106,10 @@ class _FakeSuperviseResolver: if self.bottle_id is None or self.poll_none: return None try: - response = _sv.read_response(self.bottle_id, proposal_id) + response = _SV.read_response(self.bottle_id, proposal_id) except FileNotFoundError: try: - _sv.read_proposal(self.bottle_id, proposal_id) + _SV.read_proposal(self.bottle_id, proposal_id) except FileNotFoundError: return {"status": _sv.POLL_STATUS_UNKNOWN} return {"status": _sv.POLL_STATUS_PENDING} @@ -377,10 +382,10 @@ class TestHandleToolsCall(unittest.TestCase): matching response — the operator half, out of band.""" def runner(): for _ in range(200): - pending = _sv.list_pending_proposals("dev") + pending = _SV.list_pending_proposals("dev") if pending: p = pending[0] - _sv.write_response("dev", _sv.Response( + _SV.write_response("dev", _sv.Response( proposal_id=p.id, status=status, notes=notes, )) return @@ -487,7 +492,7 @@ class TestHandleToolsCall(unittest.TestCase): responder.join() # A decided proposal drops off the operator's pending list (a response # row exists) — poll itself no longer archives (issue #469 review). - self.assertEqual([], _sv.list_pending_proposals("dev")) + self.assertEqual([], _SV.list_pending_proposals("dev")) def test_pending_response_times_out_without_archive(self): result = _tools_call( @@ -505,7 +510,7 @@ class TestHandleToolsCall(unittest.TestCase): text = result["content"][0]["text"] # type: ignore[index] self.assertIn("status: pending", text) self.assertIn("proposal remains queued", text) - self.assertEqual(1, len(_sv.list_pending_proposals("dev"))) + self.assertEqual(1, len(_SV.list_pending_proposals("dev"))) _ALLOW: dict[str, object] = { "name": _sv.TOOL_EGRESS_ALLOW, @@ -747,7 +752,7 @@ class TestNonBlockingSupervise(unittest.TestCase): justification="need example.com", current_file_hash=_sv.sha256_hex(self._ROUTES), ) - _sv.write_proposal(p) + _SV.write_proposal(p) return p # --- pending response carries the id --- @@ -771,7 +776,7 @@ class TestNonBlockingSupervise(unittest.TestCase): self.assertFalse(result["isError"]) # type: ignore[index] text = result["content"][0]["text"] # type: ignore[index] self.assertIn("status: pending", text) - pending = _sv.list_pending_proposals("dev") + pending = _SV.list_pending_proposals("dev") self.assertEqual(1, len(pending)) # still queued, not archived self.assertIn(pending[0].id, text) # agent got the id to poll @@ -779,7 +784,7 @@ class TestNonBlockingSupervise(unittest.TestCase): def test_check_returns_approved_idempotently(self): p = self._seed_proposal() - _sv.write_response("dev", _sv.Response(proposal_id=p.id, status=_sv.STATUS_APPROVED, notes="ok")) + _SV.write_response("dev", _sv.Response(proposal_id=p.id, status=_sv.STATUS_APPROVED, notes="ok")) result = _check(self.resolver, {"arguments": {"proposal_id": p.id}}) self.assertFalse(result["isError"]) text = result["content"][0]["text"] # type: ignore[index] @@ -792,7 +797,7 @@ class TestNonBlockingSupervise(unittest.TestCase): def test_check_rejected_sets_isError(self): p = self._seed_proposal() - _sv.write_response("dev", _sv.Response(proposal_id=p.id, status=_sv.STATUS_REJECTED, notes="no")) + _SV.write_response("dev", _sv.Response(proposal_id=p.id, status=_sv.STATUS_REJECTED, notes="no")) result = _check(self.resolver, {"arguments": {"proposal_id": p.id}}) self.assertTrue(result["isError"]) self.assertIn("status: rejected", result["content"][0]["text"]) # type: ignore[index] @@ -804,7 +809,7 @@ class TestNonBlockingSupervise(unittest.TestCase): text = result["content"][0]["text"] # type: ignore[index] self.assertIn("status: pending", text) self.assertIn(p.id, text) - self.assertEqual(1, len(_sv.list_pending_proposals("dev"))) # not archived + self.assertEqual(1, len(_SV.list_pending_proposals("dev"))) # not archived def test_check_unknown_id_is_error(self): result = _check(self.resolver, {"arguments": {"proposal_id": "no-such-proposal"}}) @@ -848,15 +853,15 @@ class TestNonBlockingSupervise(unittest.TestCase): }, ServerConfig(response_timeout_seconds=0.05), ) - pid = _sv.list_pending_proposals("dev")[0].id + pid = _SV.list_pending_proposals("dev")[0].id self.assertIn(pid, result["content"][0]["text"]) # type: ignore[index] # 2. operator decides out-of-band - _sv.write_response("dev", _sv.Response(proposal_id=pid, status=_sv.STATUS_APPROVED, notes="ok")) + _SV.write_response("dev", _sv.Response(proposal_id=pid, status=_sv.STATUS_APPROVED, notes="ok")) # 3. agent resumes by polling — no re-proposing poll = _check(self.resolver, {"arguments": {"proposal_id": pid}}) self.assertFalse(poll["isError"]) self.assertIn("status: approved", poll["content"][0]["text"]) # type: ignore[index] - self.assertEqual([], _sv.list_pending_proposals("dev")) # resolved (response exists) + self.assertEqual([], _SV.list_pending_proposals("dev")) # resolved (response exists) if __name__ == "__main__": -- 2.52.0 From db6a151803b0b61239f84e72bad14792a3178d83 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 15:03:37 -0400 Subject: [PATCH 09/30] =?UTF-8?q?refactor(supervise):=20hash=20the=20propo?= =?UTF-8?q?sed=20file=20inside=20Proposal.new;=20sha256=5Fhex=20=E2=86=92?= =?UTF-8?q?=20root=20util?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move sha256_hex out of orchestrator/supervisor/util.py to the root bot_bottle.util (it's a generic string hash, not supervise-specific), and have Proposal.new take the proposed file contents and compute current_file_hash itself instead of every caller passing sha256_hex(proposed_file). Every call site set current_file_hash to the hash of the proposed file, and nothing reads the field except the DB round-trip (schema + _row_to_proposal + from_dict, all untouched), so folding the hash into the factory is behavior-preserving and removes the boilerplate. Callers now pass just the file; the orchestrator no longer imports sha256_hex at all. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/orchestrator/service.py | 2 -- bot_bottle/orchestrator/supervisor/__init__.py | 6 +++--- bot_bottle/orchestrator/supervisor/util.py | 13 ++++--------- bot_bottle/supervisor/types.py | 5 +++-- bot_bottle/util.py | 6 ++++++ tests/unit/test_orchestrator_control_plane.py | 3 +-- tests/unit/test_orchestrator_service.py | 3 +-- tests/unit/test_supervise.py | 6 +----- tests/unit/test_supervise_cli.py | 3 +-- tests/unit/test_supervise_edge.py | 1 - tests/unit/test_supervise_server.py | 3 +-- 11 files changed, 21 insertions(+), 30 deletions(-) diff --git a/bot_bottle/orchestrator/service.py b/bot_bottle/orchestrator/service.py index 452af25a..d35169e4 100644 --- a/bot_bottle/orchestrator/service.py +++ b/bot_bottle/orchestrator/service.py @@ -41,7 +41,6 @@ from .supervisor import ( TOOL_EGRESS_BLOCK, Supervisor, render_diff, - sha256_hex, ) @@ -216,7 +215,6 @@ class Orchestrator: tool=tool, proposed_file=proposed_file, justification=justification, - current_file_hash=sha256_hex(proposed_file), ) self._supervisor.write_proposal(proposal) return proposal.id diff --git a/bot_bottle/orchestrator/supervisor/__init__.py b/bot_bottle/orchestrator/supervisor/__init__.py index 8a2d9127..40d4c507 100644 --- a/bot_bottle/orchestrator/supervisor/__init__.py +++ b/bot_bottle/orchestrator/supervisor/__init__.py @@ -8,8 +8,8 @@ dependency explicit and injectable in tests. For convenience this package also re-exports the neutral vocabulary (`bot_bottle.supervisor`'s types + `SupervisePlan`) and the pure `render_diff` -/ `sha256_hex` helpers (`util`), so orchestrator-side callers import from one -place. The data plane must NOT import this package — it imports +helper (`util`), so orchestrator-side callers import from one place. The data +plane must NOT import this package — it imports `bot_bottle.supervisor` (neutral) directly and reaches the queue over the control-plane RPC. """ @@ -24,7 +24,7 @@ from ..store.store_manager import StoreManager from ..store.queue_store import QueueStore from ...store.audit_store import AuditStore from ...supervisor.types import AuditEntry, Proposal, Response -from .util import render_diff, sha256_hex # noqa: F401 — re-exported for callers +from .util import render_diff # noqa: F401 — re-exported for callers class Supervisor: diff --git a/bot_bottle/orchestrator/supervisor/util.py b/bot_bottle/orchestrator/supervisor/util.py index 3776b566..4c02aaa8 100644 --- a/bot_bottle/orchestrator/supervisor/util.py +++ b/bot_bottle/orchestrator/supervisor/util.py @@ -1,14 +1,13 @@ -"""Pure supervise helpers — diff rendering and content hashing. +"""Pure supervise helpers — diff rendering. -Stateless utilities used alongside the `Supervisor` service. Kept as plain -functions (not methods) because they carry no state and are also handy to -callers that only need to hash or diff. +Stateless utility used alongside the `Supervisor` service. Kept as a plain +function (not a method) because it carries no state. (Content hashing is a +generic helper — `bot_bottle.util.sha256_hex`.) """ from __future__ import annotations import difflib -import hashlib def render_diff(before: str, after: str, *, label: str = "config") -> str: @@ -25,7 +24,3 @@ def render_diff(before: str, after: str, *, label: str = "config") -> str: if not parts: return "" return "".join(p if p.endswith("\n") else p + "\n" for p in parts).rstrip("\n") - - -def sha256_hex(content: str) -> str: - return hashlib.sha256(content.encode("utf-8")).hexdigest() diff --git a/bot_bottle/supervisor/types.py b/bot_bottle/supervisor/types.py index e09fbde2..cd6c3913 100644 --- a/bot_bottle/supervisor/types.py +++ b/bot_bottle/supervisor/types.py @@ -15,6 +15,8 @@ import uuid from dataclasses import dataclass from datetime import datetime, timezone +from ..util import sha256_hex + TOOL_EGRESS_BLOCK = "egress-block" TOOL_EGRESS_ALLOW = "egress-allow" TOOL_GITLEAKS_ALLOW = "gitleaks-allow" @@ -94,7 +96,6 @@ class Proposal: tool: str, proposed_file: str, justification: str, - current_file_hash: str, now: datetime | None = None, ) -> "Proposal": ts = (now or datetime.now(timezone.utc)).isoformat() @@ -105,7 +106,7 @@ class Proposal: proposed_file=proposed_file, justification=justification, arrival_timestamp=ts, - current_file_hash=current_file_hash, + current_file_hash=sha256_hex(proposed_file), ) def to_dict(self) -> dict[str, object]: diff --git a/bot_bottle/util.py b/bot_bottle/util.py index 8cc09200..62741636 100644 --- a/bot_bottle/util.py +++ b/bot_bottle/util.py @@ -5,11 +5,17 @@ level deeper, under their backend package.""" from __future__ import annotations +import hashlib import ipaddress import os import sys +def sha256_hex(content: str) -> str: + """Hex SHA-256 of a UTF-8 string.""" + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + def is_ip_literal(value: str) -> bool: try: ipaddress.ip_address(value) diff --git a/tests/unit/test_orchestrator_control_plane.py b/tests/unit/test_orchestrator_control_plane.py index 66119c3e..8210df51 100644 --- a/tests/unit/test_orchestrator_control_plane.py +++ b/tests/unit/test_orchestrator_control_plane.py @@ -28,7 +28,6 @@ from bot_bottle.orchestrator.store.store_manager import StoreManager from bot_bottle.orchestrator.supervisor import ( Proposal, TOOL_EGRESS_ALLOW, - sha256_hex, Supervisor, ) @@ -420,7 +419,7 @@ class TestDispatchSupervise(unittest.TestCase): "10.243.0.1", metadata=json.dumps({"slug": slug}), policy="routes: []\n") p = Proposal.new( bottle_slug=slug, tool=TOOL_EGRESS_ALLOW, proposed_file=proposed, - justification="need it", current_file_hash=sha256_hex(proposed)) + justification="need it") Supervisor().write_proposal(p) return p.id diff --git a/tests/unit/test_orchestrator_service.py b/tests/unit/test_orchestrator_service.py index d090f3ab..a203af3a 100644 --- a/tests/unit/test_orchestrator_service.py +++ b/tests/unit/test_orchestrator_service.py @@ -20,7 +20,6 @@ from bot_bottle.orchestrator.supervisor import ( Proposal, STATUS_APPROVED, TOOL_EGRESS_ALLOW, - sha256_hex, Supervisor, ) @@ -183,7 +182,7 @@ class TestOrchestratorSupervise(unittest.TestCase): def _queue(self, slug: str, proposed: str) -> str: p = Proposal.new( bottle_slug=slug, tool=TOOL_EGRESS_ALLOW, proposed_file=proposed, - justification="need it", current_file_hash=sha256_hex(proposed)) + justification="need it") Supervisor().write_proposal(p) return p.id diff --git a/tests/unit/test_supervise.py b/tests/unit/test_supervise.py index c609dbe6..8edfe44e 100644 --- a/tests/unit/test_supervise.py +++ b/tests/unit/test_supervise.py @@ -7,6 +7,7 @@ from pathlib import Path from bot_bottle.orchestrator import supervisor as supervise from bot_bottle.paths import host_db_path +from bot_bottle.util import sha256_hex from tests.unit import use_bottle_root from bot_bottle.store.audit_store import AuditStore from bot_bottle.orchestrator.store.queue_store import QueueStore @@ -21,7 +22,6 @@ from bot_bottle.orchestrator.supervisor import ( TOOL_EGRESS_ALLOW, TOOL_GITLEAKS_ALLOW, render_diff, - sha256_hex, ) _SV = Supervisor() @@ -40,7 +40,6 @@ def _proposal( tool=tool, proposed_file=proposed, justification=justification, - current_file_hash=sha256_hex(proposed), now=FIXED_TS, ) @@ -145,13 +144,11 @@ class TestQueueIO(unittest.TestCase): a = Proposal.new( bottle_slug="dev", tool=TOOL_EGRESS_ALLOW, proposed_file="routes:\n - host: early.example.com\n", justification="early", - current_file_hash="x", now=datetime(2026, 5, 25, 10, 0, 0, tzinfo=timezone.utc), ) b = Proposal.new( bottle_slug="dev", tool=TOOL_EGRESS_ALLOW, proposed_file="routes:\n - host: late.example.com\n", justification="late", - current_file_hash="x", now=datetime(2026, 5, 25, 14, 0, 0, tzinfo=timezone.utc), ) # Write in reverse order. @@ -288,7 +285,6 @@ class TestToolConstants(unittest.TestCase): tool=supervise.TOOL_EGRESS_TOKEN_ALLOW, proposed_file="host: api.example.com\n", justification="false positive", - current_file_hash="h", ) self.assertEqual(p, Proposal.from_dict(p.to_dict())) diff --git a/tests/unit/test_supervise_cli.py b/tests/unit/test_supervise_cli.py index f35105b4..f3f44770 100644 --- a/tests/unit/test_supervise_cli.py +++ b/tests/unit/test_supervise_cli.py @@ -19,7 +19,6 @@ from bot_bottle.orchestrator.supervisor import ( TOOL_EGRESS_BLOCK, TOOL_GITLEAKS_ALLOW, TOOL_EGRESS_TOKEN_ALLOW, - sha256_hex, ) @@ -37,7 +36,7 @@ def _proposal(slug: str = "dev", tool: str = TOOL_EGRESS_ALLOW, payload = payloads.get(tool, "") return Proposal.new( bottle_slug=slug, tool=tool, proposed_file=payload, - justification=f"needed for {slug}", current_file_hash=sha256_hex(payload), + justification=f"needed for {slug}", now=now, ) diff --git a/tests/unit/test_supervise_edge.py b/tests/unit/test_supervise_edge.py index d29135b7..82929e45 100644 --- a/tests/unit/test_supervise_edge.py +++ b/tests/unit/test_supervise_edge.py @@ -30,7 +30,6 @@ def _proposal() -> Proposal: tool=TOOL_EGRESS_ALLOW, proposed_file="x", justification="j", - current_file_hash="h", ) diff --git a/tests/unit/test_supervise_server.py b/tests/unit/test_supervise_server.py index fdaff909..0ae9ce50 100644 --- a/tests/unit/test_supervise_server.py +++ b/tests/unit/test_supervise_server.py @@ -92,7 +92,7 @@ class _FakeSuperviseResolver: return None proposal = _sv.Proposal.new( bottle_slug=self.bottle_id, tool=tool, proposed_file=proposed_file, - justification=justification, current_file_hash=_sv.sha256_hex(proposed_file), + justification=justification, ) _SV.write_proposal(proposal) return proposal.id @@ -750,7 +750,6 @@ class TestNonBlockingSupervise(unittest.TestCase): tool=_sv.TOOL_EGRESS_ALLOW, proposed_file=self._ROUTES, justification="need example.com", - current_file_hash=_sv.sha256_hex(self._ROUTES), ) _SV.write_proposal(p) return p -- 2.52.0 From d41236c376e7e7f324ea9a6cb43f746c0404be52 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 15:09:27 -0400 Subject: [PATCH 10/30] refactor(util): make render_diff generic (caller supplies side titles); move to root util MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generalize render_diff so it isn't supervise-flavored: the "(current)" / "(proposed)" side titles are no longer baked in — callers pass `before_title` and `after_label` (required). Move it from orchestrator/supervisor/util.py to the base bot_bottle.util alongside sha256_hex, and delete the now-empty supervisor/util.py. The supervise callsite (Orchestrator.supervise_respond) assigns before_title="current", after_label="proposed"; the facade no longer re-exports render_diff (callers import from bot_bottle.util). Behavior at the supervise callsite is unchanged. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/orchestrator/service.py | 7 +++-- .../orchestrator/supervisor/__init__.py | 8 +++--- bot_bottle/orchestrator/supervisor/util.py | 26 ------------------- bot_bottle/util.py | 26 +++++++++++++++++++ tests/unit/test_supervise.py | 7 +++-- 5 files changed, 38 insertions(+), 36 deletions(-) delete mode 100644 bot_bottle/orchestrator/supervisor/util.py diff --git a/bot_bottle/orchestrator/service.py b/bot_bottle/orchestrator/service.py index d35169e4..740a6fa4 100644 --- a/bot_bottle/orchestrator/service.py +++ b/bot_bottle/orchestrator/service.py @@ -40,8 +40,8 @@ from .supervisor import ( TOOL_EGRESS_ALLOW, TOOL_EGRESS_BLOCK, Supervisor, - render_diff, ) +from ..util import render_diff # Operator decision → Response.status. The apply half (egress tools) runs @@ -350,7 +350,10 @@ class Orchestrator: operator_action=status, operator_notes=notes, justification=proposal.justification, - diff=render_diff(diff_before, diff_after, label=component), + diff=render_diff( + diff_before, diff_after, label=component, + before_title="current", after_label="proposed", + ), )) return True, "" diff --git a/bot_bottle/orchestrator/supervisor/__init__.py b/bot_bottle/orchestrator/supervisor/__init__.py index 40d4c507..de4cf24e 100644 --- a/bot_bottle/orchestrator/supervisor/__init__.py +++ b/bot_bottle/orchestrator/supervisor/__init__.py @@ -7,9 +7,10 @@ object — optionally scoped to a `db_path` — makes the orchestrator's supervi dependency explicit and injectable in tests. For convenience this package also re-exports the neutral vocabulary -(`bot_bottle.supervisor`'s types + `SupervisePlan`) and the pure `render_diff` -helper (`util`), so orchestrator-side callers import from one place. The data -plane must NOT import this package — it imports +(`bot_bottle.supervisor`'s types + `SupervisePlan`), so orchestrator-side +callers import from one place. (Generic helpers like `render_diff` / +`sha256_hex` live in `bot_bottle.util`.) The data plane must NOT import this +package — it imports `bot_bottle.supervisor` (neutral) directly and reaches the queue over the control-plane RPC. """ @@ -24,7 +25,6 @@ from ..store.store_manager import StoreManager from ..store.queue_store import QueueStore from ...store.audit_store import AuditStore from ...supervisor.types import AuditEntry, Proposal, Response -from .util import render_diff # noqa: F401 — re-exported for callers class Supervisor: diff --git a/bot_bottle/orchestrator/supervisor/util.py b/bot_bottle/orchestrator/supervisor/util.py deleted file mode 100644 index 4c02aaa8..00000000 --- a/bot_bottle/orchestrator/supervisor/util.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Pure supervise helpers — diff rendering. - -Stateless utility used alongside the `Supervisor` service. Kept as a plain -function (not a method) because it carries no state. (Content hashing is a -generic helper — `bot_bottle.util.sha256_hex`.) -""" - -from __future__ import annotations - -import difflib - - -def render_diff(before: str, after: str, *, label: str = "config") -> str: - """Unified diff suitable for the audit log + TUI. Empty diff (no changes) - renders as the empty string.""" - diff = difflib.unified_diff( - before.splitlines(keepends=True), - after.splitlines(keepends=True), - fromfile=f"{label} (current)", - tofile=f"{label} (proposed)", - lineterm="", - ) - parts = list(diff) - if not parts: - return "" - return "".join(p if p.endswith("\n") else p + "\n" for p in parts).rstrip("\n") diff --git a/bot_bottle/util.py b/bot_bottle/util.py index 62741636..767e365f 100644 --- a/bot_bottle/util.py +++ b/bot_bottle/util.py @@ -5,6 +5,7 @@ level deeper, under their backend package.""" from __future__ import annotations +import difflib import hashlib import ipaddress import os @@ -16,6 +17,31 @@ def sha256_hex(content: str) -> str: return hashlib.sha256(content.encode("utf-8")).hexdigest() +def render_diff( + before: str, + after: str, + *, + label: str = "config", + before_title: str, + after_label: str, +) -> str: + """Unified diff of `before` vs `after`, with the two sides headed + `{label} ({before_title})` and `{label} ({after_label})`. Empty diff (no + changes) renders as the empty string. The side titles are the caller's — + e.g. "current"/"proposed" for a supervise proposal.""" + diff = difflib.unified_diff( + before.splitlines(keepends=True), + after.splitlines(keepends=True), + fromfile=f"{label} ({before_title})", + tofile=f"{label} ({after_label})", + lineterm="", + ) + parts = list(diff) + if not parts: + return "" + return "".join(p if p.endswith("\n") else p + "\n" for p in parts).rstrip("\n") + + def is_ip_literal(value: str) -> bool: try: ipaddress.ip_address(value) diff --git a/tests/unit/test_supervise.py b/tests/unit/test_supervise.py index 8edfe44e..13957058 100644 --- a/tests/unit/test_supervise.py +++ b/tests/unit/test_supervise.py @@ -7,7 +7,7 @@ from pathlib import Path from bot_bottle.orchestrator import supervisor as supervise from bot_bottle.paths import host_db_path -from bot_bottle.util import sha256_hex +from bot_bottle.util import render_diff, sha256_hex from tests.unit import use_bottle_root from bot_bottle.store.audit_store import AuditStore from bot_bottle.orchestrator.store.queue_store import QueueStore @@ -21,7 +21,6 @@ from bot_bottle.orchestrator.supervisor import ( Supervisor, TOOL_EGRESS_ALLOW, TOOL_GITLEAKS_ALLOW, - render_diff, ) _SV = Supervisor() @@ -249,10 +248,10 @@ class TestAuditLog(unittest.TestCase): class TestDiffAndHash(unittest.TestCase): def test_render_diff_returns_empty_when_unchanged(self): - self.assertEqual("", render_diff("a\nb\n", "a\nb\n")) + self.assertEqual("", render_diff("a\nb\n", "a\nb\n", before_title="current", after_label="proposed")) def test_render_diff_shows_changes(self): - diff = render_diff("a\nb\nc\n", "a\nB\nc\n", label="routes.yaml") + diff = render_diff("a\nb\nc\n", "a\nB\nc\n", label="routes.yaml", before_title="current", after_label="proposed") self.assertIn("routes.yaml (current)", diff) self.assertIn("routes.yaml (proposed)", diff) self.assertIn("-b", diff) -- 2.52.0 From b96a8b44e0829164548ed77ebc2b83a5aab0fc33 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 15:26:52 -0400 Subject: [PATCH 11/30] refactor(docker): rename OrchestratorService -> DockerInfraService, move to backend/docker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OrchestratorService wasn't the orchestrator — it's the host-side lifecycle of the docker infra *container* (the one that runs the Orchestrator). It read as "the orchestrator as a service" and lived in orchestrator/lifecycle.py, while its siblings (macOS MacosInfraService, Firecracker infra_vm) live under their backend package. Rename it DockerInfraService and move it to backend/docker/infra.py alongside the docker backend, with its docker-only constants (INFRA_*/ORCHESTRATOR_* image + container names, daemon list, mount paths). orchestrator/lifecycle.py keeps only the backend-neutral pieces the other infra services share — DEFAULT_PORT, DEFAULT_STARTUP_TIMEOUT_SECONDS, OrchestratorStartError, source_hash — which macOS / firecracker / client still import from there. backend/docker/infra.py imports those (backend -> orchestrator is an allowed direction). Renamed the unit test to test_docker_infra.py. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/backend/docker/backend.py | 4 +- .../backend/docker/consolidated_launch.py | 6 +- bot_bottle/backend/docker/infra.py | 261 ++++++++++++++++++ bot_bottle/backend/docker/launch.py | 2 +- bot_bottle/orchestrator/lifecycle.py | 253 +---------------- bot_bottle/orchestrator/rotate_ca.py | 2 +- .../integration/test_multitenant_isolation.py | 4 +- ..._orchestrator_docker_control_plane_auth.py | 6 +- tests/unit/test_backend_selection.py | 4 +- ...ator_lifecycle.py => test_docker_infra.py} | 20 +- tests/unit/test_orchestrator_rotate_ca.py | 2 +- 11 files changed, 296 insertions(+), 268 deletions(-) create mode 100644 bot_bottle/backend/docker/infra.py rename tests/unit/{test_orchestrator_lifecycle.py => test_docker_infra.py} (95%) diff --git a/bot_bottle/backend/docker/backend.py b/bot_bottle/backend/docker/backend.py index 8cab6d77..eff4d5d9 100644 --- a/bot_bottle/backend/docker/backend.py +++ b/bot_bottle/backend/docker/backend.py @@ -112,8 +112,8 @@ class DockerBottleBackend(BottleBackend["DockerBottlePlan", "DockerBottleCleanup yield bottle def ensure_orchestrator(self) -> str: - from ...orchestrator.lifecycle import OrchestratorService - return OrchestratorService().ensure_running() + from .infra import DockerInfraService + return DockerInfraService().ensure_running() def supervise_mcp_url(self, plan: DockerBottlePlan) -> str: """Docker bottles reach the supervise daemon via the diff --git a/bot_bottle/backend/docker/consolidated_launch.py b/bot_bottle/backend/docker/consolidated_launch.py index 2f57efb9..2b7097fc 100644 --- a/bot_bottle/backend/docker/consolidated_launch.py +++ b/bot_bottle/backend/docker/consolidated_launch.py @@ -21,7 +21,7 @@ from ...egress import EgressPlan from ...git_gate import GitGatePlan from ...orchestrator.client import OrchestratorClient from ...gateway import GATEWAY_NETWORK -from ...orchestrator.lifecycle import INFRA_NAME, OrchestratorService +from .infra import INFRA_NAME, DockerInfraService from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME from ...orchestrator.reprovision import reprovision_bottles from ..consolidated_util import provision_bottle @@ -144,7 +144,7 @@ def launch_consolidated( *, image_ref: str = "", tokens: dict[str, str] | None = None, - service: OrchestratorService | None = None, + service: DockerInfraService | None = None, infra_name: str = INFRA_NAME, network: str = GATEWAY_NETWORK, ) -> LaunchContext: @@ -154,7 +154,7 @@ def launch_consolidated( Also reprovisiones egress tokens for any already-running bottles that lost their in-memory credentials (e.g. after an infra container restart), so they regain egress access before the new bottle is registered.""" - service = service or OrchestratorService() + service = service or DockerInfraService() url = service.ensure_running() _reprovision_running_bottles(url, network=network, infra_name=infra_name) client = OrchestratorClient(url) diff --git a/bot_bottle/backend/docker/infra.py b/bot_bottle/backend/docker/infra.py new file mode 100644 index 00000000..2471082e --- /dev/null +++ b/bot_bottle/backend/docker/infra.py @@ -0,0 +1,261 @@ +"""The per-host infra container for the docker backend (PRD 0070). + +Runs both the orchestrator control plane and the gateway data plane inside a +single `bot-bottle-infra` container on the shared gateway network — the docker +analogue of the macOS infra container (`backend/macos_container/infra.py`) and +the Firecracker infra VM (`backend/firecracker/infra_vm.py`). `gateway_init` is +PID 1 and supervises both; the infra container is an idempotent per-host +singleton. + +The combined container replaces the prior two-container split +(bot-bottle-orchestrator + bot-bottle-orch-gateway). The host CLI reaches the +control plane via a published loopback port; gateway daemons reach it over +127.0.0.1 (same container). + +The shared, backend-neutral pieces (control-plane port, startup timeout, +`OrchestratorStartError`, `source_hash`) live in `orchestrator.lifecycle`. +""" + +from __future__ import annotations + +import os +import time +import urllib.error +import urllib.request +from pathlib import Path + +from ... import log +from ...control_auth import ROLE_GATEWAY, mint +from ...docker_cmd import run_docker +from ...paths import ( + CONTROL_AUTH_JWT_ENV, + CONTROL_PLANE_TOKEN_ENV, + bot_bottle_root, + host_control_plane_token, + host_gateway_ca_dir, +) +from ...gateway import ( + GATEWAY_DOCKERFILE, + GATEWAY_IMAGE, + GATEWAY_NETWORK, + GatewayError, + MITMPROXY_HOME, +) +from ...orchestrator.lifecycle import ( + DEFAULT_PORT, + DEFAULT_STARTUP_TIMEOUT_SECONDS, + OrchestratorStartError, + source_hash, +) + +INFRA_NAME = "bot-bottle-infra" +INFRA_LABEL = "bot-bottle-infra=1" +# The combined infra image: gateway data plane + orchestrator content. +# Built from Dockerfile.infra (FROM gateway + COPY --from orchestrator). +INFRA_IMAGE = os.environ.get("BOT_BOTTLE_INFRA_IMAGE", "bot-bottle-infra:latest") +INFRA_DOCKERFILE = "Dockerfile.infra" +# Baked as a container label so `ensure_running` can detect whether the +# running container is executing the current bind-mounted source. +INFRA_SOURCE_HASH_LABEL = "bot-bottle-infra-source-hash" + +# Orchestrator image: the single canonical definition of the control-plane +# content (lean: python:3.12-slim + bot_bottle package, no mitmproxy/git). +# Used as a build intermediate: `Dockerfile.infra` COPY --from this image. +ORCHESTRATOR_IMAGE = os.environ.get( + "BOT_BOTTLE_ORCHESTRATOR_IMAGE", "bot-bottle-orchestrator:latest" +) +ORCHESTRATOR_DOCKERFILE = "Dockerfile.orchestrator" + +# The gateway daemons + orchestrator the infra container runs. +# BOT_BOTTLE_GATEWAY_DAEMONS listing `orchestrator` opts it in to +# gateway_init's supervise tree (see gateway_init._OPT_IN_DAEMONS). +_INFRA_DAEMONS = "egress,git-http,supervise,orchestrator" + +# The bind-mount path for the live control-plane source inside the +# container. Separate from /app so the gateway's baked scripts +# (egress_addon.py, egress-entrypoint.sh) are not overlaid. +_SRC_IN_CONTAINER = "/bot-bottle-src" +# Bot-bottle host-root bind-mount inside the container (DB + state). The +# orchestrator control plane opens bot-bottle.db under here (via BOT_BOTTLE_ROOT +# -> host_db_path()); it is the ONLY process in the container with a file +# handle on it (PRD 0070 / issue #469). +_ROOT_IN_CONTAINER = "/bot-bottle-root" + +_HEALTH_POLL_SECONDS = 0.25 +_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0 + +_REPO_ROOT = Path(__file__).resolve().parents[3] + + +class DockerInfraService: + """Manages the single per-host docker infra container (control plane + + gateway). Callers only need `ensure_running()` + `url`. + + `infra_name` / `infra_label` let callers run independent infra containers + on the same host without name collisions (e.g. isolated integration tests + that can't share the production INFRA_NAME singleton).""" + + def __init__( + self, + *, + port: int = DEFAULT_PORT, + network: str = GATEWAY_NETWORK, + image: str = INFRA_IMAGE, + repo_root: Path = _REPO_ROOT, + host_root: Path | None = None, + infra_name: str = INFRA_NAME, + infra_label: str = INFRA_LABEL, + ) -> None: + self.port = port + self.network = network + self.image = image + self._repo_root = repo_root + self._host_root = host_root or bot_bottle_root() + self._infra_name = infra_name + self._infra_label = infra_label + + @property + def url(self) -> str: + """Host-side control-plane URL (published loopback port).""" + return f"http://127.0.0.1:{self.port}" + + def is_healthy(self, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS) -> bool: + try: + with urllib.request.urlopen(f"{self.url}/health", timeout=timeout) as resp: + return resp.status == 200 + except (urllib.error.URLError, TimeoutError, OSError): + return False + + def _container_running(self, name: str) -> bool: + proc = run_docker(["docker", "ps", "--filter", f"name=^/{name}$", "--format", "{{.Names}}"]) + return name in proc.stdout.split() + + def _infra_source_current(self, current_hash: str) -> bool: + """True iff the running infra container was started from the current + bind-mounted source. Mirrors the macOS backend's `_source_current`.""" + if not self._container_running(self._infra_name): + return False + proc = run_docker([ + "docker", "inspect", "--format", + "{{ index .Config.Labels \"" + INFRA_SOURCE_HASH_LABEL + "\" }}", + self._infra_name, + ]) + if proc.returncode != 0: + return True # can't compare → don't churn a working container + return proc.stdout.strip() == current_hash + + def _ensure_network(self) -> None: + if run_docker(["docker", "network", "inspect", self.network]).returncode == 0: + return + proc = run_docker(["docker", "network", "create", self.network]) + if proc.returncode != 0 and "already exists" not in proc.stderr: + raise GatewayError( + f"gateway network {self.network} failed to create: {proc.stderr.strip()}" + ) + + def _build_images(self) -> None: + """Build the gateway base, the orchestrator intermediate, then the + infra image. All are cache-aware: a no-op when nothing changed.""" + for tag, dockerfile in ( + (GATEWAY_IMAGE, GATEWAY_DOCKERFILE), + (ORCHESTRATOR_IMAGE, ORCHESTRATOR_DOCKERFILE), + (self.image, INFRA_DOCKERFILE), + ): + argv = ["docker", "build", "-t", tag, + "-f", str(self._repo_root / dockerfile), + str(self._repo_root)] + if os.environ.get("BOT_BOTTLE_NO_CACHE"): + argv.insert(2, "--no-cache") + proc = run_docker(argv) + if proc.returncode != 0: + raise GatewayError(f"{dockerfile} build failed: {proc.stderr.strip()}") + + def _run_infra_container(self, current_hash: str) -> None: + """Start the combined infra container (idempotent: clears a stale + fixed-name container first). Labels the container with `current_hash` + so a later `ensure_running` can detect a real code change.""" + self._ensure_network() + run_docker(["docker", "rm", "--force", self._infra_name]) + _signing_key = host_control_plane_token() + proc = run_docker([ + "docker", "run", "--detach", + "--name", self._infra_name, + "--label", self._infra_label, + "--label", f"{INFRA_SOURCE_HASH_LABEL}={current_hash}", + "--network", self.network, + # Host CLI reaches the control plane here (loopback only). + # gateway_init always starts the orchestrator on DEFAULT_PORT (8099) + # inside the container; self.port is the host-side published port. + "--publish", f"127.0.0.1:{self.port}:{DEFAULT_PORT}", + # Persist the mitmproxy CA on the host so it survives container + # recreation AND docker volume pruning (issue #450): every agent + # trusts this one CA, so a fresh one would break all running bottles. + "--volume", f"{host_gateway_ca_dir()}:{MITMPROXY_HOME}", + # Live control-plane source, mounted to a path that does not + # overlay the gateway's baked /app scripts. + "--volume", f"{self._repo_root}:{_SRC_IN_CONTAINER}:ro", + # PYTHONPATH lets the orchestrator (and other Python daemons) + # import the live source ahead of the installed package. + "--env", f"PYTHONPATH={_SRC_IN_CONTAINER}", + # Orchestrator registry DB on the host (sole writer: control plane). + "--volume", f"{self._host_root}:{_ROOT_IN_CONTAINER}", + "--env", f"BOT_BOTTLE_ROOT={_ROOT_IN_CONTAINER}", + # Control-plane signing key (orchestrator: verifies tokens) + the + # pre-minted `gateway` JWT (data-plane daemons: present it). gateway_init + # scopes each to its process, so a compromised data-plane daemon never + # sees the key and can't mint a `cli` token (issue #469 review). + "--env", CONTROL_PLANE_TOKEN_ENV, + "--env", CONTROL_AUTH_JWT_ENV, + # Gateway daemons reach the orchestrator over loopback at its + # fixed internal port (DEFAULT_PORT), independent of self.port. + "--env", f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{DEFAULT_PORT}", + # Opt the orchestrator into gateway_init's supervise tree. + "--env", f"BOT_BOTTLE_GATEWAY_DAEMONS={_INFRA_DAEMONS}", + self.image, + ], env={ + **os.environ, + CONTROL_PLANE_TOKEN_ENV: _signing_key, + CONTROL_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key), + }) + if proc.returncode != 0: + raise OrchestratorStartError( + f"infra container failed to start: {proc.stderr.strip()}" + ) + + def ensure_running( + self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS, + ) -> str: + """Ensure the infra container (control plane + gateway) is up; return + the host control-plane URL. Idempotent — a healthy container on current + source is left untouched. Raises `OrchestratorStartError` on timeout.""" + self._build_images() + + current_hash = source_hash(self._repo_root) + if self.is_healthy() and self._infra_source_current(current_hash): + return self.url + + log.info("starting infra container", context={"name": self._infra_name}) + self._run_infra_container(current_hash) + + deadline = time.monotonic() + startup_timeout + while time.monotonic() < deadline: + if self.is_healthy(): + log.info("infra container healthy", context={"url": self.url}) + return self.url + time.sleep(_HEALTH_POLL_SECONDS) + raise OrchestratorStartError( + f"infra container at {self.url} did not become healthy within {startup_timeout:g}s" + ) + + def stop(self) -> None: + """Remove the infra container (idempotent).""" + run_docker(["docker", "rm", "--force", self._infra_name]) + + +__all__ = [ + "DockerInfraService", + "INFRA_NAME", + "INFRA_IMAGE", + "INFRA_SOURCE_HASH_LABEL", + "ORCHESTRATOR_IMAGE", +] diff --git a/bot_bottle/backend/docker/launch.py b/bot_bottle/backend/docker/launch.py index 78c87855..068b7c41 100644 --- a/bot_bottle/backend/docker/launch.py +++ b/bot_bottle/backend/docker/launch.py @@ -66,7 +66,7 @@ from .compose import ( from .consolidated_compose import consolidated_agent_compose from ...orchestrator.store.config_store import resolve_teardown_timeout from .consolidated_launch import launch_consolidated, teardown_consolidated -from ...orchestrator.lifecycle import INFRA_NAME +from .infra import INFRA_NAME from .gateway import DockerGateway diff --git a/bot_bottle/orchestrator/lifecycle.py b/bot_bottle/orchestrator/lifecycle.py index 5c7b9c61..d57b14da 100644 --- a/bot_bottle/orchestrator/lifecycle.py +++ b/bot_bottle/orchestrator/lifecycle.py @@ -1,94 +1,29 @@ -"""Orchestrator + gateway lifecycle (PRD 0070, docker slice). +"""Shared orchestrator-process constants + helpers (PRD 0070). -Runs both the orchestrator control plane and the gateway data plane inside -a single `bot-bottle-infra` container on the shared gateway network — -matching the structure already used by the macOS and Firecracker backends. -`gateway_init` is PID 1 and supervises both; the infra container is an -idempotent per-host singleton. - -The combined container replaces the prior two-container split -(bot-bottle-orchestrator + bot-bottle-orch-gateway). The host CLI reaches -the control plane via a published loopback port; gateway daemons reach it -over 127.0.0.1 (same container). +Backend-neutral pieces the per-backend infra services build on: the +control-plane port, the startup timeout, the start-error, and the source hash +used to detect a code change. The per-backend infra lifecycle lives with each +backend — docker: `backend.docker.infra.DockerInfraService`; macOS: +`backend.macos_container.infra`; firecracker: `backend.firecracker.infra_vm`. """ from __future__ import annotations import hashlib -import os -import time -import urllib.error -import urllib.request from pathlib import Path -from .. import log -from ..control_auth import ROLE_GATEWAY, mint -from ..docker_cmd import run_docker -from ..paths import ( - CONTROL_AUTH_JWT_ENV, - CONTROL_PLANE_TOKEN_ENV, - bot_bottle_root, - host_control_plane_token, - host_gateway_ca_dir, -) -from ..gateway import ( - GATEWAY_DOCKERFILE, - GATEWAY_IMAGE, - GATEWAY_NETWORK, - GatewayError, - MITMPROXY_HOME, -) - DEFAULT_PORT = 8099 DEFAULT_STARTUP_TIMEOUT_SECONDS = 45.0 -INFRA_NAME = "bot-bottle-infra" -INFRA_LABEL = "bot-bottle-infra=1" -# The combined infra image: gateway data plane + orchestrator content. -# Built from Dockerfile.infra (FROM gateway + COPY --from orchestrator). -INFRA_IMAGE = os.environ.get("BOT_BOTTLE_INFRA_IMAGE", "bot-bottle-infra:latest") -INFRA_DOCKERFILE = "Dockerfile.infra" -# Baked as a container label so `ensure_running` can detect whether the -# running container is executing the current bind-mounted source. -INFRA_SOURCE_HASH_LABEL = "bot-bottle-infra-source-hash" - -# Orchestrator image: the single canonical definition of the control-plane -# content (lean: python:3.12-slim + bot_bottle package, no mitmproxy/git). -# Used as a build intermediate: `Dockerfile.infra` COPY --from this image. -ORCHESTRATOR_IMAGE = os.environ.get( - "BOT_BOTTLE_ORCHESTRATOR_IMAGE", "bot-bottle-orchestrator:latest" -) -ORCHESTRATOR_DOCKERFILE = "Dockerfile.orchestrator" - -# The gateway daemons + orchestrator the infra container runs. -# BOT_BOTTLE_GATEWAY_DAEMONS listing `orchestrator` opts it in to -# gateway_init's supervise tree (see gateway_init._OPT_IN_DAEMONS). -_INFRA_DAEMONS = "egress,git-http,supervise,orchestrator" - -# The bind-mount path for the live control-plane source inside the -# container. Separate from /app so the gateway's baked scripts -# (egress_addon.py, egress-entrypoint.sh) are not overlaid. -_SRC_IN_CONTAINER = "/bot-bottle-src" -# Bot-bottle host-root bind-mount inside the container (DB + state). The -# orchestrator control plane opens bot-bottle.db under here (via BOT_BOTTLE_ROOT -# -> host_db_path()); it is the ONLY process in the container with a file -# handle on it (PRD 0070 / issue #469). -_ROOT_IN_CONTAINER = "/bot-bottle-root" - -_HEALTH_POLL_SECONDS = 0.25 -_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0 - -_REPO_ROOT = Path(__file__).resolve().parents[2] - class OrchestratorStartError(RuntimeError): - """The infra container did not become healthy within the timeout.""" + """The infra container/VM did not become healthy within the timeout.""" def source_hash(repo_root: Path) -> str: """Content hash of the orchestrator's bind-mounted Python source (the `bot_bottle` package the control-plane process imports). Changes only - when the code that would actually run changes — `ensure_running` + when the code that would actually run changes — a backend's `ensure_running` recreates the container on a mismatch so a code change takes effect, but leaves a healthy up-to-date container alone to preserve in-memory egress tokens.""" @@ -99,179 +34,9 @@ def source_hash(repo_root: Path) -> str: return h.hexdigest() -class OrchestratorService: - """Manages the single per-host infra container (control plane + gateway). - Callers only need `ensure_running()` + `url`. - - `infra_name` / `infra_label` let backends run independent infra containers - on the same host without name collisions (e.g. isolated integration tests - that can't share the production INFRA_NAME singleton).""" - - def __init__( - self, - *, - port: int = DEFAULT_PORT, - network: str = GATEWAY_NETWORK, - image: str = INFRA_IMAGE, - repo_root: Path = _REPO_ROOT, - host_root: Path | None = None, - infra_name: str = INFRA_NAME, - infra_label: str = INFRA_LABEL, - ) -> None: - self.port = port - self.network = network - self.image = image - self._repo_root = repo_root - self._host_root = host_root or bot_bottle_root() - self._infra_name = infra_name - self._infra_label = infra_label - - @property - def url(self) -> str: - """Host-side control-plane URL (published loopback port).""" - return f"http://127.0.0.1:{self.port}" - - def is_healthy(self, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS) -> bool: - try: - with urllib.request.urlopen(f"{self.url}/health", timeout=timeout) as resp: - return resp.status == 200 - except (urllib.error.URLError, TimeoutError, OSError): - return False - - def _container_running(self, name: str) -> bool: - proc = run_docker(["docker", "ps", "--filter", f"name=^/{name}$", "--format", "{{.Names}}"]) - return name in proc.stdout.split() - - def _infra_source_current(self, current_hash: str) -> bool: - """True iff the running infra container was started from the current - bind-mounted source. Mirrors the macOS backend's `_source_current`.""" - if not self._container_running(self._infra_name): - return False - proc = run_docker([ - "docker", "inspect", "--format", - "{{ index .Config.Labels \"" + INFRA_SOURCE_HASH_LABEL + "\" }}", - self._infra_name, - ]) - if proc.returncode != 0: - return True # can't compare → don't churn a working container - return proc.stdout.strip() == current_hash - - def _ensure_network(self) -> None: - if run_docker(["docker", "network", "inspect", self.network]).returncode == 0: - return - proc = run_docker(["docker", "network", "create", self.network]) - if proc.returncode != 0 and "already exists" not in proc.stderr: - raise GatewayError( - f"gateway network {self.network} failed to create: {proc.stderr.strip()}" - ) - - def _build_images(self) -> None: - """Build the gateway base, the orchestrator intermediate, then the - infra image. All are cache-aware: a no-op when nothing changed.""" - for tag, dockerfile in ( - (GATEWAY_IMAGE, GATEWAY_DOCKERFILE), - (ORCHESTRATOR_IMAGE, ORCHESTRATOR_DOCKERFILE), - (self.image, INFRA_DOCKERFILE), - ): - argv = ["docker", "build", "-t", tag, - "-f", str(self._repo_root / dockerfile), - str(self._repo_root)] - if os.environ.get("BOT_BOTTLE_NO_CACHE"): - argv.insert(2, "--no-cache") - proc = run_docker(argv) - if proc.returncode != 0: - raise GatewayError(f"{dockerfile} build failed: {proc.stderr.strip()}") - - def _run_infra_container(self, current_hash: str) -> None: - """Start the combined infra container (idempotent: clears a stale - fixed-name container first). Labels the container with `current_hash` - so a later `ensure_running` can detect a real code change.""" - self._ensure_network() - run_docker(["docker", "rm", "--force", self._infra_name]) - _signing_key = host_control_plane_token() - proc = run_docker([ - "docker", "run", "--detach", - "--name", self._infra_name, - "--label", self._infra_label, - "--label", f"{INFRA_SOURCE_HASH_LABEL}={current_hash}", - "--network", self.network, - # Host CLI reaches the control plane here (loopback only). - # gateway_init always starts the orchestrator on DEFAULT_PORT (8099) - # inside the container; self.port is the host-side published port. - "--publish", f"127.0.0.1:{self.port}:{DEFAULT_PORT}", - # Persist the mitmproxy CA on the host so it survives container - # recreation AND docker volume pruning (issue #450): every agent - # trusts this one CA, so a fresh one would break all running bottles. - "--volume", f"{host_gateway_ca_dir()}:{MITMPROXY_HOME}", - # Live control-plane source, mounted to a path that does not - # overlay the gateway's baked /app scripts. - "--volume", f"{self._repo_root}:{_SRC_IN_CONTAINER}:ro", - # PYTHONPATH lets the orchestrator (and other Python daemons) - # import the live source ahead of the installed package. - "--env", f"PYTHONPATH={_SRC_IN_CONTAINER}", - # Orchestrator registry DB on the host (sole writer: control plane). - "--volume", f"{self._host_root}:{_ROOT_IN_CONTAINER}", - "--env", f"BOT_BOTTLE_ROOT={_ROOT_IN_CONTAINER}", - # Control-plane signing key (orchestrator: verifies tokens) + the - # pre-minted `gateway` JWT (data-plane daemons: present it). gateway_init - # scopes each to its process, so a compromised data-plane daemon never - # sees the key and can't mint a `cli` token (issue #469 review). - "--env", CONTROL_PLANE_TOKEN_ENV, - "--env", CONTROL_AUTH_JWT_ENV, - # Gateway daemons reach the orchestrator over loopback at its - # fixed internal port (DEFAULT_PORT), independent of self.port. - "--env", f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{DEFAULT_PORT}", - # Opt the orchestrator into gateway_init's supervise tree. - "--env", f"BOT_BOTTLE_GATEWAY_DAEMONS={_INFRA_DAEMONS}", - self.image, - ], env={ - **os.environ, - CONTROL_PLANE_TOKEN_ENV: _signing_key, - CONTROL_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key), - }) - if proc.returncode != 0: - raise OrchestratorStartError( - f"infra container failed to start: {proc.stderr.strip()}" - ) - - def ensure_running( - self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS, - ) -> str: - """Ensure the infra container (control plane + gateway) is up; return - the host control-plane URL. Idempotent — a healthy container on current - source is left untouched. Raises `OrchestratorStartError` on timeout.""" - self._build_images() - - current_hash = source_hash(self._repo_root) - if self.is_healthy() and self._infra_source_current(current_hash): - return self.url - - log.info("starting infra container", context={"name": self._infra_name}) - self._run_infra_container(current_hash) - - deadline = time.monotonic() + startup_timeout - while time.monotonic() < deadline: - if self.is_healthy(): - log.info("infra container healthy", context={"url": self.url}) - return self.url - time.sleep(_HEALTH_POLL_SECONDS) - raise OrchestratorStartError( - f"infra container at {self.url} did not become healthy within {startup_timeout:g}s" - ) - - def stop(self) -> None: - """Remove the infra container (idempotent).""" - run_docker(["docker", "rm", "--force", self._infra_name]) - - __all__ = [ - "OrchestratorService", - "OrchestratorStartError", - "INFRA_NAME", - "INFRA_IMAGE", - "INFRA_SOURCE_HASH_LABEL", - "ORCHESTRATOR_IMAGE", "DEFAULT_PORT", "DEFAULT_STARTUP_TIMEOUT_SECONDS", + "OrchestratorStartError", "source_hash", ] diff --git a/bot_bottle/orchestrator/rotate_ca.py b/bot_bottle/orchestrator/rotate_ca.py index 5ae7d2d2..0cae3004 100644 --- a/bot_bottle/orchestrator/rotate_ca.py +++ b/bot_bottle/orchestrator/rotate_ca.py @@ -26,7 +26,7 @@ from pathlib import Path from ..docker_cmd import run_docker from ..paths import host_gateway_ca_dir from ..gateway import GATEWAY_NAME, rotate_gateway_ca -from .lifecycle import INFRA_NAME +from ..backend.docker.infra import INFRA_NAME # The containers whose mitmproxy would still be serving the old CA from memory: # the consolidated infra container and the standalone per-host gateway. diff --git a/tests/integration/test_multitenant_isolation.py b/tests/integration/test_multitenant_isolation.py index fdd8ea51..e32f9e2f 100644 --- a/tests/integration/test_multitenant_isolation.py +++ b/tests/integration/test_multitenant_isolation.py @@ -30,7 +30,7 @@ from bot_bottle.backend.docker.egress import EGRESS_PORT from bot_bottle.backend.docker.gateway_net import next_free_ip from bot_bottle.orchestrator.client import OrchestratorClient from bot_bottle.gateway import GATEWAY_IMAGE, GATEWAY_NAME, GATEWAY_NETWORK -from bot_bottle.orchestrator.lifecycle import OrchestratorService +from bot_bottle.backend.docker.infra import DockerInfraService from tests._docker import skip_unless_docker # One upstream reachable under two names; reflects the Authorization header so @@ -85,7 +85,7 @@ class TestMultitenantIsolation(unittest.TestCase): self._tmp = tempfile.TemporaryDirectory() self.addCleanup(self._tmp.cleanup) # Throwaway root → a clean registry DB, independent of the host's. - self.svc = OrchestratorService(host_root=Path(self._tmp.name)) + self.svc = DockerInfraService(host_root=Path(self._tmp.name)) self.addCleanup(self._teardown_docker) # ensure_running builds the bundle image (slow on a cold cache) and # brings up the shared network + gateway + orchestrator. diff --git a/tests/integration/test_orchestrator_docker_control_plane_auth.py b/tests/integration/test_orchestrator_docker_control_plane_auth.py index aedb8caa..08a6c620 100644 --- a/tests/integration/test_orchestrator_docker_control_plane_auth.py +++ b/tests/integration/test_orchestrator_docker_control_plane_auth.py @@ -27,7 +27,7 @@ from pathlib import Path from bot_bottle.control_auth import ROLE_CLI, ROLE_GATEWAY, mint from bot_bottle.orchestrator.client import OrchestratorClient -from bot_bottle.orchestrator.lifecycle import OrchestratorService +from bot_bottle.backend.docker.infra import DockerInfraService from bot_bottle.paths import host_control_plane_token from tests._docker import skip_unless_docker @@ -54,7 +54,7 @@ class TestDockerControlPlaneAuthIntegration(unittest.TestCase): cls.addClassCleanup(cls._tmp.cleanup) # host_control_plane_token() — both the token read below and the one - # OrchestratorService injects into the container's env — resolves its + # DockerInfraService injects into the container's env — resolves its # path via the *ambient* BOT_BOTTLE_ROOT env var, not the host_root # kwarg passed to the constructor (that kwarg only controls the DB # bind-mount destination). Without pointing the env var at the same @@ -78,7 +78,7 @@ class TestDockerControlPlaneAuthIntegration(unittest.TestCase): cls._teardown_docker, infra_name, network, host_root ) - cls.svc = OrchestratorService( + cls.svc = DockerInfraService( infra_name=infra_name, network=network, image=_TEST_INFRA_IMAGE, diff --git a/tests/unit/test_backend_selection.py b/tests/unit/test_backend_selection.py index a5a4c187..bb549d13 100644 --- a/tests/unit/test_backend_selection.py +++ b/tests/unit/test_backend_selection.py @@ -390,10 +390,10 @@ class TestEnsureOrchestrator(unittest.TestCase): the orchestrator + gateway containers; firecracker boots the infra VM; macos-container starts the infra container.""" - def test_docker_delegates_to_orchestrator_service(self): + def test_docker_delegates_to_infra_service(self): b = get_bottle_backend("docker") with patch( - "bot_bottle.orchestrator.lifecycle.OrchestratorService" + "bot_bottle.backend.docker.infra.DockerInfraService" ) as service_cls: service_cls.return_value.ensure_running.return_value = ( "http://127.0.0.1:8099" diff --git a/tests/unit/test_orchestrator_lifecycle.py b/tests/unit/test_docker_infra.py similarity index 95% rename from tests/unit/test_orchestrator_lifecycle.py rename to tests/unit/test_docker_infra.py index 1a659084..bf85eb41 100644 --- a/tests/unit/test_orchestrator_lifecycle.py +++ b/tests/unit/test_docker_infra.py @@ -9,20 +9,22 @@ from pathlib import Path from unittest.mock import MagicMock, Mock, patch from bot_bottle.gateway import GatewayError -from bot_bottle.orchestrator.lifecycle import ( +from bot_bottle.backend.docker.infra import ( INFRA_NAME, INFRA_SOURCE_HASH_LABEL, - OrchestratorService, + DockerInfraService, +) +from bot_bottle.orchestrator.lifecycle import ( OrchestratorStartError, source_hash, ) from bot_bottle.paths import GATEWAY_CA_DIRNAME from tests.unit import use_bottle_root -_URLOPEN = "bot_bottle.orchestrator.lifecycle.urllib.request.urlopen" -_RUN = "bot_bottle.orchestrator.lifecycle.run_docker" -_SLEEP = "bot_bottle.orchestrator.lifecycle.time.sleep" -_MONOTONIC = "bot_bottle.orchestrator.lifecycle.time.monotonic" +_URLOPEN = "bot_bottle.backend.docker.infra.urllib.request.urlopen" +_RUN = "bot_bottle.backend.docker.infra.run_docker" +_SLEEP = "bot_bottle.backend.docker.infra.time.sleep" +_MONOTONIC = "bot_bottle.backend.docker.infra.time.monotonic" def _health(status: int) -> MagicMock: @@ -35,12 +37,12 @@ def _proc(returncode: int = 0, stdout: str = "", stderr: str = "") -> Mock: return Mock(returncode=returncode, stdout=stdout, stderr=stderr) -class TestOrchestratorService(unittest.TestCase): +class TestDockerInfraService(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() self.addCleanup(self._tmp.cleanup) self.addCleanup(use_bottle_root(Path(self._tmp.name))) - self.svc = OrchestratorService(port=8099) + self.svc = DockerInfraService(port=8099) def test_url(self) -> None: self.assertEqual("http://127.0.0.1:8099", self.svc.url) @@ -159,7 +161,7 @@ class TestOrchestratorService(unittest.TestCase): return _proc(stdout="") return _proc() - svc = OrchestratorService(port=20001) + svc = DockerInfraService(port=20001) with patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \ patch(_RUN, side_effect=fake), patch(_SLEEP): svc.ensure_running() diff --git a/tests/unit/test_orchestrator_rotate_ca.py b/tests/unit/test_orchestrator_rotate_ca.py index 3c2db894..e3ee60b7 100644 --- a/tests/unit/test_orchestrator_rotate_ca.py +++ b/tests/unit/test_orchestrator_rotate_ca.py @@ -9,7 +9,7 @@ from unittest.mock import Mock, patch from bot_bottle.orchestrator import rotate_ca from bot_bottle.gateway import GATEWAY_NAME -from bot_bottle.orchestrator.lifecycle import INFRA_NAME +from bot_bottle.backend.docker.infra import INFRA_NAME from bot_bottle.paths import host_gateway_ca_dir from tests.unit import use_bottle_root -- 2.52.0 From 3e62f31d8be8f27cf4c12d0b0a3c751cf903fe7e Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 15:47:29 -0400 Subject: [PATCH 12/30] refactor(backend): split the heavy backend/__init__ into base + selection, thin the package init MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit backend/__init__.py eagerly imported the whole framework (manifest, egress, git-gate, env, workspace, agent-provider), so importing ANY backend.* submodule paid ~32 modules just to run the package init. Split it: - backend/base.py — the abstract contract (BottleSpec, BottlePlan, BottleCleanupPlan, ExecResult, ActiveAgent, Bottle, BottleImages, BottleBackend). Carries the framework imports; loaded only when a caller needs the contract. - backend/selection.py — backend registry / selection / enumeration (get_bottle_backend, _get_backends, _auto_select_backend, has_backend, known_backend_names, enumerate_active_agents). Concrete backends still imported lazily inside. - backend/__init__.py — thin: a __getattr__ that resolves the public names from those submodules on first access (+ a TYPE_CHECKING block for checkers). Existing `from bot_bottle.backend import X` and patch.object call-sites keep working. `import bot_bottle.backend` drops from 32 -> 2 bot_bottle modules. Repointed the backend-selection test's patch targets to the `selection` module (the functions reference each other there now; the FirecrackerBottleBackend class-method patches are unchanged — same class object). Note: backend.docker.util is still heavy because backend/docker/__init__ eagerly imports DockerBottleBackend — slimming the three sub-package inits is the follow-up that makes the leaves cheap. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/backend/__init__.py | 893 +++------------------------ bot_bottle/backend/base.py | 607 ++++++++++++++++++ bot_bottle/backend/selection.py | 188 ++++++ tests/unit/test_backend_selection.py | 51 +- 4 files changed, 895 insertions(+), 844 deletions(-) create mode 100644 bot_bottle/backend/base.py create mode 100644 bot_bottle/backend/selection.py diff --git a/bot_bottle/backend/__init__.py b/bot_bottle/backend/__init__.py index 79cd7af3..0b74d807 100644 --- a/bot_bottle/backend/__init__.py +++ b/bot_bottle/backend/__init__.py @@ -1,836 +1,87 @@ -"""Per-backend bottle factories. +"""The bottle-backend package: abstract contract + backend selection. -A bottle is a running, isolated environment with claude inside. Each -backend exposes five methods: +Thin by design — nothing framework-heavy is imported at package init, so +importing any `backend.*` submodule (e.g. `backend.docker.util`) doesn't drag +the manifest / egress / git-gate framework into memory. The public names are +re-exported lazily: - prepare(spec, stage_dir=...) -> BottlePlan - Resolves names, validates host-side prerequisites, and writes - scratch files. No remote/runtime resources are created yet. - Safe to call before the y/N preflight. + * the abstract contract (`BottleBackend`, `BottleSpec`, `Bottle`, …) lives in + `backend.base`; + * backend selection / enumeration (`get_bottle_backend`, + `enumerate_active_agents`, …) in `backend.selection`; + * the concrete backends and the freeze helpers in their own submodules. - launch(plan) -> ContextManager[Bottle] - Brings up the container (or VM, or remote machine), provisions - it, yields a Bottle handle, and tears everything down on exit. - - prepare_cleanup() -> BottleCleanupPlan - Enumerates orphaned resources left behind by previous bottles - (containers, networks, ...). Idempotent; no side effects. - - cleanup(plan) -> None - Actually removes everything described by the cleanup plan. - - enumerate_active() -> Sequence[ActiveAgent] - Return every currently-running bottle on this backend, with - enough metadata for callers (CLI `list active`, dashboard - agents pane) to render a row. - -Selection is driven by `--backend` on `start` or BOT_BOTTLE_BACKEND -(env var). When neither is set, compatible macOS hosts default to -`macos-container`; Linux hosts with KVM default to `firecracker`; -otherwise `docker`. Per PRD 0003 the manifest does not carry a -backend field; the host picks. +`from bot_bottle.backend import X` resolves X on first access via `__getattr__` +and caches it at package level, so existing call-sites (and +`patch.object(backend_mod, X, …)`) keep working. """ from __future__ import annotations -import os -import shlex -import sys -from abc import ABC, abstractmethod -from contextlib import AbstractContextManager, contextmanager -from dataclasses import dataclass -from pathlib import Path -from typing import TYPE_CHECKING, Any, Generator, Generic, Sequence, TypeVar - -from ..agent_provider import AgentProvisionPlan, get_provider, build_agent_provision_plan -from ..egress import EgressPlan -from ..git_gate import GitGatePlan -from ..log import die, info, warn -from ..util import read_tty_line -from ..manifest import Manifest, ManifestIndex -from ..supervisor.plan import SupervisePlan -from ..util import expand_tilde -from ..env import resolve_env, ResolvedEnv -from ..workspace import WorkspacePlan, workspace_plan -from .print_util import print_multi, visible_agent_env_names -from .util import host_skill_dir +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: + from .base import ( + ActiveAgent, + Bottle, + BottleBackend, + BottleCleanupPlan, + BottleImages, + BottlePlan, + BottleSpec, + ExecResult, + ) + from .selection import ( + enumerate_active_agents, + get_bottle_backend, + has_backend, + known_backend_names, + ) + from .docker import DockerBottleBackend + from .firecracker import FirecrackerBottleBackend + from .macos_container import MacosContainerBottleBackend from .freeze import CommitCancelled, Freezer, get_freezer -@dataclass(frozen=True) -class BottleSpec: - """CLI-supplied intent. Backend-agnostic — each backend's prepare - step consumes it and produces its own backend-specific plan. - Resolved values (image names, container name, scratch paths, runsc - availability) live on the plan, not the spec.""" - - manifest: ManifestIndex - agent_name: str - copy_cwd: bool - user_cwd: str - # PRD 0016 follow-up: when set, the backend's prepare step uses - # this identity instead of minting a fresh one — the resume path - # (`cli.py resume `) sets this to continue an existing - # bottle's state. Empty string for a fresh `start`. - identity: str = "" - label: str = "" - color: str = "" - # Ordered bottle names selected at launch (issue #269). When non-empty - # they are merged in order and replace the agent's `bottle:` field. - bottle_names: tuple[str, ...] = () - # True when launched via --headless (no TTY, no interactive prompts). - # The git-gate host-key preflight uses this to error rather than prompt. - headless: bool = False - # Image startup policy. "fresh" preserves the normal build path; - # "cached" reuses the current local image/artifact without rebuilding. - image_policy: str = "fresh" - - -@dataclass(frozen=True) -class BottlePlan(ABC): - """Base output of a backend's prepare step. Concrete subclasses - (e.g. DockerBottlePlan) add backend-specific resolved fields.""" - - spec: BottleSpec - manifest: Manifest - stage_dir: Path - git_gate_plan: GitGatePlan - - @property - def guest_home(self) -> str: - return self.agent_provision.guest_home - - @property - def git_gate_insteadof_host(self) -> str: - """Host (and optional port) used in git-gate insteadOf URLs. - Docker uses the compose-network DNS alias; VM backends may - override with an IP:port when the guest has no DNS.""" - return "git-gate" - - @property - def git_gate_insteadof_scheme(self) -> str: - """URL scheme for git-gate insteadOf rewrites. 'git' for - Docker (git daemon); VM backends may override (e.g. 'http' - over a published host port).""" - return "git" - egress_plan: EgressPlan - supervise_plan: SupervisePlan | None - agent_provision: AgentProvisionPlan - - @property - def workspace_plan(self) -> WorkspacePlan: - return workspace_plan(self.spec, guest_home=self.guest_home) - - def print(self) -> None: - """Render the y/N preflight summary to stderr.""" - spec = self.spec - manifest = self.manifest - agent = manifest.agent - bottle = manifest.bottle - - env_names = visible_agent_env_names( - sorted( - set(bottle.env.keys()) - | set(self.agent_provision.guest_env.keys()) - ), - hidden_env_names=self.agent_provision.hidden_env_names, - ) - - print(file=sys.stderr) - info(f"agent : {spec.agent_name}") - info(f"provider : {self.agent_provision.template}") - print_multi("env ", env_names) - print_multi("skills ", list(agent.skills)) - effective_bottles = ( - list(spec.bottle_names) if spec.bottle_names - else ([agent.bottle] if agent.bottle else []) - ) - print_multi("bottle ", effective_bottles) - - identity = manifest.git_identity_summary() - if identity: - info(f" git identity : {identity}") - - git_lines = [ - f"{u.name} → {u.upstream_host}:{u.upstream_port}" - for u in self.git_gate_plan.upstreams - ] - if git_lines: - print_multi(" git gate ", git_lines) - - if self.egress_plan.routes: - egress_lines = [] - for r in self.egress_plan.routes: - auth = f" [auth:{r.auth_scheme}]" if r.auth_scheme else "" - egress_lines.append(f"{r.host}{auth}") - print_multi(" egress ", egress_lines) - print(file=sys.stderr) - - -@dataclass(frozen=True) -class BottleCleanupPlan(ABC): - """Base output of a backend's prepare_cleanup step. Concrete - subclasses (e.g. DockerBottleCleanupPlan) carry backend-specific - lists of resources to be removed and implement `print` + `empty`.""" - - @abstractmethod - def print(self) -> None: - """Render the cleanup y/N summary to stderr.""" - - @property - @abstractmethod - def empty(self) -> bool: - """True iff there is nothing to clean up; the CLI uses this to - short-circuit before showing the y/N.""" - - -@dataclass(frozen=True) -class ExecResult: - """Captured result of `Bottle.exec`. Backend-neutral: the Docker - impl populates it from a `subprocess.CompletedProcess`, but a - VM backend could populate it from any source that produces a - returncode + captured streams.""" - - returncode: int - stdout: str - stderr: str - - -@dataclass(frozen=True) -class ActiveAgent: - """One currently-running agent, as the CLI `list active` and - dashboard agents pane render it. ("Agent" is the project's - consistent name for the thing running inside a bottle — the - bottle is the container, the agent is what runs in it.) - - Fields are deliberately backend-neutral. `services` is the set - of gateway daemons currently up for this bottle (`egress`, - `git-gate`, `supervise`); the dashboard uses it to - gate edit verbs. `backend_name` is the matching key in - `_BACKENDS` (`docker` / `firecracker` / `macos-container`) — used by the active- - list rendering to disambiguate and by the dashboard's - re-attach path.""" - - backend_name: str - slug: str - agent_name: str # from metadata.json; "?" if missing - started_at: str # ISO 8601 from metadata.json; "" if missing - services: tuple[str, ...] # alphabetical - label: str = "" - color: str = "" - - -class Bottle(ABC): - """Handle to a running bottle. Yielded by a backend's launch step. - - `exec_agent` runs the selected agent CLI inside the bottle and - blocks until the session ends. `exec` runs a POSIX shell script inside the bottle - and returns the captured result. `cp_in` copies a host path into - the bottle. `close` is an idempotent alias for context-manager - teardown. - """ - - name: str - - @abstractmethod - def agent_argv( - self, argv: list[str], *, tty: bool = True, - ) -> list[str]: - """Return the host-side argv that runs the selected agent - inside the bottle. Used by `exec_agent` for foreground - handoffs and by the dashboard's tmux `respawn-pane` flow, - which needs the argv up front (it spawns claude in a tmux - pane rather than as a child of the current process). - - Implementations transparently inject - `--append-system-prompt-file` when the bottle was launched - with a provisioned prompt path.""" - ... - - @abstractmethod - def exec_agent(self, argv: list[str], *, tty: bool = True) -> int: ... - - @abstractmethod - def exec(self, script: str, *, user: str = "node") -> ExecResult: - """Run `script` as a POSIX shell script inside the bottle as - `user` (default `node`, matching the agent image's USER - directive) and return the captured stdout/stderr/returncode. - The bottle's environment (including HTTPS_PROXY pointing at - the egress daemon) is inherited by the child. Non-zero - exit does not raise — callers inspect `returncode` - themselves. - - Pass `user="root"` for shell-outs that need privileged file - writes / package install — provisioning calls that need root - bypass `Bottle.exec` and use the backend-specific raw - machine-exec helper, but the tests have a legitimate use - case for arbitrary-user runs.""" - - @abstractmethod - def cp_in(self, host_path: str, container_path: str) -> None: ... - - @abstractmethod - def close(self) -> None: ... - - - - -PlanT = TypeVar("PlanT", bound=BottlePlan) -CleanupT = TypeVar("CleanupT", bound=BottleCleanupPlan) - - -@dataclass(frozen=True) -class BottleImages: - """Resolved image references (or artifact paths) for a bottle launch. - - For Docker/macOS-container backends, `agent` and `sidecar` are string - image refs. For the smolmachines backend they are Path objects pointing - to pre-built `.smolmachine` artifacts.""" - - agent: str | Path - sidecar: str | Path = "" - - -class BottleBackend(ABC, Generic[PlanT, CleanupT]): - """Abstract base for selectable bottle backends. Concrete subclasses - (e.g. DockerBottleBackend) own their own prepare/launch impls. - Parameterized over the backend's concrete plan + cleanup-plan types - so subclass methods get the narrow type without isinstance - boilerplate.""" - - name: str - - # Whether this backend can run a container engine *inside* the bottle. - # Backends that cannot must reject `nested_containers: true` rather than - # reach for a host daemon socket (issue #392). - supports_nested_containers: bool = False - - def prepare(self, spec: BottleSpec, stage_dir: Path) -> PlanT: - """Template method: run cross-backend host-side validation, then - delegate to the subclass's `_resolve_plan` for the - backend-specific resolution (names, scratch files, etc.). The - validation step is enforced here so a future backend cannot - accidentally skip it. No remote/runtime resources are created.""" - from .resolve_common import ( - merge_provision_env_vars, - mint_slug, - prepare_agent_state_dir, - prepare_egress, - prepare_git_gate, - prepare_supervise, - reject_nested_containers, - resolve_manifest_dockerfile, - write_launch_metadata, - ) - - manifest = self._validate(spec) - - if not self.supports_nested_containers: - reject_nested_containers(self.name, manifest) - - self._preflight() - - from ..git_gate_host_key import preflight_host_keys - manifest = preflight_host_keys( - manifest, - headless=spec.headless, - home_md=spec.manifest.home_md, - ) - - manifest_bottle = manifest.bottle - manifest_agent_provider = manifest_bottle.agent_provider - agent_provider = get_provider(manifest_agent_provider.template) - resolved_env = resolve_env(manifest) - workspace = workspace_plan(spec, guest_home=agent_provider.guest_home) - - slug = mint_slug(spec) - write_launch_metadata(slug, spec, compose_project="", backend=self.name) - - # Manifest may override the Dockerfile per-bottle; otherwise fall - # back to the provider plugin's bundled Dockerfile (next to its - # agent_provider.py module). - if manifest_agent_provider.dockerfile: - agent_dockerfile_path = resolve_manifest_dockerfile( - manifest_agent_provider.dockerfile, spec, - ) - else: - agent_dockerfile_path = str(agent_provider.dockerfile) - - agent_dir, prompt_file = prepare_agent_state_dir(slug, manifest) - - agent_provision_plan = build_agent_provision_plan( - template=manifest_agent_provider.template, - dockerfile=agent_dockerfile_path, - state_dir=agent_dir, - instance_name=f"bot-bottle-{slug}", - prompt_file=prompt_file, - guest_env=self._build_guest_env(resolved_env), - forward_host_credentials=manifest_agent_provider.forward_host_credentials, - auth_token=manifest_agent_provider.auth_token, - host_env=dict(os.environ), - trusted_project_path=workspace.workdir, - label=spec.label, - color=spec.color, - provider_settings=manifest_agent_provider.settings, - ) - agent_provision_plan = merge_provision_env_vars(agent_provision_plan) - egress_plan = prepare_egress(manifest_bottle, slug, agent_provision_plan) - supervise_plan = prepare_supervise(manifest_bottle, slug) - git_gate_plan = prepare_git_gate(manifest_bottle, slug) - - return self._resolve_plan( - spec, - manifest=manifest, - slug=slug, - resolved_env=resolved_env, - agent_provision_plan=agent_provision_plan, - egress_plan=egress_plan, - supervise_plan=supervise_plan, - git_gate_plan=git_gate_plan, - stage_dir=stage_dir, - ) - - def _build_guest_env(self, resolved_env: ResolvedEnv) -> dict[str, str]: - return {} - - def _preflight(self) -> None: - """ - tasks to do before resolving a plan - """ - pass - - def _validate(self, spec: BottleSpec) -> Manifest: - """Cross-backend pre-launch checks. Parses the selected agent and - its bottle (raising ManifestError on invalid content), confirms - skills are present on the host, and every git IdentityFile resolves. - - Returns the loaded Manifest for the selected agent. Subclasses with - additional preconditions should override and call - `super()._validate(spec)` first.""" - manifest = spec.manifest.load_for_agent(spec.agent_name, spec.bottle_names) - self._validate_skills(manifest.agent.skills) - self._validate_agent_provider_dockerfile(spec, manifest) - return manifest - - def _validate_skills(self, skills: Sequence[str]) -> None: - """Each named skill must be a directory under the host's - `~/.claude/skills/`. The check is purely host-side, so the - default impl covers every backend.""" - for name in skills: - path = host_skill_dir(name) - if not os.path.isdir(path): - die( - f"skill '{name}' not found on host at {path}. " - f"Create it under ~/.claude/skills/, then re-run." - ) - - def _validate_agent_provider_dockerfile(self, spec: BottleSpec, manifest: Manifest) -> None: - bottle = manifest.bottle - dockerfile = bottle.agent_provider.dockerfile - if not dockerfile: - return - path = Path(expand_tilde(dockerfile)) - if not path.is_absolute(): - path = Path(spec.user_cwd) / path - if not path.is_file(): - effective = ( - ", ".join(spec.bottle_names) if spec.bottle_names else manifest.agent.bottle - ) - die( - f"agent_provider.dockerfile for bottle " - f"'{effective}' not found: {path}" - ) - - @abstractmethod - def _resolve_plan(self, - spec: BottleSpec, - *, - manifest: Manifest, - slug: str, - resolved_env: ResolvedEnv, - agent_provision_plan: AgentProvisionPlan, - egress_plan: EgressPlan, - git_gate_plan: GitGatePlan, - supervise_plan: SupervisePlan | None, - stage_dir: Path) -> PlanT: - """Backend-specific plan resolution: image/container names, - env-file, prompt-file, proxy plan, runtime detection. Called by - `prepare` after `_validate` succeeds. Instance name, image, - prompt file, Dockerfile path, and guest home all live on - `agent_provision_plan` — the source of truth.""" - - def prelaunch_checks(self, plan: PlanT) -> None: - """Raise StaleImageError if any cached image used by this plan is stale. - No-op default; backends override to call the shared check_stale* - helpers on their image/artifact timestamps. Called by the CLI before - launch so the operator can be prompted outside the launch context.""" - - @contextmanager - def launch(self, plan: PlanT) -> Generator[Bottle, None, None]: - """Template: build or load images, then delegate to _launch_impl.""" - images = self._build_or_load_images(plan) - with self._launch_impl(plan, images) as bottle: - yield bottle - - @abstractmethod - def _build_or_load_images(self, plan: PlanT) -> BottleImages: - """Return the agent and sidecar image references (or artifact paths) - for this plan, building fresh images when the policy requires it.""" - - @abstractmethod - def _launch_impl(self, plan: PlanT, images: BottleImages) -> AbstractContextManager[Bottle]: - """Bring up the bottle using pre-resolved images; yield a handle; tear down on exit.""" - - def provision(self, plan: PlanT, bottle: "Bottle") -> str | None: - """Copy host-side files (CA cert, prompt, skills, .git) into - the running bottle. Called from `launch` after the container - / machine is up. Returns the in-container prompt path if a - prompt was provisioned, else None — the Bottle handle uses it - to decide whether to add provider-specific prompt args to the - agent's argv. - - Default orchestration: ca → prompt → provider apply → skills - → workspace → git → supervise-mcp. CA install runs first so - the agent's trust store is rebuilt before anything inside the - agent makes a TLS call. - - Per PRD 0050 the per-provider steps (prompt, skills, - declarative provision-plan apply, supervise MCP registration) - live on the `AgentProvider` plugin. The backend only owns the - steps that are about backend infrastructure (CA, workspace, - git) and surfaces the supervise daemon URL its launch step - knows about via `supervise_mcp_url`. - - PRD 0017: cred-proxy's agent-side dotfile rewrites (~/.npmrc, - ~/.gitconfig insteadOf, tea config) are gone. Egress-proxy is - on the agent's HTTP_PROXY path so every tool that respects - HTTPS_PROXY (claude-code, git over HTTPS, npm, curl) is - intercepted without per-tool reconfiguration.""" - provider = get_provider(plan.agent_provision.template) - provider.provision_ca(bottle, plan) - prompt_path = provider.provision_prompt(plan, bottle) - provider.provision(plan, bottle) - provider.provision_skills(plan, bottle) - self.provision_workspace(plan, bottle) - provider.provision_git(bottle, plan) - provider.provision_supervise_mcp( - plan, bottle, self.supervise_mcp_url(plan), - ) - return prompt_path - - def provision_workspace(self, plan: PlanT, bottle: "Bottle") -> None: - """Copy the operator workspace into the running bottle. - - This is the only supported workspace-provisioning path: Docker - does not build a derived image containing the current - workspace.""" - workspace = plan.workspace_plan - if not (workspace.enabled and workspace.copy_contents): - return - - guest_parent = workspace.guest_path.rsplit("/", 1)[0] or "/" - guest_path = shlex.quote(workspace.guest_path) - guest_parent = shlex.quote(guest_parent) - owner = shlex.quote(workspace.owner) - mode = shlex.quote(workspace.mode) - info(f"copying {workspace.host_path} -> {bottle.name}:{workspace.guest_path}") - bottle.exec( - f"rm -rf {guest_path} && mkdir -p {guest_parent}", - user="root", - ) - bottle.cp_in(str(workspace.host_path), workspace.guest_path) - bottle.exec( - f"chown -R {owner} {guest_path} && chmod {mode} {guest_path}", - user="root", - ) - - def supervise_mcp_url(self, plan: PlanT) -> str: - """Return the agent-side URL of the per-bottle supervise - gateway, or "" when this bottle has no gateway. The provider - plugin's `provision_supervise_mcp` uses it to register the - MCP entry inside the guest. - - Default returns "" so backends without supervise support - don't have to implement it. Docker and firecracker override.""" - del plan - return "" - - def ensure_orchestrator(self) -> str: - """Bring up this backend's per-host orchestrator + shared gateway - (idempotent) and return the host-reachable control-plane URL. - - This is the backend-agnostic bring-up entry point: `launch` calls - it as part of starting a bottle, and operator tools (`supervise`) - call it to start the control plane on demand when none is running - yet. Docker starts the orchestrator + gateway containers; - firecracker boots the infra VM. Backends with no orchestrator - (macos-container) die with a pointer — the default here.""" - die(f"backend {self.name!r} has no orchestrator control plane") - - @abstractmethod - def prepare_cleanup(self) -> CleanupT: - """Enumerate orphaned resources from previous bottles. No side - effects; safe to call before the y/N.""" - - @abstractmethod - def cleanup(self, plan: CleanupT) -> None: - """Remove everything described by the cleanup plan.""" - - @abstractmethod - def enumerate_active(self) -> Sequence[ActiveAgent]: - """Return every currently-running agent on this backend. - Empty when none. Backend-specific: docker queries `docker - compose ls`; firecracker cross-references its running gateway - containers against per-bottle metadata.""" - - @classmethod - @abstractmethod - def is_available(cls) -> bool: - """Whether this backend's runtime prerequisites are satisfied - on the current host. Docker → `docker` on PATH; firecracker → - Linux + KVM. Used by the cross-backend - `enumerate_active_agents` / `cmd_cleanup` to skip backends - the operator hasn't installed, so a docker-only host - doesn't fail when `cli.py list active` walks past - firecracker.""" - - @classmethod - @abstractmethod - def setup(cls) -> int: - """Emit this backend's one-time host setup — privileged network - pool, daemon bring-up, install pointers, etc. — as - host-appropriate config or commands. Prints to stdout/stderr and - returns a shell exit code (0 = nothing to report / success). A - backend that needs no host setup prints a short note and returns - 0. Invoked generically by `./cli.py backend setup [--backend=…]` - so operators can provision any backend without a - backend-specific command. Classmethod (like `is_available`) — - it's a host query, not per-bottle state.""" - - @classmethod - @abstractmethod - def status(cls) -> int: - """Report whether this backend's prerequisites are satisfied on - the host — binaries, daemon reachability, network pool, range - conflicts, etc. Prints a human-readable summary; returns 0 when - the backend is ready to launch and non-zero when something is - missing. Invoked by `./cli.py backend status [--backend=…]`.""" - - @classmethod - @abstractmethod - def teardown(cls) -> int: - """Undo `setup()` — the inverse operation, surfaced as - `./cli.py backend teardown [--backend=…]` (uninstall). Symmetric - with setup: where setup is advisory (prints the privileged - commands / declarative config to apply), teardown prints the - commands / config change to remove the host prerequisites. A - backend with no host setup prints a short note and returns 0. - Not called by the launch path or the test suite.""" - - -# _backends is None until the first call to _get_backends(), at which -# point all three concrete backend classes are imported and instantiated. -# Keeping the imports out of module scope means that importing any -# backend sub-module (e.g. `backend.docker.util`) no longer drags the -# firecracker and macos-container implementations into memory. -# -# Tests may replace _backends with a {name: fake} dict via patch.object; -# _get_backends() returns the current module-level value as-is when it -# is not None, so test fakes take effect without triggering real imports. -_backends: dict[str, BottleBackend[Any, Any]] | None = None - - -def _get_backends() -> dict[str, BottleBackend[Any, Any]]: - """Return the registry of all backend instances, loading lazily on first call.""" - global _backends # pylint: disable=global-statement - if _backends is None: - from .docker import DockerBottleBackend - from .firecracker import FirecrackerBottleBackend - from .macos_container import MacosContainerBottleBackend - _backends = { - "docker": DockerBottleBackend(), - "firecracker": FirecrackerBottleBackend(), - "macos-container": MacosContainerBottleBackend(), - } - return _backends +# Public name -> submodule that defines it. Contract types resolve from `base`, +# selection/enumeration from `selection`, the concrete backends + freeze helpers +# from their own subpackages. +_LAZY_MODULES: dict[str, str] = { + "BottleSpec": "base", + "BottlePlan": "base", + "BottleCleanupPlan": "base", + "ExecResult": "base", + "ActiveAgent": "base", + "Bottle": "base", + "BottleImages": "base", + "BottleBackend": "base", + "get_bottle_backend": "selection", + "known_backend_names": "selection", + "has_backend": "selection", + "enumerate_active_agents": "selection", + "_print_vm_install_instructions": "selection", + "DockerBottleBackend": "docker", + "FirecrackerBottleBackend": "firecracker", + "MacosContainerBottleBackend": "macos_container", + "CommitCancelled": "freeze", + "Freezer": "freeze", + "get_freezer": "freeze", +} def __getattr__(name: str) -> Any: - """Lazily surface concrete backend classes and freeze symbols at the - package level so existing `from bot_bottle.backend import X` and - `patch.object(backend_mod, X, ...)` call-sites keep working without - forcing an import of every backend at module-init time.""" - if name == "DockerBottleBackend": - from .docker import DockerBottleBackend - globals()[name] = DockerBottleBackend - return DockerBottleBackend - if name == "FirecrackerBottleBackend": - from .firecracker import FirecrackerBottleBackend - globals()[name] = FirecrackerBottleBackend - return FirecrackerBottleBackend - if name == "MacosContainerBottleBackend": - from .macos_container import MacosContainerBottleBackend - globals()[name] = MacosContainerBottleBackend - return MacosContainerBottleBackend - if name == "CommitCancelled": - from .freeze import CommitCancelled - globals()[name] = CommitCancelled - return CommitCancelled - if name == "Freezer": - from .freeze import Freezer - globals()[name] = Freezer - return Freezer - if name == "get_freezer": - from .freeze import get_freezer - globals()[name] = get_freezer - return get_freezer - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + """Lazily surface the package's public names from their submodules and + cache them at package level — so `from bot_bottle.backend import X` and + `patch.object(backend_mod, X, …)` keep working without importing the + framework (or every backend) at package-init time.""" + mod = _LAZY_MODULES.get(name) + if mod is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module - -def get_bottle_backend( - name: str | None = None, - *, - prompt: bool = True, -) -> BottleBackend[Any, Any]: - """Resolve the bottle backend. - - `name` precedence: - 1. explicit arg (e.g. resume passes the recorded backend name) - 2. BOT_BOTTLE_BACKEND env var - 3. auto-selection: VM backend first, docker fallback with prompt - - `prompt` controls whether auto-selection may block on an interactive - [i/d/q] prompt when falling back to docker. Pass `prompt=False` in - non-interactive contexts (headless launches, CI) so the call dies - with an actionable message instead of hanging. - - Dies with a pointer at the known backends if the chosen name - isn't implemented.""" - resolved = name or os.environ.get("BOT_BOTTLE_BACKEND") - if resolved is None: - resolved = _auto_select_backend(prompt=prompt) - backends = _get_backends() - if resolved not in backends: - known = ", ".join(sorted(backends)) - die(f"unknown backend {resolved!r}; known backends: {known}") - return backends[resolved] - - -def _platform_vm_suggestion() -> str: - """Platform-appropriate VM backend name for install suggestions.""" - return "macos-container" if sys.platform == "darwin" else "firecracker" - - -def _print_vm_install_instructions() -> None: - """Print platform-appropriate VM backend install instructions to stderr.""" - vm = _platform_vm_suggestion() - if vm == "macos-container": - info("Install Apple Container: https://github.com/apple/container/releases") - info("Then start the service: container system start") - else: - info("Install Firecracker: https://github.com/firecracker-microvm/firecracker/releases") - info("Configure the host: ./cli.py backend setup") - - -def _auto_select_backend(prompt: bool = True) -> str: - """Tier-1 / tier-2 backend auto-selection. - - Tier 1: VM backend — macos-container on macOS when Apple Container is - installed; firecracker on KVM-capable Linux even before the binary is - present (its preflight prints an install pointer). - - Tier 2: docker, with a security warning and an interactive prompt. - When `prompt=False` (headless / CI), dies with an actionable message - instead of blocking on a TTY read. When docker is also absent, prints - VM install instructions and exits. - """ - # --- Tier 1: VM backend ----------------------------------------- - if has_backend("macos-container"): - return "macos-container" - # A KVM-capable Linux host defaults to firecracker even when the - # `firecracker` binary isn't installed yet: selecting it here routes - # start through firecracker's preflight, which prints an install - # pointer, instead of silently falling back to docker. - from .firecracker import FirecrackerBottleBackend - if FirecrackerBottleBackend.is_host_capable(): - return "firecracker" - - # --- Tier 2: docker fallback ------------------------------------ - if not has_backend("docker"): - info("No backend available on this host.") - _print_vm_install_instructions() - die("no backend available; install a VM backend and re-run") - - vm = _platform_vm_suggestion() - warn( - "docker is less secure than VM backends — " - "containers share the host kernel." - ) - if not prompt: - die( - f"no VM backend available; set BOT_BOTTLE_BACKEND=docker to proceed " - f"with docker, or install the {vm!r} backend." - ) - sys.stderr.write( - f"bot-bottle: For better isolation, install the {vm!r} backend.\n" - f" [i] show {vm} install instructions and exit\n" - " [d] use docker anyway\n" - " [q] quit\n" - "bot-bottle: choice [i/d/q]: " - ) - sys.stderr.flush() - reply = read_tty_line().strip().lower() - if reply == "d": - return "docker" - if reply == "i": - _print_vm_install_instructions() - die("not proceeding with docker; install a VM backend or set BOT_BOTTLE_BACKEND=docker") - - -def known_backend_names() -> tuple[str, ...]: - """Sorted tuple of all backend keys in `_get_backends()`. Used by - argparse (`--backend` choices) and the dashboard's backend - picker.""" - return tuple(sorted(_get_backends())) - - -def has_backend(name: str) -> bool: - """Whether the named backend's runtime prerequisites are - available on the current host. Cross-backend callers (list, - cleanup) skip unavailable backends so a docker-only host - doesn't fail when the firecracker backend isn't usable, - and vice versa. - - Returns False for unknown names so callers can pass - arbitrary input without separate validation.""" - backends = _get_backends() - if name not in backends: - return False - return backends[name].is_available() - - -def enumerate_active_agents() -> list[ActiveAgent]: - """All currently-running agents, across every available - backend. Used by CLI `list active` and the dashboard's agents - pane so neither has to know which backends exist. Skips - backends whose `is_available()` reports False. - - Sorted by `(started_at, slug)` so the list is stable across - dashboard refresh ticks — agents don't shift position while - the operator navigates with arrow keys. ISO 8601 timestamps - sort lexicographically in chronological order; `slug` is the - deterministic tiebreaker. Agents with missing metadata - (`started_at == ""`) sort first.""" - out: list[ActiveAgent] = [] - backends = _get_backends() - for name in sorted(backends): - if not backends[name].is_available(): - continue - out.extend(backends[name].enumerate_active()) - out.sort(key=lambda a: (a.started_at, a.slug)) - return out + value = getattr(import_module(f"{__name__}.{mod}"), name) + globals()[name] = value + return value __all__ = [ @@ -838,14 +89,18 @@ __all__ = [ "Bottle", "BottleBackend", "BottleCleanupPlan", + "BottleImages", "BottlePlan", "BottleSpec", - "CommitCancelled", "ExecResult", + "CommitCancelled", "Freezer", + "get_freezer", + "DockerBottleBackend", + "FirecrackerBottleBackend", + "MacosContainerBottleBackend", "enumerate_active_agents", "get_bottle_backend", - "get_freezer", "has_backend", "known_backend_names", ] diff --git a/bot_bottle/backend/base.py b/bot_bottle/backend/base.py new file mode 100644 index 00000000..bb8a2abc --- /dev/null +++ b/bot_bottle/backend/base.py @@ -0,0 +1,607 @@ +"""The abstract backend contract (PRD 0018 / 0070). + +The backend-neutral types every bottle backend implements: the launch +`BottleSpec`, the `BottlePlan` / `BottleCleanupPlan` ABCs, the running-`Bottle` ++ `ExecResult` shapes, `BottleImages`, and the `BottleBackend` ABC itself. + +This carries the framework imports (manifest, egress, git-gate, env, workspace, +agent-provider) the contract's signatures and helpers need — which is why it +lives here rather than in `backend/__init__.py`: touching an unrelated +`backend.*` module then doesn't drag the whole framework into memory. The +thin package `__init__` re-exports these names lazily. +""" + +from __future__ import annotations + +import os +import shlex +import sys +from abc import ABC, abstractmethod +from contextlib import AbstractContextManager, contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Generator, Generic, Sequence, TypeVar + +from ..agent_provider import AgentProvisionPlan, get_provider, build_agent_provision_plan +from ..egress import EgressPlan +from ..git_gate import GitGatePlan +from ..log import die, info +from ..util import expand_tilde +from ..manifest import Manifest, ManifestIndex +from ..supervisor.plan import SupervisePlan +from ..env import resolve_env, ResolvedEnv +from ..workspace import WorkspacePlan, workspace_plan +from .print_util import print_multi, visible_agent_env_names +from .util import host_skill_dir + + + +@dataclass(frozen=True) +class BottleSpec: + """CLI-supplied intent. Backend-agnostic — each backend's prepare + step consumes it and produces its own backend-specific plan. + Resolved values (image names, container name, scratch paths, runsc + availability) live on the plan, not the spec.""" + + manifest: ManifestIndex + agent_name: str + copy_cwd: bool + user_cwd: str + # PRD 0016 follow-up: when set, the backend's prepare step uses + # this identity instead of minting a fresh one — the resume path + # (`cli.py resume `) sets this to continue an existing + # bottle's state. Empty string for a fresh `start`. + identity: str = "" + label: str = "" + color: str = "" + # Ordered bottle names selected at launch (issue #269). When non-empty + # they are merged in order and replace the agent's `bottle:` field. + bottle_names: tuple[str, ...] = () + # True when launched via --headless (no TTY, no interactive prompts). + # The git-gate host-key preflight uses this to error rather than prompt. + headless: bool = False + # Image startup policy. "fresh" preserves the normal build path; + # "cached" reuses the current local image/artifact without rebuilding. + image_policy: str = "fresh" + + +@dataclass(frozen=True) +class BottlePlan(ABC): + """Base output of a backend's prepare step. Concrete subclasses + (e.g. DockerBottlePlan) add backend-specific resolved fields.""" + + spec: BottleSpec + manifest: Manifest + stage_dir: Path + git_gate_plan: GitGatePlan + + @property + def guest_home(self) -> str: + return self.agent_provision.guest_home + + @property + def git_gate_insteadof_host(self) -> str: + """Host (and optional port) used in git-gate insteadOf URLs. + Docker uses the compose-network DNS alias; VM backends may + override with an IP:port when the guest has no DNS.""" + return "git-gate" + + @property + def git_gate_insteadof_scheme(self) -> str: + """URL scheme for git-gate insteadOf rewrites. 'git' for + Docker (git daemon); VM backends may override (e.g. 'http' + over a published host port).""" + return "git" + egress_plan: EgressPlan + supervise_plan: SupervisePlan | None + agent_provision: AgentProvisionPlan + + @property + def workspace_plan(self) -> WorkspacePlan: + return workspace_plan(self.spec, guest_home=self.guest_home) + + def print(self) -> None: + """Render the y/N preflight summary to stderr.""" + spec = self.spec + manifest = self.manifest + agent = manifest.agent + bottle = manifest.bottle + + env_names = visible_agent_env_names( + sorted( + set(bottle.env.keys()) + | set(self.agent_provision.guest_env.keys()) + ), + hidden_env_names=self.agent_provision.hidden_env_names, + ) + + print(file=sys.stderr) + info(f"agent : {spec.agent_name}") + info(f"provider : {self.agent_provision.template}") + print_multi("env ", env_names) + print_multi("skills ", list(agent.skills)) + effective_bottles = ( + list(spec.bottle_names) if spec.bottle_names + else ([agent.bottle] if agent.bottle else []) + ) + print_multi("bottle ", effective_bottles) + + identity = manifest.git_identity_summary() + if identity: + info(f" git identity : {identity}") + + git_lines = [ + f"{u.name} → {u.upstream_host}:{u.upstream_port}" + for u in self.git_gate_plan.upstreams + ] + if git_lines: + print_multi(" git gate ", git_lines) + + if self.egress_plan.routes: + egress_lines = [] + for r in self.egress_plan.routes: + auth = f" [auth:{r.auth_scheme}]" if r.auth_scheme else "" + egress_lines.append(f"{r.host}{auth}") + print_multi(" egress ", egress_lines) + print(file=sys.stderr) + + +@dataclass(frozen=True) +class BottleCleanupPlan(ABC): + """Base output of a backend's prepare_cleanup step. Concrete + subclasses (e.g. DockerBottleCleanupPlan) carry backend-specific + lists of resources to be removed and implement `print` + `empty`.""" + + @abstractmethod + def print(self) -> None: + """Render the cleanup y/N summary to stderr.""" + + @property + @abstractmethod + def empty(self) -> bool: + """True iff there is nothing to clean up; the CLI uses this to + short-circuit before showing the y/N.""" + + +@dataclass(frozen=True) +class ExecResult: + """Captured result of `Bottle.exec`. Backend-neutral: the Docker + impl populates it from a `subprocess.CompletedProcess`, but a + VM backend could populate it from any source that produces a + returncode + captured streams.""" + + returncode: int + stdout: str + stderr: str + + +@dataclass(frozen=True) +class ActiveAgent: + """One currently-running agent, as the CLI `list active` and + dashboard agents pane render it. ("Agent" is the project's + consistent name for the thing running inside a bottle — the + bottle is the container, the agent is what runs in it.) + + Fields are deliberately backend-neutral. `services` is the set + of gateway daemons currently up for this bottle (`egress`, + `git-gate`, `supervise`); the dashboard uses it to + gate edit verbs. `backend_name` is the matching key in + `_BACKENDS` (`docker` / `firecracker` / `macos-container`) — used by the active- + list rendering to disambiguate and by the dashboard's + re-attach path.""" + + backend_name: str + slug: str + agent_name: str # from metadata.json; "?" if missing + started_at: str # ISO 8601 from metadata.json; "" if missing + services: tuple[str, ...] # alphabetical + label: str = "" + color: str = "" + + +class Bottle(ABC): + """Handle to a running bottle. Yielded by a backend's launch step. + + `exec_agent` runs the selected agent CLI inside the bottle and + blocks until the session ends. `exec` runs a POSIX shell script inside the bottle + and returns the captured result. `cp_in` copies a host path into + the bottle. `close` is an idempotent alias for context-manager + teardown. + """ + + name: str + + @abstractmethod + def agent_argv( + self, argv: list[str], *, tty: bool = True, + ) -> list[str]: + """Return the host-side argv that runs the selected agent + inside the bottle. Used by `exec_agent` for foreground + handoffs and by the dashboard's tmux `respawn-pane` flow, + which needs the argv up front (it spawns claude in a tmux + pane rather than as a child of the current process). + + Implementations transparently inject + `--append-system-prompt-file` when the bottle was launched + with a provisioned prompt path.""" + ... + + @abstractmethod + def exec_agent(self, argv: list[str], *, tty: bool = True) -> int: ... + + @abstractmethod + def exec(self, script: str, *, user: str = "node") -> ExecResult: + """Run `script` as a POSIX shell script inside the bottle as + `user` (default `node`, matching the agent image's USER + directive) and return the captured stdout/stderr/returncode. + The bottle's environment (including HTTPS_PROXY pointing at + the egress daemon) is inherited by the child. Non-zero + exit does not raise — callers inspect `returncode` + themselves. + + Pass `user="root"` for shell-outs that need privileged file + writes / package install — provisioning calls that need root + bypass `Bottle.exec` and use the backend-specific raw + machine-exec helper, but the tests have a legitimate use + case for arbitrary-user runs.""" + + @abstractmethod + def cp_in(self, host_path: str, container_path: str) -> None: ... + + @abstractmethod + def close(self) -> None: ... + + + + +PlanT = TypeVar("PlanT", bound=BottlePlan) +CleanupT = TypeVar("CleanupT", bound=BottleCleanupPlan) + + +@dataclass(frozen=True) +class BottleImages: + """Resolved image references (or artifact paths) for a bottle launch. + + For Docker/macOS-container backends, `agent` and `sidecar` are string + image refs. For the smolmachines backend they are Path objects pointing + to pre-built `.smolmachine` artifacts.""" + + agent: str | Path + sidecar: str | Path = "" + + +class BottleBackend(ABC, Generic[PlanT, CleanupT]): + """Abstract base for selectable bottle backends. Concrete subclasses + (e.g. DockerBottleBackend) own their own prepare/launch impls. + Parameterized over the backend's concrete plan + cleanup-plan types + so subclass methods get the narrow type without isinstance + boilerplate.""" + + name: str + + # Whether this backend can run a container engine *inside* the bottle. + # Backends that cannot must reject `nested_containers: true` rather than + # reach for a host daemon socket (issue #392). + supports_nested_containers: bool = False + + def prepare(self, spec: BottleSpec, stage_dir: Path) -> PlanT: + """Template method: run cross-backend host-side validation, then + delegate to the subclass's `_resolve_plan` for the + backend-specific resolution (names, scratch files, etc.). The + validation step is enforced here so a future backend cannot + accidentally skip it. No remote/runtime resources are created.""" + from .resolve_common import ( + merge_provision_env_vars, + mint_slug, + prepare_agent_state_dir, + prepare_egress, + prepare_git_gate, + prepare_supervise, + reject_nested_containers, + resolve_manifest_dockerfile, + write_launch_metadata, + ) + + manifest = self._validate(spec) + + if not self.supports_nested_containers: + reject_nested_containers(self.name, manifest) + + self._preflight() + + from ..git_gate_host_key import preflight_host_keys + manifest = preflight_host_keys( + manifest, + headless=spec.headless, + home_md=spec.manifest.home_md, + ) + + manifest_bottle = manifest.bottle + manifest_agent_provider = manifest_bottle.agent_provider + agent_provider = get_provider(manifest_agent_provider.template) + resolved_env = resolve_env(manifest) + workspace = workspace_plan(spec, guest_home=agent_provider.guest_home) + + slug = mint_slug(spec) + write_launch_metadata(slug, spec, compose_project="", backend=self.name) + + # Manifest may override the Dockerfile per-bottle; otherwise fall + # back to the provider plugin's bundled Dockerfile (next to its + # agent_provider.py module). + if manifest_agent_provider.dockerfile: + agent_dockerfile_path = resolve_manifest_dockerfile( + manifest_agent_provider.dockerfile, spec, + ) + else: + agent_dockerfile_path = str(agent_provider.dockerfile) + + agent_dir, prompt_file = prepare_agent_state_dir(slug, manifest) + + agent_provision_plan = build_agent_provision_plan( + template=manifest_agent_provider.template, + dockerfile=agent_dockerfile_path, + state_dir=agent_dir, + instance_name=f"bot-bottle-{slug}", + prompt_file=prompt_file, + guest_env=self._build_guest_env(resolved_env), + forward_host_credentials=manifest_agent_provider.forward_host_credentials, + auth_token=manifest_agent_provider.auth_token, + host_env=dict(os.environ), + trusted_project_path=workspace.workdir, + label=spec.label, + color=spec.color, + provider_settings=manifest_agent_provider.settings, + ) + agent_provision_plan = merge_provision_env_vars(agent_provision_plan) + egress_plan = prepare_egress(manifest_bottle, slug, agent_provision_plan) + supervise_plan = prepare_supervise(manifest_bottle, slug) + git_gate_plan = prepare_git_gate(manifest_bottle, slug) + + return self._resolve_plan( + spec, + manifest=manifest, + slug=slug, + resolved_env=resolved_env, + agent_provision_plan=agent_provision_plan, + egress_plan=egress_plan, + supervise_plan=supervise_plan, + git_gate_plan=git_gate_plan, + stage_dir=stage_dir, + ) + + def _build_guest_env(self, resolved_env: ResolvedEnv) -> dict[str, str]: + return {} + + def _preflight(self) -> None: + """ + tasks to do before resolving a plan + """ + pass + + def _validate(self, spec: BottleSpec) -> Manifest: + """Cross-backend pre-launch checks. Parses the selected agent and + its bottle (raising ManifestError on invalid content), confirms + skills are present on the host, and every git IdentityFile resolves. + + Returns the loaded Manifest for the selected agent. Subclasses with + additional preconditions should override and call + `super()._validate(spec)` first.""" + manifest = spec.manifest.load_for_agent(spec.agent_name, spec.bottle_names) + self._validate_skills(manifest.agent.skills) + self._validate_agent_provider_dockerfile(spec, manifest) + return manifest + + def _validate_skills(self, skills: Sequence[str]) -> None: + """Each named skill must be a directory under the host's + `~/.claude/skills/`. The check is purely host-side, so the + default impl covers every backend.""" + for name in skills: + path = host_skill_dir(name) + if not os.path.isdir(path): + die( + f"skill '{name}' not found on host at {path}. " + f"Create it under ~/.claude/skills/, then re-run." + ) + + def _validate_agent_provider_dockerfile(self, spec: BottleSpec, manifest: Manifest) -> None: + bottle = manifest.bottle + dockerfile = bottle.agent_provider.dockerfile + if not dockerfile: + return + path = Path(expand_tilde(dockerfile)) + if not path.is_absolute(): + path = Path(spec.user_cwd) / path + if not path.is_file(): + effective = ( + ", ".join(spec.bottle_names) if spec.bottle_names else manifest.agent.bottle + ) + die( + f"agent_provider.dockerfile for bottle " + f"'{effective}' not found: {path}" + ) + + @abstractmethod + def _resolve_plan(self, + spec: BottleSpec, + *, + manifest: Manifest, + slug: str, + resolved_env: ResolvedEnv, + agent_provision_plan: AgentProvisionPlan, + egress_plan: EgressPlan, + git_gate_plan: GitGatePlan, + supervise_plan: SupervisePlan | None, + stage_dir: Path) -> PlanT: + """Backend-specific plan resolution: image/container names, + env-file, prompt-file, proxy plan, runtime detection. Called by + `prepare` after `_validate` succeeds. Instance name, image, + prompt file, Dockerfile path, and guest home all live on + `agent_provision_plan` — the source of truth.""" + + def prelaunch_checks(self, plan: PlanT) -> None: + """Raise StaleImageError if any cached image used by this plan is stale. + No-op default; backends override to call the shared check_stale* + helpers on their image/artifact timestamps. Called by the CLI before + launch so the operator can be prompted outside the launch context.""" + + @contextmanager + def launch(self, plan: PlanT) -> Generator[Bottle, None, None]: + """Template: build or load images, then delegate to _launch_impl.""" + images = self._build_or_load_images(plan) + with self._launch_impl(plan, images) as bottle: + yield bottle + + @abstractmethod + def _build_or_load_images(self, plan: PlanT) -> BottleImages: + """Return the agent and sidecar image references (or artifact paths) + for this plan, building fresh images when the policy requires it.""" + + @abstractmethod + def _launch_impl(self, plan: PlanT, images: BottleImages) -> AbstractContextManager[Bottle]: + """Bring up the bottle using pre-resolved images; yield a handle; tear down on exit.""" + + def provision(self, plan: PlanT, bottle: "Bottle") -> str | None: + """Copy host-side files (CA cert, prompt, skills, .git) into + the running bottle. Called from `launch` after the container + / machine is up. Returns the in-container prompt path if a + prompt was provisioned, else None — the Bottle handle uses it + to decide whether to add provider-specific prompt args to the + agent's argv. + + Default orchestration: ca → prompt → provider apply → skills + → workspace → git → supervise-mcp. CA install runs first so + the agent's trust store is rebuilt before anything inside the + agent makes a TLS call. + + Per PRD 0050 the per-provider steps (prompt, skills, + declarative provision-plan apply, supervise MCP registration) + live on the `AgentProvider` plugin. The backend only owns the + steps that are about backend infrastructure (CA, workspace, + git) and surfaces the supervise daemon URL its launch step + knows about via `supervise_mcp_url`. + + PRD 0017: cred-proxy's agent-side dotfile rewrites (~/.npmrc, + ~/.gitconfig insteadOf, tea config) are gone. Egress-proxy is + on the agent's HTTP_PROXY path so every tool that respects + HTTPS_PROXY (claude-code, git over HTTPS, npm, curl) is + intercepted without per-tool reconfiguration.""" + provider = get_provider(plan.agent_provision.template) + provider.provision_ca(bottle, plan) + prompt_path = provider.provision_prompt(plan, bottle) + provider.provision(plan, bottle) + provider.provision_skills(plan, bottle) + self.provision_workspace(plan, bottle) + provider.provision_git(bottle, plan) + provider.provision_supervise_mcp( + plan, bottle, self.supervise_mcp_url(plan), + ) + return prompt_path + + def provision_workspace(self, plan: PlanT, bottle: "Bottle") -> None: + """Copy the operator workspace into the running bottle. + + This is the only supported workspace-provisioning path: Docker + does not build a derived image containing the current + workspace.""" + workspace = plan.workspace_plan + if not (workspace.enabled and workspace.copy_contents): + return + + guest_parent = workspace.guest_path.rsplit("/", 1)[0] or "/" + guest_path = shlex.quote(workspace.guest_path) + guest_parent = shlex.quote(guest_parent) + owner = shlex.quote(workspace.owner) + mode = shlex.quote(workspace.mode) + info(f"copying {workspace.host_path} -> {bottle.name}:{workspace.guest_path}") + bottle.exec( + f"rm -rf {guest_path} && mkdir -p {guest_parent}", + user="root", + ) + bottle.cp_in(str(workspace.host_path), workspace.guest_path) + bottle.exec( + f"chown -R {owner} {guest_path} && chmod {mode} {guest_path}", + user="root", + ) + + def supervise_mcp_url(self, plan: PlanT) -> str: + """Return the agent-side URL of the per-bottle supervise + gateway, or "" when this bottle has no gateway. The provider + plugin's `provision_supervise_mcp` uses it to register the + MCP entry inside the guest. + + Default returns "" so backends without supervise support + don't have to implement it. Docker and firecracker override.""" + del plan + return "" + + def ensure_orchestrator(self) -> str: + """Bring up this backend's per-host orchestrator + shared gateway + (idempotent) and return the host-reachable control-plane URL. + + This is the backend-agnostic bring-up entry point: `launch` calls + it as part of starting a bottle, and operator tools (`supervise`) + call it to start the control plane on demand when none is running + yet. Docker starts the orchestrator + gateway containers; + firecracker boots the infra VM. Backends with no orchestrator + (macos-container) die with a pointer — the default here.""" + die(f"backend {self.name!r} has no orchestrator control plane") + + @abstractmethod + def prepare_cleanup(self) -> CleanupT: + """Enumerate orphaned resources from previous bottles. No side + effects; safe to call before the y/N.""" + + @abstractmethod + def cleanup(self, plan: CleanupT) -> None: + """Remove everything described by the cleanup plan.""" + + @abstractmethod + def enumerate_active(self) -> Sequence[ActiveAgent]: + """Return every currently-running agent on this backend. + Empty when none. Backend-specific: docker queries `docker + compose ls`; firecracker cross-references its running gateway + containers against per-bottle metadata.""" + + @classmethod + @abstractmethod + def is_available(cls) -> bool: + """Whether this backend's runtime prerequisites are satisfied + on the current host. Docker → `docker` on PATH; firecracker → + Linux + KVM. Used by the cross-backend + `enumerate_active_agents` / `cmd_cleanup` to skip backends + the operator hasn't installed, so a docker-only host + doesn't fail when `cli.py list active` walks past + firecracker.""" + + @classmethod + @abstractmethod + def setup(cls) -> int: + """Emit this backend's one-time host setup — privileged network + pool, daemon bring-up, install pointers, etc. — as + host-appropriate config or commands. Prints to stdout/stderr and + returns a shell exit code (0 = nothing to report / success). A + backend that needs no host setup prints a short note and returns + 0. Invoked generically by `./cli.py backend setup [--backend=…]` + so operators can provision any backend without a + backend-specific command. Classmethod (like `is_available`) — + it's a host query, not per-bottle state.""" + + @classmethod + @abstractmethod + def status(cls) -> int: + """Report whether this backend's prerequisites are satisfied on + the host — binaries, daemon reachability, network pool, range + conflicts, etc. Prints a human-readable summary; returns 0 when + the backend is ready to launch and non-zero when something is + missing. Invoked by `./cli.py backend status [--backend=…]`.""" + + @classmethod + @abstractmethod + def teardown(cls) -> int: + """Undo `setup()` — the inverse operation, surfaced as + `./cli.py backend teardown [--backend=…]` (uninstall). Symmetric + with setup: where setup is advisory (prints the privileged + commands / declarative config to apply), teardown prints the + commands / config change to remove the host prerequisites. A + backend with no host setup prints a short note and returns 0. + Not called by the launch path or the test suite.""" diff --git a/bot_bottle/backend/selection.py b/bot_bottle/backend/selection.py new file mode 100644 index 00000000..60e2fa54 --- /dev/null +++ b/bot_bottle/backend/selection.py @@ -0,0 +1,188 @@ +"""Backend registry, selection, and active-agent enumeration. + +Resolves which bottle backend to use (explicit name / `BOT_BOTTLE_BACKEND` / +auto-select), and enumerates running agents across every available backend. The +three concrete backends are imported lazily inside `_get_backends` so this +module — and anything that only needs to *select* a backend — stays cheap. +""" + +from __future__ import annotations + +import os +import sys +from typing import Any + +from ..log import die, info, warn +from ..util import read_tty_line +from .base import ActiveAgent, BottleBackend + + +# _backends is None until the first call to _get_backends(), at which +# point all three concrete backend classes are imported and instantiated. +# Keeping the imports out of module scope means that importing any +# backend sub-module (e.g. `backend.docker.util`) no longer drags the +# firecracker and macos-container implementations into memory. +# +# Tests may replace _backends with a {name: fake} dict via patch.object; +# _get_backends() returns the current module-level value as-is when it +# is not None, so test fakes take effect without triggering real imports. +_backends: dict[str, BottleBackend[Any, Any]] | None = None + + +def _get_backends() -> dict[str, BottleBackend[Any, Any]]: + """Return the registry of all backend instances, loading lazily on first call.""" + global _backends # pylint: disable=global-statement + if _backends is None: + from .docker import DockerBottleBackend + from .firecracker import FirecrackerBottleBackend + from .macos_container import MacosContainerBottleBackend + _backends = { + "docker": DockerBottleBackend(), + "firecracker": FirecrackerBottleBackend(), + "macos-container": MacosContainerBottleBackend(), + } + return _backends + + +def get_bottle_backend( + name: str | None = None, + *, + prompt: bool = True, +) -> BottleBackend[Any, Any]: + """Resolve the bottle backend. + + `name` precedence: + 1. explicit arg (e.g. resume passes the recorded backend name) + 2. BOT_BOTTLE_BACKEND env var + 3. auto-selection: VM backend first, docker fallback with prompt + + `prompt` controls whether auto-selection may block on an interactive + [i/d/q] prompt when falling back to docker. Pass `prompt=False` in + non-interactive contexts (headless launches, CI) so the call dies + with an actionable message instead of hanging. + + Dies with a pointer at the known backends if the chosen name + isn't implemented.""" + resolved = name or os.environ.get("BOT_BOTTLE_BACKEND") + if resolved is None: + resolved = _auto_select_backend(prompt=prompt) + backends = _get_backends() + if resolved not in backends: + known = ", ".join(sorted(backends)) + die(f"unknown backend {resolved!r}; known backends: {known}") + return backends[resolved] + + +def _platform_vm_suggestion() -> str: + """Platform-appropriate VM backend name for install suggestions.""" + return "macos-container" if sys.platform == "darwin" else "firecracker" + + +def _print_vm_install_instructions() -> None: + """Print platform-appropriate VM backend install instructions to stderr.""" + vm = _platform_vm_suggestion() + if vm == "macos-container": + info("Install Apple Container: https://github.com/apple/container/releases") + info("Then start the service: container system start") + else: + info("Install Firecracker: https://github.com/firecracker-microvm/firecracker/releases") + info("Configure the host: ./cli.py backend setup") + + +def _auto_select_backend(prompt: bool = True) -> str: + """Tier-1 / tier-2 backend auto-selection. + + Tier 1: VM backend — macos-container on macOS when Apple Container is + installed; firecracker on KVM-capable Linux even before the binary is + present (its preflight prints an install pointer). + + Tier 2: docker, with a security warning and an interactive prompt. + When `prompt=False` (headless / CI), dies with an actionable message + instead of blocking on a TTY read. When docker is also absent, prints + VM install instructions and exits. + """ + # --- Tier 1: VM backend ----------------------------------------- + if has_backend("macos-container"): + return "macos-container" + # A KVM-capable Linux host defaults to firecracker even when the + # `firecracker` binary isn't installed yet: selecting it here routes + # start through firecracker's preflight, which prints an install + # pointer, instead of silently falling back to docker. + from .firecracker import FirecrackerBottleBackend + if FirecrackerBottleBackend.is_host_capable(): + return "firecracker" + + # --- Tier 2: docker fallback ------------------------------------ + if not has_backend("docker"): + info("No backend available on this host.") + _print_vm_install_instructions() + die("no backend available; install a VM backend and re-run") + + vm = _platform_vm_suggestion() + warn( + "docker is less secure than VM backends — " + "containers share the host kernel." + ) + if not prompt: + die( + f"no VM backend available; set BOT_BOTTLE_BACKEND=docker to proceed " + f"with docker, or install the {vm!r} backend." + ) + sys.stderr.write( + f"bot-bottle: For better isolation, install the {vm!r} backend.\n" + f" [i] show {vm} install instructions and exit\n" + " [d] use docker anyway\n" + " [q] quit\n" + "bot-bottle: choice [i/d/q]: " + ) + sys.stderr.flush() + reply = read_tty_line().strip().lower() + if reply == "d": + return "docker" + if reply == "i": + _print_vm_install_instructions() + die("not proceeding with docker; install a VM backend or set BOT_BOTTLE_BACKEND=docker") + + +def known_backend_names() -> tuple[str, ...]: + """Sorted tuple of all backend keys in `_get_backends()`. Used by + argparse (`--backend` choices) and the dashboard's backend + picker.""" + return tuple(sorted(_get_backends())) + + +def has_backend(name: str) -> bool: + """Whether the named backend's runtime prerequisites are + available on the current host. Cross-backend callers (list, + cleanup) skip unavailable backends so a docker-only host + doesn't fail when the firecracker backend isn't usable, + and vice versa. + + Returns False for unknown names so callers can pass + arbitrary input without separate validation.""" + backends = _get_backends() + if name not in backends: + return False + return backends[name].is_available() + + +def enumerate_active_agents() -> list[ActiveAgent]: + """All currently-running agents, across every available + backend. Used by CLI `list active` and the dashboard's agents + pane so neither has to know which backends exist. Skips + backends whose `is_available()` reports False. + + Sorted by `(started_at, slug)` so the list is stable across + dashboard refresh ticks — agents don't shift position while + the operator navigates with arrow keys. ISO 8601 timestamps + sort lexicographically in chronological order; `slug` is the + deterministic tiebreaker. Agents with missing metadata + (`started_at == ""`) sort first.""" + out: list[ActiveAgent] = [] + backends = _get_backends() + for name in sorted(backends): + if not backends[name].is_available(): + continue + out.extend(backends[name].enumerate_active()) + out.sort(key=lambda a: (a.started_at, a.slug)) + return out diff --git a/tests/unit/test_backend_selection.py b/tests/unit/test_backend_selection.py index bb549d13..1a2304d6 100644 --- a/tests/unit/test_backend_selection.py +++ b/tests/unit/test_backend_selection.py @@ -13,6 +13,7 @@ import unittest from unittest.mock import patch from bot_bottle import backend as backend_mod +from bot_bottle.backend import selection as _sel from bot_bottle.backend import ( ActiveAgent, enumerate_active_agents, @@ -40,7 +41,7 @@ class TestGetBottleBackend(unittest.TestCase): return True with patch.dict(os.environ, {}, clear=True), \ - patch.object(backend_mod, "_backends", { + patch.object(_sel, "_backends", { "macos-container": _FakeBackend(), "docker": _FakeBackend(), }): @@ -62,11 +63,11 @@ class TestGetBottleBackend(unittest.TestCase): with patch.dict(os.environ, {}, clear=True), \ patch.object(backend_mod.FirecrackerBottleBackend, "is_host_capable", classmethod(lambda cls: False)), \ - patch.object(backend_mod, "_backends", { + patch.object(_sel, "_backends", { "macos-container": _FakeBackend("macos-container", False), "docker": _FakeBackend("docker", True), }), \ - patch.object(backend_mod, "read_tty_line", return_value="d"): + patch.object(_sel, "read_tty_line", return_value="d"): b = get_bottle_backend() self.assertEqual("docker", b.name) @@ -85,7 +86,7 @@ class TestGetBottleBackend(unittest.TestCase): with patch.dict(os.environ, {}, clear=True), \ patch.object(backend_mod.FirecrackerBottleBackend, "is_host_capable", classmethod(lambda cls: True)), \ - patch.object(backend_mod, "_backends", { + patch.object(_sel, "_backends", { "macos-container": _FakeBackend("macos-container", False), "firecracker": _FakeBackend("firecracker", False), "docker": _FakeBackend("docker", True), @@ -94,7 +95,7 @@ class TestGetBottleBackend(unittest.TestCase): self.assertEqual("firecracker", b.name) def test_unknown_dies(self): - with patch.object(backend_mod, "die", side_effect=SystemExit("die")): + with patch.object(_sel, "die", side_effect=SystemExit("die")): with self.assertRaises(SystemExit): get_bottle_backend("nonexistent") @@ -111,11 +112,11 @@ class TestGetBottleBackend(unittest.TestCase): with patch.dict(os.environ, {}, clear=True), \ patch.object(backend_mod.FirecrackerBottleBackend, "is_host_capable", classmethod(lambda cls: False)), \ - patch.object(backend_mod, "_backends", { + patch.object(_sel, "_backends", { "macos-container": _FakeBackend("macos-container", False), "docker": _FakeBackend("docker", False), }), \ - patch.object(backend_mod, "die", side_effect=SystemExit("die")): + patch.object(_sel, "die", side_effect=SystemExit("die")): with self.assertRaises(SystemExit): get_bottle_backend() @@ -132,12 +133,12 @@ class TestGetBottleBackend(unittest.TestCase): with patch.dict(os.environ, {}, clear=True), \ patch.object(backend_mod.FirecrackerBottleBackend, "is_host_capable", classmethod(lambda cls: False)), \ - patch.object(backend_mod, "_backends", { + patch.object(_sel, "_backends", { "macos-container": _FakeBackend("macos-container", False), "docker": _FakeBackend("docker", True), }), \ - patch.object(backend_mod, "read_tty_line", return_value="q"), \ - patch.object(backend_mod, "die", side_effect=SystemExit("die")): + patch.object(_sel, "read_tty_line", return_value="q"), \ + patch.object(_sel, "die", side_effect=SystemExit("die")): with self.assertRaises(SystemExit): get_bottle_backend() @@ -155,11 +156,11 @@ class TestGetBottleBackend(unittest.TestCase): with patch.dict(os.environ, {}, clear=True), \ patch.object(backend_mod.FirecrackerBottleBackend, "is_host_capable", classmethod(lambda cls: False)), \ - patch.object(backend_mod, "_backends", { + patch.object(_sel, "_backends", { "macos-container": _FakeBackend("macos-container", False), "docker": _FakeBackend("docker", True), }), \ - patch.object(backend_mod, "die", side_effect=SystemExit("die")): + patch.object(_sel, "die", side_effect=SystemExit("die")): with self.assertRaises(SystemExit): get_bottle_backend(prompt=False) @@ -176,13 +177,13 @@ class TestGetBottleBackend(unittest.TestCase): with patch.dict(os.environ, {}, clear=True), \ patch.object(backend_mod.FirecrackerBottleBackend, "is_host_capable", classmethod(lambda cls: False)), \ - patch.object(backend_mod, "_backends", { + patch.object(_sel, "_backends", { "macos-container": _FakeBackend("macos-container", False), "docker": _FakeBackend("docker", True), }), \ - patch.object(backend_mod, "read_tty_line", return_value="i"), \ - patch.object(backend_mod, "_print_vm_install_instructions") as mock_inst, \ - patch.object(backend_mod, "die", side_effect=SystemExit("die")): + patch.object(_sel, "read_tty_line", return_value="i"), \ + patch.object(_sel, "_print_vm_install_instructions") as mock_inst, \ + patch.object(_sel, "die", side_effect=SystemExit("die")): with self.assertRaises(SystemExit): get_bottle_backend() mock_inst.assert_called_once() @@ -218,9 +219,9 @@ class TestPrintVmInstallInstructions(unittest.TestCase): def test_linux_prints_firecracker_instructions(self): from bot_bottle.backend import _print_vm_install_instructions - with patch.object(backend_mod, "_platform_vm_suggestion", + with patch.object(_sel, "_platform_vm_suggestion", return_value="firecracker"), \ - patch.object(backend_mod, "info") as mock_info: + patch.object(_sel, "info") as mock_info: _print_vm_install_instructions() messages = [str(c[0][0]) for c in mock_info.call_args_list] @@ -229,9 +230,9 @@ class TestPrintVmInstallInstructions(unittest.TestCase): def test_macos_prints_apple_container_instructions(self): from bot_bottle.backend import _print_vm_install_instructions - with patch.object(backend_mod, "_platform_vm_suggestion", + with patch.object(_sel, "_platform_vm_suggestion", return_value="macos-container"), \ - patch.object(backend_mod, "info") as mock_info: + patch.object(_sel, "info") as mock_info: _print_vm_install_instructions() messages = [str(c[0][0]) for c in mock_info.call_args_list] @@ -274,7 +275,7 @@ class TestEnumerateActiveAgents(unittest.TestCase): return self._items with patch.object( - backend_mod, "_backends", + _sel, "_backends", {"docker": _FakeBackend([a]), "firecracker": _FakeBackend([b])}, ): self.assertEqual([a, b], enumerate_active_agents()) @@ -308,7 +309,7 @@ class TestEnumerateActiveAgents(unittest.TestCase): return self._items with patch.object( - backend_mod, "_backends", + _sel, "_backends", { "docker": _FakeBackend([newer, tie_b]), "firecracker": _FakeBackend([missing_metadata, tie_a]), @@ -328,7 +329,7 @@ class TestEnumerateActiveAgents(unittest.TestCase): return [] with patch.object( - backend_mod, "_backends", + _sel, "_backends", {"docker": _FakeBackend(), "firecracker": _FakeBackend()}, ): self.assertEqual([], enumerate_active_agents()) @@ -359,7 +360,7 @@ class TestEnumerateActiveAgents(unittest.TestCase): return self._items with patch.object( - backend_mod, "_backends", + _sel, "_backends", { "docker": _FakeBackend([present], available=True), "firecracker": _FakeBackend([hidden], available=False), @@ -375,7 +376,7 @@ class TestHasBackend(unittest.TestCase): return False with patch.object( - backend_mod, "_backends", {"docker": _FakeBackend()}, + _sel, "_backends", {"docker": _FakeBackend()}, ): from bot_bottle.backend import has_backend self.assertFalse(has_backend("docker")) -- 2.52.0 From 923d44bc096099fb778d90f47f9dd8603b8c10c2 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 15:53:48 -0400 Subject: [PATCH 13/30] refactor(backend): thin the docker/firecracker/macos-container package inits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three concrete-backend package inits eagerly imported their `*BottleBackend` (which pulls the whole framework), so importing any leaf under them (e.g. `backend.docker.util`) paid ~78 modules. Give each the same thin `__getattr__` treatment as the top-level backend package: re-export the public class(es) lazily on first access, with a TYPE_CHECKING block for checkers. `import bot_bottle.backend.docker.util` drops from 78 -> 6 bot_bottle modules; the docker/firecracker/macos-container packages themselves are ~3 each. All `from bot_bottle.backend.docker import DockerBottleBackend` call-sites and the lazy selection path keep working. This is what makes backend leaves cheap — and unblocks moving docker_cmd into backend/docker/util if we want. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/backend/docker/__init__.py | 38 +++++++++++++++---- bot_bottle/backend/firecracker/__init__.py | 22 ++++++++++- .../backend/macos_container/__init__.py | 27 +++++++++++-- 3 files changed, 74 insertions(+), 13 deletions(-) diff --git a/bot_bottle/backend/docker/__init__.py b/bot_bottle/backend/docker/__init__.py index dc2340b8..8da7dc85 100644 --- a/bot_bottle/backend/docker/__init__.py +++ b/bot_bottle/backend/docker/__init__.py @@ -11,18 +11,42 @@ The bulk of the implementation lives in sibling modules: - launch: bring-up + teardown context manager - cleanup: orphan enumeration, removal, active listing - backend: DockerBottleBackend façade wiring the above + - infra: DockerInfraService (the per-host infra container) -This file only re-exports the public names so -`from bot_bottle.backend.docker import DockerBottleBackend` keeps -working. +Thin by design: the public names are re-exported lazily via `__getattr__`, so +importing a leaf like `backend.docker.util` doesn't drag `DockerBottleBackend` +(and the whole framework it pulls) into memory. """ from __future__ import annotations -from .backend import DockerBottleBackend -from .bottle import DockerBottle -from .bottle_cleanup_plan import DockerBottleCleanupPlan -from .bottle_plan import DockerBottlePlan +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .backend import DockerBottleBackend + from .bottle import DockerBottle + from .bottle_cleanup_plan import DockerBottleCleanupPlan + from .bottle_plan import DockerBottlePlan + + +_LAZY_MODULES: dict[str, str] = { + "DockerBottleBackend": "backend", + "DockerBottle": "bottle", + "DockerBottleCleanupPlan": "bottle_cleanup_plan", + "DockerBottlePlan": "bottle_plan", +} + + +def __getattr__(name: str) -> Any: + mod = _LAZY_MODULES.get(name) + if mod is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module + + value = getattr(import_module(f"{__name__}.{mod}"), name) + globals()[name] = value + return value + __all__ = [ "DockerBottle", diff --git a/bot_bottle/backend/firecracker/__init__.py b/bot_bottle/backend/firecracker/__init__.py index 57b58404..cae12486 100644 --- a/bot_bottle/backend/firecracker/__init__.py +++ b/bot_bottle/backend/firecracker/__init__.py @@ -1,7 +1,25 @@ -"""Firecracker backend: Linux KVM microVM isolation (issue #342).""" +"""Firecracker backend: Linux KVM microVM isolation (issue #342). + +Thin by design: `FirecrackerBottleBackend` is re-exported lazily via +`__getattr__`, so importing a leaf module under this package doesn't drag the +backend (and the framework it pulls) into memory. +""" from __future__ import annotations -from .backend import FirecrackerBottleBackend +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .backend import FirecrackerBottleBackend + + +def __getattr__(name: str) -> Any: + if name == "FirecrackerBottleBackend": + from .backend import FirecrackerBottleBackend + + globals()[name] = FirecrackerBottleBackend + return FirecrackerBottleBackend + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + __all__ = ["FirecrackerBottleBackend"] diff --git a/bot_bottle/backend/macos_container/__init__.py b/bot_bottle/backend/macos_container/__init__.py index 6d56f95a..c655d15c 100644 --- a/bot_bottle/backend/macos_container/__init__.py +++ b/bot_bottle/backend/macos_container/__init__.py @@ -1,10 +1,29 @@ """macOS Apple Container backend. -Selectable via `BOT_BOTTLE_BACKEND=macos-container`. This package owns -the Apple `container` CLI integration; launch remains gated until the -gateway network enforcement shape is implemented. +Selectable via `BOT_BOTTLE_BACKEND=macos-container`. This package owns the Apple +`container` CLI integration; launch remains gated until the gateway network +enforcement shape is implemented. + +Thin by design: `MacosContainerBottleBackend` is re-exported lazily via +`__getattr__`, so importing a leaf module under this package doesn't drag the +backend (and the framework it pulls) into memory. """ -from .backend import MacosContainerBottleBackend +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .backend import MacosContainerBottleBackend + + +def __getattr__(name: str) -> Any: + if name == "MacosContainerBottleBackend": + from .backend import MacosContainerBottleBackend + + globals()[name] = MacosContainerBottleBackend + return MacosContainerBottleBackend + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + __all__ = ["MacosContainerBottleBackend"] -- 2.52.0 From eab9d15130d8f579ec94960d4b98fc29027c62b3 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 15:58:23 -0400 Subject: [PATCH 14/30] refactor(docker): fold run_docker into backend/docker/util; drop the docker_cmd shim docker_cmd.py existed as a top-level module solely so the orchestrator's docker components could share run_docker without dragging the (then-heavy) backend layer in. Now that backend/__init__ and backend/docker/__init__ are thin, importing backend.docker.util costs 6 modules instead of 76, so the shim's whole reason to exist is gone. Move run_docker into backend/docker/util.py (where the other docker subprocess primitives live) and delete docker_cmd.py. Backend siblings import it via `from .util import run_docker`; the two docker-specific orchestrator modules (docker_broker, rotate_ca) import `from ..backend.docker.util import run_docker`. No import cycle (backend.docker.util pulls nothing from orchestrator); orchestrator.__main__ stays lean at 28 modules. run_docker patch targets are unchanged (tests patch it in the importing module). Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- .../backend/docker/consolidated_launch.py | 2 +- bot_bottle/backend/docker/gateway.py | 2 +- .../backend/docker/gateway_provision.py | 2 +- bot_bottle/backend/docker/infra.py | 2 +- bot_bottle/backend/docker/util.py | 25 +++++++++++--- bot_bottle/docker_cmd.py | 34 ------------------- bot_bottle/orchestrator/docker_broker.py | 2 +- bot_bottle/orchestrator/rotate_ca.py | 2 +- 8 files changed, 26 insertions(+), 45 deletions(-) delete mode 100644 bot_bottle/docker_cmd.py diff --git a/bot_bottle/backend/docker/consolidated_launch.py b/bot_bottle/backend/docker/consolidated_launch.py index 2b7097fc..7efabc61 100644 --- a/bot_bottle/backend/docker/consolidated_launch.py +++ b/bot_bottle/backend/docker/consolidated_launch.py @@ -16,7 +16,7 @@ from __future__ import annotations from dataclasses import dataclass from ... import log -from ...docker_cmd import run_docker +from .util import run_docker from ...egress import EgressPlan from ...git_gate import GitGatePlan from ...orchestrator.client import OrchestratorClient diff --git a/bot_bottle/backend/docker/gateway.py b/bot_bottle/backend/docker/gateway.py index edea8e7d..c45ff8ee 100644 --- a/bot_bottle/backend/docker/gateway.py +++ b/bot_bottle/backend/docker/gateway.py @@ -5,7 +5,7 @@ import time from pathlib import Path from ...control_auth import ROLE_GATEWAY, mint -from ...docker_cmd import run_docker +from .util import run_docker from ...paths import ( CONTROL_AUTH_JWT_ENV, host_control_plane_token, diff --git a/bot_bottle/backend/docker/gateway_provision.py b/bot_bottle/backend/docker/gateway_provision.py index d1094dcc..3eaadcbc 100644 --- a/bot_bottle/backend/docker/gateway_provision.py +++ b/bot_bottle/backend/docker/gateway_provision.py @@ -17,7 +17,7 @@ from __future__ import annotations import re from typing import Protocol -from ...docker_cmd import run_docker +from .util import run_docker from ...git_gate import GitGatePlan, git_gate_render_provision # bottle ids index the gateway's per-bottle repo + creds dirs; they land in diff --git a/bot_bottle/backend/docker/infra.py b/bot_bottle/backend/docker/infra.py index 2471082e..5a1709eb 100644 --- a/bot_bottle/backend/docker/infra.py +++ b/bot_bottle/backend/docker/infra.py @@ -26,7 +26,7 @@ from pathlib import Path from ... import log from ...control_auth import ROLE_GATEWAY, mint -from ...docker_cmd import run_docker +from .util import run_docker from ...paths import ( CONTROL_AUTH_JWT_ENV, CONTROL_PLANE_TOKEN_ENV, diff --git a/bot_bottle/backend/docker/util.py b/bot_bottle/backend/docker/util.py index fb2091b8..b05cbd6e 100644 --- a/bot_bottle/backend/docker/util.py +++ b/bot_bottle/backend/docker/util.py @@ -1,6 +1,6 @@ -"""Docker host-side primitives used by DockerBottleBackend: probing -for docker on PATH, slugifying agent names, checking image/container -existence, and building images.""" +"""Docker host-side primitives used by DockerBottleBackend: the lean +`run_docker` subprocess wrapper, probing for docker on PATH, slugifying agent +names, checking image/container existence, and building images.""" from __future__ import annotations @@ -11,9 +11,24 @@ import shutil import subprocess from typing import Iterator -from ...docker_cmd import run_docker from ...log import die, info -# from ...workspace import WorkspacePlan + + +def run_docker( + argv: list[str], *, env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[str]: + """Run a `docker` command, capturing stdout/stderr as text. Never raises on + a non-zero exit — callers inspect `returncode` / `stderr` so they can stay + fail-closed or tolerate idempotent no-ops (e.g. removing an already-absent + container). + + `env` sets the child process environment — used to hand a secret to a bare + `--env NAME` flag (docker inherits its value from this process) so the value + never lands on argv or in `docker inspect`'s recorded command line.""" + return subprocess.run( + argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + check=False, env=env, + ) # Cap on the suffix the container-name conflict logic will try before diff --git a/bot_bottle/docker_cmd.py b/bot_bottle/docker_cmd.py deleted file mode 100644 index 32a617ed..00000000 --- a/bot_bottle/docker_cmd.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Lean, framework-free `docker` subprocess primitive. - -Deliberately a top-level module with a single stdlib import so it can be -reused from anywhere without cost. It is intentionally *not* placed in -`backend.docker.util`: importing that module runs `backend/__init__.py`, -which eagerly loads all three bottle backends (docker + firecracker + -macos) plus the manifest/egress/git-gate/supervise framework — ~76 modules -— which would drag the whole backend layer into the deliberately-lean -orchestrator. This primitive stays free of that so both the orchestrator's -docker components and (in time) `backend.docker.util` can share it.""" - -from __future__ import annotations - -import subprocess - - -def run_docker( - argv: list[str], *, env: dict[str, str] | None = None, -) -> subprocess.CompletedProcess[str]: - """Run a `docker` command, capturing stdout/stderr as text. Never raises - on a non-zero exit — callers inspect `returncode` / `stderr` so they can - stay fail-closed or tolerate idempotent no-ops (e.g. removing an - already-absent container). - - `env` sets the child process environment — used to hand a secret to a bare - `--env NAME` flag (docker inherits its value from this process) so the - value never lands on argv or in `docker inspect`'s recorded command line.""" - return subprocess.run( - argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, - check=False, env=env, - ) - - -__all__ = ["run_docker"] diff --git a/bot_bottle/orchestrator/docker_broker.py b/bot_bottle/orchestrator/docker_broker.py index e5af91c5..2e22845e 100644 --- a/bot_bottle/orchestrator/docker_broker.py +++ b/bot_bottle/orchestrator/docker_broker.py @@ -15,7 +15,7 @@ from __future__ import annotations import subprocess -from ..docker_cmd import run_docker +from ..backend.docker.util import run_docker from .broker import LaunchBroker, LaunchRequest CONTAINER_PREFIX = "bot-bottle-orch-" diff --git a/bot_bottle/orchestrator/rotate_ca.py b/bot_bottle/orchestrator/rotate_ca.py index 0cae3004..93030054 100644 --- a/bot_bottle/orchestrator/rotate_ca.py +++ b/bot_bottle/orchestrator/rotate_ca.py @@ -23,7 +23,7 @@ from __future__ import annotations import sys from pathlib import Path -from ..docker_cmd import run_docker +from ..backend.docker.util import run_docker from ..paths import host_gateway_ca_dir from ..gateway import GATEWAY_NAME, rotate_gateway_ca from ..backend.docker.infra import INFRA_NAME -- 2.52.0 From 14c28946a7f0b0bb105040f83338f00a399ea7bc Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 16:04:15 -0400 Subject: [PATCH 15/30] refactor(manifest): move Manifest/ManifestIndex to manifest.index; thin the package init MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit manifest/__init__.py was a facade that eagerly imported every piece (agent, bottle, egress, git, loader, schema, util) and defined the aggregate Manifest / ManifestIndex, so importing a leaf like `manifest.util` paid 19 modules. Move the aggregate model — Manifest, ManifestIndex, and the bottle-resolution helpers — into manifest/index.py (which keeps the framework imports it needs), and make __init__ a thin __getattr__ facade over the 12 __all__ names (mapping each to its submodule), with a TYPE_CHECKING block. The package docstring is preserved. `import bot_bottle.manifest.util` drops 19 -> 3; the package itself is 2. All `from bot_bottle.manifest import Manifest/ManifestIndex/ManifestError/…` call-sites keep working via the lazy facade; no importer changes needed. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/manifest/__init__.py | 452 ++++---------------------------- bot_bottle/manifest/index.py | 413 +++++++++++++++++++++++++++++ 2 files changed, 459 insertions(+), 406 deletions(-) create mode 100644 bot_bottle/manifest/index.py diff --git a/bot_bottle/manifest/__init__.py b/bot_bottle/manifest/__init__.py index 8289298f..1dde0285 100644 --- a/bot_bottle/manifest/__init__.py +++ b/bot_bottle/manifest/__init__.py @@ -58,421 +58,61 @@ useful for building manifests without on-disk files. from __future__ import annotations -import os -from dataclasses import dataclass, field, replace -from pathlib import Path -from typing import Mapping +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .index import Manifest, ManifestIndex + from .util import ManifestError + from .agent import ManifestAgent, ManifestAgentProvider + from .bottle import ManifestBottle + from .egress import EGRESS_AUTH_SCHEMES, ManifestEgressConfig, ManifestEgressRoute + from .git import ManifestGitEntry, ManifestGitUser, ManifestKeyConfig + + +# Facade name -> submodule that defines it. The aggregate model (`Manifest`, +# `ManifestIndex`) lives in `index`; the piece types in their own modules. +_LAZY_MODULES: dict[str, str] = { + "Manifest": "index", + "ManifestIndex": "index", + "ManifestError": "util", + "ManifestAgent": "agent", + "ManifestAgentProvider": "agent", + "ManifestBottle": "bottle", + "EGRESS_AUTH_SCHEMES": "egress", + "ManifestEgressRoute": "egress", + "ManifestEgressConfig": "egress", + "ManifestGitEntry": "git", + "ManifestGitUser": "git", + "ManifestKeyConfig": "git", +} + + +def __getattr__(name: str) -> Any: + """Lazily surface the manifest facade names from their submodules and cache + them at package level — so importing a leaf (e.g. `manifest.util`) doesn't + build the whole manifest model, while `from bot_bottle.manifest import + Manifest` keeps working.""" + mod = _LAZY_MODULES.get(name) + if mod is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module + + value = getattr(import_module(f"{__name__}.{mod}"), name) + globals()[name] = value + return value -from ..log import warn -from .util import ManifestError, as_json_object -from .agent import ManifestAgent, ManifestAgentProvider -from .bottle import ManifestBottle -from .egress import ( - EGRESS_AUTH_SCHEMES, - ManifestEgressConfig, - ManifestEgressRoute, -) -from .extends import merge_bottles_runtime, resolve_bottles -from .git import ManifestGitEntry, ManifestGitUser, ManifestKeyConfig -from .loader import ( - check_stale_json, - load_bottle_chain_from_dir, - scan_agent_names, - scan_bottle_names, -) -from .schema import validate_agent_frontmatter_keys -from ..yaml_subset import YamlSubsetError, parse_frontmatter -# Re-export everything that callers currently import from this module. __all__ = [ + "Manifest", + "ManifestIndex", "ManifestError", "ManifestGitEntry", "ManifestGitUser", "ManifestKeyConfig", "ManifestAgentProvider", + "ManifestAgent", + "ManifestBottle", "EGRESS_AUTH_SCHEMES", "ManifestEgressRoute", "ManifestEgressConfig", - "ManifestAgent", - "ManifestBottle", - "ManifestIndex", - "Manifest", ] - - -def _section_dict(value: object, label: str) -> dict[str, object]: - """Like as_json_object but treats absent/null as an empty section.""" - if value is None: - return {} - return as_json_object(value, label) - - -def _merge_git_user( - agent_user: ManifestGitUser, base_user: ManifestGitUser -) -> ManifestGitUser: - """Merge the agent's git.user over the bottle's, agent-wins-on-non-empty.""" - if agent_user.is_empty(): - return base_user - return ManifestGitUser( - name=agent_user.name or base_user.name, - email=agent_user.email or base_user.email, - ) - - -def _manifest_with_merged_git_user( - agent: "ManifestAgent", raw_bottle: "ManifestBottle" -) -> "Manifest": - """Build the single-value Manifest, overlaying the agent's git-gate.user - onto the bottle (agent wins on non-empty, per-field). Shared by the eager - and lazy load_for_agent paths.""" - merged = _merge_git_user(agent.git_user, raw_bottle.git_user) - bottle = ( - raw_bottle if merged == raw_bottle.git_user - else replace(raw_bottle, git_user=merged) - ) - return Manifest(agent=agent, bottle=bottle) - - -def _resolve_effective_bottle_eager( - agent_name: str, - agent: "ManifestAgent", - bottle_names: "tuple[str, ...]", - bottles: "Mapping[str, ManifestBottle]", -) -> "ManifestBottle": - """Return the effective ManifestBottle for the eager (from_json_obj) path. - - When bottle_names is non-empty they are merged in order. When empty, falls - back to agent.bottle. Raises ManifestError when neither is set.""" - if bottle_names: - resolved: list[ManifestBottle] = [] - for bn in bottle_names: - if bn not in bottles: - available = ", ".join(sorted(bottles.keys())) or "(none)" - raise ManifestError( - f"bottle '{bn}' not defined. Available: {available}" - ) - resolved.append(bottles[bn]) - return merge_bottles_runtime(resolved) - - if not agent.bottle: - raise ManifestError( - f"agent '{agent_name}' has no 'bottle' field and no bottles were " - f"selected at launch. Select at least one bottle or add " - f"'bottle: ' to the agent manifest." - ) - return bottles[agent.bottle] - - -def _resolve_effective_bottle_lazy( - agent_name: str, - agent_bottle: str, - bottle_names: "tuple[str, ...]", - bottles_dir: "Path", -) -> "ManifestBottle": - """Return the effective ManifestBottle for the lazy (from_md_dirs) path. - - When bottle_names is non-empty they are resolved from disk and merged in - order. When empty, falls back to agent_bottle. Raises ManifestError when - neither is set.""" - if bottle_names: - resolved = [load_bottle_chain_from_dir(bn, bottles_dir) for bn in bottle_names] - return merge_bottles_runtime(resolved) - - if not agent_bottle: - raise ManifestError( - f"agent '{agent_name}' has no 'bottle' field and no bottles were " - f"selected at launch. Select at least one bottle or add " - f"'bottle: ' to the agent manifest." - ) - return load_bottle_chain_from_dir(agent_bottle, bottles_dir) - - -@dataclass(frozen=True) -class Manifest: - """Single-agent/bottle value type. Returned by ManifestIndex.load_for_agent(). - - `bottle` is the effective bottle with the agent's git-gate.user already - overlaid per-field (agent wins on non-empty). Backends and provisioners - use this directly — no agent_name lookup needed.""" - - agent: ManifestAgent - bottle: ManifestBottle - - def git_identity_summary(self) -> str | None: - """One-line effective git identity with per-field provenance, e.g. - `name=claude (agent), email=eric@dideric.is (bottle)`. - Returns None when neither agent nor bottle sets an identity.""" - over = self.agent.git_user # agent's declared git_user (pre-merge) - merged = self.bottle.git_user # effective git_user (post-merge) - if merged.is_empty(): - return None - parts: list[str] = [] - if merged.name: - parts.append(f"name={merged.name} ({'agent' if over.name else 'bottle'})") - if merged.email: - parts.append(f"email={merged.email} ({'agent' if over.email else 'bottle'})") - return ", ".join(parts) - - -@dataclass(frozen=True) -class ManifestIndex: - """Multi-agent/bottle collection. The pre-preflight form. - - In lazy mode (from resolve()/from_md_dirs()) only filenames are scanned; - no file content is read. In eager mode (from from_json_obj()) all agents - and bottles are pre-parsed. Call load_for_agent() to get a single-value - Manifest ready for backend use.""" - - bottles: Mapping[str, ManifestBottle] - agents: Mapping[str, ManifestAgent] - # Set by from_md_dirs; None in from_json_obj (test/programmatic) mode. - # Stores the manifest root dirs so load_for_agent can locate files later. - home_md: Path | None = field(default=None) - cwd_md: Path | None = field(default=None) - - @classmethod - def resolve(cls, cwd: str, *, missing_ok: bool = False) -> "ManifestIndex": - """Walk the per-file manifest tree and build a ManifestIndex. - - Layout (PRD 0011): - $HOME/.bot-bottle/bottles/.md — bottles (home-only) - $HOME/.bot-bottle/agents/.md — home agents - $CWD/.bot-bottle/agents/.md — cwd agents - - Cwd agents merge into the home agents on the same name - (cwd wins). A bottles/ subdir under $CWD is logged as a - warning and ignored — the filesystem layout IS the trust - boundary. - - If `missing_ok` is true, a missing `$HOME/.bot-bottle/` - returns an empty index instead of dying. This is for - passive UI surfaces like the dashboard, which can still - monitor already-running agents without launch config. - - If `bot-bottle.json` exists alongside a missing - `.bot-bottle/` directory at either side, dies with a - clear pointer at the README's manifest section — the - manifest format changed in PRD 0011 and we don't silently - fall back.""" - home_dir = Path(os.environ["HOME"]) - cwd_dir = Path(cwd) - home_md = home_dir / ".bot-bottle" - cwd_md = cwd_dir / ".bot-bottle" - - check_stale_json(home_dir, home_md, "$HOME") - if cwd_dir.resolve() != home_dir.resolve(): - check_stale_json(cwd_dir, cwd_md, "$CWD") - - if not home_md.is_dir(): - if missing_ok: - return cls.from_json_obj({"bottles": {}, "agents": {}}) - raise ManifestError( - f"no manifest found: {home_md} does not exist. " - f"See README.md for the per-file Markdown layout " - f"(PRD 0011)." - ) - - # When CWD == HOME (running from $HOME directly), pass the - # same dir for both — _load_md_dirs will dedupe. - cwd_md_arg = cwd_md if cwd_md.is_dir() and cwd_dir.resolve() != home_dir.resolve() else None - return cls.from_md_dirs(home_md, cwd_md_arg) - - @classmethod - def from_md_dirs( - cls, - home_dir: Path, - cwd_dir: Path | None, - ) -> "ManifestIndex": - """Return a names-only ManifestIndex. No file content is read; only - filenames are scanned for the agent selector. Full parsing happens - later, per-agent, via `load_for_agent`. - - A `bottles/` subdir under `cwd_dir` is logged as a warning and - ignored — the filesystem layout IS the trust boundary. - - Used by tests to build a ManifestIndex from fixture directories - without touching `os.environ`.""" - if cwd_dir is not None: - stale_bottles = cwd_dir / "bottles" - if stale_bottles.is_dir(): - files = sorted(stale_bottles.glob("*.md")) - if files: - names = ", ".join(p.name for p in files) - warn( - f"ignoring bottle file(s) under " - f"{stale_bottles}: {names}. Bottles can only " - f"live under $HOME/.bot-bottle/bottles/ " - f"(PRD 0011). Move them or delete." - ) - return cls(bottles={}, agents={}, home_md=home_dir, cwd_md=cwd_dir) - - @classmethod - def from_json_obj(cls, obj: object) -> "ManifestIndex": - """Validate and build a ManifestIndex from a raw JSON-like dict.""" - d = as_json_object(obj, "manifest") - raw_bottles_obj = _section_dict(d.get("bottles"), "manifest 'bottles'") - raw_agents = _section_dict(d.get("agents"), "manifest 'agents'") - - # Coerce each bottle's raw to dict[str, object] so the - # PRD 0025 resolver can apply extends-merge rules - # consistently with the md-loader path. - raw_bottles: dict[str, dict[str, object]] = {} - for n, b in raw_bottles_obj.items(): - raw_bottles[n] = as_json_object(b, f"bottle '{n}'") - - bottles = resolve_bottles(raw_bottles) - - bottle_names = set(bottles.keys()) - agents: dict[str, ManifestAgent] = { - n: ManifestAgent.from_dict(n, a, bottle_names) for n, a in raw_agents.items() - } - return cls(bottles=bottles, agents=agents) - - @property - def all_bottle_names(self) -> list[str]: - """Sorted list of all discoverable bottle names. - - In names-only mode (from resolve/from_md_dirs) this scans bottle - filenames without reading their content. In eager mode (from - from_json_obj) it returns the pre-parsed bottles' names.""" - if self.home_md is not None: - return scan_bottle_names(self.home_md / "bottles") - return sorted(self.bottles.keys()) - - @property - def all_agent_names(self) -> list[str]: - """Sorted list of all discoverable agent names. - - In names-only mode (from resolve/from_md_dirs) this scans agent - filenames without reading their content. In eager mode (from - from_json_obj) it returns the pre-parsed agents' names.""" - if self.home_md is not None: - home_names = set(scan_agent_names(self.home_md / "agents").keys()) - cwd_names: set[str] = set() - if self.cwd_md is not None: - cwd_names = set(scan_agent_names(self.cwd_md / "agents").keys()) - return sorted(home_names | cwd_names) - return sorted(self.agents.keys()) - - def load_for_agent( - self, - agent_name: str, - bottle_names: "tuple[str, ...] | None" = None, - ) -> "Manifest": - """Parse the named agent and its bottle; return a single-value Manifest. - - `bottle_names` is an ordered list of bottles selected at launch time. - When non-empty they are resolved and merged in order (index 0 = base; - later entries override). When empty or None, falls back to the agent's - own `bottle:` field. Raises ManifestError when neither is set. - - In lazy mode (from resolve/from_md_dirs) the agent file and its - bottle chain are read from disk for the first time here. In eager - mode (from_json_obj) the data is already parsed; this just filters - down to the requested agent and its bottle. - - The returned Manifest.bottle has the agent's git-gate.user already - overlaid (agent wins on non-empty, per-field). - - Always raises ManifestError if the agent is unknown or invalid. - Backends call this at preflight inside _validate.""" - effective_bottle_names: tuple[str, ...] = bottle_names or () - if self.home_md is None: - return self._load_for_agent_eager(agent_name, effective_bottle_names) - return self._load_for_agent_lazy(agent_name, effective_bottle_names) - - def _load_for_agent_eager( - self, agent_name: str, bottle_names: tuple[str, ...] - ) -> "Manifest": - """Eager path (from_json_obj): data is already parsed; filter to the one - requested agent and its bottle so the returned Manifest always holds - exactly one agent and one bottle regardless of path.""" - if agent_name not in self.agents: - available = ", ".join(sorted(self.agents.keys())) or "(none)" - raise ManifestError( - f"agent '{agent_name}' not defined. Available: {available}" - ) - agent = self.agents[agent_name] - raw_bottle = _resolve_effective_bottle_eager( - agent_name, agent, bottle_names, self.bottles - ) - return _manifest_with_merged_git_user(agent, raw_bottle) - - def _load_for_agent_lazy( - self, agent_name: str, bottle_names: tuple[str, ...] - ) -> "Manifest": - """Lazy path (resolve/from_md_dirs): read and parse the agent file and - its bottle chain from disk for the first time here.""" - assert self.home_md is not None # guaranteed by load_for_agent dispatch - # Locate the agent file; cwd wins over home on name collision. - home_agents = scan_agent_names(self.home_md / "agents") - cwd_agents: dict[str, Path] = {} - if self.cwd_md is not None: - cwd_agents = scan_agent_names(self.cwd_md / "agents") - merged_agents = {**home_agents, **cwd_agents} - - if agent_name not in merged_agents: - available = ", ".join(sorted(merged_agents.keys())) or "(none)" - raise ManifestError( - f"agent '{agent_name}' not defined. Available: {available}" - ) - - agent_path = merged_agents[agent_name] - try: - fm, body = parse_frontmatter(agent_path.read_text()) - except OSError as e: - raise ManifestError(f"could not read {agent_path}: {e}") from e - except YamlSubsetError as e: - raise ManifestError(f"{agent_path}: {e}") from e - - validate_agent_frontmatter_keys(agent_path, fm.keys()) - - # Determine the effective bottle name(s). - agent_bottle = fm.get("bottle") or "" - bottles_dir = self.home_md / "bottles" - raw_bottle = _resolve_effective_bottle_lazy( - agent_name, str(agent_bottle), bottle_names, bottles_dir - ) - effective_bottle_name = ( - bottle_names[-1] if bottle_names else str(agent_bottle) - ) - - # Build and validate the full ManifestAgent. - agent_dict: dict[str, object] = { - "skills": fm.get("skills", []), - "prompt": body.strip(), - } - if agent_bottle: - agent_dict["bottle"] = agent_bottle - if "git-gate" in fm: - agent_dict["git-gate"] = fm["git-gate"] - # Pass the effective bottle name as the known-bottles set so agents - # that have bottle: set are validated; agents without bottle: pass {} - # since bottle_names were already resolved above. - known = {effective_bottle_name} if effective_bottle_name else set() - agent = ManifestAgent.from_dict(agent_name, agent_dict, known) - - return _manifest_with_merged_git_user(agent, raw_bottle) - - def has_agent(self, name: str) -> bool: - return name in self.agents - - def require_agent(self, name: str) -> None: - """Check that `name` is a discoverable agent. In names-only mode - this checks whether the .md file exists; in eager mode it checks - the pre-parsed agents dict. Does NOT parse file content.""" - if self.has_agent(name): - return - if self.home_md is not None: - # Names-only mode: check file existence without parsing. - home_path = self.home_md / "agents" / f"{name}.md" - cwd_path = ( - self.cwd_md / "agents" / f"{name}.md" - if self.cwd_md else None - ) - if home_path.is_file() or (cwd_path and cwd_path.is_file()): - return - available = ", ".join(self.all_agent_names) or "(none)" - raise ManifestError( - f"agent '{name}' not defined. Available: {available}" - ) diff --git a/bot_bottle/manifest/index.py b/bot_bottle/manifest/index.py new file mode 100644 index 00000000..35966b47 --- /dev/null +++ b/bot_bottle/manifest/index.py @@ -0,0 +1,413 @@ +"""The `Manifest` / `ManifestIndex` aggregate model. + +The parsed manifest document (`Manifest`) and the multi-agent index over it +(`ManifestIndex`), plus the bottle-resolution helpers that stitch the pieces +(`agent`, `bottle`, `egress`, `git`, `extends`, `loader`, `schema`) into an +effective manifest. This is the heavy aggregate — the thin package `__init__` +re-exports `Manifest` / `ManifestIndex` lazily. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field, replace +from pathlib import Path +from typing import Mapping + +from ..log import warn +from .util import ManifestError, as_json_object +from .agent import ManifestAgent, ManifestAgentProvider +from .bottle import ManifestBottle +from .egress import ( + EGRESS_AUTH_SCHEMES, + ManifestEgressConfig, + ManifestEgressRoute, +) +from .extends import merge_bottles_runtime, resolve_bottles +from .git import ManifestGitEntry, ManifestGitUser, ManifestKeyConfig +from .loader import ( + check_stale_json, + load_bottle_chain_from_dir, + scan_agent_names, + scan_bottle_names, +) +from .schema import validate_agent_frontmatter_keys +from ..yaml_subset import YamlSubsetError, parse_frontmatter + + +def _section_dict(value: object, label: str) -> dict[str, object]: + """Like as_json_object but treats absent/null as an empty section.""" + if value is None: + return {} + return as_json_object(value, label) + + +def _merge_git_user( + agent_user: ManifestGitUser, base_user: ManifestGitUser +) -> ManifestGitUser: + """Merge the agent's git.user over the bottle's, agent-wins-on-non-empty.""" + if agent_user.is_empty(): + return base_user + return ManifestGitUser( + name=agent_user.name or base_user.name, + email=agent_user.email or base_user.email, + ) + + +def _manifest_with_merged_git_user( + agent: "ManifestAgent", raw_bottle: "ManifestBottle" +) -> "Manifest": + """Build the single-value Manifest, overlaying the agent's git-gate.user + onto the bottle (agent wins on non-empty, per-field). Shared by the eager + and lazy load_for_agent paths.""" + merged = _merge_git_user(agent.git_user, raw_bottle.git_user) + bottle = ( + raw_bottle if merged == raw_bottle.git_user + else replace(raw_bottle, git_user=merged) + ) + return Manifest(agent=agent, bottle=bottle) + + +def _resolve_effective_bottle_eager( + agent_name: str, + agent: "ManifestAgent", + bottle_names: "tuple[str, ...]", + bottles: "Mapping[str, ManifestBottle]", +) -> "ManifestBottle": + """Return the effective ManifestBottle for the eager (from_json_obj) path. + + When bottle_names is non-empty they are merged in order. When empty, falls + back to agent.bottle. Raises ManifestError when neither is set.""" + if bottle_names: + resolved: list[ManifestBottle] = [] + for bn in bottle_names: + if bn not in bottles: + available = ", ".join(sorted(bottles.keys())) or "(none)" + raise ManifestError( + f"bottle '{bn}' not defined. Available: {available}" + ) + resolved.append(bottles[bn]) + return merge_bottles_runtime(resolved) + + if not agent.bottle: + raise ManifestError( + f"agent '{agent_name}' has no 'bottle' field and no bottles were " + f"selected at launch. Select at least one bottle or add " + f"'bottle: ' to the agent manifest." + ) + return bottles[agent.bottle] + + +def _resolve_effective_bottle_lazy( + agent_name: str, + agent_bottle: str, + bottle_names: "tuple[str, ...]", + bottles_dir: "Path", +) -> "ManifestBottle": + """Return the effective ManifestBottle for the lazy (from_md_dirs) path. + + When bottle_names is non-empty they are resolved from disk and merged in + order. When empty, falls back to agent_bottle. Raises ManifestError when + neither is set.""" + if bottle_names: + resolved = [load_bottle_chain_from_dir(bn, bottles_dir) for bn in bottle_names] + return merge_bottles_runtime(resolved) + + if not agent_bottle: + raise ManifestError( + f"agent '{agent_name}' has no 'bottle' field and no bottles were " + f"selected at launch. Select at least one bottle or add " + f"'bottle: ' to the agent manifest." + ) + return load_bottle_chain_from_dir(agent_bottle, bottles_dir) + + +@dataclass(frozen=True) +class Manifest: + """Single-agent/bottle value type. Returned by ManifestIndex.load_for_agent(). + + `bottle` is the effective bottle with the agent's git-gate.user already + overlaid per-field (agent wins on non-empty). Backends and provisioners + use this directly — no agent_name lookup needed.""" + + agent: ManifestAgent + bottle: ManifestBottle + + def git_identity_summary(self) -> str | None: + """One-line effective git identity with per-field provenance, e.g. + `name=claude (agent), email=eric@dideric.is (bottle)`. + Returns None when neither agent nor bottle sets an identity.""" + over = self.agent.git_user # agent's declared git_user (pre-merge) + merged = self.bottle.git_user # effective git_user (post-merge) + if merged.is_empty(): + return None + parts: list[str] = [] + if merged.name: + parts.append(f"name={merged.name} ({'agent' if over.name else 'bottle'})") + if merged.email: + parts.append(f"email={merged.email} ({'agent' if over.email else 'bottle'})") + return ", ".join(parts) + + +@dataclass(frozen=True) +class ManifestIndex: + """Multi-agent/bottle collection. The pre-preflight form. + + In lazy mode (from resolve()/from_md_dirs()) only filenames are scanned; + no file content is read. In eager mode (from from_json_obj()) all agents + and bottles are pre-parsed. Call load_for_agent() to get a single-value + Manifest ready for backend use.""" + + bottles: Mapping[str, ManifestBottle] + agents: Mapping[str, ManifestAgent] + # Set by from_md_dirs; None in from_json_obj (test/programmatic) mode. + # Stores the manifest root dirs so load_for_agent can locate files later. + home_md: Path | None = field(default=None) + cwd_md: Path | None = field(default=None) + + @classmethod + def resolve(cls, cwd: str, *, missing_ok: bool = False) -> "ManifestIndex": + """Walk the per-file manifest tree and build a ManifestIndex. + + Layout (PRD 0011): + $HOME/.bot-bottle/bottles/.md — bottles (home-only) + $HOME/.bot-bottle/agents/.md — home agents + $CWD/.bot-bottle/agents/.md — cwd agents + + Cwd agents merge into the home agents on the same name + (cwd wins). A bottles/ subdir under $CWD is logged as a + warning and ignored — the filesystem layout IS the trust + boundary. + + If `missing_ok` is true, a missing `$HOME/.bot-bottle/` + returns an empty index instead of dying. This is for + passive UI surfaces like the dashboard, which can still + monitor already-running agents without launch config. + + If `bot-bottle.json` exists alongside a missing + `.bot-bottle/` directory at either side, dies with a + clear pointer at the README's manifest section — the + manifest format changed in PRD 0011 and we don't silently + fall back.""" + home_dir = Path(os.environ["HOME"]) + cwd_dir = Path(cwd) + home_md = home_dir / ".bot-bottle" + cwd_md = cwd_dir / ".bot-bottle" + + check_stale_json(home_dir, home_md, "$HOME") + if cwd_dir.resolve() != home_dir.resolve(): + check_stale_json(cwd_dir, cwd_md, "$CWD") + + if not home_md.is_dir(): + if missing_ok: + return cls.from_json_obj({"bottles": {}, "agents": {}}) + raise ManifestError( + f"no manifest found: {home_md} does not exist. " + f"See README.md for the per-file Markdown layout " + f"(PRD 0011)." + ) + + # When CWD == HOME (running from $HOME directly), pass the + # same dir for both — _load_md_dirs will dedupe. + cwd_md_arg = cwd_md if cwd_md.is_dir() and cwd_dir.resolve() != home_dir.resolve() else None + return cls.from_md_dirs(home_md, cwd_md_arg) + + @classmethod + def from_md_dirs( + cls, + home_dir: Path, + cwd_dir: Path | None, + ) -> "ManifestIndex": + """Return a names-only ManifestIndex. No file content is read; only + filenames are scanned for the agent selector. Full parsing happens + later, per-agent, via `load_for_agent`. + + A `bottles/` subdir under `cwd_dir` is logged as a warning and + ignored — the filesystem layout IS the trust boundary. + + Used by tests to build a ManifestIndex from fixture directories + without touching `os.environ`.""" + if cwd_dir is not None: + stale_bottles = cwd_dir / "bottles" + if stale_bottles.is_dir(): + files = sorted(stale_bottles.glob("*.md")) + if files: + names = ", ".join(p.name for p in files) + warn( + f"ignoring bottle file(s) under " + f"{stale_bottles}: {names}. Bottles can only " + f"live under $HOME/.bot-bottle/bottles/ " + f"(PRD 0011). Move them or delete." + ) + return cls(bottles={}, agents={}, home_md=home_dir, cwd_md=cwd_dir) + + @classmethod + def from_json_obj(cls, obj: object) -> "ManifestIndex": + """Validate and build a ManifestIndex from a raw JSON-like dict.""" + d = as_json_object(obj, "manifest") + raw_bottles_obj = _section_dict(d.get("bottles"), "manifest 'bottles'") + raw_agents = _section_dict(d.get("agents"), "manifest 'agents'") + + # Coerce each bottle's raw to dict[str, object] so the + # PRD 0025 resolver can apply extends-merge rules + # consistently with the md-loader path. + raw_bottles: dict[str, dict[str, object]] = {} + for n, b in raw_bottles_obj.items(): + raw_bottles[n] = as_json_object(b, f"bottle '{n}'") + + bottles = resolve_bottles(raw_bottles) + + bottle_names = set(bottles.keys()) + agents: dict[str, ManifestAgent] = { + n: ManifestAgent.from_dict(n, a, bottle_names) for n, a in raw_agents.items() + } + return cls(bottles=bottles, agents=agents) + + @property + def all_bottle_names(self) -> list[str]: + """Sorted list of all discoverable bottle names. + + In names-only mode (from resolve/from_md_dirs) this scans bottle + filenames without reading their content. In eager mode (from + from_json_obj) it returns the pre-parsed bottles' names.""" + if self.home_md is not None: + return scan_bottle_names(self.home_md / "bottles") + return sorted(self.bottles.keys()) + + @property + def all_agent_names(self) -> list[str]: + """Sorted list of all discoverable agent names. + + In names-only mode (from resolve/from_md_dirs) this scans agent + filenames without reading their content. In eager mode (from + from_json_obj) it returns the pre-parsed agents' names.""" + if self.home_md is not None: + home_names = set(scan_agent_names(self.home_md / "agents").keys()) + cwd_names: set[str] = set() + if self.cwd_md is not None: + cwd_names = set(scan_agent_names(self.cwd_md / "agents").keys()) + return sorted(home_names | cwd_names) + return sorted(self.agents.keys()) + + def load_for_agent( + self, + agent_name: str, + bottle_names: "tuple[str, ...] | None" = None, + ) -> "Manifest": + """Parse the named agent and its bottle; return a single-value Manifest. + + `bottle_names` is an ordered list of bottles selected at launch time. + When non-empty they are resolved and merged in order (index 0 = base; + later entries override). When empty or None, falls back to the agent's + own `bottle:` field. Raises ManifestError when neither is set. + + In lazy mode (from resolve/from_md_dirs) the agent file and its + bottle chain are read from disk for the first time here. In eager + mode (from_json_obj) the data is already parsed; this just filters + down to the requested agent and its bottle. + + The returned Manifest.bottle has the agent's git-gate.user already + overlaid (agent wins on non-empty, per-field). + + Always raises ManifestError if the agent is unknown or invalid. + Backends call this at preflight inside _validate.""" + effective_bottle_names: tuple[str, ...] = bottle_names or () + if self.home_md is None: + return self._load_for_agent_eager(agent_name, effective_bottle_names) + return self._load_for_agent_lazy(agent_name, effective_bottle_names) + + def _load_for_agent_eager( + self, agent_name: str, bottle_names: tuple[str, ...] + ) -> "Manifest": + """Eager path (from_json_obj): data is already parsed; filter to the one + requested agent and its bottle so the returned Manifest always holds + exactly one agent and one bottle regardless of path.""" + if agent_name not in self.agents: + available = ", ".join(sorted(self.agents.keys())) or "(none)" + raise ManifestError( + f"agent '{agent_name}' not defined. Available: {available}" + ) + agent = self.agents[agent_name] + raw_bottle = _resolve_effective_bottle_eager( + agent_name, agent, bottle_names, self.bottles + ) + return _manifest_with_merged_git_user(agent, raw_bottle) + + def _load_for_agent_lazy( + self, agent_name: str, bottle_names: tuple[str, ...] + ) -> "Manifest": + """Lazy path (resolve/from_md_dirs): read and parse the agent file and + its bottle chain from disk for the first time here.""" + assert self.home_md is not None # guaranteed by load_for_agent dispatch + # Locate the agent file; cwd wins over home on name collision. + home_agents = scan_agent_names(self.home_md / "agents") + cwd_agents: dict[str, Path] = {} + if self.cwd_md is not None: + cwd_agents = scan_agent_names(self.cwd_md / "agents") + merged_agents = {**home_agents, **cwd_agents} + + if agent_name not in merged_agents: + available = ", ".join(sorted(merged_agents.keys())) or "(none)" + raise ManifestError( + f"agent '{agent_name}' not defined. Available: {available}" + ) + + agent_path = merged_agents[agent_name] + try: + fm, body = parse_frontmatter(agent_path.read_text()) + except OSError as e: + raise ManifestError(f"could not read {agent_path}: {e}") from e + except YamlSubsetError as e: + raise ManifestError(f"{agent_path}: {e}") from e + + validate_agent_frontmatter_keys(agent_path, fm.keys()) + + # Determine the effective bottle name(s). + agent_bottle = fm.get("bottle") or "" + bottles_dir = self.home_md / "bottles" + raw_bottle = _resolve_effective_bottle_lazy( + agent_name, str(agent_bottle), bottle_names, bottles_dir + ) + effective_bottle_name = ( + bottle_names[-1] if bottle_names else str(agent_bottle) + ) + + # Build and validate the full ManifestAgent. + agent_dict: dict[str, object] = { + "skills": fm.get("skills", []), + "prompt": body.strip(), + } + if agent_bottle: + agent_dict["bottle"] = agent_bottle + if "git-gate" in fm: + agent_dict["git-gate"] = fm["git-gate"] + # Pass the effective bottle name as the known-bottles set so agents + # that have bottle: set are validated; agents without bottle: pass {} + # since bottle_names were already resolved above. + known = {effective_bottle_name} if effective_bottle_name else set() + agent = ManifestAgent.from_dict(agent_name, agent_dict, known) + + return _manifest_with_merged_git_user(agent, raw_bottle) + + def has_agent(self, name: str) -> bool: + return name in self.agents + + def require_agent(self, name: str) -> None: + """Check that `name` is a discoverable agent. In names-only mode + this checks whether the .md file exists; in eager mode it checks + the pre-parsed agents dict. Does NOT parse file content.""" + if self.has_agent(name): + return + if self.home_md is not None: + # Names-only mode: check file existence without parsing. + home_path = self.home_md / "agents" / f"{name}.md" + cwd_path = ( + self.cwd_md / "agents" / f"{name}.md" + if self.cwd_md else None + ) + if home_path.is_file() or (cwd_path and cwd_path.is_file()): + return + available = ", ".join(self.all_agent_names) or "(none)" + raise ManifestError( + f"agent '{name}' not defined. Available: {available}" + ) -- 2.52.0 From 450037b7e9c04f3b59f3093e0e10a674f086f8c5 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 16:27:56 -0400 Subject: [PATCH 16/30] refactor(git-gate): make GitGate a service class in a git_gate package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turn the git_gate module into a package with GitGate as a concrete service class (dropping the ABC), mirroring the Supervisor shape. The host-side git-gate operations are now methods the backend drives: git_gate/ __init__.py — thin __getattr__ facade (keeps `from bot_bottle.git_gate import …` working; render/hook names lazily forwarded to gateway.git_gate_render) plan.py — GitGatePlan (the launch DTO the backend contract references) service.py — GitGate: prepare / provision_dynamic_keys / revoke_provisioned_keys / preflight_host_keys provision.py — deploy-key lifecycle (was git_gate_provision.py) host_key.py — host-key preflight (was git_gate_host_key.py) The backend launch + base.py preflight now call GitGate() methods instead of the free functions. The runtime rendering / in-gateway hook execution stay in gateway/git_gate_render.py (data plane) — only the host-side service moved. Because the facade forwards names lazily, the ~25 `from bot_bottle.git_gate import GitGatePlan` call-sites are unchanged, and importing git_gate.plan (the contract's dependency) no longer drags in the provisioning / forge-API code. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/backend/base.py | 4 +- bot_bottle/backend/docker/launch.py | 9 +- bot_bottle/backend/firecracker/launch.py | 9 +- bot_bottle/backend/macos_container/launch.py | 9 +- bot_bottle/gateway/git_gate_render.py | 6 +- bot_bottle/git_gate.py | 169 ------------------ bot_bottle/git_gate/__init__.py | 98 ++++++++++ .../host_key.py} | 6 +- bot_bottle/git_gate/plan.py | 36 ++++ .../provision.py} | 16 +- bot_bottle/git_gate/service.py | 114 ++++++++++++ tests/unit/test_git_gate.py | 6 +- tests/unit/test_git_gate_host_key.py | 38 ++-- 13 files changed, 295 insertions(+), 225 deletions(-) delete mode 100644 bot_bottle/git_gate.py create mode 100644 bot_bottle/git_gate/__init__.py rename bot_bottle/{git_gate_host_key.py => git_gate/host_key.py} (98%) create mode 100644 bot_bottle/git_gate/plan.py rename bot_bottle/{git_gate_provision.py => git_gate/provision.py} (93%) create mode 100644 bot_bottle/git_gate/service.py diff --git a/bot_bottle/backend/base.py b/bot_bottle/backend/base.py index bb8a2abc..a58ad4f2 100644 --- a/bot_bottle/backend/base.py +++ b/bot_bottle/backend/base.py @@ -309,8 +309,8 @@ class BottleBackend(ABC, Generic[PlanT, CleanupT]): self._preflight() - from ..git_gate_host_key import preflight_host_keys - manifest = preflight_host_keys( + from ..git_gate import GitGate + manifest = GitGate().preflight_host_keys( manifest, headless=spec.headless, home_md=spec.manifest.home_md, diff --git a/bot_bottle/backend/docker/launch.py b/bot_bottle/backend/docker/launch.py index 068b7c41..665fa5e5 100644 --- a/bot_bottle/backend/docker/launch.py +++ b/bot_bottle/backend/docker/launch.py @@ -38,10 +38,7 @@ from typing import Callable, Generator from ...agent_provider import runtime_for from ...egress import egress_resolve_token_values -from ...git_gate import ( - provision_git_gate_dynamic_keys, - revoke_git_gate_provisioned_keys, -) +from ...git_gate import GitGate from ...image_cache import check_stale from ...log import die, info, warn from .. import BottleImages @@ -126,7 +123,7 @@ def launch( f"teardown failed for container {plan.container_name}" f" (compose-down): {exc!r}" ) - revoke_git_gate_provisioned_keys( + GitGate().revoke_provisioned_keys( _bottle_for_revoke, _git_gate_dir_for_revoke ) @@ -135,7 +132,7 @@ def launch( # provisioning the bottle's repos into the shared gateway. git_gate_plan = plan.git_gate_plan if git_gate_plan.upstreams: - git_gate_plan = provision_git_gate_dynamic_keys( + git_gate_plan = GitGate().provision_dynamic_keys( plan.manifest.bottle, git_gate_plan, git_gate_state_dir(plan.slug), ) diff --git a/bot_bottle/backend/firecracker/launch.py b/bot_bottle/backend/firecracker/launch.py index 37bfb2d6..b5fc00d6 100644 --- a/bot_bottle/backend/firecracker/launch.py +++ b/bot_bottle/backend/firecracker/launch.py @@ -42,10 +42,7 @@ from ...egress import ( egress_agent_env_entries, egress_resolve_token_values, ) -from ...git_gate import ( - provision_git_gate_dynamic_keys, - revoke_git_gate_provisioned_keys, -) +from ...git_gate import GitGate from ...image_cache import check_stale_path from ...log import die, info, warn from ...supervisor.types import SUPERVISE_PORT @@ -86,7 +83,7 @@ def launch( except BaseException as exc: # noqa: W0718 - teardown must continue teardown_exc = exc warn(f"firecracker teardown failed: {exc!r}") - revoke_git_gate_provisioned_keys(bottle_for_revoke, git_gate_dir_for_revoke) + GitGate().revoke_provisioned_keys(bottle_for_revoke, git_gate_dir_for_revoke) if teardown_exc is not None: raise teardown_exc @@ -97,7 +94,7 @@ def launch( # Step 2: mint the git-gate dynamic (gitea) deploy keys, if any. git_gate_plan = plan.git_gate_plan if git_gate_plan.upstreams: - git_gate_plan = provision_git_gate_dynamic_keys( + git_gate_plan = GitGate().provision_dynamic_keys( plan.manifest.bottle, git_gate_plan, git_gate_state_dir(plan.slug), ) diff --git a/bot_bottle/backend/macos_container/launch.py b/bot_bottle/backend/macos_container/launch.py index 95f5d611..bb2fc798 100644 --- a/bot_bottle/backend/macos_container/launch.py +++ b/bot_bottle/backend/macos_container/launch.py @@ -48,10 +48,7 @@ from ...egress import ( egress_agent_env_entries, egress_resolve_token_values, ) -from ...git_gate import ( - provision_git_gate_dynamic_keys, - revoke_git_gate_provisioned_keys, -) +from ...git_gate import GitGate from ...gateway.git_http_backend import DEFAULT_PORT as _GIT_HTTP_PORT from ...image_cache import check_stale from ...log import die, info, warn @@ -151,7 +148,7 @@ def launch( except BaseException as exc: # noqa: W0718 - teardown must continue teardown_exc = exc warn(f"macos-container teardown failed: {exc!r}") - revoke_git_gate_provisioned_keys(bottle_for_revoke, git_gate_dir_for_revoke) + GitGate().revoke_provisioned_keys(bottle_for_revoke, git_gate_dir_for_revoke) if teardown_exc is not None: raise teardown_exc @@ -282,7 +279,7 @@ def _provision_git_gate_keys( ) -> MacosContainerBottlePlan: if not plan.git_gate_plan.upstreams: return plan - git_gate_plan = provision_git_gate_dynamic_keys( + git_gate_plan = GitGate().provision_dynamic_keys( plan.manifest.bottle, plan.git_gate_plan, git_gate_state_dir(plan.slug), diff --git a/bot_bottle/gateway/git_gate_render.py b/bot_bottle/gateway/git_gate_render.py index d6fe825c..67f055ab 100644 --- a/bot_bottle/gateway/git_gate_render.py +++ b/bot_bottle/gateway/git_gate_render.py @@ -3,9 +3,9 @@ Builds the agent's `.gitconfig` insteadOf rewrites, the known_hosts line, and the entrypoint / pre-receive / access-hook scripts the gateway runs. No docker or forge calls — exposed for tests and reuse across -backends. Split out of `git_gate.py` so the control surface (`GitGate`) -and the deploy-key lifecycle (`git_gate_provision`) each read on their -own; `git_gate` re-exports these names for API stability.""" +backends. The host-side service (`git_gate.GitGate`) and the deploy-key +lifecycle (`git_gate.provision`) each read on their own; `git_gate` +re-exports these names for API stability.""" from __future__ import annotations diff --git a/bot_bottle/git_gate.py b/bot_bottle/git_gate.py deleted file mode 100644 index cfb8213a..00000000 --- a/bot_bottle/git_gate.py +++ /dev/null @@ -1,169 +0,0 @@ -"""Per-agent git-gate (PRD 0008). - -A third per-agent daemon that fronts the bottle's declared git -upstreams as a transparent mirror. Each `bottle.git` entry maps to -a bare repo on the gate; `git daemon` serves the bare repos over -`git:///.git`. Two hooks make the mirror bidirectional: - -- **`pre-receive`** (push path) — gitleaks-scans incoming refs and, - on clean, forwards them to the real upstream with the - gate-resident credential. -- **`--access-hook`** (fetch path) — runs `git fetch origin --prune` - against the real upstream before every `upload-pack`, so an - agent fetch returns whatever the upstream has *now*. Fail-closed - if the upstream is unreachable. - -The agent never sees the upstream credential under either path. - -Why a separate daemon (not folded into egress or ssh-gate): the -gate is the only one of the three that holds upstream push -credentials. Mixing it with egress would put push creds in the -same blast radius as internet-facing TLS interception; mixing it -with ssh-gate would force ssh-gate above L4 and into git-protocol -land. See `docs/prds/0008-git-gate.md`. - -This module defines the abstract gate (`GitGate`) and its plan -dataclass (`GitGatePlan`). The gateway's start/stop lifecycle is -backend-specific and lives on concrete subclasses (see -`bot_bottle/backend/docker/git_gate.py`).""" - - -from __future__ import annotations - -from abc import ABC -from dataclasses import dataclass -from pathlib import Path - -from .manifest import ManifestBottle - -# Rendering and the deploy-key lifecycle live in sibling modules; the -# names are re-exported here (see __all__) so existing -# `from bot_bottle.git_gate import …` callers are unchanged. -from .gateway.git_gate_render import ( - GIT_GATE_HOSTNAME, - GIT_GATE_TIMEOUT_SECS, - GitGateUpstream, - git_gate_known_hosts_line, - git_gate_render_access_hook, - git_gate_render_entrypoint, - git_gate_render_provision, - git_gate_render_gitconfig, - git_gate_render_hook, - git_gate_upstreams_for_bottle, - _gitconfig_validate_value, -) -from .git_gate_provision import ( - provision_git_gate_dynamic_keys, - revoke_git_gate_provisioned_keys, - _provision_dynamic_key, - _resolve_identity_file, -) - -@dataclass(frozen=True) -class GitGatePlan: - """Output of GitGate.prepare; consumed by .start. - - The script + slug + upstream fields are filled at prepare time - (host-side, side-effect-free on docker). The network fields are - populated by the backend's launch step via `dataclasses.replace` - once those networks exist. Empty defaults are sentinels meaning - "not yet set"; `.start` validates that they are populated. - - `hook_script` is the shared `pre-receive` for push-time gating; - `access_hook_script` is `git daemon`'s `--access-hook` for the - fetch-time upstream refresh.""" - - slug: str - entrypoint_script: Path - hook_script: Path - access_hook_script: Path - upstreams: tuple[GitGateUpstream, ...] - internal_network: str = "" - egress_network: str = "" - - - - -class GitGate(ABC): - """The per-agent git-gate. Encapsulates the host-side prepare - (upstream lift + entrypoint/hook render); the gateway's - start/stop lifecycle is backend-specific and lives on concrete - subclasses.""" - - def prepare(self, bottle: ManifestBottle, slug: str, stage_dir: Path) -> GitGatePlan: - """Compute the upstream table from `bottle.git` and write the - entrypoint, pre-receive hook, and access-hook scripts (mode - 600) under `stage_dir`. Pure host-side, no docker subprocess. - - For `gitea` key entries, the returned upstream intentionally - has an empty identity file. Backend launch fills that in after - the operator confirms the preflight. - - Returned plan is incomplete: the launch step must fill - `internal_network` / `egress_network` via `dataclasses.replace` - before passing the plan to `.start`.""" - upstreams = git_gate_upstreams_for_bottle(bottle) - entrypoint = stage_dir / "git_gate_entrypoint.sh" - entrypoint.write_text(git_gate_render_entrypoint(upstreams)) - entrypoint.chmod(0o600) - hook = stage_dir / "git_gate_pre_receive.sh" - hook.write_text(git_gate_render_hook()) - hook.chmod(0o600) - access_hook = stage_dir / "git_gate_access_hook.sh" - access_hook.write_text(git_gate_render_access_hook()) - # 0o700 (not 0o600): git daemon execs --access-hook directly, - # not via `sh`, so the script needs the x bit. The gateway copy - # does not necessarily preserve this mode (`docker cp` does, the - # Apple `container cp` does not), so provision_git_gate re-applies - # +x on the gateway side — see backend/docker/gateway_provision.py. - access_hook.chmod(0o700) - upstreams_with_files: list[GitGateUpstream] = [] - for u in upstreams: - known_hosts_file = Path() - if u.known_host_key: - known_hosts_file = stage_dir / f"{u.name}-known_hosts" - known_hosts_file.write_text( - git_gate_known_hosts_line( - u.upstream_host, u.upstream_port, u.known_host_key, - ) - ) - known_hosts_file.chmod(0o600) - upstreams_with_files.append( - GitGateUpstream( - name=u.name, - upstream_url=u.upstream_url, - upstream_host=u.upstream_host, - upstream_port=u.upstream_port, - identity_file=u.identity_file, - known_host_key=u.known_host_key, - known_hosts_file=known_hosts_file, - ) - ) - return GitGatePlan( - slug=slug, - entrypoint_script=entrypoint, - hook_script=hook, - access_hook_script=access_hook, - upstreams=tuple(upstreams_with_files), - ) - - -__all__ = [ - "GIT_GATE_HOSTNAME", - "GIT_GATE_TIMEOUT_SECS", - "GitGateUpstream", - "GitGatePlan", - "GitGate", - "git_gate_upstreams_for_bottle", - "git_gate_render_gitconfig", - "git_gate_known_hosts_line", - "git_gate_render_entrypoint", - "git_gate_render_provision", - "git_gate_render_hook", - "git_gate_render_access_hook", - "provision_git_gate_dynamic_keys", - "revoke_git_gate_provisioned_keys", - "_gitconfig_validate_value", - "_provision_dynamic_key", - "_resolve_identity_file", -] diff --git a/bot_bottle/git_gate/__init__.py b/bot_bottle/git_gate/__init__.py new file mode 100644 index 00000000..c902a1f0 --- /dev/null +++ b/bot_bottle/git_gate/__init__.py @@ -0,0 +1,98 @@ +"""Per-agent git-gate (PRD 0008). + +The git-gate fronts a bottle's declared git upstreams as a transparent mirror: +a `pre-receive` hook gitleaks-scans pushes and forwards clean refs to the real +upstream with a gate-resident credential; an `--access-hook` refreshes from the +upstream before fetches. The agent never sees the upstream credential. + +Layout: + + * `service` — the `GitGate` host-side service (prepare / provision / revoke / + preflight) the backend drives at launch. + * `plan` — `GitGatePlan`, the launch DTO (in the backend contract). + * `provision`, `host_key` — the deploy-key + host-key host-side helpers the + service delegates to. + +The rendering + the in-gateway hook execution live in +`bot_bottle.gateway.git_gate_render`. The public names are re-exported lazily +via `__getattr__`, so `from bot_bottle.git_gate import …` keeps working and +importing `git_gate.plan` (the contract's dependency) doesn't drag in the +provisioning / forge-API code. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .service import GitGate + from .plan import GitGatePlan + from .provision import ( + provision_git_gate_dynamic_keys, + revoke_git_gate_provisioned_keys, + ) + from ..gateway.git_gate_render import ( + GIT_GATE_HOSTNAME, + GIT_GATE_TIMEOUT_SECS, + GitGateUpstream, + git_gate_known_hosts_line, + git_gate_render_access_hook, + git_gate_render_entrypoint, + git_gate_render_gitconfig, + git_gate_render_hook, + git_gate_render_provision, + git_gate_upstreams_for_bottle, + ) + + +# Public name -> relative module that defines it. Render/hook names come from +# the gateway (where the in-gateway execution lives); the service, plan, and +# provisioning are this package's own submodules. +_LAZY: dict[str, str] = { + "GitGate": ".service", + "GitGatePlan": ".plan", + "provision_git_gate_dynamic_keys": ".provision", + "revoke_git_gate_provisioned_keys": ".provision", + "_provision_dynamic_key": ".provision", + "_resolve_identity_file": ".provision", + "GIT_GATE_HOSTNAME": "..gateway.git_gate_render", + "GIT_GATE_TIMEOUT_SECS": "..gateway.git_gate_render", + "GitGateUpstream": "..gateway.git_gate_render", + "git_gate_upstreams_for_bottle": "..gateway.git_gate_render", + "git_gate_render_gitconfig": "..gateway.git_gate_render", + "git_gate_known_hosts_line": "..gateway.git_gate_render", + "git_gate_render_entrypoint": "..gateway.git_gate_render", + "git_gate_render_provision": "..gateway.git_gate_render", + "git_gate_render_hook": "..gateway.git_gate_render", + "git_gate_render_access_hook": "..gateway.git_gate_render", + "_gitconfig_validate_value": "..gateway.git_gate_render", +} + + +def __getattr__(name: str) -> Any: + src = _LAZY.get(name) + if src is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module + + value = getattr(import_module(src, __name__), name) + globals()[name] = value + return value + + +__all__ = [ + "GitGate", + "GitGatePlan", + "GitGateUpstream", + "GIT_GATE_HOSTNAME", + "GIT_GATE_TIMEOUT_SECS", + "git_gate_upstreams_for_bottle", + "git_gate_render_gitconfig", + "git_gate_known_hosts_line", + "git_gate_render_entrypoint", + "git_gate_render_provision", + "git_gate_render_hook", + "git_gate_render_access_hook", + "provision_git_gate_dynamic_keys", + "revoke_git_gate_provisioned_keys", +] diff --git a/bot_bottle/git_gate_host_key.py b/bot_bottle/git_gate/host_key.py similarity index 98% rename from bot_bottle/git_gate_host_key.py rename to bot_bottle/git_gate/host_key.py index 2210e8e6..5c2ba788 100644 --- a/bot_bottle/git_gate_host_key.py +++ b/bot_bottle/git_gate/host_key.py @@ -16,9 +16,9 @@ import sys from pathlib import Path from typing import cast -from .log import die, info -from .manifest import Manifest -from .yaml_subset import YamlSubsetError, parse_frontmatter, serialize_yaml_subset +from ..log import die, info +from ..manifest import Manifest +from ..yaml_subset import YamlSubsetError, parse_frontmatter, serialize_yaml_subset # Preferred key types, most secure first. diff --git a/bot_bottle/git_gate/plan.py b/bot_bottle/git_gate/plan.py new file mode 100644 index 00000000..c5fe3bba --- /dev/null +++ b/bot_bottle/git_gate/plan.py @@ -0,0 +1,36 @@ +"""`GitGatePlan` — the git-gate launch plan (DTO). + +The output of `GitGate.prepare`, consumed by the backend launch step and the +`BottlePlan` contract. A pure value type (its `upstreams` element type +`GitGateUpstream` is the neutral render dataclass). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from ..gateway.git_gate_render import GitGateUpstream + + +@dataclass(frozen=True) +class GitGatePlan: + """Output of GitGate.prepare; consumed by .start. + + The script + slug + upstream fields are filled at prepare time (host-side, + side-effect-free on docker). The network fields are populated by the + backend's launch step via `dataclasses.replace` once those networks exist. + Empty defaults are sentinels meaning "not yet set"; `.start` validates that + they are populated. + + `hook_script` is the shared `pre-receive` for push-time gating; + `access_hook_script` is `git daemon`'s `--access-hook` for the fetch-time + upstream refresh.""" + + slug: str + entrypoint_script: Path + hook_script: Path + access_hook_script: Path + upstreams: tuple[GitGateUpstream, ...] + internal_network: str = "" + egress_network: str = "" diff --git a/bot_bottle/git_gate_provision.py b/bot_bottle/git_gate/provision.py similarity index 93% rename from bot_bottle/git_gate_provision.py rename to bot_bottle/git_gate/provision.py index ac9eebdd..007abd3f 100644 --- a/bot_bottle/git_gate_provision.py +++ b/bot_bottle/git_gate/provision.py @@ -13,14 +13,14 @@ import dataclasses from pathlib import Path from typing import TYPE_CHECKING -from .bottle_state import globalize_slug -from .errors import MissingEnvVarError -from .log import info -from .manifest import ManifestBottle, ManifestGitEntry -from .gateway.git_gate_render import GitGateUpstream +from ..bottle_state import globalize_slug +from ..errors import MissingEnvVarError +from ..log import info +from ..manifest import ManifestBottle, ManifestGitEntry +from ..gateway.git_gate_render import GitGateUpstream if TYPE_CHECKING: - from .git_gate import GitGatePlan + from .plan import GitGatePlan def _provision_dynamic_key( entry: ManifestGitEntry, @@ -32,7 +32,7 @@ def _provision_dynamic_key( Returns the host-side path to the private key file so the caller can inject it into the GitGateUpstream as `identity_file`.""" - from .deploy_key_provisioner import get_provisioner + from ..deploy_key_provisioner import get_provisioner pk = entry.Key token = os.environ.get(pk.forge_token_env) if token is None: @@ -70,7 +70,7 @@ def revoke_git_gate_provisioned_keys(bottle: ManifestBottle, stage_dir: Path) -> Called at teardown after containers stop. Raises if any revocation fails — a stranded key is a security concern that the operator must address manually.""" - from .deploy_key_provisioner import get_provisioner + from ..deploy_key_provisioner import get_provisioner for entry in bottle.git: if entry.Key.provider != "gitea": continue diff --git a/bot_bottle/git_gate/service.py b/bot_bottle/git_gate/service.py new file mode 100644 index 00000000..edd592b0 --- /dev/null +++ b/bot_bottle/git_gate/service.py @@ -0,0 +1,114 @@ +"""The `GitGate` host-side service (PRD 0008). + +The git-gate fronts a bottle's declared git upstreams as a transparent mirror: +a `pre-receive` hook gitleaks-scans pushes and forwards clean refs to the real +upstream with a gate-resident credential; an `--access-hook` refreshes from the +upstream before fetches. The agent never sees the upstream credential. + +`GitGate` is the host-side service the backend drives at launch: build the plan +(`prepare`), provision / revoke the per-upstream deploy keys, and preflight the +upstream host keys. The rendering it emits + the in-gateway hook execution live +in `bot_bottle.gateway.git_gate_render`. +""" + +from __future__ import annotations + +from pathlib import Path + +from ..manifest import Manifest, ManifestBottle +from ..gateway.git_gate_render import ( + GitGateUpstream, + git_gate_known_hosts_line, + git_gate_render_access_hook, + git_gate_render_entrypoint, + git_gate_render_hook, + git_gate_upstreams_for_bottle, +) +from .plan import GitGatePlan +from . import provision as _provision +from . import host_key as _host_key + + +class GitGate: + """The per-agent git-gate host-side service. Stateless — the backend's + launch step drives `prepare` → `provision_dynamic_keys` and, at teardown, + `revoke_provisioned_keys`; `preflight_host_keys` gates a launch on the + upstream host keys being known.""" + + def prepare(self, bottle: ManifestBottle, slug: str, stage_dir: Path) -> GitGatePlan: + """Compute the upstream table from `bottle.git` and write the + entrypoint, pre-receive hook, and access-hook scripts (mode 600) under + `stage_dir`. Pure host-side, no docker subprocess. + + For `gitea` key entries, the returned upstream intentionally has an + empty identity file. Backend launch fills that in after the operator + confirms the preflight. + + Returned plan is incomplete: the launch step must fill + `internal_network` / `egress_network` via `dataclasses.replace` before + passing the plan to `.start`.""" + upstreams = git_gate_upstreams_for_bottle(bottle) + entrypoint = stage_dir / "git_gate_entrypoint.sh" + entrypoint.write_text(git_gate_render_entrypoint(upstreams)) + entrypoint.chmod(0o600) + hook = stage_dir / "git_gate_pre_receive.sh" + hook.write_text(git_gate_render_hook()) + hook.chmod(0o600) + access_hook = stage_dir / "git_gate_access_hook.sh" + access_hook.write_text(git_gate_render_access_hook()) + # 0o700 (not 0o600): git daemon execs --access-hook directly, not via + # `sh`, so the script needs the x bit. The gateway copy does not + # necessarily preserve this mode (`docker cp` does, the Apple `container + # cp` does not), so provisioning re-applies +x on the gateway side — see + # backend/docker/gateway_provision.py. + access_hook.chmod(0o700) + upstreams_with_files: list[GitGateUpstream] = [] + for u in upstreams: + known_hosts_file = Path() + if u.known_host_key: + known_hosts_file = stage_dir / f"{u.name}-known_hosts" + known_hosts_file.write_text( + git_gate_known_hosts_line( + u.upstream_host, u.upstream_port, u.known_host_key, + ) + ) + known_hosts_file.chmod(0o600) + upstreams_with_files.append( + GitGateUpstream( + name=u.name, + upstream_url=u.upstream_url, + upstream_host=u.upstream_host, + upstream_port=u.upstream_port, + identity_file=u.identity_file, + known_host_key=u.known_host_key, + known_hosts_file=known_hosts_file, + ) + ) + return GitGatePlan( + slug=slug, + entrypoint_script=entrypoint, + hook_script=hook, + access_hook_script=access_hook, + upstreams=tuple(upstreams_with_files), + ) + + def provision_dynamic_keys( + self, bottle: ManifestBottle, plan: GitGatePlan, stage_dir: Path, + ) -> GitGatePlan: + """Mint the `gitea` upstreams' ephemeral deploy keys via the forge API + and return the plan with their identity files filled in.""" + return _provision.provision_git_gate_dynamic_keys(bottle, plan, stage_dir) + + def revoke_provisioned_keys(self, bottle: ManifestBottle, stage_dir: Path) -> None: + """Revoke every deploy key provisioned for `bottle` (teardown).""" + _provision.revoke_git_gate_provisioned_keys(bottle, stage_dir) + + def preflight_host_keys( + self, manifest: Manifest, *, headless: bool, home_md: Path | None, + ) -> Manifest: + """Ensure every git-gate upstream has a known host key, populating + missing ones (or erroring in headless mode). Returns the manifest, + possibly updated with fetched keys.""" + return _host_key.preflight_host_keys( + manifest, headless=headless, home_md=home_md, + ) diff --git a/tests/unit/test_git_gate.py b/tests/unit/test_git_gate.py index 4befb5de..bf7a891d 100644 --- a/tests/unit/test_git_gate.py +++ b/tests/unit/test_git_gate.py @@ -385,13 +385,13 @@ class TestDynamicKeyProvisioning(unittest.TestCase): def test_resolve_identity_file_gitea_provisions_key(self): entry = self._gitea_manifest().bottles["dev"].git[0] - with patch("bot_bottle.git_gate_provision._provision_dynamic_key", return_value="/tmp/provisioned-key") as mock_provision: + with patch("bot_bottle.git_gate.provision._provision_dynamic_key", return_value="/tmp/provisioned-key") as mock_provision: self.assertEqual("/tmp/provisioned-key", _resolve_identity_file(entry, "demo", self.stage)) mock_provision.assert_called_once() def test_prepare_defers_gitea_key_provisioning(self): bottle = self._gitea_manifest().bottles["dev"] - with patch("bot_bottle.git_gate_provision._provision_dynamic_key") as mock_provision: + with patch("bot_bottle.git_gate.provision._provision_dynamic_key") as mock_provision: plan = _StubGate().prepare(bottle, "demo", self.stage) mock_provision.assert_not_called() @@ -402,7 +402,7 @@ class TestDynamicKeyProvisioning(unittest.TestCase): plan = _StubGate().prepare(bottle, "demo", self.stage) with patch( - "bot_bottle.git_gate_provision._provision_dynamic_key", + "bot_bottle.git_gate.provision._provision_dynamic_key", return_value="/tmp/provisioned-key", ) as mock_provision: updated = provision_git_gate_dynamic_keys(bottle, plan, self.stage) diff --git a/tests/unit/test_git_gate_host_key.py b/tests/unit/test_git_gate_host_key.py index 3835d140..4a6541cd 100644 --- a/tests/unit/test_git_gate_host_key.py +++ b/tests/unit/test_git_gate_host_key.py @@ -7,7 +7,7 @@ import unittest from pathlib import Path from unittest.mock import MagicMock, patch -from bot_bottle.git_gate_host_key import ( +from bot_bottle.git_gate.host_key import ( add_host_key_to_frontmatter, find_and_update_bottle_file, find_repo_bottle_file, @@ -358,7 +358,7 @@ git-gate: with tempfile.TemporaryDirectory() as d: path = self._write_bottle(d, "dev", content) with patch( - "bot_bottle.git_gate_host_key.find_repo_bottle_file", + "bot_bottle.git_gate.host_key.find_repo_bottle_file", return_value=path, ), patch.object(Path, "read_text", side_effect=OSError("Permission denied")): result = find_and_update_bottle_file(Path(d), "myrepo", "ssh-ed25519 AAAA") @@ -382,7 +382,7 @@ git-gate: with tempfile.TemporaryDirectory() as d: path = self._write_bottle(d, "dev", content) with patch( - "bot_bottle.git_gate_host_key.add_host_key_to_frontmatter", + "bot_bottle.git_gate.host_key.add_host_key_to_frontmatter", return_value=content, # no change ): result = find_and_update_bottle_file(Path(d), "myrepo", "ssh-ed25519 AAAA") @@ -467,10 +467,10 @@ git-gate: mock_prompt = MagicMock(side_effect=["y", "n"]) # confirm key, skip save with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", return_value="ssh-ed25519 FETCHED", ), patch( - "bot_bottle.git_gate_host_key.prompt_tty", mock_prompt, + "bot_bottle.git_gate.host_key.prompt_tty", mock_prompt, ), patch("sys.stderr"): preflight_host_keys(manifest, headless=False, home_md=Path(home)) @@ -520,10 +520,10 @@ class TestPreflightHostKeys(unittest.TestCase): manifest = _manifest_with_git() with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", return_value="ssh-ed25519 FETCHED", ), patch( - "bot_bottle.git_gate_host_key.prompt_tty", + "bot_bottle.git_gate.host_key.prompt_tty", side_effect=["y", "n"], # confirm key, don't persist ), patch("sys.stderr"): result = preflight_host_keys(manifest, headless=False, home_md=None) @@ -534,10 +534,10 @@ class TestPreflightHostKeys(unittest.TestCase): manifest = _manifest_with_git() with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", return_value="ssh-ed25519 FETCHED", ), patch( - "bot_bottle.git_gate_host_key.prompt_tty", + "bot_bottle.git_gate.host_key.prompt_tty", return_value="n", ), patch("sys.stderr"), self.assertRaises(SystemExit): preflight_host_keys(manifest, headless=False, home_md=None) @@ -557,10 +557,10 @@ class TestPreflightHostKeys(unittest.TestCase): ) with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", return_value="ssh-ed25519 FETCHED", ), patch( - "bot_bottle.git_gate_host_key.prompt_tty", + "bot_bottle.git_gate.host_key.prompt_tty", side_effect=["y", "y"], # confirm key, yes persist ), patch("sys.stderr"): result = preflight_host_keys( @@ -586,10 +586,10 @@ class TestPreflightHostKeys(unittest.TestCase): ) with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", return_value="ssh-ed25519 FETCHED", ), patch( - "bot_bottle.git_gate_host_key.prompt_tty", + "bot_bottle.git_gate.host_key.prompt_tty", side_effect=["y", "n"], # confirm key, no persist ), patch("sys.stderr"): result = preflight_host_keys( @@ -604,7 +604,7 @@ class TestPreflightHostKeys(unittest.TestCase): manifest = _manifest_with_git() with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", side_effect=RuntimeError("connection refused"), ), patch("sys.stderr"), self.assertRaises(SystemExit): preflight_host_keys(manifest, headless=False, home_md=None) @@ -620,10 +620,10 @@ class TestPreflightHostKeys(unittest.TestCase): import io buf = io.StringIO() with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", return_value="ssh-ed25519 FETCHED", ), patch( - "bot_bottle.git_gate_host_key.prompt_tty", + "bot_bottle.git_gate.host_key.prompt_tty", side_effect=["y"], # confirm key only; no save prompt when no file found ), patch("sys.stderr", buf): result = preflight_host_keys( @@ -651,13 +651,13 @@ class TestPreflightHostKeys(unittest.TestCase): import io buf = io.StringIO() with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", return_value="ssh-ed25519 FETCHED", ), patch( - "bot_bottle.git_gate_host_key.prompt_tty", + "bot_bottle.git_gate.host_key.prompt_tty", side_effect=["y", "y"], # confirm key, yes persist ), patch( - "bot_bottle.git_gate_host_key.find_and_update_bottle_file", + "bot_bottle.git_gate.host_key.find_and_update_bottle_file", return_value=False, ), patch("sys.stderr", buf): result = preflight_host_keys( -- 2.52.0 From a446551acb0dbacd416b95c813ddf7b11931e603 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 16:43:59 -0400 Subject: [PATCH 17/30] refactor(egress): make Egress a service class in an egress package Split the flat egress.py into an egress/ package: neutral DTOs (EgressRoute, EgressPlan) in plan.py, and the concrete Egress service class plus rendering/env helpers in service.py, behind a thin __getattr__ facade so leaf imports stay cheap. Egress drops ABC and becomes a concrete service the backends call: resolve_token_values / agent_env_entries / prepare are now methods. Backend launch paths (docker, firecracker, macos_container) call Egress().(...) instead of the module-level functions. Co-Authored-By: Claude Opus 4.8 --- .../backend/docker/consolidated_compose.py | 4 +- bot_bottle/backend/docker/launch.py | 4 +- bot_bottle/backend/firecracker/launch.py | 9 +- bot_bottle/backend/macos_container/launch.py | 9 +- bot_bottle/egress/__init__.py | 90 +++++++++++++++++++ bot_bottle/egress/plan.py | 48 ++++++++++ bot_bottle/{egress.py => egress/service.py} | 90 ++++++------------- tests/unit/test_egress.py | 2 +- 8 files changed, 176 insertions(+), 80 deletions(-) create mode 100644 bot_bottle/egress/__init__.py create mode 100644 bot_bottle/egress/plan.py rename bot_bottle/{egress.py => egress/service.py} (86%) diff --git a/bot_bottle/backend/docker/consolidated_compose.py b/bot_bottle/backend/docker/consolidated_compose.py index 45296b5c..d3b7996e 100644 --- a/bot_bottle/backend/docker/consolidated_compose.py +++ b/bot_bottle/backend/docker/consolidated_compose.py @@ -16,7 +16,7 @@ from __future__ import annotations from typing import Any -from ...egress import egress_agent_env_entries +from ...egress import Egress from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH from .bottle_plan import DockerBottlePlan @@ -63,7 +63,7 @@ def consolidated_agent_compose( # env (set in launch.py) and is never written to the compose file on disk. if getattr(plan, "env_var_secret", ""): env.append(ENV_VAR_SECRET_NAME) - env.extend(egress_agent_env_entries(plan.egress_plan)) + env.extend(Egress().agent_env_entries(plan.egress_plan)) service: dict[str, Any] = { "image": plan.image, diff --git a/bot_bottle/backend/docker/launch.py b/bot_bottle/backend/docker/launch.py index 665fa5e5..4f68303c 100644 --- a/bot_bottle/backend/docker/launch.py +++ b/bot_bottle/backend/docker/launch.py @@ -37,7 +37,7 @@ from pathlib import Path from typing import Callable, Generator from ...agent_provider import runtime_for -from ...egress import egress_resolve_token_values +from ...egress import Egress from ...git_gate import GitGate from ...image_cache import check_stale from ...log import die, info, warn @@ -143,7 +143,7 @@ def launch( # handed to the orchestrator (in memory) for the gateway to inject — # the agent never sees them. effective_env = {**os.environ, **plan.agent_provision.provisioned_env} - token_values = egress_resolve_token_values( + token_values = Egress().resolve_token_values( plan.egress_plan.token_env_map, effective_env, ) teardown_timeout = resolve_teardown_timeout() diff --git a/bot_bottle/backend/firecracker/launch.py b/bot_bottle/backend/firecracker/launch.py index b5fc00d6..007724c4 100644 --- a/bot_bottle/backend/firecracker/launch.py +++ b/bot_bottle/backend/firecracker/launch.py @@ -38,10 +38,7 @@ from ...bottle_state import ( git_gate_state_dir, read_committed_image, ) -from ...egress import ( - egress_agent_env_entries, - egress_resolve_token_values, -) +from ...egress import Egress from ...git_gate import GitGate from ...image_cache import check_stale_path from ...log import die, info, warn @@ -110,7 +107,7 @@ def launch( # (in memory) for the gateway to inject — the agent never sees them. # Attribution is by the VM's guest IP (unspoofable via /31 TAP + nft). effective_env = {**os.environ, **plan.agent_provision.provisioned_env} - token_values = egress_resolve_token_values( + token_values = Egress().resolve_token_values( plan.egress_plan.token_env_map, effective_env, ) teardown_timeout = resolve_teardown_timeout() @@ -284,7 +281,7 @@ def _agent_guest_env(plan: FirecrackerBottlePlan, host_ip: str) -> dict[str, str env["MCP_SUPERVISE_URL"] = plan.agent_supervise_url if plan.env_var_secret: env[ENV_VAR_SECRET_NAME] = plan.env_var_secret - for entry in egress_agent_env_entries(plan.egress_plan): + for entry in Egress().agent_env_entries(plan.egress_plan): key, _, value = entry.partition("=") env[key] = value env.update(plan.agent_provision.guest_env) diff --git a/bot_bottle/backend/macos_container/launch.py b/bot_bottle/backend/macos_container/launch.py index bb2fc798..d6e331db 100644 --- a/bot_bottle/backend/macos_container/launch.py +++ b/bot_bottle/backend/macos_container/launch.py @@ -44,10 +44,7 @@ from ...bottle_state import ( git_gate_state_dir, read_committed_image, ) -from ...egress import ( - egress_agent_env_entries, - egress_resolve_token_values, -) +from ...egress import Egress from ...git_gate import GitGate from ...gateway.git_http_backend import DEFAULT_PORT as _GIT_HTTP_PORT from ...image_cache import check_stale @@ -191,7 +188,7 @@ def launch( f"{endpoint.network}" ) effective_env = {**os.environ, **plan.agent_provision.provisioned_env} - token_values = egress_resolve_token_values( + token_values = Egress().resolve_token_values( plan.egress_plan.token_env_map, effective_env, ) teardown_timeout = resolve_teardown_timeout() @@ -451,7 +448,7 @@ def _agent_env_entries( # so the secret value never lands on argv. for name in sorted(plan.forwarded_env.keys()): env.append(name) - env.extend(egress_agent_env_entries(plan.egress_plan)) + env.extend(Egress().agent_env_entries(plan.egress_plan)) return tuple(env) diff --git a/bot_bottle/egress/__init__.py b/bot_bottle/egress/__init__.py new file mode 100644 index 00000000..2e392e37 --- /dev/null +++ b/bot_bottle/egress/__init__.py @@ -0,0 +1,90 @@ +"""Per-agent egress (PRD 0017). + +The egress gateway is a TLS-terminating forward proxy that allow-lists a +bottle's outbound HTTP(S), scans payloads for secret exfil, and injects +per-route upstream credentials the agent never sees. + +Layout: + + * `service` — the `Egress` host-side service (`prepare` the launch plan, + `resolve_token_values`, `agent_env_entries`) + the route-building / + rendering functions. + * `plan` — `EgressPlan` / `EgressRoute`, the launch DTOs (in the backend + contract). + +The runtime enforcement (the mitmproxy addon) lives in +`bot_bottle.gateway.egress_addon*`. Public names are re-exported lazily via +`__getattr__`, so `from bot_bottle.egress import …` keeps working and importing +`egress.plan` (the contract's dependency) stays light. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .plan import EgressPlan, EgressRoute + from .service import ( + CLAUDE_HOST_CREDENTIAL_TOKEN_REF, + CODEX_HOST_CREDENTIAL_TOKEN_REF, + EGRESS_HOSTNAME, + EGRESS_ROUTES_FILENAME, + EGRESS_ROUTES_IN_CONTAINER, + Egress, + egress_agent_env_entries, + egress_gateway_env_entries, + egress_manifest_routes, + egress_render_routes, + egress_resolve_token_values, + egress_routes_for_bottle, + egress_token_env_map, + ) + + +_LAZY: dict[str, str] = { + "EgressPlan": ".plan", + "EgressRoute": ".plan", + "Egress": ".service", + "CLAUDE_HOST_CREDENTIAL_TOKEN_REF": ".service", + "CODEX_HOST_CREDENTIAL_TOKEN_REF": ".service", + "EGRESS_HOSTNAME": ".service", + "EGRESS_ROUTES_FILENAME": ".service", + "EGRESS_ROUTES_IN_CONTAINER": ".service", + "egress_agent_env_entries": ".service", + "egress_gateway_env_entries": ".service", + "egress_manifest_routes": ".service", + "egress_render_routes": ".service", + "egress_resolve_token_values": ".service", + "egress_routes_for_bottle": ".service", + "egress_token_env_map": ".service", +} + + +def __getattr__(name: str) -> Any: + src = _LAZY.get(name) + if src is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module + + value = getattr(import_module(src, __name__), name) + globals()[name] = value + return value + + +__all__ = [ + "CLAUDE_HOST_CREDENTIAL_TOKEN_REF", + "CODEX_HOST_CREDENTIAL_TOKEN_REF", + "EGRESS_HOSTNAME", + "EGRESS_ROUTES_FILENAME", + "EGRESS_ROUTES_IN_CONTAINER", + "Egress", + "EgressPlan", + "EgressRoute", + "egress_manifest_routes", + "egress_render_routes", + "egress_resolve_token_values", + "egress_routes_for_bottle", + "egress_agent_env_entries", + "egress_gateway_env_entries", + "egress_token_env_map", +] diff --git a/bot_bottle/egress/plan.py b/bot_bottle/egress/plan.py new file mode 100644 index 00000000..d62f0681 --- /dev/null +++ b/bot_bottle/egress/plan.py @@ -0,0 +1,48 @@ +"""Egress launch DTOs (PRD 0017). + +`EgressRoute` (the host-side extension of the addon's wire `Route`) and +`EgressPlan` (the launch plan the backend contract references). Pure value +types — the route-building / rendering logic and the `Egress` service live in +`egress.service`. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from ..gateway.egress_addon_core import Route + + +@dataclass(frozen=True) +class EgressRoute(Route): + """Host-side extension of the addon's `Route`. + + Inherits `host`, `matches`, `auth_scheme`, and `token_env` + from `egress_addon_core.Route` — those are the fields that cross the + YAML wire into the gateway. The fields below are host-only and + are never serialised to the addon. + + `token_ref` is the host env var the CLI reads at launch and forwards + into the container's environ under `token_env`. + + `roles` carries the manifest route's role tuple (reserved for + future use; always empty today).""" + + token_ref: str = "" + roles: tuple[str, ...] = () + + +@dataclass(frozen=True) +class EgressPlan: + slug: str + routes_path: Path + routes: tuple[EgressRoute, ...] + token_env_map: dict[str, str] + internal_network: str = "" + egress_network: str = "" + mitmproxy_ca_host_path: Path = Path() + mitmproxy_ca_cert_only_host_path: Path = Path() + log: int = 0 + canary: str = "" + canary_env: str = "" diff --git a/bot_bottle/egress.py b/bot_bottle/egress/service.py similarity index 86% rename from bot_bottle/egress.py rename to bot_bottle/egress/service.py index b933bad6..c720bf27 100644 --- a/bot_bottle/egress.py +++ b/bot_bottle/egress/service.py @@ -1,33 +1,33 @@ -"""Per-bottle egress proxy (PRD 0017, PRD 0053). +"""The `Egress` host-side service (PRD 0017). -This module defines the abstract proxy (`Egress`), its plan -dataclass (`EgressPlan`), and the resolved per-route shape -(`EgressRoute`). The gateway's start/stop lifecycle is backend- -specific and lives on concrete subclasses (see -`bot_bottle/backend/docker/egress.py`). +`Egress` builds a bottle's egress plan at launch: resolve the manifest's egress +routes, render the gateway's `routes.yaml`, assign per-route token slots, and +plant the exfil canary. The service also resolves the launch-time token values +and the agent/gateway env entries the backend injects. The runtime enforcement +(the mitmproxy addon) lives in `bot_bottle.gateway.egress_addon*`. """ from __future__ import annotations import dataclasses import secrets -from abc import ABC -from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING -from .gateway.egress_addon_core import ( +from ..gateway.egress_addon_core import ( ON_MATCH_REDACT, HeaderMatch as CoreHeaderMatch, MatchEntry as CoreMatchEntry, PathMatch as CorePathMatch, Route, ) -from .errors import MissingEnvVarError -from .log import die +from ..errors import MissingEnvVarError +from ..log import die +from .plan import EgressPlan, EgressRoute if TYPE_CHECKING: - from .manifest import ManifestBottle + from ..manifest import ManifestBottle + CODEX_HOST_CREDENTIAL_TOKEN_REF = "BOT_BOTTLE_CODEX_HOST_ACCESS_TOKEN" CLAUDE_HOST_CREDENTIAL_TOKEN_REF = "BOT_BOTTLE_CLAUDE_HOST_ACCESS_TOKEN" @@ -82,39 +82,6 @@ def egress_agent_env_entries(plan: "EgressPlan") -> tuple[str, ...]: return () -@dataclass(frozen=True) -class EgressRoute(Route): - """Host-side extension of the addon's `Route`. - - Inherits `host`, `matches`, `auth_scheme`, and `token_env` - from `egress_addon_core.Route` — those are the fields that cross the - YAML wire into the gateway. The fields below are host-only and - are never serialised to the addon. - - `token_ref` is the host env var the CLI reads at launch and forwards - into the container's environ under `token_env`. - - `roles` carries the manifest route's role tuple (reserved for - future use; always empty today).""" - - token_ref: str = "" - roles: tuple[str, ...] = () - - -@dataclass(frozen=True) -class EgressPlan: - slug: str - routes_path: Path - routes: tuple[EgressRoute, ...] - token_env_map: dict[str, str] - internal_network: str = "" - egress_network: str = "" - mitmproxy_ca_host_path: Path = Path() - mitmproxy_ca_cert_only_host_path: Path = Path() - log: int = 0 - canary: str = "" - canary_env: str = "" - def egress_manifest_routes( bottle: ManifestBottle, @@ -389,7 +356,11 @@ def egress_resolve_token_values( return out -class Egress(ABC): +class Egress: + """The host-side egress service. The backend drives `prepare` at launch, + then `resolve_token_values` to resolve the routes' upstream credentials and + `agent_env_entries` for the agent's egress env. Stateless.""" + def prepare( self, bottle: ManifestBottle, @@ -416,20 +387,13 @@ class Egress(ABC): canary_env=_random_canary_env(), ) -__all__ = [ - "CLAUDE_HOST_CREDENTIAL_TOKEN_REF", - "CODEX_HOST_CREDENTIAL_TOKEN_REF", - "EGRESS_HOSTNAME", - "EGRESS_ROUTES_FILENAME", - "EGRESS_ROUTES_IN_CONTAINER", - "Egress", - "EgressPlan", - "EgressRoute", - "egress_manifest_routes", - "egress_render_routes", - "egress_resolve_token_values", - "egress_routes_for_bottle", - "egress_agent_env_entries", - "egress_gateway_env_entries", - "egress_token_env_map", -] + def resolve_token_values( + self, token_env_map: dict[str, str], host_env: dict[str, str], + ) -> dict[str, str]: + """Resolve each route's upstream credential from the host env at launch. + Raises `MissingEnvVarError` for an unset/empty referenced host var.""" + return egress_resolve_token_values(token_env_map, host_env) + + def agent_env_entries(self, plan: EgressPlan) -> tuple[str, ...]: + """The agent-visible egress env entries (the exfil canary).""" + return egress_agent_env_entries(plan) diff --git a/tests/unit/test_egress.py b/tests/unit/test_egress.py index 027df828..9447549d 100644 --- a/tests/unit/test_egress.py +++ b/tests/unit/test_egress.py @@ -10,7 +10,6 @@ from bot_bottle.egress import ( Egress, EgressPlan, EgressRoute, - _yaml_str_escape, egress_agent_env_entries, egress_manifest_routes, egress_render_routes, @@ -19,6 +18,7 @@ from bot_bottle.egress import ( egress_gateway_env_entries, egress_token_env_map, ) +from bot_bottle.egress.service import _yaml_str_escape from bot_bottle.errors import MissingEnvVarError from bot_bottle.manifest import ManifestIndex from bot_bottle.yaml_subset import parse_yaml_subset -- 2.52.0 From ce744a85c45685159a3bd1bef389b9096188c46d Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 17:00:51 -0400 Subject: [PATCH 18/30] refactor(gateway): split data-plane files into egress/supervisor/git_gate services Group the gateway's data-plane modules into three service sub-packages mirroring the host-side trio (bot_bottle.egress / .supervisor / .git_gate): gateway/egress/ addon_core, addon, dlp_config, dlp_detectors gateway/supervisor/ server (was supervise_server) gateway/git_gate/ render, http_backend Prefix-stripped filenames now that the package namespaces them; each sub-package has a thin docstring __init__ (no eager imports, cheap leaf loads). The two cross-cutting files stay at the gateway root: policy_resolver (shared per-client lookup) and gateway_init, renamed to bootstrap now that gateway/ already namespaces it. Updated all importers (bot_bottle + tests), the in-VM/container `-m` launch strings, the Dockerfile.gateway addon shim + ENTRYPOINT, and the five gateway entries in scripts/critical-modules.txt. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- Dockerfile.gateway | 8 +++--- bot_bottle/backend/egress_apply.py | 2 +- bot_bottle/backend/firecracker/infra_vm.py | 2 +- bot_bottle/backend/macos_container/infra.py | 2 +- bot_bottle/backend/macos_container/launch.py | 2 +- bot_bottle/egress/__init__.py | 2 +- bot_bottle/egress/plan.py | 2 +- bot_bottle/egress/service.py | 4 +-- .../gateway/{gateway_init.py => bootstrap.py} | 4 +-- bot_bottle/gateway/egress/__init__.py | 9 +++++++ .../{egress_addon.py => egress/addon.py} | 4 +-- .../addon_core.py} | 4 +-- .../dlp_config.py} | 0 .../gateway/{ => egress}/dlp_detectors.py | 2 +- bot_bottle/gateway/git_gate/__init__.py | 7 +++++ .../http_backend.py} | 0 .../render.py} | 4 +-- bot_bottle/gateway/supervisor/__init__.py | 7 +++++ .../server.py} | 2 +- bot_bottle/git_gate/__init__.py | 26 +++++++++---------- bot_bottle/git_gate/plan.py | 2 +- bot_bottle/git_gate/provision.py | 2 +- bot_bottle/git_gate/service.py | 4 +-- scripts/critical-modules.txt | 10 +++---- tests/integration/test_gateway_image.py | 2 +- tests/unit/test_dlp_detectors.py | 8 +++--- tests/unit/test_egress.py | 22 ++++++++-------- tests/unit/test_egress_addon_core.py | 8 +++--- tests/unit/test_egress_addon_log_redaction.py | 2 +- tests/unit/test_egress_addon_request_flow.py | 8 +++--- tests/unit/test_egress_core_parsing.py | 2 +- tests/unit/test_egress_multitenant.py | 2 +- tests/unit/test_firecracker_infra_vm.py | 2 +- tests/unit/test_gateway_init.py | 8 +++--- tests/unit/test_git_gate_provision_render.py | 2 +- tests/unit/test_git_http_backend.py | 16 ++++++------ tests/unit/test_git_http_multitenant.py | 2 +- tests/unit/test_macos_infra.py | 2 +- tests/unit/test_orchestrator_registration.py | 2 +- tests/unit/test_supervise_server.py | 4 +-- 40 files changed, 113 insertions(+), 90 deletions(-) rename bot_bottle/gateway/{gateway_init.py => bootstrap.py} (99%) create mode 100644 bot_bottle/gateway/egress/__init__.py rename bot_bottle/gateway/{egress_addon.py => egress/addon.py} (99%) rename bot_bottle/gateway/{egress_addon_core.py => egress/addon_core.py} (99%) rename bot_bottle/gateway/{egress_dlp_config.py => egress/dlp_config.py} (100%) rename bot_bottle/gateway/{ => egress}/dlp_detectors.py (99%) create mode 100644 bot_bottle/gateway/git_gate/__init__.py rename bot_bottle/gateway/{git_http_backend.py => git_gate/http_backend.py} (100%) rename bot_bottle/gateway/{git_gate_render.py => git_gate/render.py} (99%) create mode 100644 bot_bottle/gateway/supervisor/__init__.py rename bot_bottle/gateway/{supervise_server.py => supervisor/server.py} (99%) diff --git a/Dockerfile.gateway b/Dockerfile.gateway index 91d0c52e..47ff5840 100644 --- a/Dockerfile.gateway +++ b/Dockerfile.gateway @@ -8,8 +8,8 @@ # this image's mitmproxy / git / gitleaks payload. # # Collapses the prior per-daemon images (egress, git-gate, -# supervise) into one. A small stdlib-Python init supervisor at -# /app/gateway_init.py spawns all daemons, forwards SIGTERM, and +# supervise) into one. A small stdlib-Python init supervisor +# (`bot_bottle.gateway.bootstrap`) spawns all daemons, forwards SIGTERM, and # propagates per-daemon stdout/stderr to the container log with a # `[name]` prefix. See PRD 0024 for the rationale. # @@ -102,7 +102,7 @@ RUN pip install --no-cache-dir /src/ # WORKDIR here also creates /app so the shim + COPYs below can write into it # (nothing created /app before this point). WORKDIR /app -RUN printf 'from bot_bottle.gateway.egress_addon import addons\n' > /app/egress_addon.py +RUN printf 'from bot_bottle.gateway.egress.addon import addons\n' > /app/egress_addon.py COPY bot_bottle/egress_entrypoint.sh /app/egress-entrypoint.sh RUN chmod +x /app/egress-entrypoint.sh @@ -123,4 +123,4 @@ EXPOSE 8888 9099 9418 9420 9100 # PID 1 is the supervisor. It owns signal handling and exit-code # propagation; no `exec` chain in the entrypoint itself. -ENTRYPOINT ["python3", "-m", "bot_bottle.gateway.gateway_init"] +ENTRYPOINT ["python3", "-m", "bot_bottle.gateway.bootstrap"] diff --git a/bot_bottle/backend/egress_apply.py b/bot_bottle/backend/egress_apply.py index a1981073..e174af3a 100644 --- a/bot_bottle/backend/egress_apply.py +++ b/bot_bottle/backend/egress_apply.py @@ -11,7 +11,7 @@ from pathlib import Path from ..bottle_state import egress_state_dir from ..egress import EGRESS_ROUTES_FILENAME -from ..gateway.egress_addon_core import LOG_OFF, load_config +from ..gateway.egress.addon_core import LOG_OFF, load_config class EgressApplyError(RuntimeError): diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py index a7ab12b7..1542a600 100644 --- a/bot_bottle/backend/firecracker/infra_vm.py +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -542,7 +542,7 @@ BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_CONTROL_PLANE_TOKEN="$CP_KEY" pyt BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise \\ BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{CONTROL_PLANE_PORT} \\ BOT_BOTTLE_CONTROL_AUTH_JWT="$GW_JWT" \\ - python3 -m bot_bottle.gateway.gateway_init & + python3 -m bot_bottle.gateway.bootstrap & # Reap as PID 1; children are backgrounded, so `wait` blocks. while : ; do wait ; done diff --git a/bot_bottle/backend/macos_container/infra.py b/bot_bottle/backend/macos_container/infra.py index 85bc3f06..5bb7f757 100644 --- a/bot_bottle/backend/macos_container/infra.py +++ b/bot_bottle/backend/macos_container/infra.py @@ -107,7 +107,7 @@ def _init_script(port: int) -> str: # control-plane RPC and never opens bot-bottle.db (PRD 0070 / #469). f"( cd /app && BOT_BOTTLE_GATEWAY_DAEMONS={_GATEWAY_DAEMONS} " f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{port} " - f"python3 -m bot_bottle.gateway.gateway_init ) &\n" + f"python3 -m bot_bottle.gateway.bootstrap ) &\n" "while : ; do wait ; done\n" ) diff --git a/bot_bottle/backend/macos_container/launch.py b/bot_bottle/backend/macos_container/launch.py index d6e331db..ba34ff97 100644 --- a/bot_bottle/backend/macos_container/launch.py +++ b/bot_bottle/backend/macos_container/launch.py @@ -46,7 +46,7 @@ from ...bottle_state import ( ) from ...egress import Egress from ...git_gate import GitGate -from ...gateway.git_http_backend import DEFAULT_PORT as _GIT_HTTP_PORT +from ...gateway.git_gate.http_backend import DEFAULT_PORT as _GIT_HTTP_PORT from ...image_cache import check_stale from ...log import die, info, warn from .. import BottleImages diff --git a/bot_bottle/egress/__init__.py b/bot_bottle/egress/__init__.py index 2e392e37..37756ca0 100644 --- a/bot_bottle/egress/__init__.py +++ b/bot_bottle/egress/__init__.py @@ -13,7 +13,7 @@ Layout: contract). The runtime enforcement (the mitmproxy addon) lives in -`bot_bottle.gateway.egress_addon*`. Public names are re-exported lazily via +`bot_bottle.gateway.egress.addon*`. Public names are re-exported lazily via `__getattr__`, so `from bot_bottle.egress import …` keeps working and importing `egress.plan` (the contract's dependency) stays light. """ diff --git a/bot_bottle/egress/plan.py b/bot_bottle/egress/plan.py index d62f0681..ecca0244 100644 --- a/bot_bottle/egress/plan.py +++ b/bot_bottle/egress/plan.py @@ -11,7 +11,7 @@ from __future__ import annotations from dataclasses import dataclass from pathlib import Path -from ..gateway.egress_addon_core import Route +from ..gateway.egress.addon_core import Route @dataclass(frozen=True) diff --git a/bot_bottle/egress/service.py b/bot_bottle/egress/service.py index c720bf27..76a88298 100644 --- a/bot_bottle/egress/service.py +++ b/bot_bottle/egress/service.py @@ -4,7 +4,7 @@ routes, render the gateway's `routes.yaml`, assign per-route token slots, and plant the exfil canary. The service also resolves the launch-time token values and the agent/gateway env entries the backend injects. The runtime enforcement -(the mitmproxy addon) lives in `bot_bottle.gateway.egress_addon*`. +(the mitmproxy addon) lives in `bot_bottle.gateway.egress.addon*`. """ from __future__ import annotations @@ -14,7 +14,7 @@ import secrets from pathlib import Path from typing import TYPE_CHECKING -from ..gateway.egress_addon_core import ( +from ..gateway.egress.addon_core import ( ON_MATCH_REDACT, HeaderMatch as CoreHeaderMatch, MatchEntry as CoreMatchEntry, diff --git a/bot_bottle/gateway/gateway_init.py b/bot_bottle/gateway/bootstrap.py similarity index 99% rename from bot_bottle/gateway/gateway_init.py rename to bot_bottle/gateway/bootstrap.py index 1e23be76..cd361799 100644 --- a/bot_bottle/gateway/gateway_init.py +++ b/bot_bottle/gateway/bootstrap.py @@ -100,8 +100,8 @@ _DAEMONS: tuple[_DaemonSpec, ...] = ( )), _DaemonSpec("egress", ("/bin/sh", "/app/egress-entrypoint.sh")), _DaemonSpec("git-gate", ("/bin/sh", "/git-gate-entrypoint.sh")), - _DaemonSpec("git-http", ("python3", "-m", "bot_bottle.gateway.git_http_backend")), - _DaemonSpec("supervise", ("python3", "-m", "bot_bottle.gateway.supervise_server")), + _DaemonSpec("git-http", ("python3", "-m", "bot_bottle.gateway.git_gate.http_backend")), + _DaemonSpec("supervise", ("python3", "-m", "bot_bottle.gateway.supervisor.server")), ) diff --git a/bot_bottle/gateway/egress/__init__.py b/bot_bottle/gateway/egress/__init__.py new file mode 100644 index 00000000..e406e602 --- /dev/null +++ b/bot_bottle/gateway/egress/__init__.py @@ -0,0 +1,9 @@ +"""Gateway-side (data-plane) egress service: the mitmproxy addon and its +pure decision core, DLP detectors, and route/DLP config parsing. + +These are the long-running data-plane pieces (loaded by mitmdump inside the +`bot-bottle-gateway` image), distinct from the host-side `bot_bottle.egress` +service that renders routes and prepares env. Import the concrete modules +directly (`from bot_bottle.gateway.egress.addon_core import ...`) — this +package deliberately does no eager work so leaf imports stay cheap. +""" diff --git a/bot_bottle/gateway/egress_addon.py b/bot_bottle/gateway/egress/addon.py similarity index 99% rename from bot_bottle/gateway/egress_addon.py rename to bot_bottle/gateway/egress/addon.py index eda9402e..a5292f2c 100644 --- a/bot_bottle/gateway/egress_addon.py +++ b/bot_bottle/gateway/egress/addon.py @@ -16,8 +16,8 @@ import typing from mitmproxy import http # type: ignore[import-not-found] # pylint: disable=import-error from bot_bottle.constants import IDENTITY_HEADER -from bot_bottle.gateway.dlp_detectors import redact_tokens, strip_crlf -from bot_bottle.gateway.egress_addon_core import ( +from bot_bottle.gateway.egress.dlp_detectors import redact_tokens, strip_crlf +from bot_bottle.gateway.egress.addon_core import ( LOG_BLOCKS, LOG_FULL, DEFAULT_OUTBOUND_ON_MATCH, diff --git a/bot_bottle/gateway/egress_addon_core.py b/bot_bottle/gateway/egress/addon_core.py similarity index 99% rename from bot_bottle/gateway/egress_addon_core.py rename to bot_bottle/gateway/egress/addon_core.py index e4800d52..3d0bfca1 100644 --- a/bot_bottle/gateway/egress_addon_core.py +++ b/bot_bottle/gateway/egress/addon_core.py @@ -16,11 +16,11 @@ import re import typing from dataclasses import dataclass -from ..yaml_subset import YamlSubsetError, parse_yaml_subset +from ...yaml_subset import YamlSubsetError, parse_yaml_subset # DLP detector-config parsing lives in a sibling module. Re-exported below # so existing `from egress_addon_core import ON_MATCH_*` callers keep working. -from .egress_dlp_config import ( +from .dlp_config import ( DEFAULT_OUTBOUND_ON_MATCH, INBOUND_DETECTOR_NAMES, ON_MATCH_BLOCK, diff --git a/bot_bottle/gateway/egress_dlp_config.py b/bot_bottle/gateway/egress/dlp_config.py similarity index 100% rename from bot_bottle/gateway/egress_dlp_config.py rename to bot_bottle/gateway/egress/dlp_config.py diff --git a/bot_bottle/gateway/dlp_detectors.py b/bot_bottle/gateway/egress/dlp_detectors.py similarity index 99% rename from bot_bottle/gateway/dlp_detectors.py rename to bot_bottle/gateway/egress/dlp_detectors.py index 2ca286a7..d0c8d1b9 100644 --- a/bot_bottle/gateway/dlp_detectors.py +++ b/bot_bottle/gateway/egress/dlp_detectors.py @@ -19,7 +19,7 @@ from math import log2 from collections import Counter from urllib.parse import quote as url_quote -from .egress_addon_core import ScanResult +from .addon_core import ScanResult # --------------------------------------------------------------------------- diff --git a/bot_bottle/gateway/git_gate/__init__.py b/bot_bottle/gateway/git_gate/__init__.py new file mode 100644 index 00000000..67c9fe7d --- /dev/null +++ b/bot_bottle/gateway/git_gate/__init__.py @@ -0,0 +1,7 @@ +"""Gateway-side (data-plane) git-gate service: pure host-side hook rendering +(PRD 0008) and the smart-HTTP backend that fronts git-gate repos. + +The host-side git-gate service (provisioning, preflight) lives in +`bot_bottle.git_gate`; this is the data-plane counterpart. Import the concrete +modules directly (`from bot_bottle.gateway.git_gate.render import ...`). +""" diff --git a/bot_bottle/gateway/git_http_backend.py b/bot_bottle/gateway/git_gate/http_backend.py similarity index 100% rename from bot_bottle/gateway/git_http_backend.py rename to bot_bottle/gateway/git_gate/http_backend.py diff --git a/bot_bottle/gateway/git_gate_render.py b/bot_bottle/gateway/git_gate/render.py similarity index 99% rename from bot_bottle/gateway/git_gate_render.py rename to bot_bottle/gateway/git_gate/render.py index 67f055ab..241ed9d8 100644 --- a/bot_bottle/gateway/git_gate_render.py +++ b/bot_bottle/gateway/git_gate/render.py @@ -14,8 +14,8 @@ import shlex from dataclasses import dataclass from pathlib import Path -from ..constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER -from ..manifest import ManifestBottle, ManifestGitEntry +from ...constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER +from ...manifest import ManifestBottle, ManifestGitEntry # Short network alias for git-gate inside the gateway. The # agent's `.gitconfig` insteadOf rewrites resolve through this name. diff --git a/bot_bottle/gateway/supervisor/__init__.py b/bot_bottle/gateway/supervisor/__init__.py new file mode 100644 index 00000000..47b48b79 --- /dev/null +++ b/bot_bottle/gateway/supervisor/__init__.py @@ -0,0 +1,7 @@ +"""Gateway-side (data-plane) supervise service: the supervise daemon's HTTP +server (PRD 0013). + +The host-side supervise control lives in `bot_bottle.orchestrator.supervisor`; +this is the in-gateway daemon the agents reach. Import the concrete module +directly (`from bot_bottle.gateway.supervisor.server import ...`). +""" diff --git a/bot_bottle/gateway/supervise_server.py b/bot_bottle/gateway/supervisor/server.py similarity index 99% rename from bot_bottle/gateway/supervise_server.py rename to bot_bottle/gateway/supervisor/server.py index 520f1dae..35d53c53 100644 --- a/bot_bottle/gateway/supervise_server.py +++ b/bot_bottle/gateway/supervisor/server.py @@ -58,7 +58,7 @@ import typing from dataclasses import dataclass from bot_bottle.constants import IDENTITY_HEADER -from bot_bottle.gateway.egress_addon_core import ( +from bot_bottle.gateway.egress.addon_core import ( LOG_OFF, load_config, resolve_client_context, route_to_yaml_dict, ) from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver diff --git a/bot_bottle/git_gate/__init__.py b/bot_bottle/git_gate/__init__.py index c902a1f0..f6b6256e 100644 --- a/bot_bottle/git_gate/__init__.py +++ b/bot_bottle/git_gate/__init__.py @@ -14,7 +14,7 @@ Layout: service delegates to. The rendering + the in-gateway hook execution live in -`bot_bottle.gateway.git_gate_render`. The public names are re-exported lazily +`bot_bottle.gateway.git_gate.render`. The public names are re-exported lazily via `__getattr__`, so `from bot_bottle.git_gate import …` keeps working and importing `git_gate.plan` (the contract's dependency) doesn't drag in the provisioning / forge-API code. @@ -31,7 +31,7 @@ if TYPE_CHECKING: provision_git_gate_dynamic_keys, revoke_git_gate_provisioned_keys, ) - from ..gateway.git_gate_render import ( + from ..gateway.git_gate.render import ( GIT_GATE_HOSTNAME, GIT_GATE_TIMEOUT_SECS, GitGateUpstream, @@ -55,17 +55,17 @@ _LAZY: dict[str, str] = { "revoke_git_gate_provisioned_keys": ".provision", "_provision_dynamic_key": ".provision", "_resolve_identity_file": ".provision", - "GIT_GATE_HOSTNAME": "..gateway.git_gate_render", - "GIT_GATE_TIMEOUT_SECS": "..gateway.git_gate_render", - "GitGateUpstream": "..gateway.git_gate_render", - "git_gate_upstreams_for_bottle": "..gateway.git_gate_render", - "git_gate_render_gitconfig": "..gateway.git_gate_render", - "git_gate_known_hosts_line": "..gateway.git_gate_render", - "git_gate_render_entrypoint": "..gateway.git_gate_render", - "git_gate_render_provision": "..gateway.git_gate_render", - "git_gate_render_hook": "..gateway.git_gate_render", - "git_gate_render_access_hook": "..gateway.git_gate_render", - "_gitconfig_validate_value": "..gateway.git_gate_render", + "GIT_GATE_HOSTNAME": "..gateway.git_gate.render", + "GIT_GATE_TIMEOUT_SECS": "..gateway.git_gate.render", + "GitGateUpstream": "..gateway.git_gate.render", + "git_gate_upstreams_for_bottle": "..gateway.git_gate.render", + "git_gate_render_gitconfig": "..gateway.git_gate.render", + "git_gate_known_hosts_line": "..gateway.git_gate.render", + "git_gate_render_entrypoint": "..gateway.git_gate.render", + "git_gate_render_provision": "..gateway.git_gate.render", + "git_gate_render_hook": "..gateway.git_gate.render", + "git_gate_render_access_hook": "..gateway.git_gate.render", + "_gitconfig_validate_value": "..gateway.git_gate.render", } diff --git a/bot_bottle/git_gate/plan.py b/bot_bottle/git_gate/plan.py index c5fe3bba..edd1bd21 100644 --- a/bot_bottle/git_gate/plan.py +++ b/bot_bottle/git_gate/plan.py @@ -10,7 +10,7 @@ from __future__ import annotations from dataclasses import dataclass from pathlib import Path -from ..gateway.git_gate_render import GitGateUpstream +from ..gateway.git_gate.render import GitGateUpstream @dataclass(frozen=True) diff --git a/bot_bottle/git_gate/provision.py b/bot_bottle/git_gate/provision.py index 007abd3f..bae03a72 100644 --- a/bot_bottle/git_gate/provision.py +++ b/bot_bottle/git_gate/provision.py @@ -17,7 +17,7 @@ from ..bottle_state import globalize_slug from ..errors import MissingEnvVarError from ..log import info from ..manifest import ManifestBottle, ManifestGitEntry -from ..gateway.git_gate_render import GitGateUpstream +from ..gateway.git_gate.render import GitGateUpstream if TYPE_CHECKING: from .plan import GitGatePlan diff --git a/bot_bottle/git_gate/service.py b/bot_bottle/git_gate/service.py index edd592b0..cfcc8aab 100644 --- a/bot_bottle/git_gate/service.py +++ b/bot_bottle/git_gate/service.py @@ -8,7 +8,7 @@ upstream before fetches. The agent never sees the upstream credential. `GitGate` is the host-side service the backend drives at launch: build the plan (`prepare`), provision / revoke the per-upstream deploy keys, and preflight the upstream host keys. The rendering it emits + the in-gateway hook execution live -in `bot_bottle.gateway.git_gate_render`. +in `bot_bottle.gateway.git_gate.render`. """ from __future__ import annotations @@ -16,7 +16,7 @@ from __future__ import annotations from pathlib import Path from ..manifest import Manifest, ManifestBottle -from ..gateway.git_gate_render import ( +from ..gateway.git_gate.render import ( GitGateUpstream, git_gate_known_hosts_line, git_gate_render_access_hook, diff --git a/scripts/critical-modules.txt b/scripts/critical-modules.txt index fa542f22..845e4154 100644 --- a/scripts/critical-modules.txt +++ b/scripts/critical-modules.txt @@ -8,18 +8,18 @@ # # One module path per line, relative to the repo root. Blank lines and # `#` comments are ignored. -bot_bottle/egress_addon.py -bot_bottle/egress_addon_core.py -bot_bottle/dlp_detectors.py +bot_bottle/gateway/egress/addon.py +bot_bottle/gateway/egress/addon_core.py +bot_bottle/gateway/egress/dlp_detectors.py bot_bottle/egress.py bot_bottle/manifest.py bot_bottle/manifest_egress.py bot_bottle/manifest_agent.py bot_bottle/manifest_schema.py bot_bottle/git_gate.py -bot_bottle/git_gate_render.py +bot_bottle/gateway/git_gate/render.py bot_bottle/git_gate_provision.py -bot_bottle/git_http_backend.py +bot_bottle/gateway/git_gate/http_backend.py bot_bottle/supervise.py bot_bottle/yaml_subset.py bot_bottle/bottle_state.py diff --git a/tests/integration/test_gateway_image.py b/tests/integration/test_gateway_image.py index 213c93b6..7c02a28b 100644 --- a/tests/integration/test_gateway_image.py +++ b/tests/integration/test_gateway_image.py @@ -91,7 +91,7 @@ class TestGatewayImage(unittest.TestCase): # Probe that the package imports resolve inside the image. rc, out = self._run_in_image( "python3", "-c", - "from bot_bottle.supervisor import types; from bot_bottle.gateway import supervise_server; print('ok')", + "from bot_bottle.supervisor import types; from bot_bottle.gateway.supervisor import server as supervise_server; print('ok')", ) self.assertEqual(0, rc, msg=out) self.assertIn("ok", out) diff --git a/tests/unit/test_dlp_detectors.py b/tests/unit/test_dlp_detectors.py index d62fb4dc..5310252a 100644 --- a/tests/unit/test_dlp_detectors.py +++ b/tests/unit/test_dlp_detectors.py @@ -7,7 +7,7 @@ import base64 import gzip import unittest -from bot_bottle.gateway.dlp_detectors import ( +from bot_bottle.gateway.egress.dlp_detectors import ( ENTROPY_BLOCK_THRESHOLD, PARTIAL_MATCH_MIN_LEN, REDACT, @@ -465,17 +465,17 @@ class TestMatchedAndSafeTokens(unittest.TestCase): class TestStripCrlf(unittest.TestCase): def test_removes_url_encoded_crlf(self): - from bot_bottle.gateway.dlp_detectors import strip_crlf + from bot_bottle.gateway.egress.dlp_detectors import strip_crlf out = strip_crlf("next=%0d%0aX-Injected: evil") self.assertNotRegex(out, r"%0[dD]%0[aA]") def test_removes_literal_header_injection(self): - from bot_bottle.gateway.dlp_detectors import strip_crlf + from bot_bottle.gateway.egress.dlp_detectors import strip_crlf out = strip_crlf("value\r\nX-Injected: evil") self.assertIsNone(scan_crlf_injection(out)) def test_leaves_clean_text_unchanged(self): - from bot_bottle.gateway.dlp_detectors import strip_crlf + from bot_bottle.gateway.egress.dlp_detectors import strip_crlf self.assertEqual("/api/v1/data?q=hello", strip_crlf("/api/v1/data?q=hello")) class TestAlnumProjection(unittest.TestCase): diff --git a/tests/unit/test_egress.py b/tests/unit/test_egress.py index 9447549d..2ec776a4 100644 --- a/tests/unit/test_egress.py +++ b/tests/unit/test_egress.py @@ -344,7 +344,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertEqual([], parse_yaml_subset(rendered)["routes"]) def test_round_trip_through_addon_core(self): - from bot_bottle.gateway.egress_addon_core import load_config + from bot_bottle.gateway.egress.addon_core import load_config b = _bottle([ {"host": "api.github.com", "auth": {"scheme": "Bearer", "token_ref": "GH_PAT"}, @@ -363,7 +363,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertEqual("", addon_routes[2].auth_scheme) def test_dlp_round_trips(self): - from bot_bottle.gateway.egress_addon_core import load_config + from bot_bottle.gateway.egress.addon_core import load_config b = _bottle([{"host": "x.example", "dlp": { "outbound_detectors": ["token_patterns"], "inbound_detectors": False, @@ -375,7 +375,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertEqual((), addon_routes[0].inbound_detectors) def test_outbound_on_match_round_trips(self): - from bot_bottle.gateway.egress_addon_core import load_config + from bot_bottle.gateway.egress.addon_core import load_config b = _bottle([{"host": "logs.example", "dlp": { "outbound_on_match": "redact", }}]) @@ -392,7 +392,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertNotIn("outbound_on_match", rendered) def test_git_fetch_policy_round_trips(self): - from bot_bottle.gateway.egress_addon_core import load_config + from bot_bottle.gateway.egress.addon_core import load_config b = _bottle([{"host": "github.com", "git": {"fetch": True}}]) routes = egress_routes_for_bottle(b) rendered = egress_render_routes(routes) @@ -405,7 +405,7 @@ class TestRenderRoutes(unittest.TestCase): it, but the renderer in between dropped it — so the flag never reached the proxy and registry pulls kept failing with "unauthorized" while the config looked correct everywhere it was inspected.""" - from bot_bottle.gateway.egress_addon_core import load_config + from bot_bottle.gateway.egress.addon_core import load_config b = _bottle([{"host": "registry-1.docker.io", "preserve_auth": True}]) routes = egress_routes_for_bottle(b) rendered = egress_render_routes(routes) @@ -416,7 +416,7 @@ class TestRenderRoutes(unittest.TestCase): b = _bottle([{"host": "x.example"}]) rendered = egress_render_routes(egress_routes_for_bottle(b)) self.assertNotIn("preserve_auth", rendered) - from bot_bottle.gateway.egress_addon_core import load_config + from bot_bottle.gateway.egress.addon_core import load_config self.assertFalse(load_config(rendered).routes[0].preserve_auth) def test_log_zero_omitted_from_render(self): @@ -434,7 +434,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertTrue(rendered.startswith(f"log: {level}\n")) def test_log_level_round_trips_to_addon_core(self): - from bot_bottle.gateway.egress_addon_core import load_config, LOG_FULL + from bot_bottle.gateway.egress.addon_core import load_config, LOG_FULL b = _bottle([{"host": "x.example"}]) routes = egress_routes_for_bottle(b) rendered = egress_render_routes(routes, log=LOG_FULL) @@ -444,7 +444,7 @@ class TestRenderRoutes(unittest.TestCase): def test_log_via_manifest_flows_to_render(self): from bot_bottle.manifest import ManifestIndex - from bot_bottle.gateway.egress_addon_core import load_config, LOG_BLOCKS + from bot_bottle.gateway.egress.addon_core import load_config, LOG_BLOCKS m = ManifestIndex.from_json_obj({ "bottles": {"dev": {"egress": { "log": 1, @@ -512,7 +512,7 @@ class TestRenderRoutesEscaping(unittest.TestCase): self.assertEqual('Bear"er', parsed[0]["inspect"]["auth_scheme"]) def test_path_value_with_double_quote_round_trips(self): - from bot_bottle.gateway.egress_addon_core import PathMatch, MatchEntry + from bot_bottle.gateway.egress.addon_core import PathMatch, MatchEntry routes = (EgressRoute( host="api.example", matches=(MatchEntry(paths=(PathMatch(type="prefix", value='/v1/"quoted"/'),)),), @@ -521,7 +521,7 @@ class TestRenderRoutesEscaping(unittest.TestCase): self.assertEqual('/v1/"quoted"/', parsed[0]["inspect"]["matches"][0]["paths"][0]["value"]) def test_header_value_with_double_quote_round_trips(self): - from bot_bottle.gateway.egress_addon_core import HeaderMatch, MatchEntry + from bot_bottle.gateway.egress.addon_core import HeaderMatch, MatchEntry routes = (EgressRoute( host="api.example", matches=(MatchEntry(headers=(HeaderMatch(name="x-h", value='val"ue'),)),), @@ -598,7 +598,7 @@ class TestCanaryGeneration(unittest.TestCase): self.assertNotEqual(plan_a.canary, plan_b.canary) def test_canary_detected_by_scan_known_secrets(self): - from bot_bottle.gateway.dlp_detectors import scan_known_secrets + from bot_bottle.gateway.egress.dlp_detectors import scan_known_secrets plan = self._make_plan() env = {plan.canary_env: plan.canary} diff --git a/tests/unit/test_egress_addon_core.py b/tests/unit/test_egress_addon_core.py index 593d6ca9..a99e7831 100644 --- a/tests/unit/test_egress_addon_core.py +++ b/tests/unit/test_egress_addon_core.py @@ -12,7 +12,7 @@ import unittest from pathlib import Path from urllib.parse import urlsplit -from bot_bottle.gateway.egress_addon_core import ( +from bot_bottle.gateway.egress.addon_core import ( LOG_BLOCKS, LOG_FULL, LOG_OFF, @@ -1377,15 +1377,15 @@ class TestScanOutboundEnhanced(unittest.TestCase): class TestOutboundDetectorNames(unittest.TestCase): def test_entropy_in_outbound_detector_names(self): - from bot_bottle.gateway.egress_addon_core import OUTBOUND_DETECTOR_NAMES + from bot_bottle.gateway.egress.addon_core import OUTBOUND_DETECTOR_NAMES self.assertIn("entropy", OUTBOUND_DETECTOR_NAMES) def test_known_secrets_in_outbound_detector_names(self): - from bot_bottle.gateway.egress_addon_core import OUTBOUND_DETECTOR_NAMES + from bot_bottle.gateway.egress.addon_core import OUTBOUND_DETECTOR_NAMES self.assertIn("known_secrets", OUTBOUND_DETECTOR_NAMES) def test_token_patterns_in_outbound_detector_names(self): - from bot_bottle.gateway.egress_addon_core import OUTBOUND_DETECTOR_NAMES + from bot_bottle.gateway.egress.addon_core import OUTBOUND_DETECTOR_NAMES self.assertIn("token_patterns", OUTBOUND_DETECTOR_NAMES) diff --git a/tests/unit/test_egress_addon_log_redaction.py b/tests/unit/test_egress_addon_log_redaction.py index 44af8152..c6196d4c 100644 --- a/tests/unit/test_egress_addon_log_redaction.py +++ b/tests/unit/test_egress_addon_log_redaction.py @@ -36,7 +36,7 @@ def _ensure_shims() -> None: _ensure_shims() -from bot_bottle.gateway.egress_addon import EgressAddon # noqa: E402 (import after shims) +from bot_bottle.gateway.egress.addon import EgressAddon # noqa: E402 (import after shims) # --------------------------------------------------------------------------- diff --git a/tests/unit/test_egress_addon_request_flow.py b/tests/unit/test_egress_addon_request_flow.py index a8a06d2a..415bb8d1 100644 --- a/tests/unit/test_egress_addon_request_flow.py +++ b/tests/unit/test_egress_addon_request_flow.py @@ -194,15 +194,15 @@ def _ensure_shims() -> None: _ensure_shims() -import bot_bottle.gateway.egress_addon as _ea_mod # noqa: E402 (after shims) -from bot_bottle.gateway.egress_addon import EgressAddon # noqa: E402 (after shims) -from bot_bottle.gateway.egress_addon import ( # noqa: E402 +import bot_bottle.gateway.egress.addon as _ea_mod # noqa: E402 (after shims) +from bot_bottle.gateway.egress.addon import EgressAddon # noqa: E402 (after shims) +from bot_bottle.gateway.egress.addon import ( # noqa: E402 DEFAULT_INBOUND_SCAN_LIMIT_BYTES, DEFAULT_TOKEN_ALLOW_TIMEOUT_SECONDS, _inbound_scan_limit_from_env, _token_allow_timeout_from_env, ) -from bot_bottle.gateway.egress_addon_core import ( # noqa: E402 +from bot_bottle.gateway.egress.addon_core import ( # noqa: E402 Config, LOG_BLOCKS, LOG_FULL, diff --git a/tests/unit/test_egress_core_parsing.py b/tests/unit/test_egress_core_parsing.py index 6a56e901..161efb4b 100644 --- a/tests/unit/test_egress_core_parsing.py +++ b/tests/unit/test_egress_core_parsing.py @@ -8,7 +8,7 @@ from __future__ import annotations import unittest -from bot_bottle.gateway.egress_addon_core import ( +from bot_bottle.gateway.egress.addon_core import ( HeaderMatch, MatchEntry, PathMatch, diff --git a/tests/unit/test_egress_multitenant.py b/tests/unit/test_egress_multitenant.py index adabb3e7..2aa5c869 100644 --- a/tests/unit/test_egress_multitenant.py +++ b/tests/unit/test_egress_multitenant.py @@ -4,7 +4,7 @@ from __future__ import annotations import unittest -from bot_bottle.gateway.egress_addon_core import ( +from bot_bottle.gateway.egress.addon_core import ( DENY_RESOLVER_ERROR, DENY_UNATTRIBUTED, DENY_UNPARSEABLE, diff --git a/tests/unit/test_firecracker_infra_vm.py b/tests/unit/test_firecracker_infra_vm.py index e0ae9bb8..7dcf4ea0 100644 --- a/tests/unit/test_firecracker_infra_vm.py +++ b/tests/unit/test_firecracker_infra_vm.py @@ -72,7 +72,7 @@ class TestBuildInfraRootfs(unittest.TestCase): self.assertIn("bot_bottle.orchestrator", init) # Gateway launches via the installed package (there is no # /app/gateway_init.py file since the daemons moved into bot_bottle). - self.assertIn("bot_bottle.gateway.gateway_init", init) + self.assertIn("bot_bottle.gateway.bootstrap", init) self.assertIn("export PATH=", init) # Persistent registry volume mounted at the DB dir before the CP starts. self.assertIn("/dev/vdb", init) diff --git a/tests/unit/test_gateway_init.py b/tests/unit/test_gateway_init.py index bb987bac..d43cfe6c 100644 --- a/tests/unit/test_gateway_init.py +++ b/tests/unit/test_gateway_init.py @@ -1,6 +1,6 @@ """Unit: gateway data-plane init supervisor (PRD 0070; PRD 0024 bundle shape). -Tests both the helper functions in `bot_bottle.gateway.gateway_init` +Tests both the helper functions in `bot_bottle.gateway.bootstrap` and the supervisor's end-to-end signal / exit-code behavior. The end-to-end tests use real subprocesses (`sleep`, `/bin/sh -c '...'`) — short-lived, no docker required — so they run under `tests/unit/` @@ -18,7 +18,7 @@ import warnings from pathlib import Path from unittest.mock import patch -from bot_bottle.gateway.gateway_init import ( +from bot_bottle.gateway.bootstrap import ( _DaemonSpec, _Supervisor, _argv_for_daemon, @@ -489,7 +489,7 @@ class TestSupervisor(unittest.TestCase): time.sleep(0.3) # let `trap` register sup.request_shutdown(reason="test") - with patch("bot_bottle.gateway.gateway_init._GRACE_SECONDS", 0.3): + with patch("bot_bottle.gateway.bootstrap._GRACE_SECONDS", 0.3): rc = self._drive(sup, max_wait_s=4.0) # Process was SIGKILL'd → returncode -9 on POSIX. @@ -531,7 +531,7 @@ class TestMainEndToEnd(unittest.TestCase): helper = ( "import os, runpy, sys\n" - "from bot_bottle.gateway import gateway_init as si\n" + "from bot_bottle.gateway import bootstrap as si\n" "si._DAEMONS = (\n" f" si._DaemonSpec('alpha', ({SLEEP!r},'30')),\n" f" si._DaemonSpec('beta', ({SLEEP!r},'30')),\n" diff --git a/tests/unit/test_git_gate_provision_render.py b/tests/unit/test_git_gate_provision_render.py index b9bce15c..d2ba9a7f 100644 --- a/tests/unit/test_git_gate_provision_render.py +++ b/tests/unit/test_git_gate_provision_render.py @@ -4,7 +4,7 @@ from __future__ import annotations import unittest -from bot_bottle.gateway.git_gate_render import ( +from bot_bottle.gateway.git_gate.render import ( GitGateUpstream, git_gate_render_entrypoint, git_gate_render_provision, diff --git a/tests/unit/test_git_http_backend.py b/tests/unit/test_git_http_backend.py index 6c587726..89f162cb 100644 --- a/tests/unit/test_git_http_backend.py +++ b/tests/unit/test_git_http_backend.py @@ -10,7 +10,7 @@ from pathlib import Path from unittest import mock from bot_bottle.git_gate import GIT_GATE_TIMEOUT_SECS -from bot_bottle.gateway.git_http_backend import GitHttpHandler, MAX_BODY_BYTES +from bot_bottle.gateway.git_gate.http_backend import GitHttpHandler, MAX_BODY_BYTES # The git-http backend is resolver-only: every request is attributed to a @@ -199,7 +199,7 @@ class TestGitHttpBackend(unittest.TestCase): subprocess.CompletedProcess(["git"], 0, backend_response, b""), ] with mock.patch( - "bot_bottle.gateway.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_gate.http_backend.subprocess.run", side_effect=calls, ) as run: request = urllib.request.Request( @@ -265,7 +265,7 @@ class TestGitHttpBackend(unittest.TestCase): subprocess.CompletedProcess(["git"], 0, backend_response, b""), ] with mock.patch( - "bot_bottle.gateway.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_gate.http_backend.subprocess.run", side_effect=calls, ) as run: req = urllib.request.Request( @@ -309,7 +309,7 @@ class TestGitHttpBackend(unittest.TestCase): denial = b"git-gate: upstream fetch failed; refusing to serve stale data\n" with mock.patch( - "bot_bottle.gateway.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_gate.http_backend.subprocess.run", return_value=subprocess.CompletedProcess( ["hook"], 1, b"", denial, ), @@ -355,7 +355,7 @@ class TestGitHttpBackend(unittest.TestCase): self.addCleanup(server.server_close) with mock.patch( - "bot_bottle.gateway.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_gate.http_backend.subprocess.run", return_value=subprocess.CompletedProcess( ["hook"], 2, b"", b"", ), @@ -402,7 +402,7 @@ class TestGitHttpBackend(unittest.TestCase): self.addCleanup(server.server_close) with mock.patch( - "bot_bottle.gateway.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_gate.http_backend.subprocess.run", side_effect=PermissionError(13, "Permission denied"), ): buf = io.StringIO() @@ -461,7 +461,7 @@ class TestMalformedStatusHeader(unittest.TestCase): def _get_with_backend_response(self, cgi_response: bytes) -> int: with mock.patch( - "bot_bottle.gateway.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_gate.http_backend.subprocess.run", return_value=mock.Mock(returncode=0, stdout=cgi_response), ): req = urllib.request.Request( @@ -545,7 +545,7 @@ class TestContentLengthBounds(unittest.TestCase): # With a valid Content-Length the handler proceeds into # git http-backend; that will fail (no real git repo) but the # status won't be 400 or 413. - with mock.patch("bot_bottle.gateway.git_http_backend.subprocess.run") as run: + with mock.patch("bot_bottle.gateway.git_gate.http_backend.subprocess.run") as run: run.return_value = mock.Mock( returncode=0, stdout=( diff --git a/tests/unit/test_git_http_multitenant.py b/tests/unit/test_git_http_multitenant.py index 16acff14..229e7dfe 100644 --- a/tests/unit/test_git_http_multitenant.py +++ b/tests/unit/test_git_http_multitenant.py @@ -10,7 +10,7 @@ from __future__ import annotations import unittest from pathlib import Path -from bot_bottle.gateway.git_http_backend import resolve_sandbox_root +from bot_bottle.gateway.git_gate.http_backend import resolve_sandbox_root from bot_bottle.gateway.policy_resolver import PolicyResolveError _BASE = Path("/git") diff --git a/tests/unit/test_macos_infra.py b/tests/unit/test_macos_infra.py index 7d80840a..914ed1c4 100644 --- a/tests/unit/test_macos_infra.py +++ b/tests/unit/test_macos_infra.py @@ -48,7 +48,7 @@ class TestInfraRun(unittest.TestCase): self.assertIn("bot_bottle.orchestrator", script) # Gateway launches via the installed package (there is no # /app/gateway_init.py file since the daemons moved into bot_bottle). - self.assertIn("bot_bottle.gateway.gateway_init", script) + self.assertIn("bot_bottle.gateway.bootstrap", script) self.assertIn("127.0.0.1", script) # they reach each other on loopback def test_db_is_a_container_only_volume(self) -> None: diff --git a/tests/unit/test_orchestrator_registration.py b/tests/unit/test_orchestrator_registration.py index ffb37524..738ecf84 100644 --- a/tests/unit/test_orchestrator_registration.py +++ b/tests/unit/test_orchestrator_registration.py @@ -7,7 +7,7 @@ import unittest from pathlib import Path from bot_bottle.egress import EgressPlan, EgressRoute -from bot_bottle.gateway.egress_addon_core import LOG_BLOCKS, load_config +from bot_bottle.gateway.egress.addon_core import LOG_BLOCKS, load_config from bot_bottle.orchestrator.registration import ( RegistrationInputs, egress_policy, diff --git a/tests/unit/test_supervise_server.py b/tests/unit/test_supervise_server.py index 0ae9ce50..52c575cb 100644 --- a/tests/unit/test_supervise_server.py +++ b/tests/unit/test_supervise_server.py @@ -22,8 +22,8 @@ from bot_bottle.orchestrator import supervisor as _sv from bot_bottle.orchestrator.store import queue_store as _qs from bot_bottle.store import audit_store as _as -from bot_bottle.gateway import supervise_server # noqa: E402 -from bot_bottle.gateway.supervise_server import ( +from bot_bottle.gateway.supervisor import server as supervise_server # noqa: E402 +from bot_bottle.gateway.supervisor.server import ( ERR_INTERNAL, ERR_INVALID_PARAMS, ERR_INVALID_REQUEST, -- 2.52.0 From 74dc984cf8cb773a5a8a01f737597a965165d199 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 17:08:34 -0400 Subject: [PATCH 19/30] refactor(gateway): rename bootstrap's _Supervisor to _DaemonManager The PID-1 process manager in gateway/bootstrap.py was named _Supervisor, which collided conceptually with the supervise permissions service (Supervisor / gateway.supervisor). It manages the _DaemonSpec child set, so _DaemonManager names what it does without the "supervise" echo. Co-Authored-By: Claude Opus 4.8 --- bot_bottle/gateway/bootstrap.py | 4 ++-- tests/unit/test_gateway_init.py | 40 ++++++++++++++++----------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/bot_bottle/gateway/bootstrap.py b/bot_bottle/gateway/bootstrap.py index cd361799..060a9807 100644 --- a/bot_bottle/gateway/bootstrap.py +++ b/bot_bottle/gateway/bootstrap.py @@ -173,7 +173,7 @@ def _spawn(spec: _DaemonSpec) -> subprocess.Popen[bytes]: return proc -class _Supervisor: +class _DaemonManager: """Holds the running children + shutdown state. Pulled out so the test suite can drive it with fake commands.""" @@ -387,7 +387,7 @@ def main(argv: Sequence[str] | None = None) -> int: _log("no daemons selected; nothing to do") return 0 - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() signal.signal(signal.SIGTERM, lambda *_: sup.request_shutdown("SIGTERM")) # type: ignore diff --git a/tests/unit/test_gateway_init.py b/tests/unit/test_gateway_init.py index d43cfe6c..7f5272c4 100644 --- a/tests/unit/test_gateway_init.py +++ b/tests/unit/test_gateway_init.py @@ -20,7 +20,7 @@ from unittest.mock import patch from bot_bottle.gateway.bootstrap import ( _DaemonSpec, - _Supervisor, + _DaemonManager, _argv_for_daemon, _env_for_daemon, _selected_daemons, @@ -169,7 +169,7 @@ class TestDaemonArgv(unittest.TestCase): class TestSupervisor(unittest.TestCase): - """End-to-end: drive `_Supervisor` directly with fake commands. + """End-to-end: drive `_DaemonManager` directly with fake commands. We don't go through `main()` because main installs signal handlers process-wide, which collides with the test runner.""" @@ -177,7 +177,7 @@ class TestSupervisor(unittest.TestCase): warnings.simplefilter("error", ResourceWarning) self.addCleanup(warnings.resetwarnings) - def _drive(self, sup: _Supervisor, max_wait_s: float = 6.0) -> int: + def _drive(self, sup: _DaemonManager, max_wait_s: float = 6.0) -> int: deadline = time.monotonic() + max_wait_s while not sup.tick(): if time.monotonic() > deadline: @@ -193,7 +193,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("a", ("/bin/sh", "-c", ":")), _DaemonSpec("b", ("/bin/sh", "-c", ":")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.1) sup.request_shutdown(reason="test") @@ -209,7 +209,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("crasher", ("/bin/sh", "-c", "exit 1")), _DaemonSpec("longrun", (SLEEP, "30")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() original_pid = sup.procs[0][1].pid @@ -235,7 +235,7 @@ class TestSupervisor(unittest.TestCase): def test_single_daemon_crash_is_restarted_before_tick_completes(self): specs = [_DaemonSpec("crasher", ("/bin/sh", "-c", "exit 1"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() original_pid = sup.procs[0][1].pid time.sleep(0.1) @@ -259,7 +259,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("crasher", ("/bin/sh", "-c", "exit 1")), _DaemonSpec("longrun", (SLEEP, "30")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.3) # let crasher die sup.request_shutdown(reason="test") @@ -271,7 +271,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("a", ("/bin/sh", "-c", "exit 0")), _DaemonSpec("b", ("/bin/sh", "-c", "exit 2")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() original_pids = [p.pid for _, p in sup.procs] time.sleep(0.1) @@ -309,7 +309,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("egress", sighup_marker), _DaemonSpec("other", (SLEEP, "30")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.3) # let Python install the handler @@ -331,7 +331,7 @@ class TestSupervisor(unittest.TestCase): def test_forward_signal_unknown_daemon_no_op(self): specs = [_DaemonSpec("a", (SLEEP, "30"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() delivered = sup.forward_signal(signal.SIGHUP, "ghost") self.assertFalse(delivered) @@ -345,7 +345,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("git-gate", (SLEEP, "30")), _DaemonSpec("supervise", (SLEEP, "30")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.1) old_git_gate_pid = sup.procs[0][1].pid @@ -370,7 +370,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("git-gate", (SLEEP, "30")), _DaemonSpec("supervise", (SLEEP, "30")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.1) old_git_gate_pid = sup.procs[0][1].pid @@ -392,7 +392,7 @@ class TestSupervisor(unittest.TestCase): def test_repeated_restart_requests_coalesce(self): specs = [_DaemonSpec("git-gate", (SLEEP, "30"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.1) @@ -415,7 +415,7 @@ class TestSupervisor(unittest.TestCase): def test_request_restart_unknown_daemon_no_op(self): specs = [_DaemonSpec("a", (SLEEP, "30"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() ok = sup.request_restart("ghost") self.assertFalse(ok) @@ -425,7 +425,7 @@ class TestSupervisor(unittest.TestCase): def test_restart_unknown_daemon_no_op(self): specs = [_DaemonSpec("a", (SLEEP, "30"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() ok = sup.restart_daemon("ghost") self.assertFalse(ok) @@ -434,7 +434,7 @@ class TestSupervisor(unittest.TestCase): def test_restart_during_shutdown_is_no_op(self): specs = [_DaemonSpec("git-gate", (SLEEP, "30"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() sup.request_shutdown(reason="test") ok = sup.restart_daemon("git-gate") @@ -444,7 +444,7 @@ class TestSupervisor(unittest.TestCase): def test_pending_restart_dropped_during_shutdown(self): specs = [_DaemonSpec("git-gate", (SLEEP, "30"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.1) old_pid = sup.procs[0][1].pid @@ -464,7 +464,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("a", (SLEEP, "60")), _DaemonSpec("b", (SLEEP, "60")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.2) # let them actually start sup.request_shutdown(reason="test") @@ -484,7 +484,7 @@ class TestSupervisor(unittest.TestCase): "trap '' TERM; sleep 30", ) specs = [_DaemonSpec("stubborn", ignore_term)] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.3) # let `trap` register sup.request_shutdown(reason="test") @@ -498,7 +498,7 @@ class TestSupervisor(unittest.TestCase): def test_idempotent_shutdown_requests(self): specs = [_DaemonSpec("a", (SLEEP, "60"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.1) first_at = None -- 2.52.0 From 82d02d2c4b24714d338621f7d9e8c9632b634358 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 17:14:26 -0400 Subject: [PATCH 20/30] refactor(gateway): move egress entrypoint into gateway/egress egress_entrypoint.sh is the mitmdump launcher for the egress data plane, so it belongs with the gateway egress service, not at the package root. Moved to bot_bottle/gateway/egress/entrypoint.sh (prefix stripped now that the package namespaces it). Only the Dockerfile.gateway COPY source changes; the runtime target (/app/egress-entrypoint.sh, referenced by bootstrap's egress DaemonSpec) is unchanged, and the infra images inherit it via FROM gateway. Updated the test that resolves the script path and the infra-artifact docstring example. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- Dockerfile.gateway | 2 +- bot_bottle/backend/firecracker/infra_artifact.py | 2 +- .../{egress_entrypoint.sh => gateway/egress/entrypoint.sh} | 0 tests/unit/test_egress_entrypoint.py | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename bot_bottle/{egress_entrypoint.sh => gateway/egress/entrypoint.sh} (100%) diff --git a/Dockerfile.gateway b/Dockerfile.gateway index 47ff5840..ad7001c6 100644 --- a/Dockerfile.gateway +++ b/Dockerfile.gateway @@ -103,7 +103,7 @@ RUN pip install --no-cache-dir /src/ # (nothing created /app before this point). WORKDIR /app RUN printf 'from bot_bottle.gateway.egress.addon import addons\n' > /app/egress_addon.py -COPY bot_bottle/egress_entrypoint.sh /app/egress-entrypoint.sh +COPY bot_bottle/gateway/egress/entrypoint.sh /app/egress-entrypoint.sh RUN chmod +x /app/egress-entrypoint.sh # Pre-create runtime directories the compose renderer + start diff --git a/bot_bottle/backend/firecracker/infra_artifact.py b/bot_bottle/backend/firecracker/infra_artifact.py index 5c8ecb29..eb8396df 100644 --- a/bot_bottle/backend/firecracker/infra_artifact.py +++ b/bot_bottle/backend/firecracker/infra_artifact.py @@ -66,7 +66,7 @@ def infra_artifact_version(init_script: str, *, repo_root: Path = _REPO_ROOT) -> The package is `COPY bot_bottle /app/bot_bottle`'d wholesale into the image, so hash *every* regular file under it — not just `*.py`. Non-Python inputs - (e.g. `egress_entrypoint.sh`, `netpool.defaults.env`) are baked in too, and + (e.g. `gateway/egress/entrypoint.sh`, `netpool.defaults.env`) are baked in too, and a change to one must bump the version or a launch host could boot a stale rootfs whose code differs from its checkout. `__pycache__`/`.pyc` are the only exclusions — build artifacts, never copied.""" diff --git a/bot_bottle/egress_entrypoint.sh b/bot_bottle/gateway/egress/entrypoint.sh similarity index 100% rename from bot_bottle/egress_entrypoint.sh rename to bot_bottle/gateway/egress/entrypoint.sh diff --git a/tests/unit/test_egress_entrypoint.py b/tests/unit/test_egress_entrypoint.py index fc0bb120..0c846ac6 100644 --- a/tests/unit/test_egress_entrypoint.py +++ b/tests/unit/test_egress_entrypoint.py @@ -21,7 +21,7 @@ from pathlib import Path _SCRIPT = ( Path(__file__).resolve().parent.parent.parent - / "bot_bottle" / "egress_entrypoint.sh" + / "bot_bottle" / "gateway" / "egress" / "entrypoint.sh" ) -- 2.52.0 From 50a67c04bd3fa3018413af72f31c458a59475f8b Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 17:25:25 -0400 Subject: [PATCH 21/30] refactor(cli): group subcommand handlers under cli/commands/ Move the eleven per-command modules (backend, cleanup, commit, edit, info, init, list, login, resume, start, supervise) into a new bot_bottle/cli/commands/ package, leaving the dispatcher (__init__), entrypoint (__main__), and shared helpers (_common, tui) at the cli root. Makes the command surface obvious at a glance and separates handlers from the plumbing that registers them. Updated the dispatcher's COMMANDS imports to .commands.*, bumped the moved files' relative-import depths (.._common, .. import tui, sibling .start unchanged), and repointed test references to bot_bottle.cli.commands.*. Full unit suite green (2243); CLI dispatch verified via `python -m bot_bottle.cli --help`. Co-Authored-By: Claude Opus 4.8 --- bot_bottle/cli/__init__.py | 22 +++++----- bot_bottle/cli/commands/__init__.py | 6 +++ bot_bottle/cli/{ => commands}/backend.py | 4 +- bot_bottle/cli/{ => commands}/cleanup.py | 6 +-- bot_bottle/cli/{ => commands}/commit.py | 12 ++--- bot_bottle/cli/{ => commands}/edit.py | 4 +- bot_bottle/cli/{ => commands}/info.py | 6 +-- bot_bottle/cli/{ => commands}/init.py | 4 +- bot_bottle/cli/{ => commands}/list.py | 6 +-- bot_bottle/cli/{ => commands}/login.py | 2 +- bot_bottle/cli/{ => commands}/resume.py | 10 ++--- bot_bottle/cli/{ => commands}/start.py | 26 +++++------ bot_bottle/cli/{ => commands}/supervise.py | 12 ++--- tests/unit/test_cli_backend.py | 2 +- tests/unit/test_cli_cleanup_cross_backend.py | 2 +- tests/unit/test_cli_commit.py | 20 ++++----- tests/unit/test_cli_login.py | 44 +++++++++---------- tests/unit/test_cli_start_headless.py | 24 +++++----- tests/unit/test_cli_start_selector.py | 26 +++++------ tests/unit/test_cli_start_settle.py | 2 +- tests/unit/test_cli_start_stale.py | 10 ++--- tests/unit/test_supervise_cli.py | 2 +- .../unit/test_supervise_cli_crash_logging.py | 2 +- 23 files changed, 130 insertions(+), 124 deletions(-) create mode 100644 bot_bottle/cli/commands/__init__.py rename bot_bottle/cli/{ => commands}/backend.py (94%) rename bot_bottle/cli/{ => commands}/cleanup.py (93%) rename bot_bottle/cli/{ => commands}/commit.py (86%) rename bot_bottle/cli/{ => commands}/edit.py (95%) rename bot_bottle/cli/{ => commands}/info.py (93%) rename bot_bottle/cli/{ => commands}/init.py (98%) rename bot_bottle/cli/{ => commands}/list.py (93%) rename bot_bottle/cli/{ => commands}/login.py (99%) rename bot_bottle/cli/{ => commands}/resume.py (90%) rename bot_bottle/cli/{ => commands}/start.py (97%) rename bot_bottle/cli/{ => commands}/supervise.py (98%) diff --git a/bot_bottle/cli/__init__.py b/bot_bottle/cli/__init__.py index d1d550ac..41ffa7d4 100644 --- a/bot_bottle/cli/__init__.py +++ b/bot_bottle/cli/__init__.py @@ -12,17 +12,17 @@ from ..log import Die, die, error from ..manifest import ManifestError from ..orchestrator.store.store_manager import StoreManager from ._common import PROG -from . import list as _list_mod -from .backend import cmd_backend -from .cleanup import cmd_cleanup -from .commit import cmd_commit -from .edit import cmd_edit -from .info import cmd_info -from .init import cmd_init -from .login import cmd_login -from .resume import cmd_resume -from .start import cmd_start -from .supervise import cmd_supervise +from .commands import list as _list_mod +from .commands.backend import cmd_backend +from .commands.cleanup import cmd_cleanup +from .commands.commit import cmd_commit +from .commands.edit import cmd_edit +from .commands.info import cmd_info +from .commands.init import cmd_init +from .commands.login import cmd_login +from .commands.resume import cmd_resume +from .commands.start import cmd_start +from .commands.supervise import cmd_supervise cmd_list = _list_mod.cmd_list diff --git a/bot_bottle/cli/commands/__init__.py b/bot_bottle/cli/commands/__init__.py new file mode 100644 index 00000000..0daac76b --- /dev/null +++ b/bot_bottle/cli/commands/__init__.py @@ -0,0 +1,6 @@ +"""CLI subcommand handlers — one module per `bot-bottle `. + +Each module exposes a `cmd_(argv)` handler that the dispatcher +(`bot_bottle.cli`) registers in its COMMANDS table. Shared CLI helpers +(`_common`, `tui`) stay one level up in the `cli` package. +""" diff --git a/bot_bottle/cli/backend.py b/bot_bottle/cli/commands/backend.py similarity index 94% rename from bot_bottle/cli/backend.py rename to bot_bottle/cli/commands/backend.py index ed6808f5..2997db36 100644 --- a/bot_bottle/cli/backend.py +++ b/bot_bottle/cli/commands/backend.py @@ -15,8 +15,8 @@ from __future__ import annotations import argparse -from ..backend import get_bottle_backend, known_backend_names -from ._common import PROG +from ...backend import get_bottle_backend, known_backend_names +from .._common import PROG def cmd_backend(args: list[str]) -> int: diff --git a/bot_bottle/cli/cleanup.py b/bot_bottle/cli/commands/cleanup.py similarity index 93% rename from bot_bottle/cli/cleanup.py rename to bot_bottle/cli/commands/cleanup.py index 34d6de99..685f6bcb 100644 --- a/bot_bottle/cli/cleanup.py +++ b/bot_bottle/cli/commands/cleanup.py @@ -21,9 +21,9 @@ from __future__ import annotations import sys -from ..backend import get_bottle_backend, has_backend, known_backend_names -from ..log import info -from ._common import read_tty_line +from ...backend import get_bottle_backend, has_backend, known_backend_names +from ...log import info +from .._common import read_tty_line def cmd_cleanup(_argv: list[str]) -> int: diff --git a/bot_bottle/cli/commit.py b/bot_bottle/cli/commands/commit.py similarity index 86% rename from bot_bottle/cli/commit.py rename to bot_bottle/cli/commands/commit.py index 51d94d9b..502ae697 100644 --- a/bot_bottle/cli/commit.py +++ b/bot_bottle/cli/commands/commit.py @@ -12,12 +12,12 @@ from __future__ import annotations import argparse -from ..backend import enumerate_active_agents -from ..backend.freeze import CommitCancelled, get_freezer -from ..bottle_state import read_metadata -from ..log import die -from ._common import PROG -from . import tui +from ...backend import enumerate_active_agents +from ...backend.freeze import CommitCancelled, get_freezer +from ...bottle_state import read_metadata +from ...log import die +from .._common import PROG +from .. import tui def cmd_commit(argv: list[str]) -> int: diff --git a/bot_bottle/cli/edit.py b/bot_bottle/cli/commands/edit.py similarity index 95% rename from bot_bottle/cli/edit.py rename to bot_bottle/cli/commands/edit.py index 8c412d9b..82ca91b4 100644 --- a/bot_bottle/cli/edit.py +++ b/bot_bottle/cli/commands/edit.py @@ -7,8 +7,8 @@ import json import os from pathlib import Path -from ..log import die -from ._common import PROG, USER_CWD +from ...log import die +from .._common import PROG, USER_CWD def cmd_edit(argv: list[str]) -> int: diff --git a/bot_bottle/cli/info.py b/bot_bottle/cli/commands/info.py similarity index 93% rename from bot_bottle/cli/info.py rename to bot_bottle/cli/commands/info.py index 61b89453..d15b5612 100644 --- a/bot_bottle/cli/info.py +++ b/bot_bottle/cli/commands/info.py @@ -4,9 +4,9 @@ from __future__ import annotations import argparse -from ..log import info -from ..manifest import ManifestIndex -from ._common import PROG, USER_CWD +from ...log import info +from ...manifest import ManifestIndex +from .._common import PROG, USER_CWD def cmd_info(argv: list[str]) -> int: diff --git a/bot_bottle/cli/init.py b/bot_bottle/cli/commands/init.py similarity index 98% rename from bot_bottle/cli/init.py rename to bot_bottle/cli/commands/init.py index 21b62a02..ca169f12 100644 --- a/bot_bottle/cli/init.py +++ b/bot_bottle/cli/commands/init.py @@ -10,8 +10,8 @@ import sys from pathlib import Path from typing import Any -from ..log import die, info, warn -from ._common import PROG, USER_CWD, read_tty_line +from ...log import die, info, warn +from .._common import PROG, USER_CWD, read_tty_line def cmd_init(argv: list[str]) -> int: diff --git a/bot_bottle/cli/list.py b/bot_bottle/cli/commands/list.py similarity index 93% rename from bot_bottle/cli/list.py rename to bot_bottle/cli/commands/list.py index c7274286..5c0caf35 100644 --- a/bot_bottle/cli/list.py +++ b/bot_bottle/cli/commands/list.py @@ -6,9 +6,9 @@ import argparse import os import sys -from ..backend import enumerate_active_agents -from ..manifest import ManifestIndex -from ._common import PROG, USER_CWD +from ...backend import enumerate_active_agents +from ...manifest import ManifestIndex +from .._common import PROG, USER_CWD _ANSI_COLOR_CODES: dict[str, str] = { "red": "\033[91m", diff --git a/bot_bottle/cli/login.py b/bot_bottle/cli/commands/login.py similarity index 99% rename from bot_bottle/cli/login.py rename to bot_bottle/cli/commands/login.py index 3d00f17e..55e04bfa 100644 --- a/bot_bottle/cli/login.py +++ b/bot_bottle/cli/commands/login.py @@ -25,7 +25,7 @@ import urllib.request from pathlib import Path from typing import Any -from ..paths import bot_bottle_root +from ...paths import bot_bottle_root _CONSOLE_URL_ENV = "BB_CONSOLE_URL" _POLL_SLEEP = 2 # seconds between polls; matches console's poll_interval default diff --git a/bot_bottle/cli/resume.py b/bot_bottle/cli/commands/resume.py similarity index 90% rename from bot_bottle/cli/resume.py rename to bot_bottle/cli/commands/resume.py index 5dd421bb..6d8224af 100644 --- a/bot_bottle/cli/resume.py +++ b/bot_bottle/cli/commands/resume.py @@ -16,11 +16,11 @@ from __future__ import annotations import argparse -from ..backend import BottleSpec -from ..bottle_state import read_metadata -from ..log import die -from ..manifest import ManifestIndex -from ._common import PROG, USER_CWD +from ...backend import BottleSpec +from ...bottle_state import read_metadata +from ...log import die +from ...manifest import ManifestIndex +from .._common import PROG, USER_CWD from .start import _launch_bottle diff --git a/bot_bottle/cli/start.py b/bot_bottle/cli/commands/start.py similarity index 97% rename from bot_bottle/cli/start.py rename to bot_bottle/cli/commands/start.py index b587216a..24c2e9c4 100644 --- a/bot_bottle/cli/start.py +++ b/bot_bottle/cli/commands/start.py @@ -22,25 +22,25 @@ import tempfile from pathlib import Path from typing import Callable -from ..agent_provider import get_provider, runtime_for -from ..backend import ( +from ...agent_provider import get_provider, runtime_for +from ...backend import ( Bottle, BottleSpec, enumerate_active_agents, get_bottle_backend, ) -from ..backend.docker import util as docker_mod -from ..backend.docker.bottle_plan import DockerBottlePlan -from ..bottle_state import ( +from ...backend.docker import util as docker_mod +from ...backend.docker.bottle_plan import DockerBottlePlan +from ...bottle_state import ( cleanup_state, is_preserved, mark_preserved, ) -from ..image_cache import StaleImageError -from ..log import info, die -from ..manifest import Manifest, ManifestIndex -from ._common import PROG, USER_CWD, read_tty_line -from . import tui +from ...image_cache import StaleImageError +from ...log import info, die +from ...manifest import Manifest, ManifestIndex +from .._common import PROG, USER_CWD, read_tty_line +from .. import tui def cmd_start(argv: list[str]) -> int: @@ -380,8 +380,8 @@ def _peek_agent_bottle(manifest: ManifestIndex, agent_name: str) -> str: return manifest.agents[agent_name].bottle return "" - from ..manifest.loader import scan_agent_names - from ..yaml_subset import YamlSubsetError, parse_frontmatter + from ...manifest.loader import scan_agent_names + from ...yaml_subset import YamlSubsetError, parse_frontmatter home_agents = scan_agent_names(manifest.home_md / "agents") cwd_agents: dict[str, Path] = {} @@ -449,7 +449,7 @@ def _bottle_lineage(manifest: ManifestIndex) -> dict[str, str]: if not bottles_dir.is_dir(): return {} - from ..yaml_subset import YamlSubsetError, parse_frontmatter + from ...yaml_subset import YamlSubsetError, parse_frontmatter extends_of: dict[str, str] = {} for path in bottles_dir.glob("*.md"): diff --git a/bot_bottle/cli/supervise.py b/bot_bottle/cli/commands/supervise.py similarity index 98% rename from bot_bottle/cli/supervise.py rename to bot_bottle/cli/commands/supervise.py index abf7d70e..c2beb030 100644 --- a/bot_bottle/cli/supervise.py +++ b/bot_bottle/cli/commands/supervise.py @@ -19,22 +19,22 @@ from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path -from ..paths import bot_bottle_root -from ..log import Die, error, info -from ..orchestrator.client import ( +from ...paths import bot_bottle_root +from ...log import Die, error, info +from ...orchestrator.client import ( OrchestratorClient, OrchestratorClientError, discover_orchestrator_url, ) -from ..supervisor.types import ( +from ...supervisor.types import ( Proposal, TOOL_EGRESS_ALLOW, TOOL_EGRESS_BLOCK, TOOL_GITLEAKS_ALLOW, TOOL_EGRESS_TOKEN_ALLOW, ) -from ._common import PROG +from .._common import PROG _REFRESH_INTERVAL_MS = 1000 @@ -81,7 +81,7 @@ def _resolve_orchestrator_url() -> str: try: return discover_orchestrator_url() except OrchestratorClientError: - from ..backend import get_bottle_backend + from ...backend import get_bottle_backend backend = get_bottle_backend() info(f"no orchestrator control plane running; starting one ({backend.name})…") return backend.ensure_orchestrator() diff --git a/tests/unit/test_cli_backend.py b/tests/unit/test_cli_backend.py index 16eea2c8..40f2890c 100644 --- a/tests/unit/test_cli_backend.py +++ b/tests/unit/test_cli_backend.py @@ -12,7 +12,7 @@ import subprocess import unittest from unittest.mock import MagicMock, patch -from bot_bottle.cli import backend as cmd +from bot_bottle.cli.commands import backend as cmd from bot_bottle.backend.docker import setup as docker_setup diff --git a/tests/unit/test_cli_cleanup_cross_backend.py b/tests/unit/test_cli_cleanup_cross_backend.py index e1f4be8e..47ab1334 100644 --- a/tests/unit/test_cli_cleanup_cross_backend.py +++ b/tests/unit/test_cli_cleanup_cross_backend.py @@ -11,7 +11,7 @@ from __future__ import annotations import unittest from unittest.mock import patch, MagicMock -from bot_bottle.cli import cleanup as cmd +from bot_bottle.cli.commands import cleanup as cmd def _make_backend(empty: bool = True): diff --git a/tests/unit/test_cli_commit.py b/tests/unit/test_cli_commit.py index 6a7e7fff..9eee5423 100644 --- a/tests/unit/test_cli_commit.py +++ b/tests/unit/test_cli_commit.py @@ -7,7 +7,7 @@ import unittest from pathlib import Path from unittest.mock import MagicMock, patch -from bot_bottle.cli.commit import cmd_commit +from bot_bottle.cli.commands.commit import cmd_commit from tests.unit import use_bottle_root from bot_bottle import bottle_state from bot_bottle.backend.freeze import CommitCancelled @@ -42,7 +42,7 @@ class TestCmdCommitSlugArg(_FakeHomeMixin, unittest.TestCase): slug = "dev-abc12" self._write_meta(slug, "docker") - with patch("bot_bottle.cli.commit.get_freezer") as mock_gf: + with patch("bot_bottle.cli.commands.commit.get_freezer") as mock_gf: mock_freezer = MagicMock() mock_gf.return_value = mock_freezer rc = cmd_commit([slug]) @@ -56,7 +56,7 @@ class TestCmdCommitSlugArg(_FakeHomeMixin, unittest.TestCase): slug = "dev-abc12" self._write_meta(slug, "") - with patch("bot_bottle.cli.commit.get_freezer") as mock_gf: + with patch("bot_bottle.cli.commands.commit.get_freezer") as mock_gf: mock_freezer = MagicMock() mock_gf.return_value = mock_freezer rc = cmd_commit([slug]) @@ -68,7 +68,7 @@ class TestCmdCommitSlugArg(_FakeHomeMixin, unittest.TestCase): slug = "dev-abc12" self._write_meta(slug, "macos-container") - with patch("bot_bottle.cli.commit.get_freezer") as mock_gf: + with patch("bot_bottle.cli.commands.commit.get_freezer") as mock_gf: mock_freezer = MagicMock() mock_gf.return_value = mock_freezer rc = cmd_commit([slug]) @@ -81,7 +81,7 @@ class TestCmdCommitSlugArg(_FakeHomeMixin, unittest.TestCase): slug = "dev-abc12" self._write_meta(slug, "firecracker") - with patch("bot_bottle.cli.commit.get_freezer") as mock_gf: + with patch("bot_bottle.cli.commands.commit.get_freezer") as mock_gf: mock_freezer = MagicMock() mock_gf.return_value = mock_freezer rc = cmd_commit([slug]) @@ -93,7 +93,7 @@ class TestCmdCommitSlugArg(_FakeHomeMixin, unittest.TestCase): slug = "dev-abc12" self._write_meta(slug, "macos-container") - with patch("bot_bottle.cli.commit.get_freezer") as mock_gf: + with patch("bot_bottle.cli.commands.commit.get_freezer") as mock_gf: mock_freezer = MagicMock() mock_freezer.commit_slug.side_effect = CommitCancelled mock_gf.return_value = mock_freezer @@ -111,9 +111,9 @@ class TestCmdCommitNoActiveBottles(_FakeHomeMixin, unittest.TestCase): def test_dies_when_no_active_bottles_and_no_slug(self): with patch( - "bot_bottle.cli.commit.enumerate_active_agents", return_value=[], + "bot_bottle.cli.commands.commit.enumerate_active_agents", return_value=[], ), patch( - "bot_bottle.cli.commit.die", side_effect=SystemExit("die"), + "bot_bottle.cli.commands.commit.die", side_effect=SystemExit("die"), ) as mock_die: with self.assertRaises(SystemExit): cmd_commit([]) @@ -124,9 +124,9 @@ class TestCmdCommitNoActiveBottles(_FakeHomeMixin, unittest.TestCase): active = MagicMock() active.slug = "dev-abc12" with patch( - "bot_bottle.cli.commit.enumerate_active_agents", return_value=[active], + "bot_bottle.cli.commands.commit.enumerate_active_agents", return_value=[active], ), patch( - "bot_bottle.cli.commit.tui.filter_select", return_value=None, + "bot_bottle.cli.commands.commit.tui.filter_select", return_value=None, ): rc = cmd_commit([]) diff --git a/tests/unit/test_cli_login.py b/tests/unit/test_cli_login.py index 0d4c13c7..67c00e01 100644 --- a/tests/unit/test_cli_login.py +++ b/tests/unit/test_cli_login.py @@ -14,27 +14,27 @@ from unittest.mock import MagicMock, patch class TestFlagParsing(unittest.TestCase): def test_console_url_flag(self) -> None: - from bot_bottle.cli.login import _flag + from bot_bottle.cli.commands.login import _flag self.assertEqual(_flag(["--console-url", "http://x"], "--console-url"), "http://x") def test_console_url_equals_form(self) -> None: - from bot_bottle.cli.login import _flag + from bot_bottle.cli.commands.login import _flag self.assertEqual( _flag(["--console-url=http://x"], "--console-url"), "http://x" ) def test_label_flag(self) -> None: - from bot_bottle.cli.login import _flag + from bot_bottle.cli.commands.login import _flag self.assertEqual(_flag(["--label", "my-mac"], "--label"), "my-mac") def test_missing_flag_returns_none(self) -> None: - from bot_bottle.cli.login import _flag + from bot_bottle.cli.commands.login import _flag self.assertIsNone(_flag([], "--console-url")) class TestHttpHelpers(unittest.TestCase): def test_post_sends_json_and_decodes_response(self) -> None: - from bot_bottle.cli.login import _post + from bot_bottle.cli.commands.login import _post response = MagicMock() response.__enter__.return_value.read.return_value = b'{"ok": true}' @@ -48,7 +48,7 @@ class TestHttpHelpers(unittest.TestCase): self.assertEqual(request.get_header("Content-type"), "application/json") def test_get_decodes_success_response(self) -> None: - from bot_bottle.cli.login import _get + from bot_bottle.cli.commands.login import _get response = MagicMock() response.__enter__.return_value.status = 200 @@ -59,7 +59,7 @@ class TestHttpHelpers(unittest.TestCase): ) def test_get_returns_http_error_status(self) -> None: - from bot_bottle.cli.login import _get + from bot_bottle.cli.commands.login import _get error = urllib.error.HTTPError( "http://console/status", 410, "gone", Message(), None @@ -70,7 +70,7 @@ class TestHttpHelpers(unittest.TestCase): class TestSaveCredentials(unittest.TestCase): def test_writes_json_and_sets_perms(self) -> None: - from bot_bottle.cli.login import _save_credentials + from bot_bottle.cli.commands.login import _save_credentials with tempfile.TemporaryDirectory() as tmp: with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}): @@ -85,7 +85,7 @@ class TestSaveCredentials(unittest.TestCase): def test_temp_file_is_private_before_replace(self) -> None: """Temp file must be 0600 at the moment os.replace is called.""" - from bot_bottle.cli.login import _save_credentials + from bot_bottle.cli.commands.login import _save_credentials from pathlib import Path as _Path tmp_perms_at_replace: list[int] = [] @@ -107,7 +107,7 @@ class TestSaveCredentials(unittest.TestCase): def test_cleanup_on_write_failure(self) -> None: """Temp file is removed and no credentials remain if replace fails.""" - from bot_bottle.cli.login import _save_credentials + from bot_bottle.cli.commands.login import _save_credentials with tempfile.TemporaryDirectory() as tmp: with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}): @@ -120,12 +120,12 @@ class TestSaveCredentials(unittest.TestCase): class TestCmdLoginMissingUrl(unittest.TestCase): def test_help_returns_0(self) -> None: - from bot_bottle.cli.login import cmd_login + from bot_bottle.cli.commands.login import cmd_login self.assertEqual(cmd_login(["--help"]), 0) def test_returns_1_without_url(self) -> None: - from bot_bottle.cli.login import cmd_login + from bot_bottle.cli.commands.login import cmd_login with patch.dict(os.environ, {}, clear=True): os.environ.pop("BB_CONSOLE_URL", None) result = cmd_login([]) @@ -133,7 +133,7 @@ class TestCmdLoginMissingUrl(unittest.TestCase): def test_reads_env_var(self) -> None: """Exits 1 (network error) not because of missing URL when env var is set.""" - from bot_bottle.cli.login import cmd_login + from bot_bottle.cli.commands.login import cmd_login def _fail_post(_url: str, _payload: dict[str, Any]) -> dict[str, Any]: raise OSError("connection refused") @@ -146,7 +146,7 @@ class TestCmdLoginMissingUrl(unittest.TestCase): "BOT_BOTTLE_ROOT": tmp, }, ): - with patch("bot_bottle.cli.login._post", side_effect=_fail_post): + with patch("bot_bottle.cli.commands.login._post", side_effect=_fail_post): result = cmd_login([]) self.assertEqual(result, 1) @@ -155,7 +155,7 @@ class TestCmdLoginFlow(unittest.TestCase): def _run_with_mocks( self, poll_responses: list[dict[str, Any]], tmp: str ) -> int: - from bot_bottle.cli.login import cmd_login + from bot_bottle.cli.commands.login import cmd_login start_resp = { "device_code": "dc123", @@ -175,8 +175,8 @@ class TestCmdLoginFlow(unittest.TestCase): return 200, next(poll_iter, {"status": "pending"}) with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}): - with patch("bot_bottle.cli.login._post", side_effect=_fake_post): - with patch("bot_bottle.cli.login._get", side_effect=_fake_get): + with patch("bot_bottle.cli.commands.login._post", side_effect=_fake_post): + with patch("bot_bottle.cli.commands.login._get", side_effect=_fake_get): with patch("time.sleep"): return cmd_login(["--console-url", "http://console"]) @@ -202,7 +202,7 @@ class TestCmdLoginFlow(unittest.TestCase): self.assertEqual(result, 1) def test_timeout_returns_1(self) -> None: - from bot_bottle.cli.login import cmd_login + from bot_bottle.cli.commands.login import cmd_login start_resp = { "device_code": "dc", @@ -213,13 +213,13 @@ class TestCmdLoginFlow(unittest.TestCase): with tempfile.TemporaryDirectory() as tmp: with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}): - with patch("bot_bottle.cli.login._post", return_value=start_resp): + with patch("bot_bottle.cli.commands.login._post", return_value=start_resp): result = cmd_login(["--console-url", "http://console"]) self.assertEqual(result, 1) def test_poll_interval_from_server_is_used(self) -> None: """time.sleep must be called with the server-provided poll_interval.""" - from bot_bottle.cli.login import cmd_login + from bot_bottle.cli.commands.login import cmd_login server_interval = 7 start_resp = { @@ -241,9 +241,9 @@ class TestCmdLoginFlow(unittest.TestCase): with tempfile.TemporaryDirectory() as tmp: with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}): - with patch("bot_bottle.cli.login._post", return_value=start_resp): + with patch("bot_bottle.cli.commands.login._post", return_value=start_resp): with patch( - "bot_bottle.cli.login._get", side_effect=_fake_get + "bot_bottle.cli.commands.login._get", side_effect=_fake_get ): with patch("time.sleep") as mock_sleep: result = cmd_login(["--console-url", "http://console"]) diff --git a/tests/unit/test_cli_start_headless.py b/tests/unit/test_cli_start_headless.py index 092b26c8..e259acb0 100644 --- a/tests/unit/test_cli_start_headless.py +++ b/tests/unit/test_cli_start_headless.py @@ -14,7 +14,7 @@ import os import unittest from unittest.mock import MagicMock, patch -import bot_bottle.cli.start as start_mod +import bot_bottle.cli.commands.start as start_mod import bot_bottle.cli.tui as tui_mod from bot_bottle.backend import ActiveAgent from bot_bottle.log import Die @@ -53,15 +53,15 @@ class TestCmdStartHeadless(unittest.TestCase): ["researcher", "implementer"], ["claude", "dev"], agent_bottle="claude" ) patch( - "bot_bottle.cli.start.ManifestIndex.resolve", + "bot_bottle.cli.commands.start.ManifestIndex.resolve", return_value=self._manifest, ).start() self._launch_mock = patch( - "bot_bottle.cli.start._launch_bottle", return_value=0 + "bot_bottle.cli.commands.start._launch_bottle", return_value=0 ).start() # No bottles running by default → no label collision. patch( - "bot_bottle.cli.start.enumerate_active_agents", return_value=[] + "bot_bottle.cli.commands.start.enumerate_active_agents", return_value=[] ).start() # If any TUI picker fires in headless mode, that's a bug. self._agent_picker = patch.object(tui_mod, "filter_select").start() @@ -71,7 +71,7 @@ class TestCmdStartHeadless(unittest.TestCase): os.environ.pop("BOT_BOTTLE_BACKEND", None) # PTY check uses os.isatty(sys.stdin.fileno()); stub both so # headless unit tests aren't blocked on a real TTY. - patch("bot_bottle.cli.start.os.isatty", return_value=True).start() + patch("bot_bottle.cli.commands.start.os.isatty", return_value=True).start() self.addCleanup(patch.stopall) def _spec(self): @@ -128,7 +128,7 @@ class TestCmdStartHeadless(unittest.TestCase): def test_no_bottle_and_no_default_dies(self): manifest = _make_manifest(["researcher"], ["claude"], agent_bottle="") with patch( - "bot_bottle.cli.start.ManifestIndex.resolve", return_value=manifest + "bot_bottle.cli.commands.start.ManifestIndex.resolve", return_value=manifest ): with self.assertRaises(Die): start_mod.cmd_start( @@ -170,7 +170,7 @@ class TestCmdStartHeadless(unittest.TestCase): def test_label_collision_uniquifies(self): with patch( - "bot_bottle.cli.start.enumerate_active_agents", + "bot_bottle.cli.commands.start.enumerate_active_agents", return_value=[_active_agent("researcher")], ): start_mod.cmd_start( @@ -203,9 +203,9 @@ class TestPrepareWithPreflight(unittest.TestCase): mock_backend.name = "test-backend" render = MagicMock() - with patch("bot_bottle.cli.start.get_bottle_backend", return_value=mock_backend), \ - patch("bot_bottle.cli.start._identity_from_plan", return_value="id"), \ - patch("bot_bottle.cli.start.info"): + with patch("bot_bottle.cli.commands.start.get_bottle_backend", return_value=mock_backend), \ + patch("bot_bottle.cli.commands.start._identity_from_plan", return_value="id"), \ + patch("bot_bottle.cli.commands.start.info"): start_mod.prepare_with_preflight( MagicMock(), stage_dir=Path("/tmp"), @@ -223,7 +223,7 @@ class TestNoAgentsDefined(unittest.TestCase): def test_no_agents_defined_returns_1(self): manifest = _make_manifest([], []) with patch( - "bot_bottle.cli.start.ManifestIndex.resolve", return_value=manifest + "bot_bottle.cli.commands.start.ManifestIndex.resolve", return_value=manifest ), patch("sys.stderr", io.StringIO()) as err: rc = start_mod.cmd_start([]) self.assertEqual(1, rc) @@ -236,7 +236,7 @@ class TestTextRenderPreflight(unittest.TestCase): def test_backend_name_in_output(self): render = start_mod._text_render_preflight() plan = MagicMock() - with patch("bot_bottle.cli.start._manifest_to_yaml", return_value=""), \ + with patch("bot_bottle.cli.commands.start._manifest_to_yaml", return_value=""), \ patch("sys.stderr", io.StringIO()) as err: render(plan, "my-backend") self.assertIn("my-backend", err.getvalue()) diff --git a/tests/unit/test_cli_start_selector.py b/tests/unit/test_cli_start_selector.py index 4f41f07a..5819dac8 100644 --- a/tests/unit/test_cli_start_selector.py +++ b/tests/unit/test_cli_start_selector.py @@ -14,7 +14,7 @@ import unittest from collections.abc import Mapping, Sequence from unittest.mock import MagicMock, patch -import bot_bottle.cli.start as start_mod +import bot_bottle.cli.commands.start as start_mod import bot_bottle.cli.tui as tui_mod from bot_bottle.backend import ActiveAgent @@ -38,13 +38,13 @@ class TestCmdStartSelector(unittest.TestCase): def setUp(self): self._manifest = _make_manifest(["researcher", "implementer"], ["claude", "dev"]) self._resolve_patch = patch( - "bot_bottle.cli.start.ManifestIndex.resolve", + "bot_bottle.cli.commands.start.ManifestIndex.resolve", return_value=self._manifest, ) self._resolve_patch.start() self._launch_patch = patch( - "bot_bottle.cli.start._launch_bottle", + "bot_bottle.cli.commands.start._launch_bottle", return_value=0, ) self._launch_mock = self._launch_patch.start() @@ -66,7 +66,7 @@ class TestCmdStartSelector(unittest.TestCase): self._modal_patch.start() self._image_policy_patch = patch( - "bot_bottle.cli.start._select_image_policy", + "bot_bottle.cli.commands.start._select_image_policy", return_value="fresh", ) self._image_policy_patch.start() @@ -141,14 +141,14 @@ class TestCmdStartSelector(unittest.TestCase): self.assertEqual(("claude", "dev"), spec.bottle_names) def test_image_policy_forwarded_to_spec(self): - with patch("bot_bottle.cli.start._select_image_policy", return_value="cached"): + with patch("bot_bottle.cli.commands.start._select_image_policy", return_value="cached"): start_mod.cmd_start(["researcher"]) self._launch_mock.assert_called_once() spec = self._launch_mock.call_args[0][0] self.assertEqual("cached", spec.image_policy) def test_image_policy_cancel_returns_0(self): - with patch("bot_bottle.cli.start._select_image_policy", return_value=None): + with patch("bot_bottle.cli.commands.start._select_image_policy", return_value=None): rc = start_mod.cmd_start(["researcher"]) self.assertEqual(0, rc) self._launch_mock.assert_not_called() @@ -169,7 +169,7 @@ class TestCmdStartSelector(unittest.TestCase): ["implementer"], ["claude", "dev"], agent_bottle="claude" ) with patch( - "bot_bottle.cli.start.ManifestIndex.resolve", return_value=manifest + "bot_bottle.cli.commands.start.ManifestIndex.resolve", return_value=manifest ): start_mod.cmd_start(["implementer"]) call_kwargs = self._bottle_picker_mock.call_args @@ -178,7 +178,7 @@ class TestCmdStartSelector(unittest.TestCase): def test_no_agent_bottle_empty_initial(self): manifest = _make_manifest(["researcher"], ["claude", "dev"], agent_bottle="") with patch( - "bot_bottle.cli.start.ManifestIndex.resolve", return_value=manifest + "bot_bottle.cli.commands.start.ManifestIndex.resolve", return_value=manifest ): start_mod.cmd_start(["researcher"]) call_kwargs = self._bottle_picker_mock.call_args @@ -228,19 +228,19 @@ class TestCmdStartLabelCollision(unittest.TestCase): def setUp(self): self._manifest = _make_manifest(["researcher"], ["claude"]) - patch("bot_bottle.cli.start.ManifestIndex.resolve", return_value=self._manifest).start() + patch("bot_bottle.cli.commands.start.ManifestIndex.resolve", return_value=self._manifest).start() self._launch_mock = patch( - "bot_bottle.cli.start._launch_bottle", return_value=0, + "bot_bottle.cli.commands.start._launch_bottle", return_value=0, ).start() # Stub the bottle picker to always return a selection. patch.object(tui_mod, "filter_multiselect", return_value=["claude"]).start() - patch("bot_bottle.cli.start._select_image_policy", return_value="fresh").start() + patch("bot_bottle.cli.commands.start._select_image_policy", return_value="fresh").start() self.addCleanup(patch.stopall) def test_no_collision_proceeds_without_reprompt(self): with ( patch.object(tui_mod, "name_color_modal", return_value=("researcher", "")) as modal, - patch("bot_bottle.cli.start.enumerate_active_agents", return_value=[]), + patch("bot_bottle.cli.commands.start.enumerate_active_agents", return_value=[]), ): rc = start_mod.cmd_start(["researcher"]) self.assertEqual(0, rc) @@ -261,7 +261,7 @@ class TestCmdStartLabelCollision(unittest.TestCase): with ( patch.object(tui_mod, "name_color_modal", side_effect=_modal) as modal, patch( - "bot_bottle.cli.start.enumerate_active_agents", + "bot_bottle.cli.commands.start.enumerate_active_agents", side_effect=[[collision_agent], []], ), ): diff --git a/tests/unit/test_cli_start_settle.py b/tests/unit/test_cli_start_settle.py index 4b1c405b..55666b8f 100644 --- a/tests/unit/test_cli_start_settle.py +++ b/tests/unit/test_cli_start_settle.py @@ -10,7 +10,7 @@ from pathlib import Path from tests.unit import use_bottle_root from bot_bottle import bottle_state -from bot_bottle.cli import start as start_mod +from bot_bottle.cli.commands import start as start_mod class _FakeHomeMixin: diff --git a/tests/unit/test_cli_start_stale.py b/tests/unit/test_cli_start_stale.py index 87625ae3..97651a17 100644 --- a/tests/unit/test_cli_start_stale.py +++ b/tests/unit/test_cli_start_stale.py @@ -56,7 +56,7 @@ class TestLaunchBottleStaleHandling(unittest.TestCase): ) def _run_launch(self, **patch_kwargs: Any) -> int: - import bot_bottle.cli.start as start_mod + import bot_bottle.cli.commands.start as start_mod spec = self._spec() with patch.object(start_mod, "prepare_with_preflight", return_value=(_fake_plan(), "dev-abc")), \ @@ -71,7 +71,7 @@ class TestLaunchBottleStaleHandling(unittest.TestCase): def test_headless_stale_calls_die(self) -> None: """In headless mode (assume_yes=True), a StaleImageError from prelaunch_checks must call die().""" - import bot_bottle.cli.start as start_mod + import bot_bottle.cli.commands.start as start_mod backend_mock = MagicMock() backend_mock.prelaunch_checks.side_effect = StaleImageError("image is 5 day(s) old") @@ -85,7 +85,7 @@ class TestLaunchBottleStaleHandling(unittest.TestCase): def test_interactive_user_declines_stops_before_launch(self) -> None: """Interactive user answering 'n' → launch is never called.""" - import bot_bottle.cli.start as start_mod + import bot_bottle.cli.commands.start as start_mod backend_mock = MagicMock() backend_mock.prelaunch_checks.side_effect = StaleImageError("image is 5 day(s) old") @@ -100,7 +100,7 @@ class TestLaunchBottleStaleHandling(unittest.TestCase): def test_interactive_user_confirms_launches_once(self) -> None: """Interactive user answering 'y' → prelaunch stale error is bypassed; launch called once.""" - import bot_bottle.cli.start as start_mod + import bot_bottle.cli.commands.start as start_mod bottle_mock = MagicMock() bottle_mock.name = "dev-abc" @@ -122,7 +122,7 @@ class TestLaunchBottleStaleHandling(unittest.TestCase): def test_interactive_yes_uppercase_also_accepted(self) -> None: """'Y' or 'YES' should also be accepted as confirmation.""" - import bot_bottle.cli.start as start_mod + import bot_bottle.cli.commands.start as start_mod bottle_mock = MagicMock() bottle_mock.name = "dev-abc" diff --git a/tests/unit/test_supervise_cli.py b/tests/unit/test_supervise_cli.py index f3f44770..1d6679ba 100644 --- a/tests/unit/test_supervise_cli.py +++ b/tests/unit/test_supervise_cli.py @@ -12,7 +12,7 @@ import unittest from datetime import datetime, timezone from unittest.mock import MagicMock, patch -from bot_bottle.cli import supervise as supervise_cli +from bot_bottle.cli.commands import supervise as supervise_cli from bot_bottle.orchestrator.supervisor import ( Proposal, TOOL_EGRESS_ALLOW, diff --git a/tests/unit/test_supervise_cli_crash_logging.py b/tests/unit/test_supervise_cli_crash_logging.py index 0f07f08d..e36c0c34 100644 --- a/tests/unit/test_supervise_cli_crash_logging.py +++ b/tests/unit/test_supervise_cli_crash_logging.py @@ -18,7 +18,7 @@ from pathlib import Path from unittest import mock from tests.unit import use_bottle_root -from bot_bottle.cli import supervise as supervise_cli +from bot_bottle.cli.commands import supervise as supervise_cli from bot_bottle.log import Die, die -- 2.52.0 From a21f2358c625afaa6ba9aea6bab527fa2bbf76c3 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 17:31:14 -0400 Subject: [PATCH 22/30] refactor(cli): trim _common to a PROG constants leaf; inline os.getcwd() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _common had rotted into a junk drawer. Trim it to what actually justifies a shared leaf module and rename for legibility: * REPO_DIR: deleted (dead — nothing imported it). * read_tty_line: dropped the pointless re-export; cleanup/start/init now import it straight from bot_bottle.util. * USER_CWD: deleted. It captured os.getcwd() at import time, but nothing chdirs and no test patched it, so it was equivalent to a live os.getcwd() — inlined at the six call sites. * PROG: kept, now the sole member. It's still a leaf (both the dispatcher and the commands it imports need it) so it can't move into __init__ without a circular import. _common.py -> constants.py. Full unit suite green (2243); CLI dispatch verified. Co-Authored-By: Claude Opus 4.8 --- bot_bottle/cli/__init__.py | 2 +- bot_bottle/cli/_common.py | 12 ------------ bot_bottle/cli/commands/__init__.py | 2 +- bot_bottle/cli/commands/backend.py | 2 +- bot_bottle/cli/commands/cleanup.py | 2 +- bot_bottle/cli/commands/commit.py | 2 +- bot_bottle/cli/commands/edit.py | 4 ++-- bot_bottle/cli/commands/info.py | 5 +++-- bot_bottle/cli/commands/init.py | 5 +++-- bot_bottle/cli/commands/list.py | 4 ++-- bot_bottle/cli/commands/resume.py | 7 ++++--- bot_bottle/cli/commands/start.py | 9 +++++---- bot_bottle/cli/commands/supervise.py | 2 +- bot_bottle/cli/constants.py | 8 ++++++++ 14 files changed, 33 insertions(+), 33 deletions(-) delete mode 100644 bot_bottle/cli/_common.py create mode 100644 bot_bottle/cli/constants.py diff --git a/bot_bottle/cli/__init__.py b/bot_bottle/cli/__init__.py index 41ffa7d4..b26642a7 100644 --- a/bot_bottle/cli/__init__.py +++ b/bot_bottle/cli/__init__.py @@ -11,7 +11,7 @@ from ..errors import MissingEnvVarError from ..log import Die, die, error from ..manifest import ManifestError from ..orchestrator.store.store_manager import StoreManager -from ._common import PROG +from .constants import PROG from .commands import list as _list_mod from .commands.backend import cmd_backend from .commands.cleanup import cmd_cleanup diff --git a/bot_bottle/cli/_common.py b/bot_bottle/cli/_common.py deleted file mode 100644 index 5fc6c332..00000000 --- a/bot_bottle/cli/_common.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Shared constants and tty helper for cli subcommands.""" - -from __future__ import annotations - -import os -from pathlib import Path - -from ..util import read_tty_line as read_tty_line - -PROG = "cli.py" -USER_CWD = os.getcwd() -REPO_DIR = str(Path(__file__).resolve().parent.parent.parent) diff --git a/bot_bottle/cli/commands/__init__.py b/bot_bottle/cli/commands/__init__.py index 0daac76b..7f34d763 100644 --- a/bot_bottle/cli/commands/__init__.py +++ b/bot_bottle/cli/commands/__init__.py @@ -2,5 +2,5 @@ Each module exposes a `cmd_(argv)` handler that the dispatcher (`bot_bottle.cli`) registers in its COMMANDS table. Shared CLI helpers -(`_common`, `tui`) stay one level up in the `cli` package. +(`constants`, `tui`) stay one level up in the `cli` package. """ diff --git a/bot_bottle/cli/commands/backend.py b/bot_bottle/cli/commands/backend.py index 2997db36..a06e44f5 100644 --- a/bot_bottle/cli/commands/backend.py +++ b/bot_bottle/cli/commands/backend.py @@ -16,7 +16,7 @@ from __future__ import annotations import argparse from ...backend import get_bottle_backend, known_backend_names -from .._common import PROG +from ..constants import PROG def cmd_backend(args: list[str]) -> int: diff --git a/bot_bottle/cli/commands/cleanup.py b/bot_bottle/cli/commands/cleanup.py index 685f6bcb..d83ef14c 100644 --- a/bot_bottle/cli/commands/cleanup.py +++ b/bot_bottle/cli/commands/cleanup.py @@ -23,7 +23,7 @@ import sys from ...backend import get_bottle_backend, has_backend, known_backend_names from ...log import info -from .._common import read_tty_line +from ...util import read_tty_line def cmd_cleanup(_argv: list[str]) -> int: diff --git a/bot_bottle/cli/commands/commit.py b/bot_bottle/cli/commands/commit.py index 502ae697..fcf454d2 100644 --- a/bot_bottle/cli/commands/commit.py +++ b/bot_bottle/cli/commands/commit.py @@ -16,7 +16,7 @@ from ...backend import enumerate_active_agents from ...backend.freeze import CommitCancelled, get_freezer from ...bottle_state import read_metadata from ...log import die -from .._common import PROG +from ..constants import PROG from .. import tui diff --git a/bot_bottle/cli/commands/edit.py b/bot_bottle/cli/commands/edit.py index 82ca91b4..86075175 100644 --- a/bot_bottle/cli/commands/edit.py +++ b/bot_bottle/cli/commands/edit.py @@ -8,7 +8,7 @@ import os from pathlib import Path from ...log import die -from .._common import PROG, USER_CWD +from ..constants import PROG def cmd_edit(argv: list[str]) -> int: @@ -20,7 +20,7 @@ def cmd_edit(argv: list[str]) -> int: if args.scope == "user": target_file = Path(os.environ["HOME"]) / "bot-bottle.json" else: - target_file = Path(USER_CWD) / "bot-bottle.json" + target_file = Path(os.getcwd()) / "bot-bottle.json" if not target_file.is_file(): die(f"{target_file} does not exist") diff --git a/bot_bottle/cli/commands/info.py b/bot_bottle/cli/commands/info.py index d15b5612..7850a2d1 100644 --- a/bot_bottle/cli/commands/info.py +++ b/bot_bottle/cli/commands/info.py @@ -3,10 +3,11 @@ from __future__ import annotations import argparse +import os from ...log import info from ...manifest import ManifestIndex -from .._common import PROG, USER_CWD +from ..constants import PROG def cmd_info(argv: list[str]) -> int: @@ -14,7 +15,7 @@ def cmd_info(argv: list[str]) -> int: parser.add_argument("name", help="agent name defined in bot-bottle.json") args = parser.parse_args(argv) - names = ManifestIndex.resolve(USER_CWD) + names = ManifestIndex.resolve(os.getcwd()) names.require_agent(args.name) manifest = names.load_for_agent(args.name) diff --git a/bot_bottle/cli/commands/init.py b/bot_bottle/cli/commands/init.py index ca169f12..34441779 100644 --- a/bot_bottle/cli/commands/init.py +++ b/bot_bottle/cli/commands/init.py @@ -11,7 +11,8 @@ from pathlib import Path from typing import Any from ...log import die, info, warn -from .._common import PROG, USER_CWD, read_tty_line +from ..constants import PROG +from ...util import read_tty_line def cmd_init(argv: list[str]) -> int: @@ -22,7 +23,7 @@ def cmd_init(argv: list[str]) -> int: if args.scope == "user": target_file = Path(os.environ["HOME"]) / "bot-bottle.json" else: - target_file = Path(USER_CWD) / "bot-bottle.json" + target_file = Path(os.getcwd()) / "bot-bottle.json" print(file=sys.stderr) info(f"bot-bottle init — adding a new agent to {target_file}") diff --git a/bot_bottle/cli/commands/list.py b/bot_bottle/cli/commands/list.py index 5c0caf35..6b83c4eb 100644 --- a/bot_bottle/cli/commands/list.py +++ b/bot_bottle/cli/commands/list.py @@ -8,7 +8,7 @@ import sys from ...backend import enumerate_active_agents from ...manifest import ManifestIndex -from .._common import PROG, USER_CWD +from ..constants import PROG _ANSI_COLOR_CODES: dict[str, str] = { "red": "\033[91m", @@ -40,7 +40,7 @@ def cmd_list(argv: list[str]) -> int: args = parser.parse_args(argv) if args.scope == "available": - manifest = ManifestIndex.resolve(USER_CWD) + manifest = ManifestIndex.resolve(os.getcwd()) for name in manifest.all_agent_names: print(name) return 0 diff --git a/bot_bottle/cli/commands/resume.py b/bot_bottle/cli/commands/resume.py index 6d8224af..e3b6bc28 100644 --- a/bot_bottle/cli/commands/resume.py +++ b/bot_bottle/cli/commands/resume.py @@ -15,12 +15,13 @@ to bring up the replacement from the recorded state. from __future__ import annotations import argparse +import os from ...backend import BottleSpec from ...bottle_state import read_metadata from ...log import die from ...manifest import ManifestIndex -from .._common import PROG, USER_CWD +from ..constants import PROG from .start import _launch_bottle @@ -40,14 +41,14 @@ def cmd_resume(argv: list[str]) -> int: f"check ~/.bot-bottle/state/ or run `cli.py start` to create a new bottle" ) - manifest = ManifestIndex.resolve(USER_CWD) + manifest = ManifestIndex.resolve(os.getcwd()) manifest.require_agent(metadata.agent_name) spec = BottleSpec( manifest=manifest, agent_name=metadata.agent_name, copy_cwd=metadata.copy_cwd, - user_cwd=metadata.cwd or USER_CWD, + user_cwd=metadata.cwd or os.getcwd(), identity=metadata.identity, bottle_names=tuple(metadata.bottle_names), ) diff --git a/bot_bottle/cli/commands/start.py b/bot_bottle/cli/commands/start.py index 24c2e9c4..2082f928 100644 --- a/bot_bottle/cli/commands/start.py +++ b/bot_bottle/cli/commands/start.py @@ -39,7 +39,8 @@ from ...bottle_state import ( from ...image_cache import StaleImageError from ...log import info, die from ...manifest import Manifest, ManifestIndex -from .._common import PROG, USER_CWD, read_tty_line +from ..constants import PROG +from ...util import read_tty_line from .. import tui @@ -116,7 +117,7 @@ def cmd_start(argv: list[str]) -> int: # threading a no_cache field through every backend's plan dataclass. os.environ["BOT_BOTTLE_NO_CACHE"] = "1" - manifest = ManifestIndex.resolve(USER_CWD) + manifest = ManifestIndex.resolve(os.getcwd()) if args.headless: return _start_headless( @@ -167,7 +168,7 @@ def cmd_start(argv: list[str]) -> int: manifest=manifest, agent_name=agent_name, copy_cwd=args.cwd, - user_cwd=USER_CWD, + user_cwd=os.getcwd(), label=label, color=color, bottle_names=bottle_names, @@ -235,7 +236,7 @@ def _start_headless( manifest=manifest, agent_name=agent_name, copy_cwd=args.cwd, - user_cwd=USER_CWD, + user_cwd=os.getcwd(), label=label, color=args.color or "", bottle_names=bottle_names, diff --git a/bot_bottle/cli/commands/supervise.py b/bot_bottle/cli/commands/supervise.py index c2beb030..cbb349f4 100644 --- a/bot_bottle/cli/commands/supervise.py +++ b/bot_bottle/cli/commands/supervise.py @@ -34,7 +34,7 @@ from ...supervisor.types import ( TOOL_GITLEAKS_ALLOW, TOOL_EGRESS_TOKEN_ALLOW, ) -from .._common import PROG +from ..constants import PROG _REFRESH_INTERVAL_MS = 1000 diff --git a/bot_bottle/cli/constants.py b/bot_bottle/cli/constants.py new file mode 100644 index 00000000..4df182ea --- /dev/null +++ b/bot_bottle/cli/constants.py @@ -0,0 +1,8 @@ +"""Shared CLI constants. + +Kept as a leaf module (imports nothing from the `cli` package) so both the +dispatcher (`cli/__init__.py`) and the command modules it imports can share +`PROG` without a circular import. +""" + +PROG = "cli.py" -- 2.52.0 From 3ccd308613da31a192884362ba787aa0baffe95f Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 17:40:05 -0400 Subject: [PATCH 23/30] refactor(cli): make help a first-class command; drop usage() from dispatcher Move the top-level usage/command-list text out of the dispatcher into commands/help.py as cmd_help, registered in COMMANDS so `bb help` now works as a real command. The dispatcher's -h/--help, no-args, and unknown-command fallbacks call cmd_help() for the text while keeping their exit codes (0 for help/-h, 2 for bare no-args, Die for unknown). Added `help` to NO_MIGRATION_COMMANDS so it prints without a migrated DB, and listed it in the command summary. Full unit suite green (2243); dispatch + migration-gate tests pass, `bb help` verified. Co-Authored-By: Claude Opus 4.8 --- bot_bottle/cli/__init__.py | 39 +++++++-------------------------- bot_bottle/cli/commands/help.py | 37 +++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 31 deletions(-) create mode 100644 bot_bottle/cli/commands/help.py diff --git a/bot_bottle/cli/__init__.py b/bot_bottle/cli/__init__.py index b26642a7..6a1ea3bd 100644 --- a/bot_bottle/cli/__init__.py +++ b/bot_bottle/cli/__init__.py @@ -1,6 +1,7 @@ """Main CLI dispatcher. -Commands: backend, cleanup, commit, edit, info, init, list, resume, start, supervise +Commands: backend, cleanup, commit, edit, help, info, init, list, login, +resume, start, supervise """ from __future__ import annotations @@ -17,6 +18,7 @@ from .commands.backend import cmd_backend from .commands.cleanup import cmd_cleanup from .commands.commit import cmd_commit from .commands.edit import cmd_edit +from .commands.help import cmd_help from .commands.info import cmd_info from .commands.init import cmd_init from .commands.login import cmd_login @@ -31,6 +33,7 @@ COMMANDS = { "cleanup": cmd_cleanup, "commit": cmd_commit, "edit": cmd_edit, + "help": cmd_help, "info": cmd_info, "init": cmd_init, "list": cmd_list, @@ -45,49 +48,23 @@ COMMANDS = { # the host (TAP pool, /dev/kvm, firecracker) and never opens the store, so # gating it on the schema breaks preflight on a fresh CI runner where stdin # isn't a TTY and the migration prompt can't be answered. -NO_MIGRATION_COMMANDS = frozenset({"backend", "login"}) - - -def usage() -> None: - sys.stderr.write(f"usage: {PROG} [args...]\n\n") - sys.stderr.write("Commands:\n") - sys.stderr.write(" backend set up / check / undo a backend's host prerequisites (setup|status|teardown)\n") - sys.stderr.write(" cleanup stop and remove all active bot-bottle containers\n") - sys.stderr.write(" commit snapshot a running bottle's container state to a Docker image\n") - sys.stderr.write(" edit open an agent in vim for editing\n") - sys.stderr.write(" info print env, skills, and prompt details for a named agent\n") - sys.stderr.write(" init interactively create a new agent and add it to bot-bottle.json\n") - sys.stderr.write(" list list available agents or active containers\n") - sys.stderr.write(" login register this host with a bot-bottle console\n") - sys.stderr.write( - " resume re-launch a bottle by its identity " - "(continues state from PRD 0016)\n" - ) - sys.stderr.write( - " start boot a container for a named agent and " - "attach an interactive session\n" - ) - sys.stderr.write( - " supervise view + approve/modify/reject pending supervise " - "proposals (PRD 0013)\n\n" - ) - sys.stderr.write(f"Run '{PROG} --help' for command-specific usage.\n") +NO_MIGRATION_COMMANDS = frozenset({"backend", "help", "login"}) def main(argv: list[str] | None = None) -> int: if argv is None: argv = sys.argv[1:] if not argv: - usage() + cmd_help() return 2 command = argv[0] rest = argv[1:] if command in ("-h", "--help"): - usage() + cmd_help() return 0 handler = COMMANDS.get(command) if handler is None: - usage() + cmd_help() die(f"unknown command: {command}") mgr = StoreManager.instance() if command not in NO_MIGRATION_COMMANDS and not mgr.is_migrated(): diff --git a/bot_bottle/cli/commands/help.py b/bot_bottle/cli/commands/help.py new file mode 100644 index 00000000..d7ee6330 --- /dev/null +++ b/bot_bottle/cli/commands/help.py @@ -0,0 +1,37 @@ +"""help: print the top-level command list and usage. + +Rendered by the dispatcher for the `help` command and for its +`-h`/`--help`, no-args, and unknown-command fallbacks. The per-command +summaries live here; keep them in sync with the COMMANDS table in +`bot_bottle.cli`. +""" + +from __future__ import annotations + +import sys + +from ..constants import PROG + + +def cmd_help(argv: list[str] | None = None) -> int: + """Write the top-level usage + command list to stderr. Returns 0; + the dispatcher chooses the process exit code per entry path (0 for an + explicit `help`/`-h`, 2 for the bare no-args usage error).""" + del argv # help takes no arguments + w = sys.stderr.write + w(f"usage: {PROG} [args...]\n\n") + w("Commands:\n") + w(" backend set up / check / undo a backend's host prerequisites (setup|status|teardown)\n") + w(" cleanup stop and remove all active bot-bottle containers\n") + w(" commit snapshot a running bottle's container state to a Docker image\n") + w(" edit open an agent in vim for editing\n") + w(" help show this command list\n") + w(" info print env, skills, and prompt details for a named agent\n") + w(" init interactively create a new agent and add it to bot-bottle.json\n") + w(" list list available agents or active containers\n") + w(" login register this host with a bot-bottle console\n") + w(" resume re-launch a bottle by its identity (continues state from PRD 0016)\n") + w(" start boot a container for a named agent and attach an interactive session\n") + w(" supervise view + approve/modify/reject pending supervise proposals (PRD 0013)\n\n") + w(f"Run '{PROG} --help' for command-specific usage.\n") + return 0 -- 2.52.0 From b276227cbbf54e067e6100fd2a181d730a73ac03 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 17:52:52 -0400 Subject: [PATCH 24/30] refactor(cli): registry in commands/__init__, dispatcher main() in __main__ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split the cli package's two responsibilities out of __init__: * commands/__init__.py now assembles the COMMANDS registry and NO_MIGRATION_COMMANDS from the per-command modules — the command surface lives entirely under commands/. * __main__.py now owns main() (dispatch + migration gate + exit-code mapping) alongside the runnable entry guard. cli/__init__.py shrinks to a shim that re-exports main / COMMANDS / NO_MIGRATION_COMMANDS, so bot_bottle.cli.main (repo-root cli.py entry) and the tests' bot_bottle.cli.COMMANDS keep working unchanged. Verified `python -m bot_bottle.cli` (runpy) and `cli.py` both dispatch; full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/cli/__init__.py | 96 +++-------------------------- bot_bottle/cli/__main__.py | 60 ++++++++++++++++-- bot_bottle/cli/commands/__init__.py | 47 +++++++++++++- 3 files changed, 108 insertions(+), 95 deletions(-) diff --git a/bot_bottle/cli/__init__.py b/bot_bottle/cli/__init__.py index 6a1ea3bd..56a73d01 100644 --- a/bot_bottle/cli/__init__.py +++ b/bot_bottle/cli/__init__.py @@ -1,93 +1,15 @@ -"""Main CLI dispatcher. +"""bot-bottle CLI package. -Commands: backend, cleanup, commit, edit, help, info, init, list, login, -resume, start, supervise +The subcommand handlers live in `commands/` and are assembled into the +COMMANDS registry by `commands/__init__.py`; the dispatcher `main()` lives +in `__main__.py`. They are re-exported here so `bot_bottle.cli.main`, +`bot_bottle.cli.COMMANDS`, and `bot_bottle.cli.NO_MIGRATION_COMMANDS` stay +importable (the repo-root `cli.py` entry point and the tests use them). """ from __future__ import annotations -import sys +from .__main__ import main +from .commands import COMMANDS, NO_MIGRATION_COMMANDS -from ..errors import MissingEnvVarError -from ..log import Die, die, error -from ..manifest import ManifestError -from ..orchestrator.store.store_manager import StoreManager -from .constants import PROG -from .commands import list as _list_mod -from .commands.backend import cmd_backend -from .commands.cleanup import cmd_cleanup -from .commands.commit import cmd_commit -from .commands.edit import cmd_edit -from .commands.help import cmd_help -from .commands.info import cmd_info -from .commands.init import cmd_init -from .commands.login import cmd_login -from .commands.resume import cmd_resume -from .commands.start import cmd_start -from .commands.supervise import cmd_supervise - -cmd_list = _list_mod.cmd_list - -COMMANDS = { - "backend": cmd_backend, - "cleanup": cmd_cleanup, - "commit": cmd_commit, - "edit": cmd_edit, - "help": cmd_help, - "info": cmd_info, - "init": cmd_init, - "list": cmd_list, - "login": cmd_login, - "resume": cmd_resume, - "start": cmd_start, - "supervise": cmd_supervise, -} - -# Commands that manage host prerequisites (or are otherwise store-free) and -# must run before — or without — a migrated DB. `backend` provisions/probes -# the host (TAP pool, /dev/kvm, firecracker) and never opens the store, so -# gating it on the schema breaks preflight on a fresh CI runner where stdin -# isn't a TTY and the migration prompt can't be answered. -NO_MIGRATION_COMMANDS = frozenset({"backend", "help", "login"}) - - -def main(argv: list[str] | None = None) -> int: - if argv is None: - argv = sys.argv[1:] - if not argv: - cmd_help() - return 2 - command = argv[0] - rest = argv[1:] - if command in ("-h", "--help"): - cmd_help() - return 0 - handler = COMMANDS.get(command) - if handler is None: - cmd_help() - die(f"unknown command: {command}") - mgr = StoreManager.instance() - if command not in NO_MIGRATION_COMMANDS and not mgr.is_migrated(): - sys.stderr.write("bot-bottle: database schema is out of date\n") - sys.stderr.write("Migrate now? [y/N] ") - sys.stderr.flush() - try: - answer = sys.stdin.readline().strip().lower() - except EOFError: - answer = "" - if answer != "y": - error("migration required — re-run and confirm to migrate") - return 1 - mgr.migrate() - try: - return handler(rest) or 0 - except MissingEnvVarError as e: - error(str(e)) - return 1 - except ManifestError as e: - error(str(e)) - return 1 - except Die as e: - return e.code if isinstance(e.code, int) else 1 - except KeyboardInterrupt: - return 130 +__all__ = ["main", "COMMANDS", "NO_MIGRATION_COMMANDS"] diff --git a/bot_bottle/cli/__main__.py b/bot_bottle/cli/__main__.py index 3cf5f19f..91a7e5aa 100644 --- a/bot_bottle/cli/__main__.py +++ b/bot_bottle/cli/__main__.py @@ -1,15 +1,65 @@ -"""Entry point for `python -m bot_bottle.cli`. +"""Entry point + dispatcher for `python -m bot_bottle.cli`. -`cli.py` at the repo root is the usual way in; this makes the package -runnable too, so the CLI works from an installed copy where there is no -`cli.py` on disk to point at. +Maps `bot-bottle ` to its handler in the COMMANDS registry +(`bot_bottle.cli.commands`), enforces the schema-migration gate, and +translates handler exceptions into process exit codes. The repo-root +`cli.py` is the usual way in; this makes the package runnable too, so the +CLI works from an installed copy where there is no `cli.py` on disk. """ from __future__ import annotations import sys -from . import main +from ..errors import MissingEnvVarError +from ..log import Die, die, error +from ..manifest import ManifestError +from ..orchestrator.store.store_manager import StoreManager +from .commands import COMMANDS, NO_MIGRATION_COMMANDS +from .commands.help import cmd_help + + +def main(argv: list[str] | None = None) -> int: + if argv is None: + argv = sys.argv[1:] + if not argv: + cmd_help() + return 2 + command = argv[0] + rest = argv[1:] + if command in ("-h", "--help"): + cmd_help() + return 0 + handler = COMMANDS.get(command) + if handler is None: + cmd_help() + die(f"unknown command: {command}") + mgr = StoreManager.instance() + if command not in NO_MIGRATION_COMMANDS and not mgr.is_migrated(): + sys.stderr.write("bot-bottle: database schema is out of date\n") + sys.stderr.write("Migrate now? [y/N] ") + sys.stderr.flush() + try: + answer = sys.stdin.readline().strip().lower() + except EOFError: + answer = "" + if answer != "y": + error("migration required — re-run and confirm to migrate") + return 1 + mgr.migrate() + try: + return handler(rest) or 0 + except MissingEnvVarError as e: + error(str(e)) + return 1 + except ManifestError as e: + error(str(e)) + return 1 + except Die as e: + return e.code if isinstance(e.code, int) else 1 + except KeyboardInterrupt: + return 130 + if __name__ == "__main__": sys.exit(main()) diff --git a/bot_bottle/cli/commands/__init__.py b/bot_bottle/cli/commands/__init__.py index 7f34d763..cda230d9 100644 --- a/bot_bottle/cli/commands/__init__.py +++ b/bot_bottle/cli/commands/__init__.py @@ -1,6 +1,47 @@ -"""CLI subcommand handlers — one module per `bot-bottle `. +"""CLI subcommand registry. -Each module exposes a `cmd_(argv)` handler that the dispatcher -(`bot_bottle.cli`) registers in its COMMANDS table. Shared CLI helpers +One module per `bot-bottle `, each exposing a `cmd_(argv)` +handler. This package `__init__` assembles them into the COMMANDS table +the dispatcher (`bot_bottle.cli.__main__`) reads. Shared CLI helpers (`constants`, `tui`) stay one level up in the `cli` package. """ + +from __future__ import annotations + +from .backend import cmd_backend +from .cleanup import cmd_cleanup +from .commit import cmd_commit +from .edit import cmd_edit +from .help import cmd_help +from .info import cmd_info +from .init import cmd_init +from .list import cmd_list +from .login import cmd_login +from .resume import cmd_resume +from .start import cmd_start +from .supervise import cmd_supervise + +COMMANDS = { + "backend": cmd_backend, + "cleanup": cmd_cleanup, + "commit": cmd_commit, + "edit": cmd_edit, + "help": cmd_help, + "info": cmd_info, + "init": cmd_init, + "list": cmd_list, + "login": cmd_login, + "resume": cmd_resume, + "start": cmd_start, + "supervise": cmd_supervise, +} + +# Commands that manage host prerequisites (or are otherwise store-free) and +# must run before — or without — a migrated DB. `backend` provisions/probes +# the host (TAP pool, /dev/kvm, firecracker) and never opens the store, so +# gating it on the schema breaks preflight on a fresh CI runner where stdin +# isn't a TTY and the migration prompt can't be answered. `help` and `login` +# likewise never touch the store. +NO_MIGRATION_COMMANDS = frozenset({"backend", "help", "login"}) + +__all__ = ["COMMANDS", "NO_MIGRATION_COMMANDS"] -- 2.52.0 From 4166057abc2bcf6ba53f1c286784b4c752121e0a Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 17:57:49 -0400 Subject: [PATCH 25/30] refactor(orchestrator): rename control_plane module to server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit orchestrator/control_plane.py -> orchestrator/server.py. Within the orchestrator package the "control_plane" filename stutters (the orchestrator *is* the control plane), and `orchestrator.server` reads as "the orchestrator's HTTP server", pairing with service.py (domain logic) and matching gateway/supervisor/server.py. Scope is the module name only. The class ControlPlaneServer, the CONTROL_AUTH_HEADER constant, and the "control plane" architectural term (CONTROL_PLANE_PORT, host_control_plane_token, control_plane_url, …) are deliberately unchanged — that's the load-bearing control-plane/data-plane distinction. Test file renamed to match; full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/control_auth.py | 4 ++-- bot_bottle/orchestrator/__init__.py | 2 +- bot_bottle/orchestrator/__main__.py | 2 +- bot_bottle/orchestrator/client.py | 2 +- bot_bottle/orchestrator/{control_plane.py => server.py} | 0 bot_bottle/paths.py | 2 +- ...hestrator_control_plane.py => test_orchestrator_server.py} | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) rename bot_bottle/orchestrator/{control_plane.py => server.py} (100%) rename tests/unit/{test_orchestrator_control_plane.py => test_orchestrator_server.py} (99%) diff --git a/bot_bottle/control_auth.py b/bot_bottle/control_auth.py index 23cf3513..54f47620 100644 --- a/bot_bottle/control_auth.py +++ b/bot_bottle/control_auth.py @@ -15,8 +15,8 @@ Only the orchestrator (and the host CLI, which shares the host trust domain) holds the signing key; the gateway is handed a pre-minted ``gateway`` token it cannot rewrite into a ``cli`` token. So a compromised data-plane process can no longer approve its own supervise proposals or drive operator routes — it can -only present the ``gateway`` role it was issued (control_plane rejects it on -operator routes with 403). +only present the ``gateway`` role it was issued (the control plane rejects it +on operator routes with 403). Stdlib-only (HMAC-SHA256 over a JSON payload); no JWT dependency — the project carries no runtime pip deps. Tokens are **signed, not encrypted** (the role is diff --git a/bot_bottle/orchestrator/__init__.py b/bot_bottle/orchestrator/__init__.py index 5346c004..8a0a9da7 100644 --- a/bot_bottle/orchestrator/__init__.py +++ b/bot_bottle/orchestrator/__init__.py @@ -40,7 +40,7 @@ from .broker import ( from .docker_broker import DockerBroker, DockerBrokerError from ..gateway import Gateway, GatewayError from .service import Orchestrator -from .control_plane import ControlPlaneServer, dispatch, make_server +from .server import ControlPlaneServer, dispatch, make_server __all__ = [ "BottleRecord", diff --git a/bot_bottle/orchestrator/__main__.py b/bot_bottle/orchestrator/__main__.py index cd532f9f..0370c832 100644 --- a/bot_bottle/orchestrator/__main__.py +++ b/bot_bottle/orchestrator/__main__.py @@ -18,7 +18,7 @@ from pathlib import Path from .. import log from .store.store_manager import StoreManager from .broker import LaunchBroker, StubBroker -from .control_plane import make_server +from .server import make_server from .docker_broker import DockerBroker from .registry import RegistryStore, default_db_path from .service import Orchestrator diff --git a/bot_bottle/orchestrator/client.py b/bot_bottle/orchestrator/client.py index a349c645..fb80c5ee 100644 --- a/bot_bottle/orchestrator/client.py +++ b/bot_bottle/orchestrator/client.py @@ -20,7 +20,7 @@ from dataclasses import dataclass from ..control_auth import ROLE_CLI, mint from ..paths import host_control_plane_token -from .control_plane import CONTROL_AUTH_HEADER +from .server import CONTROL_AUTH_HEADER DEFAULT_TIMEOUT_SECONDS = 5.0 diff --git a/bot_bottle/orchestrator/control_plane.py b/bot_bottle/orchestrator/server.py similarity index 100% rename from bot_bottle/orchestrator/control_plane.py rename to bot_bottle/orchestrator/server.py diff --git a/bot_bottle/paths.py b/bot_bottle/paths.py index 629edb55..e1723c44 100644 --- a/bot_bottle/paths.py +++ b/bot_bottle/paths.py @@ -33,7 +33,7 @@ HOST_DB_FILENAME = "bot-bottle.db" # The per-host control-plane secret file, and the env var the launchers inject # its value into. The control plane requires this secret on every mutating / -# reading route (see orchestrator/control_plane.py); it is held only by the +# reading route (see orchestrator/server.py); it is held only by the # trusted callers (control plane, gateway, host CLI) and never handed to an # agent, so an agent that can reach the control-plane port still can't drive it. CONTROL_PLANE_TOKEN_FILENAME = "control-plane-token" diff --git a/tests/unit/test_orchestrator_control_plane.py b/tests/unit/test_orchestrator_server.py similarity index 99% rename from tests/unit/test_orchestrator_control_plane.py rename to tests/unit/test_orchestrator_server.py index 8210df51..2d5c7361 100644 --- a/tests/unit/test_orchestrator_control_plane.py +++ b/tests/unit/test_orchestrator_server.py @@ -21,7 +21,7 @@ from unittest.mock import patch from bot_bottle.control_auth import ROLE_CLI, ROLE_GATEWAY, mint from bot_bottle.orchestrator.broker import StubBroker -from bot_bottle.orchestrator.control_plane import dispatch, make_server +from bot_bottle.orchestrator.server import dispatch, make_server from bot_bottle.orchestrator.registry import BottleRecord, RegistryStore from bot_bottle.orchestrator.service import Orchestrator from bot_bottle.orchestrator.store.store_manager import StoreManager -- 2.52.0 From ca1d341d4ffded224c5579268b1d19b51e8b60ea Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 18:17:48 -0400 Subject: [PATCH 26/30] =?UTF-8?q?refactor:=20unify=20component=20naming=20?= =?UTF-8?q?=E2=80=94=20control=5Fplane/control=5Fauth=20->=20orchestrator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The codebase used "control plane" both as an architectural role term AND as an identifier alias for the orchestrator component, producing duplicate names for one thing (control_plane_url vs orchestrator_url, CONTROL_PLANE_PORT, host_control_plane_token, …). Going forward the concrete component is always named for what it is — Gateway or Orchestrator — and the plane vocabulary is reserved for prose (module descriptions, the security argument). Renamed (identifiers + the in-repo env/wire/file string values, all setters/getters are in this repo so the change is atomic): ControlPlaneServer -> OrchestratorServer control_plane_url -> orchestrator_url probe_control_plane_url -> probe_orchestrator_url host_control_plane_token -> host_orchestrator_token CONTROL_PLANE_PORT -> ORCHESTRATOR_PORT CONTROL_PLANE_TOKEN_ENV/FILE -> ORCHESTRATOR_TOKEN_ENV/FILENAME BOT_BOTTLE_CONTROL_PLANE_TOKEN-> BOT_BOTTLE_ORCHESTRATOR_TOKEN control-plane-token (file) -> orchestrator-token control_auth (module) -> orchestrator_auth (stays top-level; the gateway imports it and must not import the orchestrator/ package) CONTROL_AUTH_HEADER -> ORCHESTRATOR_AUTH_HEADER x-bot-bottle-control-auth -> x-bot-bottle-orchestrator-auth CONTROL_AUTH_JWT_ENV -> ORCHESTRATOR_AUTH_JWT_ENV BOT_BOTTLE_CONTROL_AUTH_JWT -> BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT _control_auth_headers -> _orchestrator_auth_headers Prose plane-terms ("control plane", "data plane") are preserved, including the test name test_data_plane_daemons_get_jwt_not_key (it names the security invariant). Gateway and orchestrator verified to agree on the renamed wire header; full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/backend/docker/gateway.py | 10 +++--- bot_bottle/backend/docker/infra.py | 18 +++++------ bot_bottle/backend/firecracker/backend.py | 2 +- .../firecracker/consolidated_launch.py | 2 +- bot_bottle/backend/firecracker/infra_vm.py | 32 +++++++++---------- bot_bottle/backend/macos_container/backend.py | 2 +- .../macos_container/consolidated_launch.py | 2 +- bot_bottle/backend/macos_container/infra.py | 26 +++++++-------- bot_bottle/gateway/bootstrap.py | 8 ++--- bot_bottle/gateway/policy_resolver.py | 12 +++---- bot_bottle/orchestrator/__init__.py | 6 ++-- bot_bottle/orchestrator/client.py | 18 +++++------ bot_bottle/orchestrator/server.py | 32 +++++++++---------- .../{control_auth.py => orchestrator_auth.py} | 0 bot_bottle/paths.py | 23 +++++++------ ...th.py => test_orchestrator_docker_auth.py} | 12 +++---- tests/unit/test_backend_selection.py | 4 +-- tests/unit/test_firecracker_infra_vm.py | 16 +++++----- tests/unit/test_gateway_init.py | 14 ++++---- tests/unit/test_macos_consolidated_launch.py | 4 +-- tests/unit/test_macos_infra.py | 10 +++--- ...trol_auth.py => test_orchestrator_auth.py} | 8 ++--- tests/unit/test_orchestrator_client.py | 6 ++-- tests/unit/test_orchestrator_server.py | 10 +++--- tests/unit/test_policy_resolver.py | 16 +++++----- 25 files changed, 146 insertions(+), 147 deletions(-) rename bot_bottle/{control_auth.py => orchestrator_auth.py} (100%) rename tests/integration/{test_orchestrator_docker_control_plane_auth.py => test_orchestrator_docker_auth.py} (95%) rename tests/unit/{test_control_auth.py => test_orchestrator_auth.py} (91%) diff --git a/bot_bottle/backend/docker/gateway.py b/bot_bottle/backend/docker/gateway.py index c45ff8ee..221de210 100644 --- a/bot_bottle/backend/docker/gateway.py +++ b/bot_bottle/backend/docker/gateway.py @@ -4,11 +4,11 @@ import os import time from pathlib import Path -from ...control_auth import ROLE_GATEWAY, mint +from ...orchestrator_auth import ROLE_GATEWAY, mint from .util import run_docker from ...paths import ( - CONTROL_AUTH_JWT_ENV, - host_control_plane_token, + ORCHESTRATOR_AUTH_JWT_ENV, + host_orchestrator_token, host_gateway_ca_dir, ) from ...gateway import ( @@ -158,8 +158,8 @@ class DockerGateway(Gateway): # a `cli` token, so a compromised data-plane process can't drive the # operator routes (issue #469 review). Bare `--env NAME` keeps the value # off argv / `docker inspect`; only the gateway (not the agent) is given it. - argv += ["--env", CONTROL_AUTH_JWT_ENV] - run_env[CONTROL_AUTH_JWT_ENV] = mint(ROLE_GATEWAY, host_control_plane_token()) + argv += ["--env", ORCHESTRATOR_AUTH_JWT_ENV] + run_env[ORCHESTRATOR_AUTH_JWT_ENV] = mint(ROLE_GATEWAY, host_orchestrator_token()) argv.append(self.image_ref) proc = run_docker(argv, env=run_env) if proc.returncode != 0: diff --git a/bot_bottle/backend/docker/infra.py b/bot_bottle/backend/docker/infra.py index 5a1709eb..d9d119d9 100644 --- a/bot_bottle/backend/docker/infra.py +++ b/bot_bottle/backend/docker/infra.py @@ -25,13 +25,13 @@ import urllib.request from pathlib import Path from ... import log -from ...control_auth import ROLE_GATEWAY, mint +from ...orchestrator_auth import ROLE_GATEWAY, mint from .util import run_docker from ...paths import ( - CONTROL_AUTH_JWT_ENV, - CONTROL_PLANE_TOKEN_ENV, + ORCHESTRATOR_AUTH_JWT_ENV, + ORCHESTRATOR_TOKEN_ENV, bot_bottle_root, - host_control_plane_token, + host_orchestrator_token, host_gateway_ca_dir, ) from ...gateway import ( @@ -176,7 +176,7 @@ class DockerInfraService: so a later `ensure_running` can detect a real code change.""" self._ensure_network() run_docker(["docker", "rm", "--force", self._infra_name]) - _signing_key = host_control_plane_token() + _signing_key = host_orchestrator_token() proc = run_docker([ "docker", "run", "--detach", "--name", self._infra_name, @@ -204,8 +204,8 @@ class DockerInfraService: # pre-minted `gateway` JWT (data-plane daemons: present it). gateway_init # scopes each to its process, so a compromised data-plane daemon never # sees the key and can't mint a `cli` token (issue #469 review). - "--env", CONTROL_PLANE_TOKEN_ENV, - "--env", CONTROL_AUTH_JWT_ENV, + "--env", ORCHESTRATOR_TOKEN_ENV, + "--env", ORCHESTRATOR_AUTH_JWT_ENV, # Gateway daemons reach the orchestrator over loopback at its # fixed internal port (DEFAULT_PORT), independent of self.port. "--env", f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{DEFAULT_PORT}", @@ -214,8 +214,8 @@ class DockerInfraService: self.image, ], env={ **os.environ, - CONTROL_PLANE_TOKEN_ENV: _signing_key, - CONTROL_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key), + ORCHESTRATOR_TOKEN_ENV: _signing_key, + ORCHESTRATOR_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key), }) if proc.returncode != 0: raise OrchestratorStartError( diff --git a/bot_bottle/backend/firecracker/backend.py b/bot_bottle/backend/firecracker/backend.py index a5281822..41193838 100644 --- a/bot_bottle/backend/firecracker/backend.py +++ b/bot_bottle/backend/firecracker/backend.py @@ -120,4 +120,4 @@ class FirecrackerBottleBackend( def ensure_orchestrator(self) -> str: from . import infra_vm - return infra_vm.ensure_running().control_plane_url + return infra_vm.ensure_running().orchestrator_url diff --git a/bot_bottle/backend/firecracker/consolidated_launch.py b/bot_bottle/backend/firecracker/consolidated_launch.py index 503d0af5..24e8232f 100644 --- a/bot_bottle/backend/firecracker/consolidated_launch.py +++ b/bot_bottle/backend/firecracker/consolidated_launch.py @@ -124,7 +124,7 @@ def launch_consolidated( provision its git-gate state into the gateway VM. Returns the context the agent-VM launch needs. Raises on failure — the caller tears down.""" infra = infra_vm.ensure_running() - url = infra.control_plane_url + url = infra.orchestrator_url client = OrchestratorClient(url) _reprovision_running_bottles(client) diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py index 1542a600..cf644ab5 100644 --- a/bot_bottle/backend/firecracker/infra_vm.py +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -33,7 +33,7 @@ from pathlib import Path from typing import Generator from ...log import die, info -from ...paths import CONTROL_PLANE_TOKEN_FILENAME, bot_bottle_root +from ...paths import ORCHESTRATOR_TOKEN_FILENAME, bot_bottle_root from .. import util as backend_util from ..docker import util as docker_mod from ..docker.gateway_provision import GatewayProvisionError @@ -42,7 +42,7 @@ from . import firecracker_vm, infra_artifact, netpool, util # Where the infra VM keeps its control-plane signing key (generated on the # persistent /dev/vdb volume mounted at BOT_BOTTLE_ROOT). The host mirrors it # back so the CLI signs `cli` tokens the VM verifies (issue #469 review). -_GUEST_SIGNING_KEY_PATH = "/var/lib/bot-bottle/control-plane-token" +_GUEST_SIGNING_KEY_PATH = "/var/lib/bot-bottle/orchestrator-token" # The single infra-VM image: gateway data plane + baked control-plane source # (Dockerfile.infra FROM the gateway image). Built from source by default; @@ -52,7 +52,7 @@ _GATEWAY_IMAGE = "bot-bottle-gateway:latest" _ORCHESTRATOR_IMAGE = "bot-bottle-orchestrator:latest" _REPO_ROOT = Path(__file__).resolve().parents[3] -CONTROL_PLANE_PORT = 8099 +ORCHESTRATOR_PORT = 8099 # Gateway data-plane ports (agent-facing): egress proxy, supervise MCP, # git-http. Reached by agent VMs over VM-to-VM routing (added next). EGRESS_PORT = 9099 @@ -84,8 +84,8 @@ class InfraVm: vm: firecracker_vm.VmHandle | None = None @property - def control_plane_url(self) -> str: - return f"http://{self.guest_ip}:{CONTROL_PLANE_PORT}" + def orchestrator_url(self) -> str: + return f"http://{self.guest_ip}:{ORCHESTRATOR_PORT}" def terminate(self) -> None: """Stop the infra VM — via the live handle if we booted it, else the @@ -168,7 +168,7 @@ def ensure_running() -> InfraVm: flock, so two simultaneous first launches don't both boot on the same rootfs/PID. The healthy fast-path takes no lock.""" slot = netpool.orch_slot() - url = f"http://{slot.guest_ip}:{CONTROL_PLANE_PORT}" + url = f"http://{slot.guest_ip}:{ORCHESTRATOR_PORT}" key = _infra_dir() / "id_ed25519" want = _expected_version() if _adoptable(key, url, want): @@ -192,7 +192,7 @@ def ensure_running() -> InfraVm: def _with_signing_key(infra: InfraVm) -> InfraVm: """Mirror the infra VM's control-plane signing key (generated on its - persistent volume) into the host's control-plane-token file, so the host CLI + persistent volume) into the host's orchestrator-token file, so the host CLI signs `cli` tokens the VM verifies (issue #469 review). Best-effort: an unreadable key is logged, not fatal — the VM still enforces auth, but the CLI may then be rejected until the key is readable. Returns `infra` for chaining.""" @@ -205,7 +205,7 @@ def _with_signing_key(infra: InfraVm) -> InfraVm: if proc.returncode != 0 or not signing_key: info("infra signing key not yet readable; control-plane auth may fail") return infra - path = bot_bottle_root() / CONTROL_PLANE_TOKEN_FILENAME + path = bot_bottle_root() / ORCHESTRATOR_TOKEN_FILENAME path.parent.mkdir(parents=True, exist_ok=True) fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) with os.fdopen(fd, "w") as f: @@ -458,7 +458,7 @@ def wait_for_health( ) -> None: """Poll the control plane's /health until it answers 200 or the deadline passes. Dies (with the console tail) if the VMM exits early.""" - url = f"{infra.control_plane_url}/health" + url = f"{infra.orchestrator_url}/health" deadline = time.monotonic() + timeout while time.monotonic() < deadline: if infra.vm is not None and not infra.vm.is_alive(): @@ -467,7 +467,7 @@ def wait_for_health( try: with urllib.request.urlopen(url, timeout=1.0) as resp: if resp.status == 200: - info(f"infra control plane healthy at {infra.control_plane_url}") + info(f"infra control plane healthy at {infra.orchestrator_url}") return except (urllib.error.URLError, TimeoutError, OSError): pass @@ -525,11 +525,11 @@ cd /app # same VM — reaching the orchestrator over 127.0.0.1, past the nft boundary that # only fences off the separate agent VM — could drive the operator routes # (approve its own supervise proposals, rewrite policy, read injected tokens). -CP_KEY=$(BOT_BOTTLE_ROOT=/var/lib/bot-bottle python3 -c 'from bot_bottle.paths import host_control_plane_token as t; print(t())') -GW_JWT=$(BB_SIGNING_KEY="$CP_KEY" python3 -c 'import os; from bot_bottle.control_auth import mint, ROLE_GATEWAY; print(mint(ROLE_GATEWAY, os.environ["BB_SIGNING_KEY"]))') +CP_KEY=$(BOT_BOTTLE_ROOT=/var/lib/bot-bottle python3 -c 'from bot_bottle.paths import host_orchestrator_token as t; print(t())') +GW_JWT=$(BB_SIGNING_KEY="$CP_KEY" python3 -c 'import os; from bot_bottle.orchestrator_auth import mint, ROLE_GATEWAY; print(mint(ROLE_GATEWAY, os.environ["BB_SIGNING_KEY"]))') -BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_CONTROL_PLANE_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator \\ - --host 0.0.0.0 --port {CONTROL_PLANE_PORT} --broker stub & +BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_ORCHESTRATOR_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator \\ + --host 0.0.0.0 --port {ORCHESTRATOR_PORT} --broker stub & # Gateway data plane, multi-tenant: each request resolves source-IP -> # policy against the local control plane. The VM backend reaches git over @@ -540,8 +540,8 @@ BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_CONTROL_PLANE_TOKEN="$CP_KEY" pyt # the pre-minted `gateway` JWT; gateway_init keeps the signing key out of the # data-plane daemons' env. BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise \\ -BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{CONTROL_PLANE_PORT} \\ -BOT_BOTTLE_CONTROL_AUTH_JWT="$GW_JWT" \\ +BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{ORCHESTRATOR_PORT} \\ +BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT="$GW_JWT" \\ python3 -m bot_bottle.gateway.bootstrap & # Reap as PID 1; children are backgrounded, so `wait` blocks. diff --git a/bot_bottle/backend/macos_container/backend.py b/bot_bottle/backend/macos_container/backend.py index 71b4d5e1..e9225eda 100644 --- a/bot_bottle/backend/macos_container/backend.py +++ b/bot_bottle/backend/macos_container/backend.py @@ -102,7 +102,7 @@ class MacosContainerBottleBackend( (`supervise`) call when no control plane is running yet. Mirrors firecracker's infra-VM bring-up.""" from .infra import MacosInfraService - return MacosInfraService().ensure_running().control_plane_url + return MacosInfraService().ensure_running().orchestrator_url def prepare_cleanup(self) -> MacosContainerBottleCleanupPlan: return _cleanup.prepare_cleanup() diff --git a/bot_bottle/backend/macos_container/consolidated_launch.py b/bot_bottle/backend/macos_container/consolidated_launch.py index 94976fb9..a641a9b6 100644 --- a/bot_bottle/backend/macos_container/consolidated_launch.py +++ b/bot_bottle/backend/macos_container/consolidated_launch.py @@ -90,7 +90,7 @@ def ensure_gateway( service = service or MacosInfraService() infra = service.ensure_running() endpoint = GatewayEndpoint( - orchestrator_url=infra.control_plane_url, + orchestrator_url=infra.orchestrator_url, gateway_ip=infra.gateway_ip, gateway_ca_pem=service.ca_cert_pem(), network=service.network, diff --git a/bot_bottle/backend/macos_container/infra.py b/bot_bottle/backend/macos_container/infra.py index 5bb7f757..c2f14cc7 100644 --- a/bot_bottle/backend/macos_container/infra.py +++ b/bot_bottle/backend/macos_container/infra.py @@ -48,12 +48,12 @@ from ...orchestrator.lifecycle import ( OrchestratorStartError, source_hash, ) -from ...control_auth import ROLE_GATEWAY, mint +from ...orchestrator_auth import ROLE_GATEWAY, mint from ...paths import ( - CONTROL_AUTH_JWT_ENV, - CONTROL_PLANE_TOKEN_ENV, + ORCHESTRATOR_AUTH_JWT_ENV, + ORCHESTRATOR_TOKEN_ENV, HOST_DB_FILENAME, - host_control_plane_token, + host_orchestrator_token, host_gateway_ca_dir, ) from .. import util as backend_util @@ -117,7 +117,7 @@ class InfraEndpoint: """How to reach the running infra container. The control plane and the gateway are the same container, so one address serves both.""" - control_plane_url: str # http://:8099 — host CLI + registration + orchestrator_url: str # http://:8099 — host CLI + registration gateway_ip: str # same container; agents' proxy / git-http / MCP target @@ -181,7 +181,7 @@ class MacosInfraService: return None url = self._resolve_url() if url and self.is_healthy(url): - return InfraEndpoint(control_plane_url=url, gateway_ip=_ip_of(url)) + return InfraEndpoint(orchestrator_url=url, gateway_ip=_ip_of(url)) return None def ensure_built(self) -> None: @@ -247,17 +247,17 @@ class MacosInfraService: # run process below, so neither lands on argv or in `container # inspect`'s command line. The agent runs in a SEPARATE container that # is never given these vars, which is the whole point. - "--env", CONTROL_PLANE_TOKEN_ENV, - "--env", CONTROL_AUTH_JWT_ENV, + "--env", ORCHESTRATOR_TOKEN_ENV, + "--env", ORCHESTRATOR_AUTH_JWT_ENV, "--entrypoint", "sh", self.image, "-c", _init_script(self.port), ] - _signing_key = host_control_plane_token() + _signing_key = host_orchestrator_token() run_env = { **os.environ, - CONTROL_PLANE_TOKEN_ENV: _signing_key, - CONTROL_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key), + ORCHESTRATOR_TOKEN_ENV: _signing_key, + ORCHESTRATOR_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key), } result = container_mod.run_container_argv(argv, env=run_env) if result.returncode != 0: @@ -272,7 +272,7 @@ class MacosInfraService: url = self._resolve_url() if url and self.is_healthy(url): log.info("infra container healthy", context={"url": url}) - return InfraEndpoint(control_plane_url=url, gateway_ip=_ip_of(url)) + return InfraEndpoint(orchestrator_url=url, gateway_ip=_ip_of(url)) if time.monotonic() >= deadline: raise OrchestratorStartError( f"infra container did not become healthy within " @@ -306,7 +306,7 @@ def _ip_of(url: str) -> str: return url.split("://", 1)[-1].rsplit(":", 1)[0] -def probe_control_plane_url(port: int = DEFAULT_PORT) -> str: +def probe_orchestrator_url(port: int = DEFAULT_PORT) -> str: """The running infra container's control-plane URL, or "" if it isn't up. Used by host-side control-plane discovery (`discover_orchestrator_url`); safe to call on any host — returns "" when the container or the `container` diff --git a/bot_bottle/gateway/bootstrap.py b/bot_bottle/gateway/bootstrap.py index 060a9807..4e1c6d16 100644 --- a/bot_bottle/gateway/bootstrap.py +++ b/bot_bottle/gateway/bootstrap.py @@ -58,10 +58,10 @@ _READY_GATED_DAEMONS: tuple[str, ...] = ("git-gate", "git-http") # The data-plane daemons instead hold the pre-minted `gateway` JWT they present. # Scoping each to its process (even in the combined infra container) keeps a # compromised data-plane daemon from reading the key and minting a `cli` token -# (issue #469 review). Values match paths.CONTROL_PLANE_TOKEN_ENV / -# CONTROL_AUTH_JWT_ENV; hardcoded here so this supervisor stays import-light. -_SIGNING_KEY_ENV = "BOT_BOTTLE_CONTROL_PLANE_TOKEN" -_GATEWAY_JWT_ENV = "BOT_BOTTLE_CONTROL_AUTH_JWT" +# (issue #469 review). Values match paths.ORCHESTRATOR_TOKEN_ENV / +# ORCHESTRATOR_AUTH_JWT_ENV; hardcoded here so this supervisor stays import-light. +_SIGNING_KEY_ENV = "BOT_BOTTLE_ORCHESTRATOR_TOKEN" +_GATEWAY_JWT_ENV = "BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT" # Daemons that must be requested explicitly via BOT_BOTTLE_GATEWAY_DAEMONS # and are NOT started in the default (env-var-unset) case. The orchestrator diff --git a/bot_bottle/gateway/policy_resolver.py b/bot_bottle/gateway/policy_resolver.py index 9298a12b..a61f3daf 100644 --- a/bot_bottle/gateway/policy_resolver.py +++ b/bot_bottle/gateway/policy_resolver.py @@ -41,16 +41,16 @@ DEFAULT_TIMEOUT_SECONDS = 2.0 # rather than imported because this module is COPYed flat into the gateway image, # free of bot-bottle imports — same rationale as IDENTITY_HEADER in egress_addon # / git_http_backend. -CONTROL_AUTH_HEADER = "x-bot-bottle-control-auth" -CONTROL_AUTH_JWT_ENV = "BOT_BOTTLE_CONTROL_AUTH_JWT" +ORCHESTRATOR_AUTH_HEADER = "x-bot-bottle-orchestrator-auth" +ORCHESTRATOR_AUTH_JWT_ENV = "BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT" -def _control_auth_headers() -> dict[str, str]: +def _orchestrator_auth_headers() -> dict[str, str]: """The auth header to send, or {} when no token is configured (an open control plane, e.g. Firecracker behind its nft boundary — sending nothing is correct there and harmlessly ignored).""" - token = os.environ.get(CONTROL_AUTH_JWT_ENV, "").strip() - return {CONTROL_AUTH_HEADER: token} if token else {} + token = os.environ.get(ORCHESTRATOR_AUTH_JWT_ENV, "").strip() + return {ORCHESTRATOR_AUTH_HEADER: token} if token else {} class PolicyResolveError(RuntimeError): @@ -74,7 +74,7 @@ class PolicyResolver: body = json.dumps(payload).encode() req = urllib.request.Request( f"{self._base}{path}", data=body, method="POST", - headers={"Content-Type": "application/json", **_control_auth_headers()}, + headers={"Content-Type": "application/json", **_orchestrator_auth_headers()}, ) try: with urllib.request.urlopen(req, timeout=self._timeout) as resp: diff --git a/bot_bottle/orchestrator/__init__.py b/bot_bottle/orchestrator/__init__.py index 8a0a9da7..06e3d18f 100644 --- a/bot_bottle/orchestrator/__init__.py +++ b/bot_bottle/orchestrator/__init__.py @@ -17,7 +17,7 @@ backend-neutral "consolidation core" that needs no VM packaging: * `service` — the `Orchestrator`: owns the registry, brokers the launch lifecycle (launch/teardown), manages the shared gateway, attributes. - * `control_plane` — the HTTP control-plane RPC (launch / teardown / + * `server` — the HTTP control-plane RPC (launch / teardown / list / attribute / gateway / health). The actual backend-native launch (a real docker/firecracker broker) and @@ -40,7 +40,7 @@ from .broker import ( from .docker_broker import DockerBroker, DockerBrokerError from ..gateway import Gateway, GatewayError from .service import Orchestrator -from .server import ControlPlaneServer, dispatch, make_server +from .server import OrchestratorServer, dispatch, make_server __all__ = [ "BottleRecord", @@ -57,7 +57,7 @@ __all__ = [ "sign_request", "verify_request", "Orchestrator", - "ControlPlaneServer", + "OrchestratorServer", "dispatch", "make_server", ] diff --git a/bot_bottle/orchestrator/client.py b/bot_bottle/orchestrator/client.py index fb80c5ee..b6c95492 100644 --- a/bot_bottle/orchestrator/client.py +++ b/bot_bottle/orchestrator/client.py @@ -18,9 +18,9 @@ import urllib.request from collections.abc import Iterable from dataclasses import dataclass -from ..control_auth import ROLE_CLI, mint -from ..paths import host_control_plane_token -from .server import CONTROL_AUTH_HEADER +from ..orchestrator_auth import ROLE_CLI, mint +from ..paths import host_orchestrator_token +from .server import ORCHESTRATOR_AUTH_HEADER DEFAULT_TIMEOUT_SECONDS = 5.0 @@ -32,7 +32,7 @@ def _host_auth_token() -> str: "" means 'send no auth header' — correct against an open (unconfigured) control plane, and harmlessly rejected by a secured one.""" try: - return mint(ROLE_CLI, host_control_plane_token()) + return mint(ROLE_CLI, host_orchestrator_token()) except (OSError, ValueError): return "" @@ -83,7 +83,7 @@ class OrchestratorClient: data = json.dumps(body).encode() if body is not None else None headers = {"Content-Type": "application/json"} if data is not None else {} if self._auth_token: - headers[CONTROL_AUTH_HEADER] = self._auth_token + headers[ORCHESTRATOR_AUTH_HEADER] = self._auth_token req = urllib.request.Request( f"{self._base}{path}", data=data, method=method, headers=headers, ) @@ -252,14 +252,14 @@ def discover_orchestrator_url(*, timeout: float = 2.0) -> str: candidates.append("http://127.0.0.1:8099") try: # firecracker: infra VM control plane on the orchestrator TAP from ..backend.firecracker import netpool - from ..backend.firecracker.infra_vm import CONTROL_PLANE_PORT + from ..backend.firecracker.infra_vm import ORCHESTRATOR_PORT candidates.append( - f"http://{netpool.orch_slot().guest_ip}:{CONTROL_PLANE_PORT}") + f"http://{netpool.orch_slot().guest_ip}:{ORCHESTRATOR_PORT}") except Exception: # noqa: BLE001 — backend optional / not firecracker pass try: # macOS: infra container control plane on its host-only address - from ..backend.macos_container.infra import probe_control_plane_url - url = probe_control_plane_url() + from ..backend.macos_container.infra import probe_orchestrator_url + url = probe_orchestrator_url() if url: candidates.append(url) except Exception: # noqa: BLE001 — backend optional / not macOS diff --git a/bot_bottle/orchestrator/server.py b/bot_bottle/orchestrator/server.py index 4daed700..a7ec7a4a 100644 --- a/bot_bottle/orchestrator/server.py +++ b/bot_bottle/orchestrator/server.py @@ -48,7 +48,7 @@ via the orchestrator. Register/deregister without a launch are internal to `Orchestrator`, not exposed here. Routing/handling is the pure function `dispatch()` so it is unit-testable -without a socket; `Handler` / `ControlPlaneServer` / `make_server` are a +without a socket; `Handler` / `OrchestratorServer` / `make_server` are a thin stdlib adapter around it. Listing redacts identity tokens — they are returned only once, to the caller that launches the bottle. """ @@ -63,8 +63,8 @@ import sys import typing from urllib.parse import urlsplit -from ..control_auth import ROLE_CLI, ROLES, verify -from ..paths import CONTROL_PLANE_TOKEN_ENV +from ..orchestrator_auth import ROLE_CLI, ROLES, verify +from ..paths import ORCHESTRATOR_TOKEN_ENV from ..supervisor.types import TOOLS from .service import Orchestrator @@ -72,13 +72,13 @@ from .service import Orchestrator Json = dict[str, object] # The request header carrying the caller's role-scoped control-plane token (a -# signed JWT naming the caller's role — see control_auth). The role gates which +# signed JWT naming the caller's role — see orchestrator_auth). The role gates which # routes the caller may reach: the data plane holds a `gateway` token good only # for the agent-facing lookups; the host CLI holds a `cli` token for the # operator/mutating routes. An agent that can merely *reach* the port holds no # token at all, and a compromised gateway holds only `gateway` — neither can # drive the operator routes (approve proposals, rewrite policy, read tokens). -CONTROL_AUTH_HEADER = "x-bot-bottle-control-auth" +ORCHESTRATOR_AUTH_HEADER = "x-bot-bottle-orchestrator-auth" # The routes the data plane (role `gateway`) is allowed to reach — exactly the # per-request lookups PolicyResolver makes. Every other authenticated route is @@ -116,7 +116,7 @@ def dispatch( # pylint: disable=too-many-return-statements,too-many-branches `role` is the caller's verified control-plane role (`gateway` or `cli`), or None for an unauthenticated request; an open-mode server (no signing key - configured — see `ControlPlaneServer`) passes `cli`. Every route except + configured — see `OrchestratorServer`) passes `cli`. Every route except `GET /health` requires a role: a missing role is 401, and a role that doesn't cover the route is 403 — so a `gateway` data-plane token can reach `/resolve` + `/supervise/{propose,poll}` but not the operator routes @@ -365,10 +365,10 @@ class Handler(http.server.BaseHTTPRequestHandler): crashing the connection, so one bad request can't take the control plane down for the caller.""" server = self.server - assert isinstance(server, ControlPlaneServer) + assert isinstance(server, OrchestratorServer) length = int(self.headers.get("Content-Length") or 0) body = self.rfile.read(length) if length > 0 else b"" - role = server.role_for(self.headers.get(CONTROL_AUTH_HEADER, "")) + role = server.role_for(self.headers.get(ORCHESTRATOR_AUTH_HEADER, "")) try: status, payload = dispatch( server.orchestrator, method, self.path, body, role=role) @@ -396,11 +396,11 @@ class Handler(http.server.BaseHTTPRequestHandler): self._serve("DELETE") -class ControlPlaneServer(socketserver.ThreadingMixIn, http.server.HTTPServer): +class OrchestratorServer(socketserver.ThreadingMixIn, http.server.HTTPServer): """Threading HTTP server that carries the orchestrator for its handlers. Holds the per-host control-plane *signing key* (from - `$BOT_BOTTLE_CONTROL_PLANE_TOKEN`, injected by the launcher into the + `$BOT_BOTTLE_ORCHESTRATOR_TOKEN`, injected by the launcher into the orchestrator process only) and verifies each request's role-scoped token against it. When a key is set, every route but `/health` requires a valid token whose role covers the route; when it is unset the server runs **open** @@ -413,11 +413,11 @@ class ControlPlaneServer(socketserver.ThreadingMixIn, http.server.HTTPServer): def __init__(self, address: tuple[str, int], orchestrator: Orchestrator) -> None: self.orchestrator = orchestrator - self._signing_key = os.environ.get(CONTROL_PLANE_TOKEN_ENV, "").strip() + self._signing_key = os.environ.get(ORCHESTRATOR_TOKEN_ENV, "").strip() if not self._signing_key: sys.stderr.write( "orchestrator: WARNING — no control-plane signing key " - f"(${CONTROL_PLANE_TOKEN_ENV}); running WITHOUT caller " + f"(${ORCHESTRATOR_TOKEN_ENV}); running WITHOUT caller " "authentication. Any client that can reach this port can drive " "it. Backends that put the control plane on an agent-reachable " "network MUST set this.\n" @@ -438,13 +438,13 @@ class ControlPlaneServer(socketserver.ThreadingMixIn, http.server.HTTPServer): def make_server( orchestrator: Orchestrator, host: str = "127.0.0.1", port: int = 0 -) -> ControlPlaneServer: +) -> OrchestratorServer: """Build (but do not start) a control-plane server. `port=0` binds an ephemeral port — read `server.server_address` for the actual one.""" - return ControlPlaneServer((host, port), orchestrator) + return OrchestratorServer((host, port), orchestrator) __all__ = [ - "dispatch", "Handler", "ControlPlaneServer", "make_server", "Json", - "CONTROL_AUTH_HEADER", + "dispatch", "Handler", "OrchestratorServer", "make_server", "Json", + "ORCHESTRATOR_AUTH_HEADER", ] diff --git a/bot_bottle/control_auth.py b/bot_bottle/orchestrator_auth.py similarity index 100% rename from bot_bottle/control_auth.py rename to bot_bottle/orchestrator_auth.py diff --git a/bot_bottle/paths.py b/bot_bottle/paths.py index e1723c44..7b991cbf 100644 --- a/bot_bottle/paths.py +++ b/bot_bottle/paths.py @@ -36,17 +36,16 @@ HOST_DB_FILENAME = "bot-bottle.db" # reading route (see orchestrator/server.py); it is held only by the # trusted callers (control plane, gateway, host CLI) and never handed to an # agent, so an agent that can reach the control-plane port still can't drive it. -CONTROL_PLANE_TOKEN_FILENAME = "control-plane-token" -# The env var carrying the control-plane *signing key* — held only by the +ORCHESTRATOR_TOKEN_FILENAME = "orchestrator-token" +# The env var carrying the orchestrator's *signing key* — held only by the # orchestrator (to verify tokens) and the host CLI (to mint its own), never by -# the data plane. Same value as the host token file; the name is unchanged for -# backward compatibility with existing launchers. -CONTROL_PLANE_TOKEN_ENV = "BOT_BOTTLE_CONTROL_PLANE_TOKEN" +# the data plane. Same value as the host token file. +ORCHESTRATOR_TOKEN_ENV = "BOT_BOTTLE_ORCHESTRATOR_TOKEN" # The env var carrying the data plane's pre-minted `gateway`-role token (a # signed JWT the launcher mints from the signing key). The gateway presents this # on /resolve + /supervise/{propose,poll}; it never holds the signing key, so it # cannot forge a higher-privilege `cli` token (issue #469 review). -CONTROL_AUTH_JWT_ENV = "BOT_BOTTLE_CONTROL_AUTH_JWT" +ORCHESTRATOR_AUTH_JWT_ENV = "BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT" # The host directory holding the gateway's persistent mitmproxy CA. Bind-mounted # into the infra/gateway container at mitmproxy's confdir so the self-generated @@ -98,7 +97,7 @@ def host_gateway_ca_dir() -> Path: return ca_dir -def host_control_plane_token() -> str: +def host_orchestrator_token() -> str: """The per-host control-plane secret, minted (256-bit, url-safe) and persisted 0600 on first use, then reused. @@ -107,7 +106,7 @@ def host_control_plane_token() -> str: *host* artifact — the file lives under the root the agent never mounts, and the env var is set only on the trusted containers — so reading it here is safe on the host launch path but the value never reaches a bottle.""" - path = bot_bottle_root() / CONTROL_PLANE_TOKEN_FILENAME + path = bot_bottle_root() / ORCHESTRATOR_TOKEN_FILENAME try: existing = path.read_text().strip() if existing: @@ -131,13 +130,13 @@ def host_control_plane_token() -> str: __all__ = [ "HOST_DB_FILENAME", - "CONTROL_PLANE_TOKEN_FILENAME", - "CONTROL_PLANE_TOKEN_ENV", - "CONTROL_AUTH_JWT_ENV", + "ORCHESTRATOR_TOKEN_FILENAME", + "ORCHESTRATOR_TOKEN_ENV", + "ORCHESTRATOR_AUTH_JWT_ENV", "GATEWAY_CA_DIRNAME", "bot_bottle_root", "host_db_path", "host_db_dir", "host_gateway_ca_dir", - "host_control_plane_token", + "host_orchestrator_token", ] diff --git a/tests/integration/test_orchestrator_docker_control_plane_auth.py b/tests/integration/test_orchestrator_docker_auth.py similarity index 95% rename from tests/integration/test_orchestrator_docker_control_plane_auth.py rename to tests/integration/test_orchestrator_docker_auth.py index 08a6c620..50ecaa58 100644 --- a/tests/integration/test_orchestrator_docker_control_plane_auth.py +++ b/tests/integration/test_orchestrator_docker_auth.py @@ -25,10 +25,10 @@ import tempfile import unittest from pathlib import Path -from bot_bottle.control_auth import ROLE_CLI, ROLE_GATEWAY, mint +from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY, mint from bot_bottle.orchestrator.client import OrchestratorClient from bot_bottle.backend.docker.infra import DockerInfraService -from bot_bottle.paths import host_control_plane_token +from bot_bottle.paths import host_orchestrator_token from tests._docker import skip_unless_docker # Fixed (not per-run-suffixed) so repeated runs reuse the same layer-cached @@ -46,20 +46,20 @@ _TEST_INFRA_IMAGE = "bot-bottle-infra:itest" "runner's /workspace — same host-bind-mount constraint as the other " "bottle-bringup integration tests", ) -class TestDockerControlPlaneAuthIntegration(unittest.TestCase): +class TestDockerOrchestratorAuthIntegration(unittest.TestCase): @classmethod def setUpClass(cls) -> None: suffix = secrets.token_hex(4) cls._tmp = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with cls.addClassCleanup(cls._tmp.cleanup) - # host_control_plane_token() — both the token read below and the one + # host_orchestrator_token() — both the token read below and the one # DockerInfraService injects into the container's env — resolves its # path via the *ambient* BOT_BOTTLE_ROOT env var, not the host_root # kwarg passed to the constructor (that kwarg only controls the DB # bind-mount destination). Without pointing the env var at the same # throwaway dir, this "isolated" test would read/write the developer's - # real ~/.bot-bottle/control-plane-token. + # real ~/.bot-bottle/orchestrator-token. previous_root = os.environ.get("BOT_BOTTLE_ROOT") def _restore_root() -> None: @@ -88,7 +88,7 @@ class TestDockerControlPlaneAuthIntegration(unittest.TestCase): cls.svc.ensure_running() # The control plane now verifies role-scoped signed tokens, not the raw # key. Mint one of each role from the host signing key (issue #469 review). - signing_key = host_control_plane_token() + signing_key = host_orchestrator_token() cls.cli_token = mint(ROLE_CLI, signing_key) cls.gateway_token = mint(ROLE_GATEWAY, signing_key) diff --git a/tests/unit/test_backend_selection.py b/tests/unit/test_backend_selection.py index 1a2304d6..25b221c4 100644 --- a/tests/unit/test_backend_selection.py +++ b/tests/unit/test_backend_selection.py @@ -408,7 +408,7 @@ class TestEnsureOrchestrator(unittest.TestCase): with patch( "bot_bottle.backend.firecracker.infra_vm.ensure_running" ) as ensure_running: - ensure_running.return_value.control_plane_url = ( + ensure_running.return_value.orchestrator_url = ( "http://10.243.255.1:8099" ) url = b.ensure_orchestrator() @@ -419,7 +419,7 @@ class TestEnsureOrchestrator(unittest.TestCase): with patch( "bot_bottle.backend.macos_container.infra.MacosInfraService" ) as service_cls: - service_cls.return_value.ensure_running.return_value.control_plane_url = ( + service_cls.return_value.ensure_running.return_value.orchestrator_url = ( "http://192.168.128.2:8099" ) url = b.ensure_orchestrator() diff --git a/tests/unit/test_firecracker_infra_vm.py b/tests/unit/test_firecracker_infra_vm.py index 7dcf4ea0..157a5c5b 100644 --- a/tests/unit/test_firecracker_infra_vm.py +++ b/tests/unit/test_firecracker_infra_vm.py @@ -31,7 +31,7 @@ class TestSigningKeySync(unittest.TestCase): patch.object(infra_vm, "bot_bottle_root", return_value=root): out = infra_vm._with_signing_key(self._infra()) self.assertIsInstance(out, infra_vm.InfraVm) # returned for chaining - token = root / infra_vm.CONTROL_PLANE_TOKEN_FILENAME + token = root / infra_vm.ORCHESTRATOR_TOKEN_FILENAME self.assertEqual("the-signing-key", token.read_text()) self.assertEqual(0o600, token.stat().st_mode & 0o777) # It cat'd the guest volume path over SSH. @@ -44,16 +44,16 @@ class TestSigningKeySync(unittest.TestCase): with patch.object(infra_vm.subprocess, "run", return_value=proc), \ patch.object(infra_vm, "bot_bottle_root", return_value=root): infra_vm._with_signing_key(self._infra()) # no raise - self.assertFalse((root / infra_vm.CONTROL_PLANE_TOKEN_FILENAME).exists()) + self.assertFalse((root / infra_vm.ORCHESTRATOR_TOKEN_FILENAME).exists()) -class TestControlPlaneUrl(unittest.TestCase): +class TestOrchestratorUrl(unittest.TestCase): def test_url_uses_guest_ip_and_port(self): infra = infra_vm.InfraVm( vm=MagicMock(), guest_ip="10.243.255.1", private_key=Path("/k")) self.assertEqual( - f"http://10.243.255.1:{infra_vm.CONTROL_PLANE_PORT}", - infra.control_plane_url, + f"http://10.243.255.1:{infra_vm.ORCHESTRATOR_PORT}", + infra.orchestrator_url, ) @@ -81,11 +81,11 @@ class TestBuildInfraRootfs(unittest.TestCase): # Role-scoped control-plane auth (issue #469 review): the orchestrator # gets the signing key, the gateway daemons get a pre-minted `gateway` # JWT — never open mode in the infra VM. - self.assertIn("host_control_plane_token", init) # key generated on the volume + self.assertIn("host_orchestrator_token", init) # key generated on the volume self.assertIn("mint, ROLE_GATEWAY", init) # gateway JWT minted from it - self.assertIn('BOT_BOTTLE_CONTROL_PLANE_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator', + self.assertIn('BOT_BOTTLE_ORCHESTRATOR_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator', init) # key -> orchestrator only - self.assertIn('BOT_BOTTLE_CONTROL_AUTH_JWT="$GW_JWT"', init) # JWT -> gateway daemons + self.assertIn('BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT="$GW_JWT"', init) # JWT -> gateway daemons class TestSshGatewayTransport(unittest.TestCase): diff --git a/tests/unit/test_gateway_init.py b/tests/unit/test_gateway_init.py index 7f5272c4..1fb471f5 100644 --- a/tests/unit/test_gateway_init.py +++ b/tests/unit/test_gateway_init.py @@ -64,7 +64,7 @@ class TestEnvForDaemon(unittest.TestCase): self.assertNotIn("X", self._BASE) -class TestControlPlaneEnvScoping(unittest.TestCase): +class TestOrchestratorEnvScoping(unittest.TestCase): """The control-plane signing key stays with the orchestrator; the pre-minted `gateway` JWT goes to the data-plane daemons (issue #469 review). Scoping them per-process keeps a compromised data-plane daemon from reading the key @@ -72,20 +72,20 @@ class TestControlPlaneEnvScoping(unittest.TestCase): _BASE = { "PATH": "/usr/bin", - "BOT_BOTTLE_CONTROL_PLANE_TOKEN": "sk-x", - "BOT_BOTTLE_CONTROL_AUTH_JWT": "gw-jwt", + "BOT_BOTTLE_ORCHESTRATOR_TOKEN": "sk-x", + "BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT": "gw-jwt", } def test_orchestrator_gets_key_not_jwt(self): env = _env_for_daemon("orchestrator", self._BASE) - self.assertEqual("sk-x", env["BOT_BOTTLE_CONTROL_PLANE_TOKEN"]) - self.assertNotIn("BOT_BOTTLE_CONTROL_AUTH_JWT", env) + self.assertEqual("sk-x", env["BOT_BOTTLE_ORCHESTRATOR_TOKEN"]) + self.assertNotIn("BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT", env) def test_data_plane_daemons_get_jwt_not_key(self): for name in ("egress", "git-gate", "git-http", "supervise"): env = _env_for_daemon(name, self._BASE) - self.assertNotIn("BOT_BOTTLE_CONTROL_PLANE_TOKEN", env, name) - self.assertEqual("gw-jwt", env["BOT_BOTTLE_CONTROL_AUTH_JWT"], name) + self.assertNotIn("BOT_BOTTLE_ORCHESTRATOR_TOKEN", env, name) + self.assertEqual("gw-jwt", env["BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT"], name) class TestSelectedDaemons(unittest.TestCase): diff --git a/tests/unit/test_macos_consolidated_launch.py b/tests/unit/test_macos_consolidated_launch.py index b0f0b840..812e14e6 100644 --- a/tests/unit/test_macos_consolidated_launch.py +++ b/tests/unit/test_macos_consolidated_launch.py @@ -58,7 +58,7 @@ class TestEnsureGateway(unittest.TestCase): from bot_bottle.backend.macos_container.infra import InfraEndpoint service = MagicMock() service.ensure_running.return_value = InfraEndpoint( - control_plane_url="http://192.168.128.2:8099", + orchestrator_url="http://192.168.128.2:8099", gateway_ip="192.168.128.2", ) service.network = "bot-bottle-mac-gateway" @@ -72,7 +72,7 @@ class TestEnsureGateway(unittest.TestCase): self.assertEqual("PEM", endpoint.gateway_ca_pem) self.assertEqual("bot-bottle-mac-gateway", endpoint.network) - def test_control_plane_and_gateway_share_one_address(self) -> None: + def test_orchestrator_and_gateway_share_one_address(self) -> None: """One infra container hosts both, so the gateway IP and the control-plane host are the same.""" endpoint = self._run(self._service()) diff --git a/tests/unit/test_macos_infra.py b/tests/unit/test_macos_infra.py index 914ed1c4..f1341ecd 100644 --- a/tests/unit/test_macos_infra.py +++ b/tests/unit/test_macos_infra.py @@ -10,7 +10,7 @@ from bot_bottle.backend.macos_container.infra import ( INFRA_DB_VOLUME, MacosInfraService, OrchestratorStartError, - probe_control_plane_url, + probe_orchestrator_url, ) _INFRA = "bot_bottle.backend.macos_container.infra" @@ -110,7 +110,7 @@ class TestInfraEnsureRunning(unittest.TestCase): mod.try_container_ipv4_on_network.return_value = "192.168.128.2" endpoint = svc.ensure_running() run.assert_not_called() - self.assertEqual("http://192.168.128.2:8099", endpoint.control_plane_url) + self.assertEqual("http://192.168.128.2:8099", endpoint.orchestrator_url) self.assertEqual("192.168.128.2", endpoint.gateway_ip) def test_changed_source_recreates(self) -> None: @@ -176,16 +176,16 @@ class TestCaCertPem(unittest.TestCase): svc.ca_cert_pem(timeout=0) -class TestProbeControlPlane(unittest.TestCase): +class TestProbeOrchestrator(unittest.TestCase): def test_returns_url_when_running(self) -> None: with patch(f"{_INFRA}.container_mod") as mod: mod.try_container_ipv4_on_network.return_value = "192.168.128.2" - self.assertEqual("http://192.168.128.2:8099", probe_control_plane_url()) + self.assertEqual("http://192.168.128.2:8099", probe_orchestrator_url()) def test_empty_when_absent(self) -> None: with patch(f"{_INFRA}.container_mod") as mod: mod.try_container_ipv4_on_network.return_value = "" - self.assertEqual("", probe_control_plane_url()) + self.assertEqual("", probe_orchestrator_url()) if __name__ == "__main__": diff --git a/tests/unit/test_control_auth.py b/tests/unit/test_orchestrator_auth.py similarity index 91% rename from tests/unit/test_control_auth.py rename to tests/unit/test_orchestrator_auth.py index bfcb7cc7..78e25596 100644 --- a/tests/unit/test_control_auth.py +++ b/tests/unit/test_orchestrator_auth.py @@ -6,7 +6,7 @@ import base64 import json import unittest -from bot_bottle.control_auth import ROLE_CLI, ROLE_GATEWAY, ROLES, mint, verify +from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY, ROLES, mint, verify _KEY = "test-key" @@ -45,14 +45,14 @@ class TestMintVerify(unittest.TestCase): def test_validly_signed_but_wrong_alg_rejected(self) -> None: # Alg-confusion: even a *correctly signed* token whose header claims a # non-HS256 alg must be rejected. - from bot_bottle.control_auth import _sign # noqa: PLC0415 + from bot_bottle.orchestrator_auth import _sign # noqa: PLC0415 signing_input = f"{_b64({'alg': 'none', 'typ': 'JWT'})}.{_b64({'role': 'cli'})}" self.assertIsNone(verify(f"{signing_input}.{_sign(_KEY, signing_input)}", _KEY)) def test_validly_signed_but_undecodable_rejected(self) -> None: # A correct signature over a header that isn't valid base64/JSON still # fails closed rather than raising. - from bot_bottle.control_auth import _sign # noqa: PLC0415 + from bot_bottle.orchestrator_auth import _sign # noqa: PLC0415 signing_input = f"!!!not-base64!!!.{_b64({'role': 'cli'})}" self.assertIsNone(verify(f"{signing_input}.{_sign(_KEY, signing_input)}", _KEY)) @@ -67,7 +67,7 @@ class TestMintVerify(unittest.TestCase): header, _p, _s = mint(ROLE_CLI, _KEY).split(".") # Re-sign a token carrying an unknown role — a valid signature but a # role the control plane doesn't recognise must still be rejected. - from bot_bottle.control_auth import _sign # noqa: PLC0415 + from bot_bottle.orchestrator_auth import _sign # noqa: PLC0415 payload = _b64({"role": "root"}) signing_input = f"{header}.{payload}" forged = f"{signing_input}.{_sign(_KEY, signing_input)}" diff --git a/tests/unit/test_orchestrator_client.py b/tests/unit/test_orchestrator_client.py index 4086adec..a9f77da2 100644 --- a/tests/unit/test_orchestrator_client.py +++ b/tests/unit/test_orchestrator_client.py @@ -7,7 +7,7 @@ import unittest import urllib.error from unittest.mock import MagicMock, patch -from bot_bottle.control_auth import ROLE_CLI, verify +from bot_bottle.orchestrator_auth import ROLE_CLI, verify from bot_bottle.orchestrator.client import ( OrchestratorClient, OrchestratorClientError, @@ -20,13 +20,13 @@ _URLOPEN = "bot_bottle.orchestrator.client.urllib.request.urlopen" class TestHostAuthToken(unittest.TestCase): def test_mints_a_cli_token_from_the_host_key(self) -> None: - with patch("bot_bottle.orchestrator.client.host_control_plane_token", + with patch("bot_bottle.orchestrator.client.host_orchestrator_token", return_value="signing-key"): tok = _host_auth_token() self.assertEqual(ROLE_CLI, verify(tok, "signing-key")) def test_returns_empty_when_key_unreadable(self) -> None: - with patch("bot_bottle.orchestrator.client.host_control_plane_token", + with patch("bot_bottle.orchestrator.client.host_orchestrator_token", side_effect=OSError("no host root")): self.assertEqual("", _host_auth_token()) diff --git a/tests/unit/test_orchestrator_server.py b/tests/unit/test_orchestrator_server.py index 2d5c7361..d55908b5 100644 --- a/tests/unit/test_orchestrator_server.py +++ b/tests/unit/test_orchestrator_server.py @@ -19,7 +19,7 @@ from contextlib import closing from pathlib import Path from unittest.mock import patch -from bot_bottle.control_auth import ROLE_CLI, ROLE_GATEWAY, mint +from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY, mint from bot_bottle.orchestrator.broker import StubBroker from bot_bottle.orchestrator.server import dispatch, make_server from bot_bottle.orchestrator.registry import BottleRecord, RegistryStore @@ -284,7 +284,7 @@ class TestServerRoundTrip(unittest.TestCase): self.assertEqual(reg["bottle_id"], attr["bottle_id"]) -class TestControlPlaneAuth(unittest.TestCase): +class TestOrchestratorAuth(unittest.TestCase): """Role-scoped control-plane tokens (issue #400 / #469 review): every route but /health needs a valid token, and the token's role gates which routes it reaches — a `gateway` data-plane token can't drive the operator routes.""" @@ -349,7 +349,7 @@ class TestControlPlaneAuth(unittest.TestCase): self.assertIsNotNone(self.orch.registry.get(rec.bottle_id)) def _server_with_key(self, signing_key: str): - with patch.dict("os.environ", {"BOT_BOTTLE_CONTROL_PLANE_TOKEN": signing_key}): + with patch.dict("os.environ", {"BOT_BOTTLE_ORCHESTRATOR_TOKEN": signing_key}): server = make_server(self.orch, "127.0.0.1", 0) self.addCleanup(server.server_close) threading.Thread(target=server.serve_forever, daemon=True).start() @@ -360,7 +360,7 @@ class TestControlPlaneAuth(unittest.TestCase): def _status(self, url: str, *, header: str | None = None) -> int: req = urllib.request.Request(url) if header is not None: - req.add_header("x-bot-bottle-control-auth", header) + req.add_header("x-bot-bottle-orchestrator-auth", header) try: return urllib.request.urlopen(req, timeout=5).status except urllib.error.HTTPError as e: @@ -384,7 +384,7 @@ class TestControlPlaneAuth(unittest.TestCase): grants full cli access, so existing round-trip behavior is unchanged.""" with patch.dict("os.environ", {}, clear=False): import os - os.environ.pop("BOT_BOTTLE_CONTROL_PLANE_TOKEN", None) + os.environ.pop("BOT_BOTTLE_ORCHESTRATOR_TOKEN", None) server = make_server(self.orch, "127.0.0.1", 0) self.addCleanup(server.server_close) self.assertEqual(ROLE_CLI, server.role_for("")) diff --git a/tests/unit/test_policy_resolver.py b/tests/unit/test_policy_resolver.py index 44178a3b..e034d238 100644 --- a/tests/unit/test_policy_resolver.py +++ b/tests/unit/test_policy_resolver.py @@ -8,11 +8,11 @@ import urllib.error from unittest.mock import MagicMock, patch from bot_bottle.gateway.policy_resolver import ( - CONTROL_AUTH_HEADER, - CONTROL_AUTH_JWT_ENV, + ORCHESTRATOR_AUTH_HEADER, + ORCHESTRATOR_AUTH_JWT_ENV, PolicyResolveError, PolicyResolver, - _control_auth_headers, + _orchestrator_auth_headers, ) _URLOPEN = "bot_bottle.gateway.policy_resolver.urllib.request.urlopen" @@ -29,16 +29,16 @@ def _http_error(code: int) -> urllib.error.HTTPError: return urllib.error.HTTPError("http://x/resolve", code, "err", {}, None) # type: ignore[arg-type] -class TestControlAuthHeaders(unittest.TestCase): +class TestOrchestratorAuthHeaders(unittest.TestCase): def test_sends_the_gateway_jwt_when_configured(self) -> None: - with patch.dict("os.environ", {CONTROL_AUTH_JWT_ENV: "gateway.jwt.tok"}): - self.assertEqual({CONTROL_AUTH_HEADER: "gateway.jwt.tok"}, _control_auth_headers()) + with patch.dict("os.environ", {ORCHESTRATOR_AUTH_JWT_ENV: "gateway.jwt.tok"}): + self.assertEqual({ORCHESTRATOR_AUTH_HEADER: "gateway.jwt.tok"}, _orchestrator_auth_headers()) def test_sends_nothing_when_unset(self) -> None: import os with patch.dict("os.environ", {}, clear=False): - os.environ.pop(CONTROL_AUTH_JWT_ENV, None) - self.assertEqual({}, _control_auth_headers()) + os.environ.pop(ORCHESTRATOR_AUTH_JWT_ENV, None) + self.assertEqual({}, _orchestrator_auth_headers()) class TestPolicyResolver(unittest.TestCase): -- 2.52.0 From cb3a74edb00db83d20dcde527d8a9531098f56fa Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 18:22:03 -0400 Subject: [PATCH 27/30] refactor(orchestrator): collapse supervisor package into supervisor.py orchestrator/supervisor/ held only __init__.py, so the package no longer earns its own directory. Moved it to orchestrator/supervisor.py and removed the dir. External imports are unchanged (orchestrator.supervisor resolves identically as a module); only the file's own relative-import depths shift up one level. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- .../{supervisor/__init__.py => supervisor.py} | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) rename bot_bottle/orchestrator/{supervisor/__init__.py => supervisor.py} (91%) diff --git a/bot_bottle/orchestrator/supervisor/__init__.py b/bot_bottle/orchestrator/supervisor.py similarity index 91% rename from bot_bottle/orchestrator/supervisor/__init__.py rename to bot_bottle/orchestrator/supervisor.py index de4cf24e..7db78398 100644 --- a/bot_bottle/orchestrator/supervisor/__init__.py +++ b/bot_bottle/orchestrator/supervisor.py @@ -19,12 +19,12 @@ from __future__ import annotations from pathlib import Path -from ...supervisor.types import * # noqa: F401,F403 — re-export the neutral vocabulary -from ...supervisor.plan import SupervisePlan -from ..store.store_manager import StoreManager -from ..store.queue_store import QueueStore -from ...store.audit_store import AuditStore -from ...supervisor.types import AuditEntry, Proposal, Response +from ..supervisor.types import * # noqa: F401,F403 — re-export the neutral vocabulary +from ..supervisor.plan import SupervisePlan +from .store.store_manager import StoreManager +from .store.queue_store import QueueStore +from ..store.audit_store import AuditStore +from ..supervisor.types import AuditEntry, Proposal, Response class Supervisor: -- 2.52.0 From 3efb014aceeed4ecc4228fffc4459db41f67c4f6 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 18:24:20 -0400 Subject: [PATCH 28/30] refactor(orchestrator): move registry into store/registry_store registry.py is a SQLite-backed store (its class is already RegistryStore), so it belongs with the other orchestrator-owned stores rather than at the package root. Moved orchestrator/registry.py -> orchestrator/store/ registry_store.py, joining queue_store / secret_store / config_store. Repointed the in-package importers (.registry -> .store.registry_store) and the tests, bumped the moved file's relative-import depths, renamed the test file to match, and listed it in the store package docstring. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/orchestrator/__init__.py | 2 +- bot_bottle/orchestrator/__main__.py | 2 +- bot_bottle/orchestrator/service.py | 2 +- bot_bottle/orchestrator/store/__init__.py | 6 +++--- .../orchestrator/{registry.py => store/registry_store.py} | 6 +++--- ...ator_registry.py => test_orchestrator_registry_store.py} | 2 +- tests/unit/test_orchestrator_server.py | 2 +- tests/unit/test_orchestrator_service.py | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) rename bot_bottle/orchestrator/{registry.py => store/registry_store.py} (99%) rename tests/unit/{test_orchestrator_registry.py => test_orchestrator_registry_store.py} (99%) diff --git a/bot_bottle/orchestrator/__init__.py b/bot_bottle/orchestrator/__init__.py index 06e3d18f..0ae241d3 100644 --- a/bot_bottle/orchestrator/__init__.py +++ b/bot_bottle/orchestrator/__init__.py @@ -28,7 +28,7 @@ orchestrator -> firecracker). from __future__ import annotations -from .registry import BottleRecord, RegistryStore, new_identity_token +from .store.registry_store import BottleRecord, RegistryStore, new_identity_token from .broker import ( BrokerAuthError, LaunchBroker, diff --git a/bot_bottle/orchestrator/__main__.py b/bot_bottle/orchestrator/__main__.py index 0370c832..6fad8364 100644 --- a/bot_bottle/orchestrator/__main__.py +++ b/bot_bottle/orchestrator/__main__.py @@ -20,7 +20,7 @@ from .store.store_manager import StoreManager from .broker import LaunchBroker, StubBroker from .server import make_server from .docker_broker import DockerBroker -from .registry import RegistryStore, default_db_path +from .store.registry_store import RegistryStore, default_db_path from .service import Orchestrator diff --git a/bot_bottle/orchestrator/service.py b/bot_bottle/orchestrator/service.py index 740a6fa4..5b7e1091 100644 --- a/bot_bottle/orchestrator/service.py +++ b/bot_bottle/orchestrator/service.py @@ -26,7 +26,7 @@ from collections.abc import Iterable from datetime import datetime, timezone from .broker import LaunchBroker, LaunchRequest, sign_request -from .registry import DEFAULT_REAP_GRACE_SECONDS, BottleRecord, RegistryStore +from .store.registry_store import DEFAULT_REAP_GRACE_SECONDS, BottleRecord, RegistryStore from .supervisor import ( AuditEntry, COMPONENT_FOR_TOOL, diff --git a/bot_bottle/orchestrator/store/__init__.py b/bot_bottle/orchestrator/store/__init__.py index 1c4a81d1..4b2b70fa 100644 --- a/bot_bottle/orchestrator/store/__init__.py +++ b/bot_bottle/orchestrator/store/__init__.py @@ -1,9 +1,9 @@ """Orchestrator-owned SQLite stores. The stores whose tables only the orchestrator (control plane) opens: the -supervise proposal/response `queue_store`, the agent-secret `secret_store`, and -the orchestrator `config_store`. They build on the shared `DbStore` / -`migrations` base in `bot_bottle.store`. +per-host bottle `registry_store`, the supervise proposal/response `queue_store`, +the agent-secret `secret_store`, and the orchestrator `config_store`. They build +on the shared `DbStore` / `migrations` base in `bot_bottle.store`. Callers import the concrete module directly, e.g. `from bot_bottle.orchestrator.store.queue_store import QueueStore`. diff --git a/bot_bottle/orchestrator/registry.py b/bot_bottle/orchestrator/store/registry_store.py similarity index 99% rename from bot_bottle/orchestrator/registry.py rename to bot_bottle/orchestrator/store/registry_store.py index a5afe4e3..05a2eb66 100644 --- a/bot_bottle/orchestrator/registry.py +++ b/bot_bottle/orchestrator/store/registry_store.py @@ -36,9 +36,9 @@ from collections.abc import Iterable from dataclasses import dataclass from pathlib import Path -from ..store.db_store import DbStore -from ..store.migrations import TableMigrations -from ..paths import host_db_path +from ...store.db_store import DbStore +from ...store.migrations import TableMigrations +from ...paths import host_db_path # 256 bits of urandom, URL-safe — unguessable per-bottle identity token. IDENTITY_TOKEN_BYTES = 32 diff --git a/tests/unit/test_orchestrator_registry.py b/tests/unit/test_orchestrator_registry_store.py similarity index 99% rename from tests/unit/test_orchestrator_registry.py rename to tests/unit/test_orchestrator_registry_store.py index 90d0ff43..8b46397a 100644 --- a/tests/unit/test_orchestrator_registry.py +++ b/tests/unit/test_orchestrator_registry_store.py @@ -9,7 +9,7 @@ import unittest from contextlib import closing from pathlib import Path -from bot_bottle.orchestrator.registry import ( +from bot_bottle.orchestrator.store.registry_store import ( BottleRecord, RegistryStore, new_identity_token, diff --git a/tests/unit/test_orchestrator_server.py b/tests/unit/test_orchestrator_server.py index d55908b5..e3dfb1be 100644 --- a/tests/unit/test_orchestrator_server.py +++ b/tests/unit/test_orchestrator_server.py @@ -22,7 +22,7 @@ from unittest.mock import patch from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY, mint from bot_bottle.orchestrator.broker import StubBroker from bot_bottle.orchestrator.server import dispatch, make_server -from bot_bottle.orchestrator.registry import BottleRecord, RegistryStore +from bot_bottle.orchestrator.store.registry_store import BottleRecord, RegistryStore from bot_bottle.orchestrator.service import Orchestrator from bot_bottle.orchestrator.store.store_manager import StoreManager from bot_bottle.orchestrator.supervisor import ( diff --git a/tests/unit/test_orchestrator_service.py b/tests/unit/test_orchestrator_service.py index a203af3a..8ba25891 100644 --- a/tests/unit/test_orchestrator_service.py +++ b/tests/unit/test_orchestrator_service.py @@ -12,7 +12,7 @@ from pathlib import Path from unittest.mock import patch from bot_bottle.orchestrator.broker import LaunchBroker, LaunchRequest, StubBroker -from bot_bottle.orchestrator.registry import RegistryStore +from bot_bottle.orchestrator.store.registry_store import RegistryStore from bot_bottle.orchestrator.service import Orchestrator from bot_bottle.orchestrator.store.secret_store import new_env_var_secret from bot_bottle.orchestrator.store.store_manager import StoreManager -- 2.52.0 From e2740842a039cbe50c6eaed189fbf4f348daaa7f Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 18:54:41 -0400 Subject: [PATCH 29/30] refactor: make the two remaining heavy __init__ modules lazy Audit of package __init__ import cost turned up two eager ones: orchestrator/__init__.py (27 -> 2): was pure `from .X import Y` re-exports (registry_store, broker, docker_broker, gateway, service, server). Because importing any submodule runs the parent __init__, the CLI migration gate (orchestrator.store.store_manager, hit on every command) transitively dragged backend.docker, docker_broker, server, and http.server. Converted to the same lazy __getattr__ + _LAZY facade used by manifest/backend/egress/git_gate; the re-export API is unchanged and the docker/http drag is gone. cli/commands/__init__.py (78 -> 23): the registry imported all twelve handlers to build COMMANDS, so importing one command pulled all of them plus their deps. COMMANDS now maps each name to a thin lazy wrapper that imports its handler's module on first dispatch. Values stay callable, so the dispatcher and the patch.dict dispatch tests are unchanged; a CLI run now loads only the one command it dispatches. The remaining 23 is the dispatcher's own baseline (store_manager migration gate + help + log). Full unit suite green (2243); `bb help` + dispatch/migration-gate tests verified. Co-Authored-By: Claude Opus 4.8 --- bot_bottle/cli/commands/__init__.py | 67 +++++++++++++--------- bot_bottle/orchestrator/__init__.py | 88 ++++++++++++++++++----------- 2 files changed, 94 insertions(+), 61 deletions(-) diff --git a/bot_bottle/cli/commands/__init__.py b/bot_bottle/cli/commands/__init__.py index cda230d9..813524f6 100644 --- a/bot_bottle/cli/commands/__init__.py +++ b/bot_bottle/cli/commands/__init__.py @@ -1,41 +1,52 @@ """CLI subcommand registry. One module per `bot-bottle `, each exposing a `cmd_(argv)` -handler. This package `__init__` assembles them into the COMMANDS table -the dispatcher (`bot_bottle.cli.__main__`) reads. Shared CLI helpers -(`constants`, `tui`) stay one level up in the `cli` package. +handler. This package `__init__` maps command names to their handlers +**lazily**: a short-lived CLI run dispatches exactly one command, so +importing all twelve handlers (and their transitive deps — backend, +manifest, orchestrator, …) up front is wasted work. Each COMMANDS value is +a thin wrapper that imports its handler's module on first call. Shared CLI +helpers (`constants`, `tui`) stay one level up in the `cli` package. """ from __future__ import annotations -from .backend import cmd_backend -from .cleanup import cmd_cleanup -from .commit import cmd_commit -from .edit import cmd_edit -from .help import cmd_help -from .info import cmd_info -from .init import cmd_init -from .list import cmd_list -from .login import cmd_login -from .resume import cmd_resume -from .start import cmd_start -from .supervise import cmd_supervise +from importlib import import_module +from typing import Callable -COMMANDS = { - "backend": cmd_backend, - "cleanup": cmd_cleanup, - "commit": cmd_commit, - "edit": cmd_edit, - "help": cmd_help, - "info": cmd_info, - "init": cmd_init, - "list": cmd_list, - "login": cmd_login, - "resume": cmd_resume, - "start": cmd_start, - "supervise": cmd_supervise, +# command name -> ":". Kept as strings so building +# the registry imports nothing; the module loads only when dispatched. +_HANDLERS: dict[str, str] = { + "backend": "backend:cmd_backend", + "cleanup": "cleanup:cmd_cleanup", + "commit": "commit:cmd_commit", + "edit": "edit:cmd_edit", + "help": "help:cmd_help", + "info": "info:cmd_info", + "init": "init:cmd_init", + "list": "list:cmd_list", + "login": "login:cmd_login", + "resume": "resume:cmd_resume", + "start": "start:cmd_start", + "supervise": "supervise:cmd_supervise", } + +def _lazy(spec: str) -> Callable[[list[str]], "int | None"]: + """Wrap a `:` handler so its module is imported only when + the command is actually dispatched, not when the registry is built.""" + module, attr = spec.split(":") + + def run(argv: list[str]) -> "int | None": + handler = getattr(import_module(f".{module}", __name__), attr) + return handler(argv) + + run.__name__ = attr + return run + + +COMMANDS = {name: _lazy(spec) for name, spec in _HANDLERS.items()} + # Commands that manage host prerequisites (or are otherwise store-free) and # must run before — or without — a migrated DB. `backend` provisions/probes # the host (TAP pool, /dev/kvm, firecracker) and never opens the store, so diff --git a/bot_bottle/orchestrator/__init__.py b/bot_bottle/orchestrator/__init__.py index 0ae241d3..0858e0e2 100644 --- a/bot_bottle/orchestrator/__init__.py +++ b/bot_bottle/orchestrator/__init__.py @@ -5,7 +5,7 @@ A single persistent per-host service that will run the gateway functions agent launches. This package is being built bottom-up, starting with the backend-neutral "consolidation core" that needs no VM packaging: - * `registry` — the SQLite runtime-state store + fail-closed + * `store.registry_store` — the SQLite runtime-state store + fail-closed attribution (source IP + per-bottle identity token). * `broker` — the signed, structured launch-request contract + a `LaunchBroker` (stub for the harness) that verifies @@ -28,36 +28,58 @@ orchestrator -> firecracker). from __future__ import annotations -from .store.registry_store import BottleRecord, RegistryStore, new_identity_token -from .broker import ( - BrokerAuthError, - LaunchBroker, - LaunchRequest, - StubBroker, - sign_request, - verify_request, -) -from .docker_broker import DockerBroker, DockerBrokerError -from ..gateway import Gateway, GatewayError -from .service import Orchestrator -from .server import OrchestratorServer, dispatch, make_server +from typing import TYPE_CHECKING, Any -__all__ = [ - "BottleRecord", - "RegistryStore", - "new_identity_token", - "BrokerAuthError", - "LaunchBroker", - "LaunchRequest", - "StubBroker", - "DockerBroker", - "DockerBrokerError", - "Gateway", - "GatewayError", - "sign_request", - "verify_request", - "Orchestrator", - "OrchestratorServer", - "dispatch", - "make_server", -] +if TYPE_CHECKING: + from .store.registry_store import BottleRecord, RegistryStore, new_identity_token + from .broker import ( + BrokerAuthError, + LaunchBroker, + LaunchRequest, + StubBroker, + sign_request, + verify_request, + ) + from .docker_broker import DockerBroker, DockerBrokerError + from ..gateway import Gateway, GatewayError + from .service import Orchestrator + from .server import OrchestratorServer, dispatch, make_server + + +# Facade name -> submodule that defines it. Lazy so importing a leaf (or the +# parent package, e.g. via `orchestrator.store.store_manager`) doesn't drag the +# whole orchestrator — server, docker_broker + the docker backend, http — while +# `from bot_bottle.orchestrator import RegistryStore` keeps working. +_LAZY: dict[str, str] = { + "BottleRecord": ".store.registry_store", + "RegistryStore": ".store.registry_store", + "new_identity_token": ".store.registry_store", + "BrokerAuthError": ".broker", + "LaunchBroker": ".broker", + "LaunchRequest": ".broker", + "StubBroker": ".broker", + "sign_request": ".broker", + "verify_request": ".broker", + "DockerBroker": ".docker_broker", + "DockerBrokerError": ".docker_broker", + "Gateway": "..gateway", + "GatewayError": "..gateway", + "Orchestrator": ".service", + "OrchestratorServer": ".server", + "dispatch": ".server", + "make_server": ".server", +} + + +def __getattr__(name: str) -> Any: + src = _LAZY.get(name) + if src is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module + + value = getattr(import_module(src, __name__), name) + globals()[name] = value + return value + + +__all__ = list(_LAZY) -- 2.52.0 From bef45348f57155d9f08726362dfe819d09045655 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 24 Jul 2026 19:11:37 -0400 Subject: [PATCH 30/30] fix(types): satisfy pyright strict on the refactored facades MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two facade issues from the reorg: * orchestrator/__init__.py used `__all__ = list(_LAZY)`, which pyright strict can't analyze (reportUnsupportedDunderAll) — and because it isn't a static literal, the TYPE_CHECKING re-export imports read as unused. Replaced with the explicit literal list the other lazy facades already use. * manifest/index.py imported six piece types (ManifestAgentProvider, EGRESS_AUTH_SCHEMES, ManifestEgressConfig/Route, ManifestGitEntry, ManifestKeyConfig) it never referenced — the package facade re-exports those straight from their submodules, so index.py needn't import them. Dropped the dead imports. pyright: 0 errors (was 24). Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 --- bot_bottle/manifest/index.py | 9 ++------- bot_bottle/orchestrator/__init__.py | 20 +++++++++++++++++++- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/bot_bottle/manifest/index.py b/bot_bottle/manifest/index.py index 35966b47..8996560d 100644 --- a/bot_bottle/manifest/index.py +++ b/bot_bottle/manifest/index.py @@ -16,15 +16,10 @@ from typing import Mapping from ..log import warn from .util import ManifestError, as_json_object -from .agent import ManifestAgent, ManifestAgentProvider +from .agent import ManifestAgent from .bottle import ManifestBottle -from .egress import ( - EGRESS_AUTH_SCHEMES, - ManifestEgressConfig, - ManifestEgressRoute, -) from .extends import merge_bottles_runtime, resolve_bottles -from .git import ManifestGitEntry, ManifestGitUser, ManifestKeyConfig +from .git import ManifestGitUser from .loader import ( check_stale_json, load_bottle_chain_from_dir, diff --git a/bot_bottle/orchestrator/__init__.py b/bot_bottle/orchestrator/__init__.py index 0858e0e2..7597a69c 100644 --- a/bot_bottle/orchestrator/__init__.py +++ b/bot_bottle/orchestrator/__init__.py @@ -82,4 +82,22 @@ def __getattr__(name: str) -> Any: return value -__all__ = list(_LAZY) +__all__ = [ + "BottleRecord", + "RegistryStore", + "new_identity_token", + "BrokerAuthError", + "LaunchBroker", + "LaunchRequest", + "StubBroker", + "sign_request", + "verify_request", + "DockerBroker", + "DockerBrokerError", + "Gateway", + "GatewayError", + "Orchestrator", + "OrchestratorServer", + "dispatch", + "make_server", +] -- 2.52.0