diff --git a/Dockerfile.gateway b/Dockerfile.gateway index 8849d214..ad7001c6 100644 --- a/Dockerfile.gateway +++ b/Dockerfile.gateway @@ -8,8 +8,8 @@ # this image's mitmproxy / git / gitleaks payload. # # Collapses the prior per-daemon images (egress, git-gate, -# supervise) into one. A small stdlib-Python init supervisor at -# /app/gateway_init.py spawns all daemons, forwards SIGTERM, and +# supervise) into one. A small stdlib-Python init supervisor +# (`bot_bottle.gateway.bootstrap`) spawns all daemons, forwards SIGTERM, and # propagates per-daemon stdout/stderr to the container log with a # `[name]` prefix. See PRD 0024 for the rationale. # @@ -102,8 +102,8 @@ RUN pip install --no-cache-dir /src/ # WORKDIR here also creates /app so the shim + COPYs below can write into it # (nothing created /app before this point). WORKDIR /app -RUN printf 'from bot_bottle.egress_addon import addons\n' > /app/egress_addon.py -COPY bot_bottle/egress_entrypoint.sh /app/egress-entrypoint.sh +RUN printf 'from bot_bottle.gateway.egress.addon import addons\n' > /app/egress_addon.py +COPY bot_bottle/gateway/egress/entrypoint.sh /app/egress-entrypoint.sh RUN chmod +x /app/egress-entrypoint.sh # Pre-create runtime directories the compose renderer + start @@ -123,4 +123,4 @@ EXPOSE 8888 9099 9418 9420 9100 # PID 1 is the supervisor. It owns signal handling and exit-code # propagation; no `exec` chain in the entrypoint itself. -ENTRYPOINT ["python3", "-m", "bot_bottle.gateway_init"] +ENTRYPOINT ["python3", "-m", "bot_bottle.gateway.bootstrap"] diff --git a/bot_bottle/backend/__init__.py b/bot_bottle/backend/__init__.py index ff5223ef..0b74d807 100644 --- a/bot_bottle/backend/__init__.py +++ b/bot_bottle/backend/__init__.py @@ -1,836 +1,87 @@ -"""Per-backend bottle factories. +"""The bottle-backend package: abstract contract + backend selection. -A bottle is a running, isolated environment with claude inside. Each -backend exposes five methods: +Thin by design — nothing framework-heavy is imported at package init, so +importing any `backend.*` submodule (e.g. `backend.docker.util`) doesn't drag +the manifest / egress / git-gate framework into memory. The public names are +re-exported lazily: - prepare(spec, stage_dir=...) -> BottlePlan - Resolves names, validates host-side prerequisites, and writes - scratch files. No remote/runtime resources are created yet. - Safe to call before the y/N preflight. + * the abstract contract (`BottleBackend`, `BottleSpec`, `Bottle`, …) lives in + `backend.base`; + * backend selection / enumeration (`get_bottle_backend`, + `enumerate_active_agents`, …) in `backend.selection`; + * the concrete backends and the freeze helpers in their own submodules. - launch(plan) -> ContextManager[Bottle] - Brings up the container (or VM, or remote machine), provisions - it, yields a Bottle handle, and tears everything down on exit. - - prepare_cleanup() -> BottleCleanupPlan - Enumerates orphaned resources left behind by previous bottles - (containers, networks, ...). Idempotent; no side effects. - - cleanup(plan) -> None - Actually removes everything described by the cleanup plan. - - enumerate_active() -> Sequence[ActiveAgent] - Return every currently-running bottle on this backend, with - enough metadata for callers (CLI `list active`, dashboard - agents pane) to render a row. - -Selection is driven by `--backend` on `start` or BOT_BOTTLE_BACKEND -(env var). When neither is set, compatible macOS hosts default to -`macos-container`; Linux hosts with KVM default to `firecracker`; -otherwise `docker`. Per PRD 0003 the manifest does not carry a -backend field; the host picks. +`from bot_bottle.backend import X` resolves X on first access via `__getattr__` +and caches it at package level, so existing call-sites (and +`patch.object(backend_mod, X, …)`) keep working. """ from __future__ import annotations -import os -import shlex -import sys -from abc import ABC, abstractmethod -from contextlib import AbstractContextManager, contextmanager -from dataclasses import dataclass -from pathlib import Path -from typing import TYPE_CHECKING, Any, Generator, Generic, Sequence, TypeVar - -from ..agent_provider import AgentProvisionPlan, get_provider, build_agent_provision_plan -from ..egress import EgressPlan -from ..git_gate import GitGatePlan -from ..log import die, info, warn -from ..util import read_tty_line -from ..manifest import Manifest, ManifestIndex -from ..supervise import SupervisePlan -from ..util import expand_tilde -from ..env import resolve_env, ResolvedEnv -from ..workspace import WorkspacePlan, workspace_plan -from .print_util import print_multi, visible_agent_env_names -from .util import host_skill_dir +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: + from .base import ( + ActiveAgent, + Bottle, + BottleBackend, + BottleCleanupPlan, + BottleImages, + BottlePlan, + BottleSpec, + ExecResult, + ) + from .selection import ( + enumerate_active_agents, + get_bottle_backend, + has_backend, + known_backend_names, + ) + from .docker import DockerBottleBackend + from .firecracker import FirecrackerBottleBackend + from .macos_container import MacosContainerBottleBackend from .freeze import CommitCancelled, Freezer, get_freezer -@dataclass(frozen=True) -class BottleSpec: - """CLI-supplied intent. Backend-agnostic — each backend's prepare - step consumes it and produces its own backend-specific plan. - Resolved values (image names, container name, scratch paths, runsc - availability) live on the plan, not the spec.""" - - manifest: ManifestIndex - agent_name: str - copy_cwd: bool - user_cwd: str - # PRD 0016 follow-up: when set, the backend's prepare step uses - # this identity instead of minting a fresh one — the resume path - # (`cli.py resume `) sets this to continue an existing - # bottle's state. Empty string for a fresh `start`. - identity: str = "" - label: str = "" - color: str = "" - # Ordered bottle names selected at launch (issue #269). When non-empty - # they are merged in order and replace the agent's `bottle:` field. - bottle_names: tuple[str, ...] = () - # True when launched via --headless (no TTY, no interactive prompts). - # The git-gate host-key preflight uses this to error rather than prompt. - headless: bool = False - # Image startup policy. "fresh" preserves the normal build path; - # "cached" reuses the current local image/artifact without rebuilding. - image_policy: str = "fresh" - - -@dataclass(frozen=True) -class BottlePlan(ABC): - """Base output of a backend's prepare step. Concrete subclasses - (e.g. DockerBottlePlan) add backend-specific resolved fields.""" - - spec: BottleSpec - manifest: Manifest - stage_dir: Path - git_gate_plan: GitGatePlan - - @property - def guest_home(self) -> str: - return self.agent_provision.guest_home - - @property - def git_gate_insteadof_host(self) -> str: - """Host (and optional port) used in git-gate insteadOf URLs. - Docker uses the compose-network DNS alias; VM backends may - override with an IP:port when the guest has no DNS.""" - return "git-gate" - - @property - def git_gate_insteadof_scheme(self) -> str: - """URL scheme for git-gate insteadOf rewrites. 'git' for - Docker (git daemon); VM backends may override (e.g. 'http' - over a published host port).""" - return "git" - egress_plan: EgressPlan - supervise_plan: SupervisePlan | None - agent_provision: AgentProvisionPlan - - @property - def workspace_plan(self) -> WorkspacePlan: - return workspace_plan(self.spec, guest_home=self.guest_home) - - def print(self) -> None: - """Render the y/N preflight summary to stderr.""" - spec = self.spec - manifest = self.manifest - agent = manifest.agent - bottle = manifest.bottle - - env_names = visible_agent_env_names( - sorted( - set(bottle.env.keys()) - | set(self.agent_provision.guest_env.keys()) - ), - hidden_env_names=self.agent_provision.hidden_env_names, - ) - - print(file=sys.stderr) - info(f"agent : {spec.agent_name}") - info(f"provider : {self.agent_provision.template}") - print_multi("env ", env_names) - print_multi("skills ", list(agent.skills)) - effective_bottles = ( - list(spec.bottle_names) if spec.bottle_names - else ([agent.bottle] if agent.bottle else []) - ) - print_multi("bottle ", effective_bottles) - - identity = manifest.git_identity_summary() - if identity: - info(f" git identity : {identity}") - - git_lines = [ - f"{u.name} → {u.upstream_host}:{u.upstream_port}" - for u in self.git_gate_plan.upstreams - ] - if git_lines: - print_multi(" git gate ", git_lines) - - if self.egress_plan.routes: - egress_lines = [] - for r in self.egress_plan.routes: - auth = f" [auth:{r.auth_scheme}]" if r.auth_scheme else "" - egress_lines.append(f"{r.host}{auth}") - print_multi(" egress ", egress_lines) - print(file=sys.stderr) - - -@dataclass(frozen=True) -class BottleCleanupPlan(ABC): - """Base output of a backend's prepare_cleanup step. Concrete - subclasses (e.g. DockerBottleCleanupPlan) carry backend-specific - lists of resources to be removed and implement `print` + `empty`.""" - - @abstractmethod - def print(self) -> None: - """Render the cleanup y/N summary to stderr.""" - - @property - @abstractmethod - def empty(self) -> bool: - """True iff there is nothing to clean up; the CLI uses this to - short-circuit before showing the y/N.""" - - -@dataclass(frozen=True) -class ExecResult: - """Captured result of `Bottle.exec`. Backend-neutral: the Docker - impl populates it from a `subprocess.CompletedProcess`, but a - VM backend could populate it from any source that produces a - returncode + captured streams.""" - - returncode: int - stdout: str - stderr: str - - -@dataclass(frozen=True) -class ActiveAgent: - """One currently-running agent, as the CLI `list active` and - dashboard agents pane render it. ("Agent" is the project's - consistent name for the thing running inside a bottle — the - bottle is the container, the agent is what runs in it.) - - Fields are deliberately backend-neutral. `services` is the set - of gateway daemons currently up for this bottle (`egress`, - `git-gate`, `supervise`); the dashboard uses it to - gate edit verbs. `backend_name` is the matching key in - `_BACKENDS` (`docker` / `firecracker` / `macos-container`) — used by the active- - list rendering to disambiguate and by the dashboard's - re-attach path.""" - - backend_name: str - slug: str - agent_name: str # from metadata.json; "?" if missing - started_at: str # ISO 8601 from metadata.json; "" if missing - services: tuple[str, ...] # alphabetical - label: str = "" - color: str = "" - - -class Bottle(ABC): - """Handle to a running bottle. Yielded by a backend's launch step. - - `exec_agent` runs the selected agent CLI inside the bottle and - blocks until the session ends. `exec` runs a POSIX shell script inside the bottle - and returns the captured result. `cp_in` copies a host path into - the bottle. `close` is an idempotent alias for context-manager - teardown. - """ - - name: str - - @abstractmethod - def agent_argv( - self, argv: list[str], *, tty: bool = True, - ) -> list[str]: - """Return the host-side argv that runs the selected agent - inside the bottle. Used by `exec_agent` for foreground - handoffs and by the dashboard's tmux `respawn-pane` flow, - which needs the argv up front (it spawns claude in a tmux - pane rather than as a child of the current process). - - Implementations transparently inject - `--append-system-prompt-file` when the bottle was launched - with a provisioned prompt path.""" - ... - - @abstractmethod - def exec_agent(self, argv: list[str], *, tty: bool = True) -> int: ... - - @abstractmethod - def exec(self, script: str, *, user: str = "node") -> ExecResult: - """Run `script` as a POSIX shell script inside the bottle as - `user` (default `node`, matching the agent image's USER - directive) and return the captured stdout/stderr/returncode. - The bottle's environment (including HTTPS_PROXY pointing at - the egress daemon) is inherited by the child. Non-zero - exit does not raise — callers inspect `returncode` - themselves. - - Pass `user="root"` for shell-outs that need privileged file - writes / package install — provisioning calls that need root - bypass `Bottle.exec` and use the backend-specific raw - machine-exec helper, but the tests have a legitimate use - case for arbitrary-user runs.""" - - @abstractmethod - def cp_in(self, host_path: str, container_path: str) -> None: ... - - @abstractmethod - def close(self) -> None: ... - - - - -PlanT = TypeVar("PlanT", bound=BottlePlan) -CleanupT = TypeVar("CleanupT", bound=BottleCleanupPlan) - - -@dataclass(frozen=True) -class BottleImages: - """Resolved image references (or artifact paths) for a bottle launch. - - For Docker/macOS-container backends, `agent` and `sidecar` are string - image refs. For the smolmachines backend they are Path objects pointing - to pre-built `.smolmachine` artifacts.""" - - agent: str | Path - sidecar: str | Path = "" - - -class BottleBackend(ABC, Generic[PlanT, CleanupT]): - """Abstract base for selectable bottle backends. Concrete subclasses - (e.g. DockerBottleBackend) own their own prepare/launch impls. - Parameterized over the backend's concrete plan + cleanup-plan types - so subclass methods get the narrow type without isinstance - boilerplate.""" - - name: str - - # Whether this backend can run a container engine *inside* the bottle. - # Backends that cannot must reject `nested_containers: true` rather than - # reach for a host daemon socket (issue #392). - supports_nested_containers: bool = False - - def prepare(self, spec: BottleSpec, stage_dir: Path) -> PlanT: - """Template method: run cross-backend host-side validation, then - delegate to the subclass's `_resolve_plan` for the - backend-specific resolution (names, scratch files, etc.). The - validation step is enforced here so a future backend cannot - accidentally skip it. No remote/runtime resources are created.""" - from .resolve_common import ( - merge_provision_env_vars, - mint_slug, - prepare_agent_state_dir, - prepare_egress, - prepare_git_gate, - prepare_supervise, - reject_nested_containers, - resolve_manifest_dockerfile, - write_launch_metadata, - ) - - manifest = self._validate(spec) - - if not self.supports_nested_containers: - reject_nested_containers(self.name, manifest) - - self._preflight() - - from ..git_gate_host_key import preflight_host_keys - manifest = preflight_host_keys( - manifest, - headless=spec.headless, - home_md=spec.manifest.home_md, - ) - - manifest_bottle = manifest.bottle - manifest_agent_provider = manifest_bottle.agent_provider - agent_provider = get_provider(manifest_agent_provider.template) - resolved_env = resolve_env(manifest) - workspace = workspace_plan(spec, guest_home=agent_provider.guest_home) - - slug = mint_slug(spec) - write_launch_metadata(slug, spec, compose_project="", backend=self.name) - - # Manifest may override the Dockerfile per-bottle; otherwise fall - # back to the provider plugin's bundled Dockerfile (next to its - # agent_provider.py module). - if manifest_agent_provider.dockerfile: - agent_dockerfile_path = resolve_manifest_dockerfile( - manifest_agent_provider.dockerfile, spec, - ) - else: - agent_dockerfile_path = str(agent_provider.dockerfile) - - agent_dir, prompt_file = prepare_agent_state_dir(slug, manifest) - - agent_provision_plan = build_agent_provision_plan( - template=manifest_agent_provider.template, - dockerfile=agent_dockerfile_path, - state_dir=agent_dir, - instance_name=f"bot-bottle-{slug}", - prompt_file=prompt_file, - guest_env=self._build_guest_env(resolved_env), - forward_host_credentials=manifest_agent_provider.forward_host_credentials, - auth_token=manifest_agent_provider.auth_token, - host_env=dict(os.environ), - trusted_project_path=workspace.workdir, - label=spec.label, - color=spec.color, - provider_settings=manifest_agent_provider.settings, - ) - agent_provision_plan = merge_provision_env_vars(agent_provision_plan) - egress_plan = prepare_egress(manifest_bottle, slug, agent_provision_plan) - supervise_plan = prepare_supervise(manifest_bottle, slug) - git_gate_plan = prepare_git_gate(manifest_bottle, slug) - - return self._resolve_plan( - spec, - manifest=manifest, - slug=slug, - resolved_env=resolved_env, - agent_provision_plan=agent_provision_plan, - egress_plan=egress_plan, - supervise_plan=supervise_plan, - git_gate_plan=git_gate_plan, - stage_dir=stage_dir, - ) - - def _build_guest_env(self, resolved_env: ResolvedEnv) -> dict[str, str]: - return {} - - def _preflight(self) -> None: - """ - tasks to do before resolving a plan - """ - pass - - def _validate(self, spec: BottleSpec) -> Manifest: - """Cross-backend pre-launch checks. Parses the selected agent and - its bottle (raising ManifestError on invalid content), confirms - skills are present on the host, and every git IdentityFile resolves. - - Returns the loaded Manifest for the selected agent. Subclasses with - additional preconditions should override and call - `super()._validate(spec)` first.""" - manifest = spec.manifest.load_for_agent(spec.agent_name, spec.bottle_names) - self._validate_skills(manifest.agent.skills) - self._validate_agent_provider_dockerfile(spec, manifest) - return manifest - - def _validate_skills(self, skills: Sequence[str]) -> None: - """Each named skill must be a directory under the host's - `~/.claude/skills/`. The check is purely host-side, so the - default impl covers every backend.""" - for name in skills: - path = host_skill_dir(name) - if not os.path.isdir(path): - die( - f"skill '{name}' not found on host at {path}. " - f"Create it under ~/.claude/skills/, then re-run." - ) - - def _validate_agent_provider_dockerfile(self, spec: BottleSpec, manifest: Manifest) -> None: - bottle = manifest.bottle - dockerfile = bottle.agent_provider.dockerfile - if not dockerfile: - return - path = Path(expand_tilde(dockerfile)) - if not path.is_absolute(): - path = Path(spec.user_cwd) / path - if not path.is_file(): - effective = ( - ", ".join(spec.bottle_names) if spec.bottle_names else manifest.agent.bottle - ) - die( - f"agent_provider.dockerfile for bottle " - f"'{effective}' not found: {path}" - ) - - @abstractmethod - def _resolve_plan(self, - spec: BottleSpec, - *, - manifest: Manifest, - slug: str, - resolved_env: ResolvedEnv, - agent_provision_plan: AgentProvisionPlan, - egress_plan: EgressPlan, - git_gate_plan: GitGatePlan, - supervise_plan: SupervisePlan | None, - stage_dir: Path) -> PlanT: - """Backend-specific plan resolution: image/container names, - env-file, prompt-file, proxy plan, runtime detection. Called by - `prepare` after `_validate` succeeds. Instance name, image, - prompt file, Dockerfile path, and guest home all live on - `agent_provision_plan` — the source of truth.""" - - def prelaunch_checks(self, plan: PlanT) -> None: - """Raise StaleImageError if any cached image used by this plan is stale. - No-op default; backends override to call the shared check_stale* - helpers on their image/artifact timestamps. Called by the CLI before - launch so the operator can be prompted outside the launch context.""" - - @contextmanager - def launch(self, plan: PlanT) -> Generator[Bottle, None, None]: - """Template: build or load images, then delegate to _launch_impl.""" - images = self._build_or_load_images(plan) - with self._launch_impl(plan, images) as bottle: - yield bottle - - @abstractmethod - def _build_or_load_images(self, plan: PlanT) -> BottleImages: - """Return the agent and sidecar image references (or artifact paths) - for this plan, building fresh images when the policy requires it.""" - - @abstractmethod - def _launch_impl(self, plan: PlanT, images: BottleImages) -> AbstractContextManager[Bottle]: - """Bring up the bottle using pre-resolved images; yield a handle; tear down on exit.""" - - def provision(self, plan: PlanT, bottle: "Bottle") -> str | None: - """Copy host-side files (CA cert, prompt, skills, .git) into - the running bottle. Called from `launch` after the container - / machine is up. Returns the in-container prompt path if a - prompt was provisioned, else None — the Bottle handle uses it - to decide whether to add provider-specific prompt args to the - agent's argv. - - Default orchestration: ca → prompt → provider apply → skills - → workspace → git → supervise-mcp. CA install runs first so - the agent's trust store is rebuilt before anything inside the - agent makes a TLS call. - - Per PRD 0050 the per-provider steps (prompt, skills, - declarative provision-plan apply, supervise MCP registration) - live on the `AgentProvider` plugin. The backend only owns the - steps that are about backend infrastructure (CA, workspace, - git) and surfaces the supervise daemon URL its launch step - knows about via `supervise_mcp_url`. - - PRD 0017: cred-proxy's agent-side dotfile rewrites (~/.npmrc, - ~/.gitconfig insteadOf, tea config) are gone. Egress-proxy is - on the agent's HTTP_PROXY path so every tool that respects - HTTPS_PROXY (claude-code, git over HTTPS, npm, curl) is - intercepted without per-tool reconfiguration.""" - provider = get_provider(plan.agent_provision.template) - provider.provision_ca(bottle, plan) - prompt_path = provider.provision_prompt(plan, bottle) - provider.provision(plan, bottle) - provider.provision_skills(plan, bottle) - self.provision_workspace(plan, bottle) - provider.provision_git(bottle, plan) - provider.provision_supervise_mcp( - plan, bottle, self.supervise_mcp_url(plan), - ) - return prompt_path - - def provision_workspace(self, plan: PlanT, bottle: "Bottle") -> None: - """Copy the operator workspace into the running bottle. - - This is the only supported workspace-provisioning path: Docker - does not build a derived image containing the current - workspace.""" - workspace = plan.workspace_plan - if not (workspace.enabled and workspace.copy_contents): - return - - guest_parent = workspace.guest_path.rsplit("/", 1)[0] or "/" - guest_path = shlex.quote(workspace.guest_path) - guest_parent = shlex.quote(guest_parent) - owner = shlex.quote(workspace.owner) - mode = shlex.quote(workspace.mode) - info(f"copying {workspace.host_path} -> {bottle.name}:{workspace.guest_path}") - bottle.exec( - f"rm -rf {guest_path} && mkdir -p {guest_parent}", - user="root", - ) - bottle.cp_in(str(workspace.host_path), workspace.guest_path) - bottle.exec( - f"chown -R {owner} {guest_path} && chmod {mode} {guest_path}", - user="root", - ) - - def supervise_mcp_url(self, plan: PlanT) -> str: - """Return the agent-side URL of the per-bottle supervise - gateway, or "" when this bottle has no gateway. The provider - plugin's `provision_supervise_mcp` uses it to register the - MCP entry inside the guest. - - Default returns "" so backends without supervise support - don't have to implement it. Docker and firecracker override.""" - del plan - return "" - - def ensure_orchestrator(self) -> str: - """Bring up this backend's per-host orchestrator + shared gateway - (idempotent) and return the host-reachable control-plane URL. - - This is the backend-agnostic bring-up entry point: `launch` calls - it as part of starting a bottle, and operator tools (`supervise`) - call it to start the control plane on demand when none is running - yet. Docker starts the orchestrator + gateway containers; - firecracker boots the infra VM. Backends with no orchestrator - (macos-container) die with a pointer — the default here.""" - die(f"backend {self.name!r} has no orchestrator control plane") - - @abstractmethod - def prepare_cleanup(self) -> CleanupT: - """Enumerate orphaned resources from previous bottles. No side - effects; safe to call before the y/N.""" - - @abstractmethod - def cleanup(self, plan: CleanupT) -> None: - """Remove everything described by the cleanup plan.""" - - @abstractmethod - def enumerate_active(self) -> Sequence[ActiveAgent]: - """Return every currently-running agent on this backend. - Empty when none. Backend-specific: docker queries `docker - compose ls`; firecracker cross-references its running gateway - containers against per-bottle metadata.""" - - @classmethod - @abstractmethod - def is_available(cls) -> bool: - """Whether this backend's runtime prerequisites are satisfied - on the current host. Docker → `docker` on PATH; firecracker → - Linux + KVM. Used by the cross-backend - `enumerate_active_agents` / `cmd_cleanup` to skip backends - the operator hasn't installed, so a docker-only host - doesn't fail when `cli.py list active` walks past - firecracker.""" - - @classmethod - @abstractmethod - def setup(cls) -> int: - """Emit this backend's one-time host setup — privileged network - pool, daemon bring-up, install pointers, etc. — as - host-appropriate config or commands. Prints to stdout/stderr and - returns a shell exit code (0 = nothing to report / success). A - backend that needs no host setup prints a short note and returns - 0. Invoked generically by `./cli.py backend setup [--backend=…]` - so operators can provision any backend without a - backend-specific command. Classmethod (like `is_available`) — - it's a host query, not per-bottle state.""" - - @classmethod - @abstractmethod - def status(cls) -> int: - """Report whether this backend's prerequisites are satisfied on - the host — binaries, daemon reachability, network pool, range - conflicts, etc. Prints a human-readable summary; returns 0 when - the backend is ready to launch and non-zero when something is - missing. Invoked by `./cli.py backend status [--backend=…]`.""" - - @classmethod - @abstractmethod - def teardown(cls) -> int: - """Undo `setup()` — the inverse operation, surfaced as - `./cli.py backend teardown [--backend=…]` (uninstall). Symmetric - with setup: where setup is advisory (prints the privileged - commands / declarative config to apply), teardown prints the - commands / config change to remove the host prerequisites. A - backend with no host setup prints a short note and returns 0. - Not called by the launch path or the test suite.""" - - -# _backends is None until the first call to _get_backends(), at which -# point all three concrete backend classes are imported and instantiated. -# Keeping the imports out of module scope means that importing any -# backend sub-module (e.g. `backend.docker.util`) no longer drags the -# firecracker and macos-container implementations into memory. -# -# Tests may replace _backends with a {name: fake} dict via patch.object; -# _get_backends() returns the current module-level value as-is when it -# is not None, so test fakes take effect without triggering real imports. -_backends: dict[str, BottleBackend[Any, Any]] | None = None - - -def _get_backends() -> dict[str, BottleBackend[Any, Any]]: - """Return the registry of all backend instances, loading lazily on first call.""" - global _backends # pylint: disable=global-statement - if _backends is None: - from .docker import DockerBottleBackend - from .firecracker import FirecrackerBottleBackend - from .macos_container import MacosContainerBottleBackend - _backends = { - "docker": DockerBottleBackend(), - "firecracker": FirecrackerBottleBackend(), - "macos-container": MacosContainerBottleBackend(), - } - return _backends +# Public name -> submodule that defines it. Contract types resolve from `base`, +# selection/enumeration from `selection`, the concrete backends + freeze helpers +# from their own subpackages. +_LAZY_MODULES: dict[str, str] = { + "BottleSpec": "base", + "BottlePlan": "base", + "BottleCleanupPlan": "base", + "ExecResult": "base", + "ActiveAgent": "base", + "Bottle": "base", + "BottleImages": "base", + "BottleBackend": "base", + "get_bottle_backend": "selection", + "known_backend_names": "selection", + "has_backend": "selection", + "enumerate_active_agents": "selection", + "_print_vm_install_instructions": "selection", + "DockerBottleBackend": "docker", + "FirecrackerBottleBackend": "firecracker", + "MacosContainerBottleBackend": "macos_container", + "CommitCancelled": "freeze", + "Freezer": "freeze", + "get_freezer": "freeze", +} def __getattr__(name: str) -> Any: - """Lazily surface concrete backend classes and freeze symbols at the - package level so existing `from bot_bottle.backend import X` and - `patch.object(backend_mod, X, ...)` call-sites keep working without - forcing an import of every backend at module-init time.""" - if name == "DockerBottleBackend": - from .docker import DockerBottleBackend - globals()[name] = DockerBottleBackend - return DockerBottleBackend - if name == "FirecrackerBottleBackend": - from .firecracker import FirecrackerBottleBackend - globals()[name] = FirecrackerBottleBackend - return FirecrackerBottleBackend - if name == "MacosContainerBottleBackend": - from .macos_container import MacosContainerBottleBackend - globals()[name] = MacosContainerBottleBackend - return MacosContainerBottleBackend - if name == "CommitCancelled": - from .freeze import CommitCancelled - globals()[name] = CommitCancelled - return CommitCancelled - if name == "Freezer": - from .freeze import Freezer - globals()[name] = Freezer - return Freezer - if name == "get_freezer": - from .freeze import get_freezer - globals()[name] = get_freezer - return get_freezer - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + """Lazily surface the package's public names from their submodules and + cache them at package level — so `from bot_bottle.backend import X` and + `patch.object(backend_mod, X, …)` keep working without importing the + framework (or every backend) at package-init time.""" + mod = _LAZY_MODULES.get(name) + if mod is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module - -def get_bottle_backend( - name: str | None = None, - *, - prompt: bool = True, -) -> BottleBackend[Any, Any]: - """Resolve the bottle backend. - - `name` precedence: - 1. explicit arg (e.g. resume passes the recorded backend name) - 2. BOT_BOTTLE_BACKEND env var - 3. auto-selection: VM backend first, docker fallback with prompt - - `prompt` controls whether auto-selection may block on an interactive - [i/d/q] prompt when falling back to docker. Pass `prompt=False` in - non-interactive contexts (headless launches, CI) so the call dies - with an actionable message instead of hanging. - - Dies with a pointer at the known backends if the chosen name - isn't implemented.""" - resolved = name or os.environ.get("BOT_BOTTLE_BACKEND") - if resolved is None: - resolved = _auto_select_backend(prompt=prompt) - backends = _get_backends() - if resolved not in backends: - known = ", ".join(sorted(backends)) - die(f"unknown backend {resolved!r}; known backends: {known}") - return backends[resolved] - - -def _platform_vm_suggestion() -> str: - """Platform-appropriate VM backend name for install suggestions.""" - return "macos-container" if sys.platform == "darwin" else "firecracker" - - -def _print_vm_install_instructions() -> None: - """Print platform-appropriate VM backend install instructions to stderr.""" - vm = _platform_vm_suggestion() - if vm == "macos-container": - info("Install Apple Container: https://github.com/apple/container/releases") - info("Then start the service: container system start") - else: - info("Install Firecracker: https://github.com/firecracker-microvm/firecracker/releases") - info("Configure the host: ./cli.py backend setup") - - -def _auto_select_backend(prompt: bool = True) -> str: - """Tier-1 / tier-2 backend auto-selection. - - Tier 1: VM backend — macos-container on macOS when Apple Container is - installed; firecracker on KVM-capable Linux even before the binary is - present (its preflight prints an install pointer). - - Tier 2: docker, with a security warning and an interactive prompt. - When `prompt=False` (headless / CI), dies with an actionable message - instead of blocking on a TTY read. When docker is also absent, prints - VM install instructions and exits. - """ - # --- Tier 1: VM backend ----------------------------------------- - if has_backend("macos-container"): - return "macos-container" - # A KVM-capable Linux host defaults to firecracker even when the - # `firecracker` binary isn't installed yet: selecting it here routes - # start through firecracker's preflight, which prints an install - # pointer, instead of silently falling back to docker. - from .firecracker import FirecrackerBottleBackend - if FirecrackerBottleBackend.is_host_capable(): - return "firecracker" - - # --- Tier 2: docker fallback ------------------------------------ - if not has_backend("docker"): - info("No backend available on this host.") - _print_vm_install_instructions() - die("no backend available; install a VM backend and re-run") - - vm = _platform_vm_suggestion() - warn( - "docker is less secure than VM backends — " - "containers share the host kernel." - ) - if not prompt: - die( - f"no VM backend available; set BOT_BOTTLE_BACKEND=docker to proceed " - f"with docker, or install the {vm!r} backend." - ) - sys.stderr.write( - f"bot-bottle: For better isolation, install the {vm!r} backend.\n" - f" [i] show {vm} install instructions and exit\n" - " [d] use docker anyway\n" - " [q] quit\n" - "bot-bottle: choice [i/d/q]: " - ) - sys.stderr.flush() - reply = read_tty_line().strip().lower() - if reply == "d": - return "docker" - if reply == "i": - _print_vm_install_instructions() - die("not proceeding with docker; install a VM backend or set BOT_BOTTLE_BACKEND=docker") - - -def known_backend_names() -> tuple[str, ...]: - """Sorted tuple of all backend keys in `_get_backends()`. Used by - argparse (`--backend` choices) and the dashboard's backend - picker.""" - return tuple(sorted(_get_backends())) - - -def has_backend(name: str) -> bool: - """Whether the named backend's runtime prerequisites are - available on the current host. Cross-backend callers (list, - cleanup) skip unavailable backends so a docker-only host - doesn't fail when the firecracker backend isn't usable, - and vice versa. - - Returns False for unknown names so callers can pass - arbitrary input without separate validation.""" - backends = _get_backends() - if name not in backends: - return False - return backends[name].is_available() - - -def enumerate_active_agents() -> list[ActiveAgent]: - """All currently-running agents, across every available - backend. Used by CLI `list active` and the dashboard's agents - pane so neither has to know which backends exist. Skips - backends whose `is_available()` reports False. - - Sorted by `(started_at, slug)` so the list is stable across - dashboard refresh ticks — agents don't shift position while - the operator navigates with arrow keys. ISO 8601 timestamps - sort lexicographically in chronological order; `slug` is the - deterministic tiebreaker. Agents with missing metadata - (`started_at == ""`) sort first.""" - out: list[ActiveAgent] = [] - backends = _get_backends() - for name in sorted(backends): - if not backends[name].is_available(): - continue - out.extend(backends[name].enumerate_active()) - out.sort(key=lambda a: (a.started_at, a.slug)) - return out + value = getattr(import_module(f"{__name__}.{mod}"), name) + globals()[name] = value + return value __all__ = [ @@ -838,14 +89,18 @@ __all__ = [ "Bottle", "BottleBackend", "BottleCleanupPlan", + "BottleImages", "BottlePlan", "BottleSpec", - "CommitCancelled", "ExecResult", + "CommitCancelled", "Freezer", + "get_freezer", + "DockerBottleBackend", + "FirecrackerBottleBackend", + "MacosContainerBottleBackend", "enumerate_active_agents", "get_bottle_backend", - "get_freezer", "has_backend", "known_backend_names", ] diff --git a/bot_bottle/backend/base.py b/bot_bottle/backend/base.py new file mode 100644 index 00000000..a58ad4f2 --- /dev/null +++ b/bot_bottle/backend/base.py @@ -0,0 +1,607 @@ +"""The abstract backend contract (PRD 0018 / 0070). + +The backend-neutral types every bottle backend implements: the launch +`BottleSpec`, the `BottlePlan` / `BottleCleanupPlan` ABCs, the running-`Bottle` ++ `ExecResult` shapes, `BottleImages`, and the `BottleBackend` ABC itself. + +This carries the framework imports (manifest, egress, git-gate, env, workspace, +agent-provider) the contract's signatures and helpers need — which is why it +lives here rather than in `backend/__init__.py`: touching an unrelated +`backend.*` module then doesn't drag the whole framework into memory. The +thin package `__init__` re-exports these names lazily. +""" + +from __future__ import annotations + +import os +import shlex +import sys +from abc import ABC, abstractmethod +from contextlib import AbstractContextManager, contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Generator, Generic, Sequence, TypeVar + +from ..agent_provider import AgentProvisionPlan, get_provider, build_agent_provision_plan +from ..egress import EgressPlan +from ..git_gate import GitGatePlan +from ..log import die, info +from ..util import expand_tilde +from ..manifest import Manifest, ManifestIndex +from ..supervisor.plan import SupervisePlan +from ..env import resolve_env, ResolvedEnv +from ..workspace import WorkspacePlan, workspace_plan +from .print_util import print_multi, visible_agent_env_names +from .util import host_skill_dir + + + +@dataclass(frozen=True) +class BottleSpec: + """CLI-supplied intent. Backend-agnostic — each backend's prepare + step consumes it and produces its own backend-specific plan. + Resolved values (image names, container name, scratch paths, runsc + availability) live on the plan, not the spec.""" + + manifest: ManifestIndex + agent_name: str + copy_cwd: bool + user_cwd: str + # PRD 0016 follow-up: when set, the backend's prepare step uses + # this identity instead of minting a fresh one — the resume path + # (`cli.py resume `) sets this to continue an existing + # bottle's state. Empty string for a fresh `start`. + identity: str = "" + label: str = "" + color: str = "" + # Ordered bottle names selected at launch (issue #269). When non-empty + # they are merged in order and replace the agent's `bottle:` field. + bottle_names: tuple[str, ...] = () + # True when launched via --headless (no TTY, no interactive prompts). + # The git-gate host-key preflight uses this to error rather than prompt. + headless: bool = False + # Image startup policy. "fresh" preserves the normal build path; + # "cached" reuses the current local image/artifact without rebuilding. + image_policy: str = "fresh" + + +@dataclass(frozen=True) +class BottlePlan(ABC): + """Base output of a backend's prepare step. Concrete subclasses + (e.g. DockerBottlePlan) add backend-specific resolved fields.""" + + spec: BottleSpec + manifest: Manifest + stage_dir: Path + git_gate_plan: GitGatePlan + + @property + def guest_home(self) -> str: + return self.agent_provision.guest_home + + @property + def git_gate_insteadof_host(self) -> str: + """Host (and optional port) used in git-gate insteadOf URLs. + Docker uses the compose-network DNS alias; VM backends may + override with an IP:port when the guest has no DNS.""" + return "git-gate" + + @property + def git_gate_insteadof_scheme(self) -> str: + """URL scheme for git-gate insteadOf rewrites. 'git' for + Docker (git daemon); VM backends may override (e.g. 'http' + over a published host port).""" + return "git" + egress_plan: EgressPlan + supervise_plan: SupervisePlan | None + agent_provision: AgentProvisionPlan + + @property + def workspace_plan(self) -> WorkspacePlan: + return workspace_plan(self.spec, guest_home=self.guest_home) + + def print(self) -> None: + """Render the y/N preflight summary to stderr.""" + spec = self.spec + manifest = self.manifest + agent = manifest.agent + bottle = manifest.bottle + + env_names = visible_agent_env_names( + sorted( + set(bottle.env.keys()) + | set(self.agent_provision.guest_env.keys()) + ), + hidden_env_names=self.agent_provision.hidden_env_names, + ) + + print(file=sys.stderr) + info(f"agent : {spec.agent_name}") + info(f"provider : {self.agent_provision.template}") + print_multi("env ", env_names) + print_multi("skills ", list(agent.skills)) + effective_bottles = ( + list(spec.bottle_names) if spec.bottle_names + else ([agent.bottle] if agent.bottle else []) + ) + print_multi("bottle ", effective_bottles) + + identity = manifest.git_identity_summary() + if identity: + info(f" git identity : {identity}") + + git_lines = [ + f"{u.name} → {u.upstream_host}:{u.upstream_port}" + for u in self.git_gate_plan.upstreams + ] + if git_lines: + print_multi(" git gate ", git_lines) + + if self.egress_plan.routes: + egress_lines = [] + for r in self.egress_plan.routes: + auth = f" [auth:{r.auth_scheme}]" if r.auth_scheme else "" + egress_lines.append(f"{r.host}{auth}") + print_multi(" egress ", egress_lines) + print(file=sys.stderr) + + +@dataclass(frozen=True) +class BottleCleanupPlan(ABC): + """Base output of a backend's prepare_cleanup step. Concrete + subclasses (e.g. DockerBottleCleanupPlan) carry backend-specific + lists of resources to be removed and implement `print` + `empty`.""" + + @abstractmethod + def print(self) -> None: + """Render the cleanup y/N summary to stderr.""" + + @property + @abstractmethod + def empty(self) -> bool: + """True iff there is nothing to clean up; the CLI uses this to + short-circuit before showing the y/N.""" + + +@dataclass(frozen=True) +class ExecResult: + """Captured result of `Bottle.exec`. Backend-neutral: the Docker + impl populates it from a `subprocess.CompletedProcess`, but a + VM backend could populate it from any source that produces a + returncode + captured streams.""" + + returncode: int + stdout: str + stderr: str + + +@dataclass(frozen=True) +class ActiveAgent: + """One currently-running agent, as the CLI `list active` and + dashboard agents pane render it. ("Agent" is the project's + consistent name for the thing running inside a bottle — the + bottle is the container, the agent is what runs in it.) + + Fields are deliberately backend-neutral. `services` is the set + of gateway daemons currently up for this bottle (`egress`, + `git-gate`, `supervise`); the dashboard uses it to + gate edit verbs. `backend_name` is the matching key in + `_BACKENDS` (`docker` / `firecracker` / `macos-container`) — used by the active- + list rendering to disambiguate and by the dashboard's + re-attach path.""" + + backend_name: str + slug: str + agent_name: str # from metadata.json; "?" if missing + started_at: str # ISO 8601 from metadata.json; "" if missing + services: tuple[str, ...] # alphabetical + label: str = "" + color: str = "" + + +class Bottle(ABC): + """Handle to a running bottle. Yielded by a backend's launch step. + + `exec_agent` runs the selected agent CLI inside the bottle and + blocks until the session ends. `exec` runs a POSIX shell script inside the bottle + and returns the captured result. `cp_in` copies a host path into + the bottle. `close` is an idempotent alias for context-manager + teardown. + """ + + name: str + + @abstractmethod + def agent_argv( + self, argv: list[str], *, tty: bool = True, + ) -> list[str]: + """Return the host-side argv that runs the selected agent + inside the bottle. Used by `exec_agent` for foreground + handoffs and by the dashboard's tmux `respawn-pane` flow, + which needs the argv up front (it spawns claude in a tmux + pane rather than as a child of the current process). + + Implementations transparently inject + `--append-system-prompt-file` when the bottle was launched + with a provisioned prompt path.""" + ... + + @abstractmethod + def exec_agent(self, argv: list[str], *, tty: bool = True) -> int: ... + + @abstractmethod + def exec(self, script: str, *, user: str = "node") -> ExecResult: + """Run `script` as a POSIX shell script inside the bottle as + `user` (default `node`, matching the agent image's USER + directive) and return the captured stdout/stderr/returncode. + The bottle's environment (including HTTPS_PROXY pointing at + the egress daemon) is inherited by the child. Non-zero + exit does not raise — callers inspect `returncode` + themselves. + + Pass `user="root"` for shell-outs that need privileged file + writes / package install — provisioning calls that need root + bypass `Bottle.exec` and use the backend-specific raw + machine-exec helper, but the tests have a legitimate use + case for arbitrary-user runs.""" + + @abstractmethod + def cp_in(self, host_path: str, container_path: str) -> None: ... + + @abstractmethod + def close(self) -> None: ... + + + + +PlanT = TypeVar("PlanT", bound=BottlePlan) +CleanupT = TypeVar("CleanupT", bound=BottleCleanupPlan) + + +@dataclass(frozen=True) +class BottleImages: + """Resolved image references (or artifact paths) for a bottle launch. + + For Docker/macOS-container backends, `agent` and `sidecar` are string + image refs. For the smolmachines backend they are Path objects pointing + to pre-built `.smolmachine` artifacts.""" + + agent: str | Path + sidecar: str | Path = "" + + +class BottleBackend(ABC, Generic[PlanT, CleanupT]): + """Abstract base for selectable bottle backends. Concrete subclasses + (e.g. DockerBottleBackend) own their own prepare/launch impls. + Parameterized over the backend's concrete plan + cleanup-plan types + so subclass methods get the narrow type without isinstance + boilerplate.""" + + name: str + + # Whether this backend can run a container engine *inside* the bottle. + # Backends that cannot must reject `nested_containers: true` rather than + # reach for a host daemon socket (issue #392). + supports_nested_containers: bool = False + + def prepare(self, spec: BottleSpec, stage_dir: Path) -> PlanT: + """Template method: run cross-backend host-side validation, then + delegate to the subclass's `_resolve_plan` for the + backend-specific resolution (names, scratch files, etc.). The + validation step is enforced here so a future backend cannot + accidentally skip it. No remote/runtime resources are created.""" + from .resolve_common import ( + merge_provision_env_vars, + mint_slug, + prepare_agent_state_dir, + prepare_egress, + prepare_git_gate, + prepare_supervise, + reject_nested_containers, + resolve_manifest_dockerfile, + write_launch_metadata, + ) + + manifest = self._validate(spec) + + if not self.supports_nested_containers: + reject_nested_containers(self.name, manifest) + + self._preflight() + + from ..git_gate import GitGate + manifest = GitGate().preflight_host_keys( + manifest, + headless=spec.headless, + home_md=spec.manifest.home_md, + ) + + manifest_bottle = manifest.bottle + manifest_agent_provider = manifest_bottle.agent_provider + agent_provider = get_provider(manifest_agent_provider.template) + resolved_env = resolve_env(manifest) + workspace = workspace_plan(spec, guest_home=agent_provider.guest_home) + + slug = mint_slug(spec) + write_launch_metadata(slug, spec, compose_project="", backend=self.name) + + # Manifest may override the Dockerfile per-bottle; otherwise fall + # back to the provider plugin's bundled Dockerfile (next to its + # agent_provider.py module). + if manifest_agent_provider.dockerfile: + agent_dockerfile_path = resolve_manifest_dockerfile( + manifest_agent_provider.dockerfile, spec, + ) + else: + agent_dockerfile_path = str(agent_provider.dockerfile) + + agent_dir, prompt_file = prepare_agent_state_dir(slug, manifest) + + agent_provision_plan = build_agent_provision_plan( + template=manifest_agent_provider.template, + dockerfile=agent_dockerfile_path, + state_dir=agent_dir, + instance_name=f"bot-bottle-{slug}", + prompt_file=prompt_file, + guest_env=self._build_guest_env(resolved_env), + forward_host_credentials=manifest_agent_provider.forward_host_credentials, + auth_token=manifest_agent_provider.auth_token, + host_env=dict(os.environ), + trusted_project_path=workspace.workdir, + label=spec.label, + color=spec.color, + provider_settings=manifest_agent_provider.settings, + ) + agent_provision_plan = merge_provision_env_vars(agent_provision_plan) + egress_plan = prepare_egress(manifest_bottle, slug, agent_provision_plan) + supervise_plan = prepare_supervise(manifest_bottle, slug) + git_gate_plan = prepare_git_gate(manifest_bottle, slug) + + return self._resolve_plan( + spec, + manifest=manifest, + slug=slug, + resolved_env=resolved_env, + agent_provision_plan=agent_provision_plan, + egress_plan=egress_plan, + supervise_plan=supervise_plan, + git_gate_plan=git_gate_plan, + stage_dir=stage_dir, + ) + + def _build_guest_env(self, resolved_env: ResolvedEnv) -> dict[str, str]: + return {} + + def _preflight(self) -> None: + """ + tasks to do before resolving a plan + """ + pass + + def _validate(self, spec: BottleSpec) -> Manifest: + """Cross-backend pre-launch checks. Parses the selected agent and + its bottle (raising ManifestError on invalid content), confirms + skills are present on the host, and every git IdentityFile resolves. + + Returns the loaded Manifest for the selected agent. Subclasses with + additional preconditions should override and call + `super()._validate(spec)` first.""" + manifest = spec.manifest.load_for_agent(spec.agent_name, spec.bottle_names) + self._validate_skills(manifest.agent.skills) + self._validate_agent_provider_dockerfile(spec, manifest) + return manifest + + def _validate_skills(self, skills: Sequence[str]) -> None: + """Each named skill must be a directory under the host's + `~/.claude/skills/`. The check is purely host-side, so the + default impl covers every backend.""" + for name in skills: + path = host_skill_dir(name) + if not os.path.isdir(path): + die( + f"skill '{name}' not found on host at {path}. " + f"Create it under ~/.claude/skills/, then re-run." + ) + + def _validate_agent_provider_dockerfile(self, spec: BottleSpec, manifest: Manifest) -> None: + bottle = manifest.bottle + dockerfile = bottle.agent_provider.dockerfile + if not dockerfile: + return + path = Path(expand_tilde(dockerfile)) + if not path.is_absolute(): + path = Path(spec.user_cwd) / path + if not path.is_file(): + effective = ( + ", ".join(spec.bottle_names) if spec.bottle_names else manifest.agent.bottle + ) + die( + f"agent_provider.dockerfile for bottle " + f"'{effective}' not found: {path}" + ) + + @abstractmethod + def _resolve_plan(self, + spec: BottleSpec, + *, + manifest: Manifest, + slug: str, + resolved_env: ResolvedEnv, + agent_provision_plan: AgentProvisionPlan, + egress_plan: EgressPlan, + git_gate_plan: GitGatePlan, + supervise_plan: SupervisePlan | None, + stage_dir: Path) -> PlanT: + """Backend-specific plan resolution: image/container names, + env-file, prompt-file, proxy plan, runtime detection. Called by + `prepare` after `_validate` succeeds. Instance name, image, + prompt file, Dockerfile path, and guest home all live on + `agent_provision_plan` — the source of truth.""" + + def prelaunch_checks(self, plan: PlanT) -> None: + """Raise StaleImageError if any cached image used by this plan is stale. + No-op default; backends override to call the shared check_stale* + helpers on their image/artifact timestamps. Called by the CLI before + launch so the operator can be prompted outside the launch context.""" + + @contextmanager + def launch(self, plan: PlanT) -> Generator[Bottle, None, None]: + """Template: build or load images, then delegate to _launch_impl.""" + images = self._build_or_load_images(plan) + with self._launch_impl(plan, images) as bottle: + yield bottle + + @abstractmethod + def _build_or_load_images(self, plan: PlanT) -> BottleImages: + """Return the agent and sidecar image references (or artifact paths) + for this plan, building fresh images when the policy requires it.""" + + @abstractmethod + def _launch_impl(self, plan: PlanT, images: BottleImages) -> AbstractContextManager[Bottle]: + """Bring up the bottle using pre-resolved images; yield a handle; tear down on exit.""" + + def provision(self, plan: PlanT, bottle: "Bottle") -> str | None: + """Copy host-side files (CA cert, prompt, skills, .git) into + the running bottle. Called from `launch` after the container + / machine is up. Returns the in-container prompt path if a + prompt was provisioned, else None — the Bottle handle uses it + to decide whether to add provider-specific prompt args to the + agent's argv. + + Default orchestration: ca → prompt → provider apply → skills + → workspace → git → supervise-mcp. CA install runs first so + the agent's trust store is rebuilt before anything inside the + agent makes a TLS call. + + Per PRD 0050 the per-provider steps (prompt, skills, + declarative provision-plan apply, supervise MCP registration) + live on the `AgentProvider` plugin. The backend only owns the + steps that are about backend infrastructure (CA, workspace, + git) and surfaces the supervise daemon URL its launch step + knows about via `supervise_mcp_url`. + + PRD 0017: cred-proxy's agent-side dotfile rewrites (~/.npmrc, + ~/.gitconfig insteadOf, tea config) are gone. Egress-proxy is + on the agent's HTTP_PROXY path so every tool that respects + HTTPS_PROXY (claude-code, git over HTTPS, npm, curl) is + intercepted without per-tool reconfiguration.""" + provider = get_provider(plan.agent_provision.template) + provider.provision_ca(bottle, plan) + prompt_path = provider.provision_prompt(plan, bottle) + provider.provision(plan, bottle) + provider.provision_skills(plan, bottle) + self.provision_workspace(plan, bottle) + provider.provision_git(bottle, plan) + provider.provision_supervise_mcp( + plan, bottle, self.supervise_mcp_url(plan), + ) + return prompt_path + + def provision_workspace(self, plan: PlanT, bottle: "Bottle") -> None: + """Copy the operator workspace into the running bottle. + + This is the only supported workspace-provisioning path: Docker + does not build a derived image containing the current + workspace.""" + workspace = plan.workspace_plan + if not (workspace.enabled and workspace.copy_contents): + return + + guest_parent = workspace.guest_path.rsplit("/", 1)[0] or "/" + guest_path = shlex.quote(workspace.guest_path) + guest_parent = shlex.quote(guest_parent) + owner = shlex.quote(workspace.owner) + mode = shlex.quote(workspace.mode) + info(f"copying {workspace.host_path} -> {bottle.name}:{workspace.guest_path}") + bottle.exec( + f"rm -rf {guest_path} && mkdir -p {guest_parent}", + user="root", + ) + bottle.cp_in(str(workspace.host_path), workspace.guest_path) + bottle.exec( + f"chown -R {owner} {guest_path} && chmod {mode} {guest_path}", + user="root", + ) + + def supervise_mcp_url(self, plan: PlanT) -> str: + """Return the agent-side URL of the per-bottle supervise + gateway, or "" when this bottle has no gateway. The provider + plugin's `provision_supervise_mcp` uses it to register the + MCP entry inside the guest. + + Default returns "" so backends without supervise support + don't have to implement it. Docker and firecracker override.""" + del plan + return "" + + def ensure_orchestrator(self) -> str: + """Bring up this backend's per-host orchestrator + shared gateway + (idempotent) and return the host-reachable control-plane URL. + + This is the backend-agnostic bring-up entry point: `launch` calls + it as part of starting a bottle, and operator tools (`supervise`) + call it to start the control plane on demand when none is running + yet. Docker starts the orchestrator + gateway containers; + firecracker boots the infra VM. Backends with no orchestrator + (macos-container) die with a pointer — the default here.""" + die(f"backend {self.name!r} has no orchestrator control plane") + + @abstractmethod + def prepare_cleanup(self) -> CleanupT: + """Enumerate orphaned resources from previous bottles. No side + effects; safe to call before the y/N.""" + + @abstractmethod + def cleanup(self, plan: CleanupT) -> None: + """Remove everything described by the cleanup plan.""" + + @abstractmethod + def enumerate_active(self) -> Sequence[ActiveAgent]: + """Return every currently-running agent on this backend. + Empty when none. Backend-specific: docker queries `docker + compose ls`; firecracker cross-references its running gateway + containers against per-bottle metadata.""" + + @classmethod + @abstractmethod + def is_available(cls) -> bool: + """Whether this backend's runtime prerequisites are satisfied + on the current host. Docker → `docker` on PATH; firecracker → + Linux + KVM. Used by the cross-backend + `enumerate_active_agents` / `cmd_cleanup` to skip backends + the operator hasn't installed, so a docker-only host + doesn't fail when `cli.py list active` walks past + firecracker.""" + + @classmethod + @abstractmethod + def setup(cls) -> int: + """Emit this backend's one-time host setup — privileged network + pool, daemon bring-up, install pointers, etc. — as + host-appropriate config or commands. Prints to stdout/stderr and + returns a shell exit code (0 = nothing to report / success). A + backend that needs no host setup prints a short note and returns + 0. Invoked generically by `./cli.py backend setup [--backend=…]` + so operators can provision any backend without a + backend-specific command. Classmethod (like `is_available`) — + it's a host query, not per-bottle state.""" + + @classmethod + @abstractmethod + def status(cls) -> int: + """Report whether this backend's prerequisites are satisfied on + the host — binaries, daemon reachability, network pool, range + conflicts, etc. Prints a human-readable summary; returns 0 when + the backend is ready to launch and non-zero when something is + missing. Invoked by `./cli.py backend status [--backend=…]`.""" + + @classmethod + @abstractmethod + def teardown(cls) -> int: + """Undo `setup()` — the inverse operation, surfaced as + `./cli.py backend teardown [--backend=…]` (uninstall). Symmetric + with setup: where setup is advisory (prints the privileged + commands / declarative config to apply), teardown prints the + commands / config change to remove the host prerequisites. A + backend with no host setup prints a short note and returns 0. + Not called by the launch path or the test suite.""" diff --git a/bot_bottle/backend/consolidated_util.py b/bot_bottle/backend/consolidated_util.py index 520ca0a0..3aa85905 100644 --- a/bot_bottle/backend/consolidated_util.py +++ b/bot_bottle/backend/consolidated_util.py @@ -13,7 +13,7 @@ from ..egress import EgressPlan from ..git_gate import GitGatePlan from ..orchestrator.client import OrchestratorClient, RegisteredBottle from ..orchestrator.registration import registration_inputs -from ..orchestrator.secret_store import new_env_var_secret +from ..orchestrator.store.secret_store import new_env_var_secret from .docker.gateway_provision import GatewayTransport, deprovision_git_gate, provision_git_gate @@ -58,7 +58,7 @@ def teardown_consolidated( ) -> None: """Deregister the bottle and remove its git-gate state. Both steps are idempotent so this is safe from a cleanup trap.""" - from ..orchestrator.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS + from ..orchestrator.store.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS OrchestratorClient( orchestrator_url, timeout=timeout if timeout is not None else DEFAULT_TEARDOWN_TIMEOUT_SECONDS, diff --git a/bot_bottle/backend/docker/__init__.py b/bot_bottle/backend/docker/__init__.py index dc2340b8..8da7dc85 100644 --- a/bot_bottle/backend/docker/__init__.py +++ b/bot_bottle/backend/docker/__init__.py @@ -11,18 +11,42 @@ The bulk of the implementation lives in sibling modules: - launch: bring-up + teardown context manager - cleanup: orphan enumeration, removal, active listing - backend: DockerBottleBackend façade wiring the above + - infra: DockerInfraService (the per-host infra container) -This file only re-exports the public names so -`from bot_bottle.backend.docker import DockerBottleBackend` keeps -working. +Thin by design: the public names are re-exported lazily via `__getattr__`, so +importing a leaf like `backend.docker.util` doesn't drag `DockerBottleBackend` +(and the whole framework it pulls) into memory. """ from __future__ import annotations -from .backend import DockerBottleBackend -from .bottle import DockerBottle -from .bottle_cleanup_plan import DockerBottleCleanupPlan -from .bottle_plan import DockerBottlePlan +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .backend import DockerBottleBackend + from .bottle import DockerBottle + from .bottle_cleanup_plan import DockerBottleCleanupPlan + from .bottle_plan import DockerBottlePlan + + +_LAZY_MODULES: dict[str, str] = { + "DockerBottleBackend": "backend", + "DockerBottle": "bottle", + "DockerBottleCleanupPlan": "bottle_cleanup_plan", + "DockerBottlePlan": "bottle_plan", +} + + +def __getattr__(name: str) -> Any: + mod = _LAZY_MODULES.get(name) + if mod is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module + + value = getattr(import_module(f"{__name__}.{mod}"), name) + globals()[name] = value + return value + __all__ = [ "DockerBottle", diff --git a/bot_bottle/backend/docker/backend.py b/bot_bottle/backend/docker/backend.py index b2ae45ba..eff4d5d9 100644 --- a/bot_bottle/backend/docker/backend.py +++ b/bot_bottle/backend/docker/backend.py @@ -24,12 +24,12 @@ from contextlib import contextmanager from pathlib import Path from typing import Generator, Sequence -from ...supervise import SUPERVISE_HOSTNAME, SUPERVISE_PORT +from ...supervisor.types import SUPERVISE_HOSTNAME, SUPERVISE_PORT from ...agent_provider import AgentProvisionPlan from ...egress import EgressPlan from ...env import ResolvedEnv from ...git_gate import GitGatePlan -from ...supervise import SupervisePlan +from ...supervisor.plan import SupervisePlan from ...manifest import Manifest from .. import ActiveAgent, BottleBackend, BottleImages, BottleSpec from . import cleanup as _cleanup @@ -112,8 +112,8 @@ class DockerBottleBackend(BottleBackend["DockerBottlePlan", "DockerBottleCleanup yield bottle def ensure_orchestrator(self) -> str: - from ...orchestrator.lifecycle import OrchestratorService - return OrchestratorService().ensure_running() + from .infra import DockerInfraService + return DockerInfraService().ensure_running() def supervise_mcp_url(self, plan: DockerBottlePlan) -> str: """Docker bottles reach the supervise daemon via the diff --git a/bot_bottle/backend/docker/consolidated_compose.py b/bot_bottle/backend/docker/consolidated_compose.py index 00652248..d3b7996e 100644 --- a/bot_bottle/backend/docker/consolidated_compose.py +++ b/bot_bottle/backend/docker/consolidated_compose.py @@ -16,8 +16,8 @@ from __future__ import annotations from typing import Any -from ...egress import egress_agent_env_entries -from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME +from ...egress import Egress +from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH from .bottle_plan import DockerBottlePlan from .egress import EGRESS_PORT @@ -63,7 +63,7 @@ def consolidated_agent_compose( # env (set in launch.py) and is never written to the compose file on disk. if getattr(plan, "env_var_secret", ""): env.append(ENV_VAR_SECRET_NAME) - env.extend(egress_agent_env_entries(plan.egress_plan)) + env.extend(Egress().agent_env_entries(plan.egress_plan)) service: dict[str, Any] = { "image": plan.image, diff --git a/bot_bottle/backend/docker/consolidated_launch.py b/bot_bottle/backend/docker/consolidated_launch.py index 9f9768a3..7efabc61 100644 --- a/bot_bottle/backend/docker/consolidated_launch.py +++ b/bot_bottle/backend/docker/consolidated_launch.py @@ -16,13 +16,13 @@ from __future__ import annotations from dataclasses import dataclass from ... import log -from ...docker_cmd import run_docker +from .util import run_docker from ...egress import EgressPlan from ...git_gate import GitGatePlan from ...orchestrator.client import OrchestratorClient -from ...orchestrator.gateway import GATEWAY_NETWORK -from ...orchestrator.lifecycle import INFRA_NAME, OrchestratorService -from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME +from ...gateway import GATEWAY_NETWORK +from .infra import INFRA_NAME, DockerInfraService +from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME from ...orchestrator.reprovision import reprovision_bottles from ..consolidated_util import provision_bottle from ..consolidated_util import teardown_consolidated as _teardown_util @@ -144,7 +144,7 @@ def launch_consolidated( *, image_ref: str = "", tokens: dict[str, str] | None = None, - service: OrchestratorService | None = None, + service: DockerInfraService | None = None, infra_name: str = INFRA_NAME, network: str = GATEWAY_NETWORK, ) -> LaunchContext: @@ -154,7 +154,7 @@ def launch_consolidated( Also reprovisiones egress tokens for any already-running bottles that lost their in-memory credentials (e.g. after an infra container restart), so they regain egress access before the new bottle is registered.""" - service = service or OrchestratorService() + service = service or DockerInfraService() url = service.ensure_running() _reprovision_running_bottles(url, network=network, infra_name=infra_name) client = OrchestratorClient(url) diff --git a/bot_bottle/orchestrator/gateway.py b/bot_bottle/backend/docker/gateway.py similarity index 61% rename from bot_bottle/orchestrator/gateway.py rename to bot_bottle/backend/docker/gateway.py index cc12330e..221de210 100644 --- a/bot_bottle/orchestrator/gateway.py +++ b/bot_bottle/backend/docker/gateway.py @@ -1,122 +1,21 @@ -"""The consolidated per-host gateway (PRD 0070). - -The core consolidation win: **one** persistent gateway per host, shared by -every bottle, instead of a gateway per bottle. It's safe to share -because the attribution invariant (source IP + identity token, see -`registry`) lets the gateway attribute each request to the right bottle — -so per-bottle policy lives in one long-lived process keyed on who's calling. - -`Gateway` is the backend-neutral lifecycle contract (mirrors `LaunchBroker`): -ensure the single instance is up, report it, tear it down. `DockerGateway` -is the docker implementation; a firecracker gateway VM slots in later. - -The defining behaviour is **idempotent singleton**: `ensure_running` starts -the instance if absent and is a no-op if it's already up, so N bottle -launches never spawn N gateways. -""" - from __future__ import annotations -import abc import os import time from pathlib import Path -from ..control_auth import ROLE_GATEWAY, mint -from ..docker_cmd import run_docker -from ..paths import ( - CONTROL_AUTH_JWT_ENV, - host_control_plane_token, +from ...orchestrator_auth import ROLE_GATEWAY, mint +from .util import run_docker +from ...paths import ( + ORCHESTRATOR_AUTH_JWT_ENV, + host_orchestrator_token, host_gateway_ca_dir, ) - -# The gateway's mitmproxy writes its CA a beat after the container starts, so -# reads poll for it rather than assuming it's there on a fresh launch. -_CA_POLL_SECONDS = 0.5 -DEFAULT_CA_TIMEOUT_SECONDS = 30.0 - -GATEWAY_NAME = "bot-bottle-orch-gateway" -GATEWAY_LABEL = "bot-bottle-orch-gateway=1" -# The single user-defined network the gateway and every agent bottle share. -# Agents attach here with a pinned IP and reach the gateway's egress / -# git-http / supervise ports by its address — no host port publishing, and -# the source IP the gateway attributes by is the address on this network. -GATEWAY_NETWORK = "bot-bottle-gateway" - -# mitmproxy's CA dir in the bundle. The host's gateway-CA dir (see -# `host_gateway_ca_dir`) is bind-mounted here so the gateway's self-generated -# CA stays STABLE across container recreation — every agent installs this one -# CA to trust the shared gateway's TLS interception, so it must not rotate when -# the gateway restarts. A host bind-mount rather than a named volume: a named -# volume is silently wiped by `docker volume prune`, minting a fresh CA that -# breaks every running bottle (issue #450). -MITMPROXY_HOME = "/home/mitmproxy/.mitmproxy" -GATEWAY_CA_CERT = f"{MITMPROXY_HOME}/mitmproxy-ca-cert.pem" - -# The CA material mitmproxy writes into its confdir. mitmproxy reuses these on -# startup when present and generates them only on first run, so persisting them -# is what makes the CA stable; deleting them (see `rotate_gateway_ca`) forces a -# fresh CA on the next start. `mitmproxy-ca.pem` (cert + private key) is the -# signing identity; the rest are derived encodings agents/clients consume. -GATEWAY_CA_GLOB = "mitmproxy-ca*" - -# The gateway data-plane image + its Dockerfile. Kept as a local constant -# rather than imported from the backend layer, which would drag -# the whole backend layer into the lean orchestrator (see #359); unify when -# that lands. Env override matches the backend's BOT_BOTTLE_GATEWAY_IMAGE. -GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest") -GATEWAY_DOCKERFILE = "Dockerfile.gateway" -_REPO_ROOT = Path(__file__).resolve().parents[2] - - -def rotate_gateway_ca(ca_dir: Path | None = None) -> list[Path]: - """Delete the persisted mitmproxy CA so the next gateway start mints a - fresh one — the explicit, deliberate CA-rollover path (issue #450). - - Persistence keeps the CA stable across restarts precisely because mitmproxy - reuses the on-disk CA; rotation is therefore just removing that material. - Returns the files removed (empty when there was no CA yet); idempotent. - - This only clears the on-disk CA. It does NOT stop the running gateway (whose - mitmproxy still holds the old CA in memory) or re-provision agents — the - caller recreates the gateway to mint the new CA and re-attaches bottles. - `rotate-ca` on the orchestrator CLI wires those steps together.""" - ca_dir = ca_dir if ca_dir is not None else host_gateway_ca_dir() - removed: list[Path] = [] - for path in sorted(ca_dir.glob(GATEWAY_CA_GLOB)): - path.unlink() - removed.append(path) - return removed - - -class GatewayError(Exception): - """The shared gateway failed to build/start/stop (non-zero `docker` exit).""" - - -class Gateway(abc.ABC): - """Lifecycle of the single per-host gateway. Backend-neutral.""" - - name: str - - def ensure_built(self) -> None: - """Ensure the gateway's image / rootfs exists, building it if needed. - Default: nothing to build (e.g. a stub or a pre-pulled image).""" - return - - @abc.abstractmethod - def ensure_running(self) -> None: - """Start the gateway if it isn't already up. Idempotent: a no-op - when it's already running (that's the whole point — one per host). - Assumes the image exists — call `ensure_built()` first.""" - - @abc.abstractmethod - def is_running(self) -> bool: - """True iff the gateway instance is currently up.""" - - @abc.abstractmethod - def stop(self) -> None: - """Remove the gateway. Idempotent — absent is success.""" - +from ...gateway import ( + Gateway, GATEWAY_IMAGE, GATEWAY_NAME, GATEWAY_NETWORK, GATEWAY_DOCKERFILE, + REPO_ROOT, GATEWAY_LABEL, MITMPROXY_HOME, DEFAULT_CA_TIMEOUT_SECONDS, + CA_POLL_SECONDS, GATEWAY_CA_CERT, GatewayError +) class DockerGateway(Gateway): """The consolidated gateway as a single, fixed-name Docker container. @@ -147,7 +46,7 @@ class DockerGateway(Gateway): # construct-then-read-CA path (`ca_cert_pem` on an already-running # container), which never launches a container. self._orchestrator_url = orchestrator_url - self._build_context = build_context or _REPO_ROOT + self._build_context = build_context or REPO_ROOT self._dockerfile = dockerfile # Ports published on the host (0.0.0.0). Used by the Firecracker # backend's dev-harness gateway so VMs can reach it via their TAP link; @@ -259,8 +158,8 @@ class DockerGateway(Gateway): # a `cli` token, so a compromised data-plane process can't drive the # operator routes (issue #469 review). Bare `--env NAME` keeps the value # off argv / `docker inspect`; only the gateway (not the agent) is given it. - argv += ["--env", CONTROL_AUTH_JWT_ENV] - run_env[CONTROL_AUTH_JWT_ENV] = mint(ROLE_GATEWAY, host_control_plane_token()) + argv += ["--env", ORCHESTRATOR_AUTH_JWT_ENV] + run_env[ORCHESTRATOR_AUTH_JWT_ENV] = mint(ROLE_GATEWAY, host_orchestrator_token()) argv.append(self.image_ref) proc = run_docker(argv, env=run_env) if proc.returncode != 0: @@ -282,16 +181,9 @@ class DockerGateway(Gateway): f"gateway CA cert not available after {timeout:g}s: " f"{proc.stderr.strip() or 'empty'}" ) - time.sleep(_CA_POLL_SECONDS) + time.sleep(CA_POLL_SECONDS) def stop(self) -> None: proc = run_docker(["docker", "rm", "--force", self.name]) if proc.returncode != 0 and "No such container" not in proc.stderr: - raise GatewayError(f"gateway failed to stop: {proc.stderr.strip()}") - - -__all__ = [ - "Gateway", "DockerGateway", "GatewayError", "rotate_gateway_ca", - "GATEWAY_NAME", "GATEWAY_LABEL", "GATEWAY_IMAGE", "GATEWAY_NETWORK", - "GATEWAY_CA_CERT", "GATEWAY_CA_GLOB", -] + raise GatewayError(f"gateway failed to stop: {proc.stderr.strip()}") \ No newline at end of file diff --git a/bot_bottle/backend/docker/gateway_provision.py b/bot_bottle/backend/docker/gateway_provision.py index d1094dcc..3eaadcbc 100644 --- a/bot_bottle/backend/docker/gateway_provision.py +++ b/bot_bottle/backend/docker/gateway_provision.py @@ -17,7 +17,7 @@ from __future__ import annotations import re from typing import Protocol -from ...docker_cmd import run_docker +from .util import run_docker from ...git_gate import GitGatePlan, git_gate_render_provision # bottle ids index the gateway's per-bottle repo + creds dirs; they land in diff --git a/bot_bottle/backend/docker/infra.py b/bot_bottle/backend/docker/infra.py new file mode 100644 index 00000000..d9d119d9 --- /dev/null +++ b/bot_bottle/backend/docker/infra.py @@ -0,0 +1,261 @@ +"""The per-host infra container for the docker backend (PRD 0070). + +Runs both the orchestrator control plane and the gateway data plane inside a +single `bot-bottle-infra` container on the shared gateway network — the docker +analogue of the macOS infra container (`backend/macos_container/infra.py`) and +the Firecracker infra VM (`backend/firecracker/infra_vm.py`). `gateway_init` is +PID 1 and supervises both; the infra container is an idempotent per-host +singleton. + +The combined container replaces the prior two-container split +(bot-bottle-orchestrator + bot-bottle-orch-gateway). The host CLI reaches the +control plane via a published loopback port; gateway daemons reach it over +127.0.0.1 (same container). + +The shared, backend-neutral pieces (control-plane port, startup timeout, +`OrchestratorStartError`, `source_hash`) live in `orchestrator.lifecycle`. +""" + +from __future__ import annotations + +import os +import time +import urllib.error +import urllib.request +from pathlib import Path + +from ... import log +from ...orchestrator_auth import ROLE_GATEWAY, mint +from .util import run_docker +from ...paths import ( + ORCHESTRATOR_AUTH_JWT_ENV, + ORCHESTRATOR_TOKEN_ENV, + bot_bottle_root, + host_orchestrator_token, + host_gateway_ca_dir, +) +from ...gateway import ( + GATEWAY_DOCKERFILE, + GATEWAY_IMAGE, + GATEWAY_NETWORK, + GatewayError, + MITMPROXY_HOME, +) +from ...orchestrator.lifecycle import ( + DEFAULT_PORT, + DEFAULT_STARTUP_TIMEOUT_SECONDS, + OrchestratorStartError, + source_hash, +) + +INFRA_NAME = "bot-bottle-infra" +INFRA_LABEL = "bot-bottle-infra=1" +# The combined infra image: gateway data plane + orchestrator content. +# Built from Dockerfile.infra (FROM gateway + COPY --from orchestrator). +INFRA_IMAGE = os.environ.get("BOT_BOTTLE_INFRA_IMAGE", "bot-bottle-infra:latest") +INFRA_DOCKERFILE = "Dockerfile.infra" +# Baked as a container label so `ensure_running` can detect whether the +# running container is executing the current bind-mounted source. +INFRA_SOURCE_HASH_LABEL = "bot-bottle-infra-source-hash" + +# Orchestrator image: the single canonical definition of the control-plane +# content (lean: python:3.12-slim + bot_bottle package, no mitmproxy/git). +# Used as a build intermediate: `Dockerfile.infra` COPY --from this image. +ORCHESTRATOR_IMAGE = os.environ.get( + "BOT_BOTTLE_ORCHESTRATOR_IMAGE", "bot-bottle-orchestrator:latest" +) +ORCHESTRATOR_DOCKERFILE = "Dockerfile.orchestrator" + +# The gateway daemons + orchestrator the infra container runs. +# BOT_BOTTLE_GATEWAY_DAEMONS listing `orchestrator` opts it in to +# gateway_init's supervise tree (see gateway_init._OPT_IN_DAEMONS). +_INFRA_DAEMONS = "egress,git-http,supervise,orchestrator" + +# The bind-mount path for the live control-plane source inside the +# container. Separate from /app so the gateway's baked scripts +# (egress_addon.py, egress-entrypoint.sh) are not overlaid. +_SRC_IN_CONTAINER = "/bot-bottle-src" +# Bot-bottle host-root bind-mount inside the container (DB + state). The +# orchestrator control plane opens bot-bottle.db under here (via BOT_BOTTLE_ROOT +# -> host_db_path()); it is the ONLY process in the container with a file +# handle on it (PRD 0070 / issue #469). +_ROOT_IN_CONTAINER = "/bot-bottle-root" + +_HEALTH_POLL_SECONDS = 0.25 +_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0 + +_REPO_ROOT = Path(__file__).resolve().parents[3] + + +class DockerInfraService: + """Manages the single per-host docker infra container (control plane + + gateway). Callers only need `ensure_running()` + `url`. + + `infra_name` / `infra_label` let callers run independent infra containers + on the same host without name collisions (e.g. isolated integration tests + that can't share the production INFRA_NAME singleton).""" + + def __init__( + self, + *, + port: int = DEFAULT_PORT, + network: str = GATEWAY_NETWORK, + image: str = INFRA_IMAGE, + repo_root: Path = _REPO_ROOT, + host_root: Path | None = None, + infra_name: str = INFRA_NAME, + infra_label: str = INFRA_LABEL, + ) -> None: + self.port = port + self.network = network + self.image = image + self._repo_root = repo_root + self._host_root = host_root or bot_bottle_root() + self._infra_name = infra_name + self._infra_label = infra_label + + @property + def url(self) -> str: + """Host-side control-plane URL (published loopback port).""" + return f"http://127.0.0.1:{self.port}" + + def is_healthy(self, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS) -> bool: + try: + with urllib.request.urlopen(f"{self.url}/health", timeout=timeout) as resp: + return resp.status == 200 + except (urllib.error.URLError, TimeoutError, OSError): + return False + + def _container_running(self, name: str) -> bool: + proc = run_docker(["docker", "ps", "--filter", f"name=^/{name}$", "--format", "{{.Names}}"]) + return name in proc.stdout.split() + + def _infra_source_current(self, current_hash: str) -> bool: + """True iff the running infra container was started from the current + bind-mounted source. Mirrors the macOS backend's `_source_current`.""" + if not self._container_running(self._infra_name): + return False + proc = run_docker([ + "docker", "inspect", "--format", + "{{ index .Config.Labels \"" + INFRA_SOURCE_HASH_LABEL + "\" }}", + self._infra_name, + ]) + if proc.returncode != 0: + return True # can't compare → don't churn a working container + return proc.stdout.strip() == current_hash + + def _ensure_network(self) -> None: + if run_docker(["docker", "network", "inspect", self.network]).returncode == 0: + return + proc = run_docker(["docker", "network", "create", self.network]) + if proc.returncode != 0 and "already exists" not in proc.stderr: + raise GatewayError( + f"gateway network {self.network} failed to create: {proc.stderr.strip()}" + ) + + def _build_images(self) -> None: + """Build the gateway base, the orchestrator intermediate, then the + infra image. All are cache-aware: a no-op when nothing changed.""" + for tag, dockerfile in ( + (GATEWAY_IMAGE, GATEWAY_DOCKERFILE), + (ORCHESTRATOR_IMAGE, ORCHESTRATOR_DOCKERFILE), + (self.image, INFRA_DOCKERFILE), + ): + argv = ["docker", "build", "-t", tag, + "-f", str(self._repo_root / dockerfile), + str(self._repo_root)] + if os.environ.get("BOT_BOTTLE_NO_CACHE"): + argv.insert(2, "--no-cache") + proc = run_docker(argv) + if proc.returncode != 0: + raise GatewayError(f"{dockerfile} build failed: {proc.stderr.strip()}") + + def _run_infra_container(self, current_hash: str) -> None: + """Start the combined infra container (idempotent: clears a stale + fixed-name container first). Labels the container with `current_hash` + so a later `ensure_running` can detect a real code change.""" + self._ensure_network() + run_docker(["docker", "rm", "--force", self._infra_name]) + _signing_key = host_orchestrator_token() + proc = run_docker([ + "docker", "run", "--detach", + "--name", self._infra_name, + "--label", self._infra_label, + "--label", f"{INFRA_SOURCE_HASH_LABEL}={current_hash}", + "--network", self.network, + # Host CLI reaches the control plane here (loopback only). + # gateway_init always starts the orchestrator on DEFAULT_PORT (8099) + # inside the container; self.port is the host-side published port. + "--publish", f"127.0.0.1:{self.port}:{DEFAULT_PORT}", + # Persist the mitmproxy CA on the host so it survives container + # recreation AND docker volume pruning (issue #450): every agent + # trusts this one CA, so a fresh one would break all running bottles. + "--volume", f"{host_gateway_ca_dir()}:{MITMPROXY_HOME}", + # Live control-plane source, mounted to a path that does not + # overlay the gateway's baked /app scripts. + "--volume", f"{self._repo_root}:{_SRC_IN_CONTAINER}:ro", + # PYTHONPATH lets the orchestrator (and other Python daemons) + # import the live source ahead of the installed package. + "--env", f"PYTHONPATH={_SRC_IN_CONTAINER}", + # Orchestrator registry DB on the host (sole writer: control plane). + "--volume", f"{self._host_root}:{_ROOT_IN_CONTAINER}", + "--env", f"BOT_BOTTLE_ROOT={_ROOT_IN_CONTAINER}", + # Control-plane signing key (orchestrator: verifies tokens) + the + # pre-minted `gateway` JWT (data-plane daemons: present it). gateway_init + # scopes each to its process, so a compromised data-plane daemon never + # sees the key and can't mint a `cli` token (issue #469 review). + "--env", ORCHESTRATOR_TOKEN_ENV, + "--env", ORCHESTRATOR_AUTH_JWT_ENV, + # Gateway daemons reach the orchestrator over loopback at its + # fixed internal port (DEFAULT_PORT), independent of self.port. + "--env", f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{DEFAULT_PORT}", + # Opt the orchestrator into gateway_init's supervise tree. + "--env", f"BOT_BOTTLE_GATEWAY_DAEMONS={_INFRA_DAEMONS}", + self.image, + ], env={ + **os.environ, + ORCHESTRATOR_TOKEN_ENV: _signing_key, + ORCHESTRATOR_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key), + }) + if proc.returncode != 0: + raise OrchestratorStartError( + f"infra container failed to start: {proc.stderr.strip()}" + ) + + def ensure_running( + self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS, + ) -> str: + """Ensure the infra container (control plane + gateway) is up; return + the host control-plane URL. Idempotent — a healthy container on current + source is left untouched. Raises `OrchestratorStartError` on timeout.""" + self._build_images() + + current_hash = source_hash(self._repo_root) + if self.is_healthy() and self._infra_source_current(current_hash): + return self.url + + log.info("starting infra container", context={"name": self._infra_name}) + self._run_infra_container(current_hash) + + deadline = time.monotonic() + startup_timeout + while time.monotonic() < deadline: + if self.is_healthy(): + log.info("infra container healthy", context={"url": self.url}) + return self.url + time.sleep(_HEALTH_POLL_SECONDS) + raise OrchestratorStartError( + f"infra container at {self.url} did not become healthy within {startup_timeout:g}s" + ) + + def stop(self) -> None: + """Remove the infra container (idempotent).""" + run_docker(["docker", "rm", "--force", self._infra_name]) + + +__all__ = [ + "DockerInfraService", + "INFRA_NAME", + "INFRA_IMAGE", + "INFRA_SOURCE_HASH_LABEL", + "ORCHESTRATOR_IMAGE", +] diff --git a/bot_bottle/backend/docker/launch.py b/bot_bottle/backend/docker/launch.py index e767d922..4f68303c 100644 --- a/bot_bottle/backend/docker/launch.py +++ b/bot_bottle/backend/docker/launch.py @@ -37,11 +37,8 @@ from pathlib import Path from typing import Callable, Generator from ...agent_provider import runtime_for -from ...egress import egress_resolve_token_values -from ...git_gate import ( - provision_git_gate_dynamic_keys, - revoke_git_gate_provisioned_keys, -) +from ...egress import Egress +from ...git_gate import GitGate from ...image_cache import check_stale from ...log import die, info, warn from .. import BottleImages @@ -64,9 +61,10 @@ from .compose import ( write_compose_file, ) from .consolidated_compose import consolidated_agent_compose -from ...orchestrator.config_store import resolve_teardown_timeout +from ...orchestrator.store.config_store import resolve_teardown_timeout from .consolidated_launch import launch_consolidated, teardown_consolidated -from ...orchestrator.gateway import DockerGateway +from .infra import INFRA_NAME +from .gateway import DockerGateway # Where the repo root lives, for `docker build` context. Computed once. @@ -125,7 +123,7 @@ def launch( f"teardown failed for container {plan.container_name}" f" (compose-down): {exc!r}" ) - revoke_git_gate_provisioned_keys( + GitGate().revoke_provisioned_keys( _bottle_for_revoke, _git_gate_dir_for_revoke ) @@ -134,7 +132,7 @@ def launch( # provisioning the bottle's repos into the shared gateway. git_gate_plan = plan.git_gate_plan if git_gate_plan.upstreams: - git_gate_plan = provision_git_gate_dynamic_keys( + git_gate_plan = GitGate().provision_dynamic_keys( plan.manifest.bottle, git_gate_plan, git_gate_state_dir(plan.slug), ) @@ -145,7 +143,7 @@ def launch( # handed to the orchestrator (in memory) for the gateway to inject — # the agent never sees them. effective_env = {**os.environ, **plan.agent_provision.provisioned_env} - token_values = egress_resolve_token_values( + token_values = Egress().resolve_token_values( plan.egress_plan.token_env_map, effective_env, ) teardown_timeout = resolve_teardown_timeout() @@ -159,11 +157,14 @@ def launch( ) # Step 4: install the SHARED gateway CA into the agent (replaces the - # per-bottle CA) — read it out of the running gateway. + # per-bottle CA) — read it out of the running gateway. The consolidated + # flow runs the gateway data plane inside the per-host infra container + # (INFRA_NAME), so read the CA from there, not the legacy standalone + # gateway container name. ca_dir = egress_state_dir(plan.slug) / "gateway-ca" ca_dir.mkdir(parents=True, exist_ok=True) ca_file = ca_dir / "gateway-ca.pem" - ca_file.write_text(DockerGateway(network=ctx.network).ca_cert_pem()) + ca_file.write_text(DockerGateway(name=INFRA_NAME).ca_cert_pem()) egress_plan = dataclasses.replace( plan.egress_plan, mitmproxy_ca_host_path=ca_file, @@ -203,7 +204,7 @@ def launch( # spec, value only in the subprocess env so it is never written to disk. compose_env: dict[str, str] = {**os.environ, **plan.forwarded_env} if plan.env_var_secret: - from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME + from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME compose_env[ENV_VAR_SECRET_NAME] = plan.env_var_secret info( f"docker compose up -d (project {project}, agent on shared " diff --git a/bot_bottle/backend/docker/resolve_plan.py b/bot_bottle/backend/docker/resolve_plan.py index c48e3acd..eac5d2ae 100644 --- a/bot_bottle/backend/docker/resolve_plan.py +++ b/bot_bottle/backend/docker/resolve_plan.py @@ -19,7 +19,7 @@ from ...env import ResolvedEnv from ...agent_provider import AgentProvisionPlan from ...egress import EgressPlan from ...manifest import Manifest -from ...supervise import SupervisePlan +from ...supervisor.plan import SupervisePlan from ...git_gate import GitGatePlan def preflight() -> None: diff --git a/bot_bottle/backend/docker/util.py b/bot_bottle/backend/docker/util.py index fb2091b8..b05cbd6e 100644 --- a/bot_bottle/backend/docker/util.py +++ b/bot_bottle/backend/docker/util.py @@ -1,6 +1,6 @@ -"""Docker host-side primitives used by DockerBottleBackend: probing -for docker on PATH, slugifying agent names, checking image/container -existence, and building images.""" +"""Docker host-side primitives used by DockerBottleBackend: the lean +`run_docker` subprocess wrapper, probing for docker on PATH, slugifying agent +names, checking image/container existence, and building images.""" from __future__ import annotations @@ -11,9 +11,24 @@ import shutil import subprocess from typing import Iterator -from ...docker_cmd import run_docker from ...log import die, info -# from ...workspace import WorkspacePlan + + +def run_docker( + argv: list[str], *, env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[str]: + """Run a `docker` command, capturing stdout/stderr as text. Never raises on + a non-zero exit — callers inspect `returncode` / `stderr` so they can stay + fail-closed or tolerate idempotent no-ops (e.g. removing an already-absent + container). + + `env` sets the child process environment — used to hand a secret to a bare + `--env NAME` flag (docker inherits its value from this process) so the value + never lands on argv or in `docker inspect`'s recorded command line.""" + return subprocess.run( + argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + check=False, env=env, + ) # Cap on the suffix the container-name conflict logic will try before diff --git a/bot_bottle/backend/egress_apply.py b/bot_bottle/backend/egress_apply.py index ed54cdf1..e174af3a 100644 --- a/bot_bottle/backend/egress_apply.py +++ b/bot_bottle/backend/egress_apply.py @@ -11,7 +11,7 @@ from pathlib import Path from ..bottle_state import egress_state_dir from ..egress import EGRESS_ROUTES_FILENAME -from ..egress_addon_core import LOG_OFF, load_config +from ..gateway.egress.addon_core import LOG_OFF, load_config class EgressApplyError(RuntimeError): diff --git a/bot_bottle/backend/firecracker/__init__.py b/bot_bottle/backend/firecracker/__init__.py index 57b58404..cae12486 100644 --- a/bot_bottle/backend/firecracker/__init__.py +++ b/bot_bottle/backend/firecracker/__init__.py @@ -1,7 +1,25 @@ -"""Firecracker backend: Linux KVM microVM isolation (issue #342).""" +"""Firecracker backend: Linux KVM microVM isolation (issue #342). + +Thin by design: `FirecrackerBottleBackend` is re-exported lazily via +`__getattr__`, so importing a leaf module under this package doesn't drag the +backend (and the framework it pulls) into memory. +""" from __future__ import annotations -from .backend import FirecrackerBottleBackend +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .backend import FirecrackerBottleBackend + + +def __getattr__(name: str) -> Any: + if name == "FirecrackerBottleBackend": + from .backend import FirecrackerBottleBackend + + globals()[name] = FirecrackerBottleBackend + return FirecrackerBottleBackend + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + __all__ = ["FirecrackerBottleBackend"] diff --git a/bot_bottle/backend/firecracker/backend.py b/bot_bottle/backend/firecracker/backend.py index a8a9df93..41193838 100644 --- a/bot_bottle/backend/firecracker/backend.py +++ b/bot_bottle/backend/firecracker/backend.py @@ -17,7 +17,7 @@ from ...egress import EgressPlan from ...env import ResolvedEnv from ...git_gate import GitGatePlan from ...manifest import Manifest -from ...supervise import SupervisePlan +from ...supervisor.plan import SupervisePlan from .. import ActiveAgent, BottleBackend, BottleImages, BottleSpec from . import cleanup as _cleanup from . import enumerate as _enumerate @@ -120,4 +120,4 @@ class FirecrackerBottleBackend( def ensure_orchestrator(self) -> str: from . import infra_vm - return infra_vm.ensure_running().control_plane_url + return infra_vm.ensure_running().orchestrator_url diff --git a/bot_bottle/backend/firecracker/consolidated_launch.py b/bot_bottle/backend/firecracker/consolidated_launch.py index cb726907..24e8232f 100644 --- a/bot_bottle/backend/firecracker/consolidated_launch.py +++ b/bot_bottle/backend/firecracker/consolidated_launch.py @@ -38,7 +38,7 @@ from ...orchestrator.lifecycle import ( OrchestratorStartError, # re-exported so callers can catch it ) from ...orchestrator.reprovision import reprovision_bottles -from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME +from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME from ..consolidated_util import ( provision_bottle, teardown_consolidated as _teardown_util, @@ -124,7 +124,7 @@ def launch_consolidated( provision its git-gate state into the gateway VM. Returns the context the agent-VM launch needs. Raises on failure — the caller tears down.""" infra = infra_vm.ensure_running() - url = infra.control_plane_url + url = infra.orchestrator_url client = OrchestratorClient(url) _reprovision_running_bottles(client) diff --git a/bot_bottle/backend/firecracker/infra_artifact.py b/bot_bottle/backend/firecracker/infra_artifact.py index 5c8ecb29..eb8396df 100644 --- a/bot_bottle/backend/firecracker/infra_artifact.py +++ b/bot_bottle/backend/firecracker/infra_artifact.py @@ -66,7 +66,7 @@ def infra_artifact_version(init_script: str, *, repo_root: Path = _REPO_ROOT) -> The package is `COPY bot_bottle /app/bot_bottle`'d wholesale into the image, so hash *every* regular file under it — not just `*.py`. Non-Python inputs - (e.g. `egress_entrypoint.sh`, `netpool.defaults.env`) are baked in too, and + (e.g. `gateway/egress/entrypoint.sh`, `netpool.defaults.env`) are baked in too, and a change to one must bump the version or a launch host could boot a stale rootfs whose code differs from its checkout. `__pycache__`/`.pyc` are the only exclusions — build artifacts, never copied.""" diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py index cb64dd28..cf644ab5 100644 --- a/bot_bottle/backend/firecracker/infra_vm.py +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -33,7 +33,7 @@ from pathlib import Path from typing import Generator from ...log import die, info -from ...paths import CONTROL_PLANE_TOKEN_FILENAME, bot_bottle_root +from ...paths import ORCHESTRATOR_TOKEN_FILENAME, bot_bottle_root from .. import util as backend_util from ..docker import util as docker_mod from ..docker.gateway_provision import GatewayProvisionError @@ -42,7 +42,7 @@ from . import firecracker_vm, infra_artifact, netpool, util # Where the infra VM keeps its control-plane signing key (generated on the # persistent /dev/vdb volume mounted at BOT_BOTTLE_ROOT). The host mirrors it # back so the CLI signs `cli` tokens the VM verifies (issue #469 review). -_GUEST_SIGNING_KEY_PATH = "/var/lib/bot-bottle/control-plane-token" +_GUEST_SIGNING_KEY_PATH = "/var/lib/bot-bottle/orchestrator-token" # The single infra-VM image: gateway data plane + baked control-plane source # (Dockerfile.infra FROM the gateway image). Built from source by default; @@ -52,7 +52,7 @@ _GATEWAY_IMAGE = "bot-bottle-gateway:latest" _ORCHESTRATOR_IMAGE = "bot-bottle-orchestrator:latest" _REPO_ROOT = Path(__file__).resolve().parents[3] -CONTROL_PLANE_PORT = 8099 +ORCHESTRATOR_PORT = 8099 # Gateway data-plane ports (agent-facing): egress proxy, supervise MCP, # git-http. Reached by agent VMs over VM-to-VM routing (added next). EGRESS_PORT = 9099 @@ -84,8 +84,8 @@ class InfraVm: vm: firecracker_vm.VmHandle | None = None @property - def control_plane_url(self) -> str: - return f"http://{self.guest_ip}:{CONTROL_PLANE_PORT}" + def orchestrator_url(self) -> str: + return f"http://{self.guest_ip}:{ORCHESTRATOR_PORT}" def terminate(self) -> None: """Stop the infra VM — via the live handle if we booted it, else the @@ -168,7 +168,7 @@ def ensure_running() -> InfraVm: flock, so two simultaneous first launches don't both boot on the same rootfs/PID. The healthy fast-path takes no lock.""" slot = netpool.orch_slot() - url = f"http://{slot.guest_ip}:{CONTROL_PLANE_PORT}" + url = f"http://{slot.guest_ip}:{ORCHESTRATOR_PORT}" key = _infra_dir() / "id_ed25519" want = _expected_version() if _adoptable(key, url, want): @@ -192,7 +192,7 @@ def ensure_running() -> InfraVm: def _with_signing_key(infra: InfraVm) -> InfraVm: """Mirror the infra VM's control-plane signing key (generated on its - persistent volume) into the host's control-plane-token file, so the host CLI + persistent volume) into the host's orchestrator-token file, so the host CLI signs `cli` tokens the VM verifies (issue #469 review). Best-effort: an unreadable key is logged, not fatal — the VM still enforces auth, but the CLI may then be rejected until the key is readable. Returns `infra` for chaining.""" @@ -205,7 +205,7 @@ def _with_signing_key(infra: InfraVm) -> InfraVm: if proc.returncode != 0 or not signing_key: info("infra signing key not yet readable; control-plane auth may fail") return infra - path = bot_bottle_root() / CONTROL_PLANE_TOKEN_FILENAME + path = bot_bottle_root() / ORCHESTRATOR_TOKEN_FILENAME path.parent.mkdir(parents=True, exist_ok=True) fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) with os.fdopen(fd, "w") as f: @@ -458,7 +458,7 @@ def wait_for_health( ) -> None: """Poll the control plane's /health until it answers 200 or the deadline passes. Dies (with the console tail) if the VMM exits early.""" - url = f"{infra.control_plane_url}/health" + url = f"{infra.orchestrator_url}/health" deadline = time.monotonic() + timeout while time.monotonic() < deadline: if infra.vm is not None and not infra.vm.is_alive(): @@ -467,7 +467,7 @@ def wait_for_health( try: with urllib.request.urlopen(url, timeout=1.0) as resp: if resp.status == 200: - info(f"infra control plane healthy at {infra.control_plane_url}") + info(f"infra control plane healthy at {infra.orchestrator_url}") return except (urllib.error.URLError, TimeoutError, OSError): pass @@ -525,11 +525,11 @@ cd /app # same VM — reaching the orchestrator over 127.0.0.1, past the nft boundary that # only fences off the separate agent VM — could drive the operator routes # (approve its own supervise proposals, rewrite policy, read injected tokens). -CP_KEY=$(BOT_BOTTLE_ROOT=/var/lib/bot-bottle python3 -c 'from bot_bottle.paths import host_control_plane_token as t; print(t())') -GW_JWT=$(BB_SIGNING_KEY="$CP_KEY" python3 -c 'import os; from bot_bottle.control_auth import mint, ROLE_GATEWAY; print(mint(ROLE_GATEWAY, os.environ["BB_SIGNING_KEY"]))') +CP_KEY=$(BOT_BOTTLE_ROOT=/var/lib/bot-bottle python3 -c 'from bot_bottle.paths import host_orchestrator_token as t; print(t())') +GW_JWT=$(BB_SIGNING_KEY="$CP_KEY" python3 -c 'import os; from bot_bottle.orchestrator_auth import mint, ROLE_GATEWAY; print(mint(ROLE_GATEWAY, os.environ["BB_SIGNING_KEY"]))') -BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_CONTROL_PLANE_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator \\ - --host 0.0.0.0 --port {CONTROL_PLANE_PORT} --broker stub & +BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_ORCHESTRATOR_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator \\ + --host 0.0.0.0 --port {ORCHESTRATOR_PORT} --broker stub & # Gateway data plane, multi-tenant: each request resolves source-IP -> # policy against the local control plane. The VM backend reaches git over @@ -540,9 +540,9 @@ BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_CONTROL_PLANE_TOKEN="$CP_KEY" pyt # the pre-minted `gateway` JWT; gateway_init keeps the signing key out of the # data-plane daemons' env. BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise \\ -BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{CONTROL_PLANE_PORT} \\ -BOT_BOTTLE_CONTROL_AUTH_JWT="$GW_JWT" \\ - python3 -m bot_bottle.gateway_init & +BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{ORCHESTRATOR_PORT} \\ +BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT="$GW_JWT" \\ + python3 -m bot_bottle.gateway.bootstrap & # Reap as PID 1; children are backgrounded, so `wait` blocks. while : ; do wait ; done diff --git a/bot_bottle/backend/firecracker/launch.py b/bot_bottle/backend/firecracker/launch.py index d72f40c9..007724c4 100644 --- a/bot_bottle/backend/firecracker/launch.py +++ b/bot_bottle/backend/firecracker/launch.py @@ -38,24 +38,18 @@ from ...bottle_state import ( git_gate_state_dir, read_committed_image, ) -from ...egress import ( - egress_agent_env_entries, - egress_resolve_token_values, -) -from ...git_gate import ( - provision_git_gate_dynamic_keys, - revoke_git_gate_provisioned_keys, -) +from ...egress import Egress +from ...git_gate import GitGate from ...image_cache import check_stale_path from ...log import die, info, warn -from ...supervise import SUPERVISE_PORT +from ...supervisor.types import SUPERVISE_PORT from ..docker.egress import EGRESS_PORT from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH from . import firecracker_vm, image_builder, isolation_probe, netpool, util from .bottle import FirecrackerBottle from .bottle_plan import FirecrackerBottlePlan -from ...orchestrator.config_store import resolve_teardown_timeout -from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME +from ...orchestrator.store.config_store import resolve_teardown_timeout +from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME from .consolidated_launch import ( launch_consolidated, persist_env_var_secret, @@ -86,7 +80,7 @@ def launch( except BaseException as exc: # noqa: W0718 - teardown must continue teardown_exc = exc warn(f"firecracker teardown failed: {exc!r}") - revoke_git_gate_provisioned_keys(bottle_for_revoke, git_gate_dir_for_revoke) + GitGate().revoke_provisioned_keys(bottle_for_revoke, git_gate_dir_for_revoke) if teardown_exc is not None: raise teardown_exc @@ -97,7 +91,7 @@ def launch( # Step 2: mint the git-gate dynamic (gitea) deploy keys, if any. git_gate_plan = plan.git_gate_plan if git_gate_plan.upstreams: - git_gate_plan = provision_git_gate_dynamic_keys( + git_gate_plan = GitGate().provision_dynamic_keys( plan.manifest.bottle, git_gate_plan, git_gate_state_dir(plan.slug), ) @@ -113,7 +107,7 @@ def launch( # (in memory) for the gateway to inject — the agent never sees them. # Attribution is by the VM's guest IP (unspoofable via /31 TAP + nft). effective_env = {**os.environ, **plan.agent_provision.provisioned_env} - token_values = egress_resolve_token_values( + token_values = Egress().resolve_token_values( plan.egress_plan.token_env_map, effective_env, ) teardown_timeout = resolve_teardown_timeout() @@ -287,7 +281,7 @@ def _agent_guest_env(plan: FirecrackerBottlePlan, host_ip: str) -> dict[str, str env["MCP_SUPERVISE_URL"] = plan.agent_supervise_url if plan.env_var_secret: env[ENV_VAR_SECRET_NAME] = plan.env_var_secret - for entry in egress_agent_env_entries(plan.egress_plan): + for entry in Egress().agent_env_entries(plan.egress_plan): key, _, value = entry.partition("=") env[key] = value env.update(plan.agent_provision.guest_env) diff --git a/bot_bottle/backend/firecracker/resolve_plan.py b/bot_bottle/backend/firecracker/resolve_plan.py index 62921756..f5b0dd60 100644 --- a/bot_bottle/backend/firecracker/resolve_plan.py +++ b/bot_bottle/backend/firecracker/resolve_plan.py @@ -9,7 +9,7 @@ from ...egress import EgressPlan from ...env import ResolvedEnv from ...git_gate import GitGatePlan from ...manifest import Manifest -from ...supervise import SupervisePlan +from ...supervisor.plan import SupervisePlan from .. import BottleSpec from . import util from .bottle_plan import FirecrackerBottlePlan diff --git a/bot_bottle/backend/macos_container/__init__.py b/bot_bottle/backend/macos_container/__init__.py index 6d56f95a..c655d15c 100644 --- a/bot_bottle/backend/macos_container/__init__.py +++ b/bot_bottle/backend/macos_container/__init__.py @@ -1,10 +1,29 @@ """macOS Apple Container backend. -Selectable via `BOT_BOTTLE_BACKEND=macos-container`. This package owns -the Apple `container` CLI integration; launch remains gated until the -gateway network enforcement shape is implemented. +Selectable via `BOT_BOTTLE_BACKEND=macos-container`. This package owns the Apple +`container` CLI integration; launch remains gated until the gateway network +enforcement shape is implemented. + +Thin by design: `MacosContainerBottleBackend` is re-exported lazily via +`__getattr__`, so importing a leaf module under this package doesn't drag the +backend (and the framework it pulls) into memory. """ -from .backend import MacosContainerBottleBackend +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .backend import MacosContainerBottleBackend + + +def __getattr__(name: str) -> Any: + if name == "MacosContainerBottleBackend": + from .backend import MacosContainerBottleBackend + + globals()[name] = MacosContainerBottleBackend + return MacosContainerBottleBackend + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + __all__ = ["MacosContainerBottleBackend"] diff --git a/bot_bottle/backend/macos_container/backend.py b/bot_bottle/backend/macos_container/backend.py index 2448c002..e9225eda 100644 --- a/bot_bottle/backend/macos_container/backend.py +++ b/bot_bottle/backend/macos_container/backend.py @@ -10,7 +10,7 @@ from ...agent_provider import AgentProvisionPlan from ...egress import EgressPlan from ...env import ResolvedEnv from ...git_gate import GitGatePlan -from ...supervise import SupervisePlan +from ...supervisor.plan import SupervisePlan from ...manifest import Manifest from .. import ActiveAgent, BottleBackend, BottleImages, BottleSpec from . import cleanup as _cleanup @@ -102,7 +102,7 @@ class MacosContainerBottleBackend( (`supervise`) call when no control plane is running yet. Mirrors firecracker's infra-VM bring-up.""" from .infra import MacosInfraService - return MacosInfraService().ensure_running().control_plane_url + return MacosInfraService().ensure_running().orchestrator_url def prepare_cleanup(self) -> MacosContainerBottleCleanupPlan: return _cleanup.prepare_cleanup() diff --git a/bot_bottle/backend/macos_container/consolidated_launch.py b/bot_bottle/backend/macos_container/consolidated_launch.py index 6c25f5e4..a641a9b6 100644 --- a/bot_bottle/backend/macos_container/consolidated_launch.py +++ b/bot_bottle/backend/macos_container/consolidated_launch.py @@ -39,7 +39,7 @@ from ...git_gate import GitGatePlan from ...log import info from ...orchestrator.client import OrchestratorClient, OrchestratorClientError from ...orchestrator.reprovision import reprovision_bottles -from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME +from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME from ..consolidated_util import ( provision_bottle, teardown_consolidated as _teardown_util, @@ -90,7 +90,7 @@ def ensure_gateway( service = service or MacosInfraService() infra = service.ensure_running() endpoint = GatewayEndpoint( - orchestrator_url=infra.control_plane_url, + orchestrator_url=infra.orchestrator_url, gateway_ip=infra.gateway_ip, gateway_ca_pem=service.ca_cert_pem(), network=service.network, diff --git a/bot_bottle/backend/macos_container/gateway.py b/bot_bottle/backend/macos_container/gateway.py index 3d296ff7..fcac8558 100644 --- a/bot_bottle/backend/macos_container/gateway.py +++ b/bot_bottle/backend/macos_container/gateway.py @@ -12,7 +12,7 @@ from __future__ import annotations import os -from ...orchestrator.gateway import GatewayError +from ...gateway import GatewayError from . import util as container_mod # The shared host-only network the infra container and every agent bottle sit diff --git a/bot_bottle/backend/macos_container/infra.py b/bot_bottle/backend/macos_container/infra.py index 731a62a0..c2f14cc7 100644 --- a/bot_bottle/backend/macos_container/infra.py +++ b/bot_bottle/backend/macos_container/infra.py @@ -41,19 +41,19 @@ from dataclasses import dataclass from pathlib import Path from ... import log -from ...orchestrator.gateway import GATEWAY_CA_CERT, MITMPROXY_HOME +from ...gateway import GATEWAY_CA_CERT, MITMPROXY_HOME from ...orchestrator.lifecycle import ( DEFAULT_PORT, DEFAULT_STARTUP_TIMEOUT_SECONDS, OrchestratorStartError, source_hash, ) -from ...control_auth import ROLE_GATEWAY, mint +from ...orchestrator_auth import ROLE_GATEWAY, mint from ...paths import ( - CONTROL_AUTH_JWT_ENV, - CONTROL_PLANE_TOKEN_ENV, + ORCHESTRATOR_AUTH_JWT_ENV, + ORCHESTRATOR_TOKEN_ENV, HOST_DB_FILENAME, - host_control_plane_token, + host_orchestrator_token, host_gateway_ca_dir, ) from .. import util as backend_util @@ -107,7 +107,7 @@ def _init_script(port: int) -> str: # control-plane RPC and never opens bot-bottle.db (PRD 0070 / #469). f"( cd /app && BOT_BOTTLE_GATEWAY_DAEMONS={_GATEWAY_DAEMONS} " f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{port} " - f"python3 -m bot_bottle.gateway_init ) &\n" + f"python3 -m bot_bottle.gateway.bootstrap ) &\n" "while : ; do wait ; done\n" ) @@ -117,7 +117,7 @@ class InfraEndpoint: """How to reach the running infra container. The control plane and the gateway are the same container, so one address serves both.""" - control_plane_url: str # http://:8099 — host CLI + registration + orchestrator_url: str # http://:8099 — host CLI + registration gateway_ip: str # same container; agents' proxy / git-http / MCP target @@ -181,7 +181,7 @@ class MacosInfraService: return None url = self._resolve_url() if url and self.is_healthy(url): - return InfraEndpoint(control_plane_url=url, gateway_ip=_ip_of(url)) + return InfraEndpoint(orchestrator_url=url, gateway_ip=_ip_of(url)) return None def ensure_built(self) -> None: @@ -247,17 +247,17 @@ class MacosInfraService: # run process below, so neither lands on argv or in `container # inspect`'s command line. The agent runs in a SEPARATE container that # is never given these vars, which is the whole point. - "--env", CONTROL_PLANE_TOKEN_ENV, - "--env", CONTROL_AUTH_JWT_ENV, + "--env", ORCHESTRATOR_TOKEN_ENV, + "--env", ORCHESTRATOR_AUTH_JWT_ENV, "--entrypoint", "sh", self.image, "-c", _init_script(self.port), ] - _signing_key = host_control_plane_token() + _signing_key = host_orchestrator_token() run_env = { **os.environ, - CONTROL_PLANE_TOKEN_ENV: _signing_key, - CONTROL_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key), + ORCHESTRATOR_TOKEN_ENV: _signing_key, + ORCHESTRATOR_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key), } result = container_mod.run_container_argv(argv, env=run_env) if result.returncode != 0: @@ -272,7 +272,7 @@ class MacosInfraService: url = self._resolve_url() if url and self.is_healthy(url): log.info("infra container healthy", context={"url": url}) - return InfraEndpoint(control_plane_url=url, gateway_ip=_ip_of(url)) + return InfraEndpoint(orchestrator_url=url, gateway_ip=_ip_of(url)) if time.monotonic() >= deadline: raise OrchestratorStartError( f"infra container did not become healthy within " @@ -306,7 +306,7 @@ def _ip_of(url: str) -> str: return url.split("://", 1)[-1].rsplit(":", 1)[0] -def probe_control_plane_url(port: int = DEFAULT_PORT) -> str: +def probe_orchestrator_url(port: int = DEFAULT_PORT) -> str: """The running infra container's control-plane URL, or "" if it isn't up. Used by host-side control-plane discovery (`discover_orchestrator_url`); safe to call on any host — returns "" when the container or the `container` diff --git a/bot_bottle/backend/macos_container/launch.py b/bot_bottle/backend/macos_container/launch.py index 89022668..ba34ff97 100644 --- a/bot_bottle/backend/macos_container/launch.py +++ b/bot_bottle/backend/macos_container/launch.py @@ -44,19 +44,13 @@ from ...bottle_state import ( git_gate_state_dir, read_committed_image, ) -from ...egress import ( - egress_agent_env_entries, - egress_resolve_token_values, -) -from ...git_gate import ( - provision_git_gate_dynamic_keys, - revoke_git_gate_provisioned_keys, -) -from ...git_http_backend import DEFAULT_PORT as _GIT_HTTP_PORT +from ...egress import Egress +from ...git_gate import GitGate +from ...gateway.git_gate.http_backend import DEFAULT_PORT as _GIT_HTTP_PORT from ...image_cache import check_stale from ...log import die, info, warn from .. import BottleImages -from ...supervise import SUPERVISE_PORT +from ...supervisor.types import SUPERVISE_PORT from ..docker.egress import EGRESS_PORT from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH from . import util as container_mod @@ -68,8 +62,8 @@ from .gateway_hosts import ( ) from . import nested_containers as nested_containers_mod from .bottle_plan import MacosContainerBottlePlan -from ...orchestrator.config_store import resolve_teardown_timeout -from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME, new_env_var_secret +from ...orchestrator.store.config_store import resolve_teardown_timeout +from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME, new_env_var_secret from .consolidated_launch import ( GatewayEndpoint, ensure_gateway, @@ -151,7 +145,7 @@ def launch( except BaseException as exc: # noqa: W0718 - teardown must continue teardown_exc = exc warn(f"macos-container teardown failed: {exc!r}") - revoke_git_gate_provisioned_keys(bottle_for_revoke, git_gate_dir_for_revoke) + GitGate().revoke_provisioned_keys(bottle_for_revoke, git_gate_dir_for_revoke) if teardown_exc is not None: raise teardown_exc @@ -194,7 +188,7 @@ def launch( f"{endpoint.network}" ) effective_env = {**os.environ, **plan.agent_provision.provisioned_env} - token_values = egress_resolve_token_values( + token_values = Egress().resolve_token_values( plan.egress_plan.token_env_map, effective_env, ) teardown_timeout = resolve_teardown_timeout() @@ -282,7 +276,7 @@ def _provision_git_gate_keys( ) -> MacosContainerBottlePlan: if not plan.git_gate_plan.upstreams: return plan - git_gate_plan = provision_git_gate_dynamic_keys( + git_gate_plan = GitGate().provision_dynamic_keys( plan.manifest.bottle, plan.git_gate_plan, git_gate_state_dir(plan.slug), @@ -454,7 +448,7 @@ def _agent_env_entries( # so the secret value never lands on argv. for name in sorted(plan.forwarded_env.keys()): env.append(name) - env.extend(egress_agent_env_entries(plan.egress_plan)) + env.extend(Egress().agent_env_entries(plan.egress_plan)) return tuple(env) diff --git a/bot_bottle/backend/macos_container/resolve_plan.py b/bot_bottle/backend/macos_container/resolve_plan.py index 4c9883ef..f8fb9758 100644 --- a/bot_bottle/backend/macos_container/resolve_plan.py +++ b/bot_bottle/backend/macos_container/resolve_plan.py @@ -8,7 +8,7 @@ from ...agent_provider import AgentProvisionPlan from ...egress import EgressPlan from ...env import ResolvedEnv from ...git_gate import GitGatePlan -from ...supervise import SupervisePlan +from ...supervisor.plan import SupervisePlan from ...manifest import Manifest from .. import BottleSpec from . import util as container_mod diff --git a/bot_bottle/backend/resolve_common.py b/bot_bottle/backend/resolve_common.py index 88f94ff9..7ea9c145 100644 --- a/bot_bottle/backend/resolve_common.py +++ b/bot_bottle/backend/resolve_common.py @@ -28,7 +28,8 @@ from ..egress import Egress, EgressPlan from ..git_gate import GitGate, GitGatePlan from ..log import die from ..manifest import Manifest, ManifestBottle -from ..supervise import Supervise, SupervisePlan +from ..supervisor.plan import SupervisePlan +from ..orchestrator.supervisor import Supervisor from . import BottleSpec @@ -101,7 +102,7 @@ def prepare_supervise(bottle: ManifestBottle, slug: str) -> SupervisePlan | None return None supervise_dir = supervise_state_dir(slug) supervise_dir.mkdir(parents=True, exist_ok=True) - return Supervise().prepare(slug, supervise_dir) + return Supervisor().prepare(slug, supervise_dir) def merge_provision_env_vars(provision: AgentProvisionPlan) -> AgentProvisionPlan: diff --git a/bot_bottle/backend/selection.py b/bot_bottle/backend/selection.py new file mode 100644 index 00000000..60e2fa54 --- /dev/null +++ b/bot_bottle/backend/selection.py @@ -0,0 +1,188 @@ +"""Backend registry, selection, and active-agent enumeration. + +Resolves which bottle backend to use (explicit name / `BOT_BOTTLE_BACKEND` / +auto-select), and enumerates running agents across every available backend. The +three concrete backends are imported lazily inside `_get_backends` so this +module — and anything that only needs to *select* a backend — stays cheap. +""" + +from __future__ import annotations + +import os +import sys +from typing import Any + +from ..log import die, info, warn +from ..util import read_tty_line +from .base import ActiveAgent, BottleBackend + + +# _backends is None until the first call to _get_backends(), at which +# point all three concrete backend classes are imported and instantiated. +# Keeping the imports out of module scope means that importing any +# backend sub-module (e.g. `backend.docker.util`) no longer drags the +# firecracker and macos-container implementations into memory. +# +# Tests may replace _backends with a {name: fake} dict via patch.object; +# _get_backends() returns the current module-level value as-is when it +# is not None, so test fakes take effect without triggering real imports. +_backends: dict[str, BottleBackend[Any, Any]] | None = None + + +def _get_backends() -> dict[str, BottleBackend[Any, Any]]: + """Return the registry of all backend instances, loading lazily on first call.""" + global _backends # pylint: disable=global-statement + if _backends is None: + from .docker import DockerBottleBackend + from .firecracker import FirecrackerBottleBackend + from .macos_container import MacosContainerBottleBackend + _backends = { + "docker": DockerBottleBackend(), + "firecracker": FirecrackerBottleBackend(), + "macos-container": MacosContainerBottleBackend(), + } + return _backends + + +def get_bottle_backend( + name: str | None = None, + *, + prompt: bool = True, +) -> BottleBackend[Any, Any]: + """Resolve the bottle backend. + + `name` precedence: + 1. explicit arg (e.g. resume passes the recorded backend name) + 2. BOT_BOTTLE_BACKEND env var + 3. auto-selection: VM backend first, docker fallback with prompt + + `prompt` controls whether auto-selection may block on an interactive + [i/d/q] prompt when falling back to docker. Pass `prompt=False` in + non-interactive contexts (headless launches, CI) so the call dies + with an actionable message instead of hanging. + + Dies with a pointer at the known backends if the chosen name + isn't implemented.""" + resolved = name or os.environ.get("BOT_BOTTLE_BACKEND") + if resolved is None: + resolved = _auto_select_backend(prompt=prompt) + backends = _get_backends() + if resolved not in backends: + known = ", ".join(sorted(backends)) + die(f"unknown backend {resolved!r}; known backends: {known}") + return backends[resolved] + + +def _platform_vm_suggestion() -> str: + """Platform-appropriate VM backend name for install suggestions.""" + return "macos-container" if sys.platform == "darwin" else "firecracker" + + +def _print_vm_install_instructions() -> None: + """Print platform-appropriate VM backend install instructions to stderr.""" + vm = _platform_vm_suggestion() + if vm == "macos-container": + info("Install Apple Container: https://github.com/apple/container/releases") + info("Then start the service: container system start") + else: + info("Install Firecracker: https://github.com/firecracker-microvm/firecracker/releases") + info("Configure the host: ./cli.py backend setup") + + +def _auto_select_backend(prompt: bool = True) -> str: + """Tier-1 / tier-2 backend auto-selection. + + Tier 1: VM backend — macos-container on macOS when Apple Container is + installed; firecracker on KVM-capable Linux even before the binary is + present (its preflight prints an install pointer). + + Tier 2: docker, with a security warning and an interactive prompt. + When `prompt=False` (headless / CI), dies with an actionable message + instead of blocking on a TTY read. When docker is also absent, prints + VM install instructions and exits. + """ + # --- Tier 1: VM backend ----------------------------------------- + if has_backend("macos-container"): + return "macos-container" + # A KVM-capable Linux host defaults to firecracker even when the + # `firecracker` binary isn't installed yet: selecting it here routes + # start through firecracker's preflight, which prints an install + # pointer, instead of silently falling back to docker. + from .firecracker import FirecrackerBottleBackend + if FirecrackerBottleBackend.is_host_capable(): + return "firecracker" + + # --- Tier 2: docker fallback ------------------------------------ + if not has_backend("docker"): + info("No backend available on this host.") + _print_vm_install_instructions() + die("no backend available; install a VM backend and re-run") + + vm = _platform_vm_suggestion() + warn( + "docker is less secure than VM backends — " + "containers share the host kernel." + ) + if not prompt: + die( + f"no VM backend available; set BOT_BOTTLE_BACKEND=docker to proceed " + f"with docker, or install the {vm!r} backend." + ) + sys.stderr.write( + f"bot-bottle: For better isolation, install the {vm!r} backend.\n" + f" [i] show {vm} install instructions and exit\n" + " [d] use docker anyway\n" + " [q] quit\n" + "bot-bottle: choice [i/d/q]: " + ) + sys.stderr.flush() + reply = read_tty_line().strip().lower() + if reply == "d": + return "docker" + if reply == "i": + _print_vm_install_instructions() + die("not proceeding with docker; install a VM backend or set BOT_BOTTLE_BACKEND=docker") + + +def known_backend_names() -> tuple[str, ...]: + """Sorted tuple of all backend keys in `_get_backends()`. Used by + argparse (`--backend` choices) and the dashboard's backend + picker.""" + return tuple(sorted(_get_backends())) + + +def has_backend(name: str) -> bool: + """Whether the named backend's runtime prerequisites are + available on the current host. Cross-backend callers (list, + cleanup) skip unavailable backends so a docker-only host + doesn't fail when the firecracker backend isn't usable, + and vice versa. + + Returns False for unknown names so callers can pass + arbitrary input without separate validation.""" + backends = _get_backends() + if name not in backends: + return False + return backends[name].is_available() + + +def enumerate_active_agents() -> list[ActiveAgent]: + """All currently-running agents, across every available + backend. Used by CLI `list active` and the dashboard's agents + pane so neither has to know which backends exist. Skips + backends whose `is_available()` reports False. + + Sorted by `(started_at, slug)` so the list is stable across + dashboard refresh ticks — agents don't shift position while + the operator navigates with arrow keys. ISO 8601 timestamps + sort lexicographically in chronological order; `slug` is the + deterministic tiebreaker. Agents with missing metadata + (`started_at == ""`) sort first.""" + out: list[ActiveAgent] = [] + backends = _get_backends() + for name in sorted(backends): + if not backends[name].is_available(): + continue + out.extend(backends[name].enumerate_active()) + out.sort(key=lambda a: (a.started_at, a.slug)) + return out diff --git a/bot_bottle/cli/__init__.py b/bot_bottle/cli/__init__.py index cf62f68c..56a73d01 100644 --- a/bot_bottle/cli/__init__.py +++ b/bot_bottle/cli/__init__.py @@ -1,116 +1,15 @@ -"""Main CLI dispatcher. +"""bot-bottle CLI package. -Commands: backend, cleanup, commit, edit, info, init, list, resume, start, supervise +The subcommand handlers live in `commands/` and are assembled into the +COMMANDS registry by `commands/__init__.py`; the dispatcher `main()` lives +in `__main__.py`. They are re-exported here so `bot_bottle.cli.main`, +`bot_bottle.cli.COMMANDS`, and `bot_bottle.cli.NO_MIGRATION_COMMANDS` stay +importable (the repo-root `cli.py` entry point and the tests use them). """ from __future__ import annotations -import sys +from .__main__ import main +from .commands import COMMANDS, NO_MIGRATION_COMMANDS -from ..errors import MissingEnvVarError -from ..log import Die, die, error -from ..manifest import ManifestError -from ..store_manager import StoreManager -from ._common import PROG -from . import list as _list_mod -from .backend import cmd_backend -from .cleanup import cmd_cleanup -from .commit import cmd_commit -from .edit import cmd_edit -from .info import cmd_info -from .init import cmd_init -from .login import cmd_login -from .resume import cmd_resume -from .start import cmd_start -from .supervise import cmd_supervise - -cmd_list = _list_mod.cmd_list - -COMMANDS = { - "backend": cmd_backend, - "cleanup": cmd_cleanup, - "commit": cmd_commit, - "edit": cmd_edit, - "info": cmd_info, - "init": cmd_init, - "list": cmd_list, - "login": cmd_login, - "resume": cmd_resume, - "start": cmd_start, - "supervise": cmd_supervise, -} - -# Commands that manage host prerequisites (or are otherwise store-free) and -# must run before — or without — a migrated DB. `backend` provisions/probes -# the host (TAP pool, /dev/kvm, firecracker) and never opens the store, so -# gating it on the schema breaks preflight on a fresh CI runner where stdin -# isn't a TTY and the migration prompt can't be answered. -NO_MIGRATION_COMMANDS = frozenset({"backend", "login"}) - - -def usage() -> None: - sys.stderr.write(f"usage: {PROG} [args...]\n\n") - sys.stderr.write("Commands:\n") - sys.stderr.write(" backend set up / check / undo a backend's host prerequisites (setup|status|teardown)\n") - sys.stderr.write(" cleanup stop and remove all active bot-bottle containers\n") - sys.stderr.write(" commit snapshot a running bottle's container state to a Docker image\n") - sys.stderr.write(" edit open an agent in vim for editing\n") - sys.stderr.write(" info print env, skills, and prompt details for a named agent\n") - sys.stderr.write(" init interactively create a new agent and add it to bot-bottle.json\n") - sys.stderr.write(" list list available agents or active containers\n") - sys.stderr.write(" login register this host with a bot-bottle console\n") - sys.stderr.write( - " resume re-launch a bottle by its identity " - "(continues state from PRD 0016)\n" - ) - sys.stderr.write( - " start boot a container for a named agent and " - "attach an interactive session\n" - ) - sys.stderr.write( - " supervise view + approve/modify/reject pending supervise " - "proposals (PRD 0013)\n\n" - ) - sys.stderr.write(f"Run '{PROG} --help' for command-specific usage.\n") - - -def main(argv: list[str] | None = None) -> int: - if argv is None: - argv = sys.argv[1:] - if not argv: - usage() - return 2 - command = argv[0] - rest = argv[1:] - if command in ("-h", "--help"): - usage() - return 0 - handler = COMMANDS.get(command) - if handler is None: - usage() - die(f"unknown command: {command}") - mgr = StoreManager.instance() - if command not in NO_MIGRATION_COMMANDS and not mgr.is_migrated(): - sys.stderr.write("bot-bottle: database schema is out of date\n") - sys.stderr.write("Migrate now? [y/N] ") - sys.stderr.flush() - try: - answer = sys.stdin.readline().strip().lower() - except EOFError: - answer = "" - if answer != "y": - error("migration required — re-run and confirm to migrate") - return 1 - mgr.migrate() - try: - return handler(rest) or 0 - except MissingEnvVarError as e: - error(str(e)) - return 1 - except ManifestError as e: - error(str(e)) - return 1 - except Die as e: - return e.code if isinstance(e.code, int) else 1 - except KeyboardInterrupt: - return 130 +__all__ = ["main", "COMMANDS", "NO_MIGRATION_COMMANDS"] diff --git a/bot_bottle/cli/__main__.py b/bot_bottle/cli/__main__.py index 3cf5f19f..91a7e5aa 100644 --- a/bot_bottle/cli/__main__.py +++ b/bot_bottle/cli/__main__.py @@ -1,15 +1,65 @@ -"""Entry point for `python -m bot_bottle.cli`. +"""Entry point + dispatcher for `python -m bot_bottle.cli`. -`cli.py` at the repo root is the usual way in; this makes the package -runnable too, so the CLI works from an installed copy where there is no -`cli.py` on disk to point at. +Maps `bot-bottle ` to its handler in the COMMANDS registry +(`bot_bottle.cli.commands`), enforces the schema-migration gate, and +translates handler exceptions into process exit codes. The repo-root +`cli.py` is the usual way in; this makes the package runnable too, so the +CLI works from an installed copy where there is no `cli.py` on disk. """ from __future__ import annotations import sys -from . import main +from ..errors import MissingEnvVarError +from ..log import Die, die, error +from ..manifest import ManifestError +from ..orchestrator.store.store_manager import StoreManager +from .commands import COMMANDS, NO_MIGRATION_COMMANDS +from .commands.help import cmd_help + + +def main(argv: list[str] | None = None) -> int: + if argv is None: + argv = sys.argv[1:] + if not argv: + cmd_help() + return 2 + command = argv[0] + rest = argv[1:] + if command in ("-h", "--help"): + cmd_help() + return 0 + handler = COMMANDS.get(command) + if handler is None: + cmd_help() + die(f"unknown command: {command}") + mgr = StoreManager.instance() + if command not in NO_MIGRATION_COMMANDS and not mgr.is_migrated(): + sys.stderr.write("bot-bottle: database schema is out of date\n") + sys.stderr.write("Migrate now? [y/N] ") + sys.stderr.flush() + try: + answer = sys.stdin.readline().strip().lower() + except EOFError: + answer = "" + if answer != "y": + error("migration required — re-run and confirm to migrate") + return 1 + mgr.migrate() + try: + return handler(rest) or 0 + except MissingEnvVarError as e: + error(str(e)) + return 1 + except ManifestError as e: + error(str(e)) + return 1 + except Die as e: + return e.code if isinstance(e.code, int) else 1 + except KeyboardInterrupt: + return 130 + if __name__ == "__main__": sys.exit(main()) diff --git a/bot_bottle/cli/_common.py b/bot_bottle/cli/_common.py deleted file mode 100644 index 5fc6c332..00000000 --- a/bot_bottle/cli/_common.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Shared constants and tty helper for cli subcommands.""" - -from __future__ import annotations - -import os -from pathlib import Path - -from ..util import read_tty_line as read_tty_line - -PROG = "cli.py" -USER_CWD = os.getcwd() -REPO_DIR = str(Path(__file__).resolve().parent.parent.parent) diff --git a/bot_bottle/cli/commands/__init__.py b/bot_bottle/cli/commands/__init__.py new file mode 100644 index 00000000..813524f6 --- /dev/null +++ b/bot_bottle/cli/commands/__init__.py @@ -0,0 +1,58 @@ +"""CLI subcommand registry. + +One module per `bot-bottle `, each exposing a `cmd_(argv)` +handler. This package `__init__` maps command names to their handlers +**lazily**: a short-lived CLI run dispatches exactly one command, so +importing all twelve handlers (and their transitive deps — backend, +manifest, orchestrator, …) up front is wasted work. Each COMMANDS value is +a thin wrapper that imports its handler's module on first call. Shared CLI +helpers (`constants`, `tui`) stay one level up in the `cli` package. +""" + +from __future__ import annotations + +from importlib import import_module +from typing import Callable + +# command name -> ":". Kept as strings so building +# the registry imports nothing; the module loads only when dispatched. +_HANDLERS: dict[str, str] = { + "backend": "backend:cmd_backend", + "cleanup": "cleanup:cmd_cleanup", + "commit": "commit:cmd_commit", + "edit": "edit:cmd_edit", + "help": "help:cmd_help", + "info": "info:cmd_info", + "init": "init:cmd_init", + "list": "list:cmd_list", + "login": "login:cmd_login", + "resume": "resume:cmd_resume", + "start": "start:cmd_start", + "supervise": "supervise:cmd_supervise", +} + + +def _lazy(spec: str) -> Callable[[list[str]], "int | None"]: + """Wrap a `:` handler so its module is imported only when + the command is actually dispatched, not when the registry is built.""" + module, attr = spec.split(":") + + def run(argv: list[str]) -> "int | None": + handler = getattr(import_module(f".{module}", __name__), attr) + return handler(argv) + + run.__name__ = attr + return run + + +COMMANDS = {name: _lazy(spec) for name, spec in _HANDLERS.items()} + +# Commands that manage host prerequisites (or are otherwise store-free) and +# must run before — or without — a migrated DB. `backend` provisions/probes +# the host (TAP pool, /dev/kvm, firecracker) and never opens the store, so +# gating it on the schema breaks preflight on a fresh CI runner where stdin +# isn't a TTY and the migration prompt can't be answered. `help` and `login` +# likewise never touch the store. +NO_MIGRATION_COMMANDS = frozenset({"backend", "help", "login"}) + +__all__ = ["COMMANDS", "NO_MIGRATION_COMMANDS"] diff --git a/bot_bottle/cli/backend.py b/bot_bottle/cli/commands/backend.py similarity index 94% rename from bot_bottle/cli/backend.py rename to bot_bottle/cli/commands/backend.py index ed6808f5..a06e44f5 100644 --- a/bot_bottle/cli/backend.py +++ b/bot_bottle/cli/commands/backend.py @@ -15,8 +15,8 @@ from __future__ import annotations import argparse -from ..backend import get_bottle_backend, known_backend_names -from ._common import PROG +from ...backend import get_bottle_backend, known_backend_names +from ..constants import PROG def cmd_backend(args: list[str]) -> int: diff --git a/bot_bottle/cli/cleanup.py b/bot_bottle/cli/commands/cleanup.py similarity index 93% rename from bot_bottle/cli/cleanup.py rename to bot_bottle/cli/commands/cleanup.py index 34d6de99..d83ef14c 100644 --- a/bot_bottle/cli/cleanup.py +++ b/bot_bottle/cli/commands/cleanup.py @@ -21,9 +21,9 @@ from __future__ import annotations import sys -from ..backend import get_bottle_backend, has_backend, known_backend_names -from ..log import info -from ._common import read_tty_line +from ...backend import get_bottle_backend, has_backend, known_backend_names +from ...log import info +from ...util import read_tty_line def cmd_cleanup(_argv: list[str]) -> int: diff --git a/bot_bottle/cli/commit.py b/bot_bottle/cli/commands/commit.py similarity index 86% rename from bot_bottle/cli/commit.py rename to bot_bottle/cli/commands/commit.py index 51d94d9b..fcf454d2 100644 --- a/bot_bottle/cli/commit.py +++ b/bot_bottle/cli/commands/commit.py @@ -12,12 +12,12 @@ from __future__ import annotations import argparse -from ..backend import enumerate_active_agents -from ..backend.freeze import CommitCancelled, get_freezer -from ..bottle_state import read_metadata -from ..log import die -from ._common import PROG -from . import tui +from ...backend import enumerate_active_agents +from ...backend.freeze import CommitCancelled, get_freezer +from ...bottle_state import read_metadata +from ...log import die +from ..constants import PROG +from .. import tui def cmd_commit(argv: list[str]) -> int: diff --git a/bot_bottle/cli/edit.py b/bot_bottle/cli/commands/edit.py similarity index 90% rename from bot_bottle/cli/edit.py rename to bot_bottle/cli/commands/edit.py index 8c412d9b..86075175 100644 --- a/bot_bottle/cli/edit.py +++ b/bot_bottle/cli/commands/edit.py @@ -7,8 +7,8 @@ import json import os from pathlib import Path -from ..log import die -from ._common import PROG, USER_CWD +from ...log import die +from ..constants import PROG def cmd_edit(argv: list[str]) -> int: @@ -20,7 +20,7 @@ def cmd_edit(argv: list[str]) -> int: if args.scope == "user": target_file = Path(os.environ["HOME"]) / "bot-bottle.json" else: - target_file = Path(USER_CWD) / "bot-bottle.json" + target_file = Path(os.getcwd()) / "bot-bottle.json" if not target_file.is_file(): die(f"{target_file} does not exist") diff --git a/bot_bottle/cli/commands/help.py b/bot_bottle/cli/commands/help.py new file mode 100644 index 00000000..d7ee6330 --- /dev/null +++ b/bot_bottle/cli/commands/help.py @@ -0,0 +1,37 @@ +"""help: print the top-level command list and usage. + +Rendered by the dispatcher for the `help` command and for its +`-h`/`--help`, no-args, and unknown-command fallbacks. The per-command +summaries live here; keep them in sync with the COMMANDS table in +`bot_bottle.cli`. +""" + +from __future__ import annotations + +import sys + +from ..constants import PROG + + +def cmd_help(argv: list[str] | None = None) -> int: + """Write the top-level usage + command list to stderr. Returns 0; + the dispatcher chooses the process exit code per entry path (0 for an + explicit `help`/`-h`, 2 for the bare no-args usage error).""" + del argv # help takes no arguments + w = sys.stderr.write + w(f"usage: {PROG} [args...]\n\n") + w("Commands:\n") + w(" backend set up / check / undo a backend's host prerequisites (setup|status|teardown)\n") + w(" cleanup stop and remove all active bot-bottle containers\n") + w(" commit snapshot a running bottle's container state to a Docker image\n") + w(" edit open an agent in vim for editing\n") + w(" help show this command list\n") + w(" info print env, skills, and prompt details for a named agent\n") + w(" init interactively create a new agent and add it to bot-bottle.json\n") + w(" list list available agents or active containers\n") + w(" login register this host with a bot-bottle console\n") + w(" resume re-launch a bottle by its identity (continues state from PRD 0016)\n") + w(" start boot a container for a named agent and attach an interactive session\n") + w(" supervise view + approve/modify/reject pending supervise proposals (PRD 0013)\n\n") + w(f"Run '{PROG} --help' for command-specific usage.\n") + return 0 diff --git a/bot_bottle/cli/info.py b/bot_bottle/cli/commands/info.py similarity index 90% rename from bot_bottle/cli/info.py rename to bot_bottle/cli/commands/info.py index 61b89453..7850a2d1 100644 --- a/bot_bottle/cli/info.py +++ b/bot_bottle/cli/commands/info.py @@ -3,10 +3,11 @@ from __future__ import annotations import argparse +import os -from ..log import info -from ..manifest import ManifestIndex -from ._common import PROG, USER_CWD +from ...log import info +from ...manifest import ManifestIndex +from ..constants import PROG def cmd_info(argv: list[str]) -> int: @@ -14,7 +15,7 @@ def cmd_info(argv: list[str]) -> int: parser.add_argument("name", help="agent name defined in bot-bottle.json") args = parser.parse_args(argv) - names = ManifestIndex.resolve(USER_CWD) + names = ManifestIndex.resolve(os.getcwd()) names.require_agent(args.name) manifest = names.load_for_agent(args.name) diff --git a/bot_bottle/cli/init.py b/bot_bottle/cli/commands/init.py similarity index 97% rename from bot_bottle/cli/init.py rename to bot_bottle/cli/commands/init.py index 21b62a02..34441779 100644 --- a/bot_bottle/cli/init.py +++ b/bot_bottle/cli/commands/init.py @@ -10,8 +10,9 @@ import sys from pathlib import Path from typing import Any -from ..log import die, info, warn -from ._common import PROG, USER_CWD, read_tty_line +from ...log import die, info, warn +from ..constants import PROG +from ...util import read_tty_line def cmd_init(argv: list[str]) -> int: @@ -22,7 +23,7 @@ def cmd_init(argv: list[str]) -> int: if args.scope == "user": target_file = Path(os.environ["HOME"]) / "bot-bottle.json" else: - target_file = Path(USER_CWD) / "bot-bottle.json" + target_file = Path(os.getcwd()) / "bot-bottle.json" print(file=sys.stderr) info(f"bot-bottle init — adding a new agent to {target_file}") diff --git a/bot_bottle/cli/list.py b/bot_bottle/cli/commands/list.py similarity index 91% rename from bot_bottle/cli/list.py rename to bot_bottle/cli/commands/list.py index c7274286..6b83c4eb 100644 --- a/bot_bottle/cli/list.py +++ b/bot_bottle/cli/commands/list.py @@ -6,9 +6,9 @@ import argparse import os import sys -from ..backend import enumerate_active_agents -from ..manifest import ManifestIndex -from ._common import PROG, USER_CWD +from ...backend import enumerate_active_agents +from ...manifest import ManifestIndex +from ..constants import PROG _ANSI_COLOR_CODES: dict[str, str] = { "red": "\033[91m", @@ -40,7 +40,7 @@ def cmd_list(argv: list[str]) -> int: args = parser.parse_args(argv) if args.scope == "available": - manifest = ManifestIndex.resolve(USER_CWD) + manifest = ManifestIndex.resolve(os.getcwd()) for name in manifest.all_agent_names: print(name) return 0 diff --git a/bot_bottle/cli/login.py b/bot_bottle/cli/commands/login.py similarity index 99% rename from bot_bottle/cli/login.py rename to bot_bottle/cli/commands/login.py index 3d00f17e..55e04bfa 100644 --- a/bot_bottle/cli/login.py +++ b/bot_bottle/cli/commands/login.py @@ -25,7 +25,7 @@ import urllib.request from pathlib import Path from typing import Any -from ..paths import bot_bottle_root +from ...paths import bot_bottle_root _CONSOLE_URL_ENV = "BB_CONSOLE_URL" _POLL_SLEEP = 2 # seconds between polls; matches console's poll_interval default diff --git a/bot_bottle/cli/resume.py b/bot_bottle/cli/commands/resume.py similarity index 85% rename from bot_bottle/cli/resume.py rename to bot_bottle/cli/commands/resume.py index 5dd421bb..e3b6bc28 100644 --- a/bot_bottle/cli/resume.py +++ b/bot_bottle/cli/commands/resume.py @@ -15,12 +15,13 @@ to bring up the replacement from the recorded state. from __future__ import annotations import argparse +import os -from ..backend import BottleSpec -from ..bottle_state import read_metadata -from ..log import die -from ..manifest import ManifestIndex -from ._common import PROG, USER_CWD +from ...backend import BottleSpec +from ...bottle_state import read_metadata +from ...log import die +from ...manifest import ManifestIndex +from ..constants import PROG from .start import _launch_bottle @@ -40,14 +41,14 @@ def cmd_resume(argv: list[str]) -> int: f"check ~/.bot-bottle/state/ or run `cli.py start` to create a new bottle" ) - manifest = ManifestIndex.resolve(USER_CWD) + manifest = ManifestIndex.resolve(os.getcwd()) manifest.require_agent(metadata.agent_name) spec = BottleSpec( manifest=manifest, agent_name=metadata.agent_name, copy_cwd=metadata.copy_cwd, - user_cwd=metadata.cwd or USER_CWD, + user_cwd=metadata.cwd or os.getcwd(), identity=metadata.identity, bottle_names=tuple(metadata.bottle_names), ) diff --git a/bot_bottle/cli/start.py b/bot_bottle/cli/commands/start.py similarity index 96% rename from bot_bottle/cli/start.py rename to bot_bottle/cli/commands/start.py index 27bcf3ac..2082f928 100644 --- a/bot_bottle/cli/start.py +++ b/bot_bottle/cli/commands/start.py @@ -22,25 +22,26 @@ import tempfile from pathlib import Path from typing import Callable -from ..agent_provider import get_provider, runtime_for -from ..backend import ( +from ...agent_provider import get_provider, runtime_for +from ...backend import ( Bottle, BottleSpec, enumerate_active_agents, get_bottle_backend, ) -from ..backend.docker import util as docker_mod -from ..backend.docker.bottle_plan import DockerBottlePlan -from ..bottle_state import ( +from ...backend.docker import util as docker_mod +from ...backend.docker.bottle_plan import DockerBottlePlan +from ...bottle_state import ( cleanup_state, is_preserved, mark_preserved, ) -from ..image_cache import StaleImageError -from ..log import info, die -from ..manifest import Manifest, ManifestIndex -from ._common import PROG, USER_CWD, read_tty_line -from . import tui +from ...image_cache import StaleImageError +from ...log import info, die +from ...manifest import Manifest, ManifestIndex +from ..constants import PROG +from ...util import read_tty_line +from .. import tui def cmd_start(argv: list[str]) -> int: @@ -116,7 +117,7 @@ def cmd_start(argv: list[str]) -> int: # threading a no_cache field through every backend's plan dataclass. os.environ["BOT_BOTTLE_NO_CACHE"] = "1" - manifest = ManifestIndex.resolve(USER_CWD) + manifest = ManifestIndex.resolve(os.getcwd()) if args.headless: return _start_headless( @@ -167,7 +168,7 @@ def cmd_start(argv: list[str]) -> int: manifest=manifest, agent_name=agent_name, copy_cwd=args.cwd, - user_cwd=USER_CWD, + user_cwd=os.getcwd(), label=label, color=color, bottle_names=bottle_names, @@ -235,7 +236,7 @@ def _start_headless( manifest=manifest, agent_name=agent_name, copy_cwd=args.cwd, - user_cwd=USER_CWD, + user_cwd=os.getcwd(), label=label, color=args.color or "", bottle_names=bottle_names, @@ -380,8 +381,8 @@ def _peek_agent_bottle(manifest: ManifestIndex, agent_name: str) -> str: return manifest.agents[agent_name].bottle return "" - from ..manifest_loader import scan_agent_names - from ..yaml_subset import YamlSubsetError, parse_frontmatter + from ...manifest.loader import scan_agent_names + from ...yaml_subset import YamlSubsetError, parse_frontmatter home_agents = scan_agent_names(manifest.home_md / "agents") cwd_agents: dict[str, Path] = {} @@ -449,7 +450,7 @@ def _bottle_lineage(manifest: ManifestIndex) -> dict[str, str]: if not bottles_dir.is_dir(): return {} - from ..yaml_subset import YamlSubsetError, parse_frontmatter + from ...yaml_subset import YamlSubsetError, parse_frontmatter extends_of: dict[str, str] = {} for path in bottles_dir.glob("*.md"): diff --git a/bot_bottle/cli/supervise.py b/bot_bottle/cli/commands/supervise.py similarity index 98% rename from bot_bottle/cli/supervise.py rename to bot_bottle/cli/commands/supervise.py index d87726c9..cbb349f4 100644 --- a/bot_bottle/cli/supervise.py +++ b/bot_bottle/cli/commands/supervise.py @@ -19,22 +19,22 @@ from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path -from ..paths import bot_bottle_root -from ..log import Die, error, info -from ..orchestrator.client import ( +from ...paths import bot_bottle_root +from ...log import Die, error, info +from ...orchestrator.client import ( OrchestratorClient, OrchestratorClientError, discover_orchestrator_url, ) -from ..supervise import ( +from ...supervisor.types import ( Proposal, TOOL_EGRESS_ALLOW, TOOL_EGRESS_BLOCK, TOOL_GITLEAKS_ALLOW, TOOL_EGRESS_TOKEN_ALLOW, ) -from ._common import PROG +from ..constants import PROG _REFRESH_INTERVAL_MS = 1000 @@ -81,7 +81,7 @@ def _resolve_orchestrator_url() -> str: try: return discover_orchestrator_url() except OrchestratorClientError: - from ..backend import get_bottle_backend + from ...backend import get_bottle_backend backend = get_bottle_backend() info(f"no orchestrator control plane running; starting one ({backend.name})…") return backend.ensure_orchestrator() diff --git a/bot_bottle/cli/constants.py b/bot_bottle/cli/constants.py new file mode 100644 index 00000000..4df182ea --- /dev/null +++ b/bot_bottle/cli/constants.py @@ -0,0 +1,8 @@ +"""Shared CLI constants. + +Kept as a leaf module (imports nothing from the `cli` package) so both the +dispatcher (`cli/__init__.py`) and the command modules it imports can share +`PROG` without a circular import. +""" + +PROG = "cli.py" diff --git a/bot_bottle/deploy_key_provisioner.py b/bot_bottle/deploy_key_provisioner.py index e733bc10..a6d90d5f 100644 --- a/bot_bottle/deploy_key_provisioner.py +++ b/bot_bottle/deploy_key_provisioner.py @@ -49,7 +49,7 @@ def get_provisioner( GiteaDeployKeyProvisioner, ) return GiteaDeployKeyProvisioner(token=token, api_url=api_url) - from .manifest_util import ManifestError + from .manifest.util import ManifestError raise ManifestError( f"unknown provisioned_key provider: {provider!r}; " f"available: gitea" diff --git a/bot_bottle/docker_cmd.py b/bot_bottle/docker_cmd.py deleted file mode 100644 index 32a617ed..00000000 --- a/bot_bottle/docker_cmd.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Lean, framework-free `docker` subprocess primitive. - -Deliberately a top-level module with a single stdlib import so it can be -reused from anywhere without cost. It is intentionally *not* placed in -`backend.docker.util`: importing that module runs `backend/__init__.py`, -which eagerly loads all three bottle backends (docker + firecracker + -macos) plus the manifest/egress/git-gate/supervise framework — ~76 modules -— which would drag the whole backend layer into the deliberately-lean -orchestrator. This primitive stays free of that so both the orchestrator's -docker components and (in time) `backend.docker.util` can share it.""" - -from __future__ import annotations - -import subprocess - - -def run_docker( - argv: list[str], *, env: dict[str, str] | None = None, -) -> subprocess.CompletedProcess[str]: - """Run a `docker` command, capturing stdout/stderr as text. Never raises - on a non-zero exit — callers inspect `returncode` / `stderr` so they can - stay fail-closed or tolerate idempotent no-ops (e.g. removing an - already-absent container). - - `env` sets the child process environment — used to hand a secret to a bare - `--env NAME` flag (docker inherits its value from this process) so the - value never lands on argv or in `docker inspect`'s recorded command line.""" - return subprocess.run( - argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, - check=False, env=env, - ) - - -__all__ = ["run_docker"] diff --git a/bot_bottle/egress/__init__.py b/bot_bottle/egress/__init__.py new file mode 100644 index 00000000..37756ca0 --- /dev/null +++ b/bot_bottle/egress/__init__.py @@ -0,0 +1,90 @@ +"""Per-agent egress (PRD 0017). + +The egress gateway is a TLS-terminating forward proxy that allow-lists a +bottle's outbound HTTP(S), scans payloads for secret exfil, and injects +per-route upstream credentials the agent never sees. + +Layout: + + * `service` — the `Egress` host-side service (`prepare` the launch plan, + `resolve_token_values`, `agent_env_entries`) + the route-building / + rendering functions. + * `plan` — `EgressPlan` / `EgressRoute`, the launch DTOs (in the backend + contract). + +The runtime enforcement (the mitmproxy addon) lives in +`bot_bottle.gateway.egress.addon*`. Public names are re-exported lazily via +`__getattr__`, so `from bot_bottle.egress import …` keeps working and importing +`egress.plan` (the contract's dependency) stays light. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .plan import EgressPlan, EgressRoute + from .service import ( + CLAUDE_HOST_CREDENTIAL_TOKEN_REF, + CODEX_HOST_CREDENTIAL_TOKEN_REF, + EGRESS_HOSTNAME, + EGRESS_ROUTES_FILENAME, + EGRESS_ROUTES_IN_CONTAINER, + Egress, + egress_agent_env_entries, + egress_gateway_env_entries, + egress_manifest_routes, + egress_render_routes, + egress_resolve_token_values, + egress_routes_for_bottle, + egress_token_env_map, + ) + + +_LAZY: dict[str, str] = { + "EgressPlan": ".plan", + "EgressRoute": ".plan", + "Egress": ".service", + "CLAUDE_HOST_CREDENTIAL_TOKEN_REF": ".service", + "CODEX_HOST_CREDENTIAL_TOKEN_REF": ".service", + "EGRESS_HOSTNAME": ".service", + "EGRESS_ROUTES_FILENAME": ".service", + "EGRESS_ROUTES_IN_CONTAINER": ".service", + "egress_agent_env_entries": ".service", + "egress_gateway_env_entries": ".service", + "egress_manifest_routes": ".service", + "egress_render_routes": ".service", + "egress_resolve_token_values": ".service", + "egress_routes_for_bottle": ".service", + "egress_token_env_map": ".service", +} + + +def __getattr__(name: str) -> Any: + src = _LAZY.get(name) + if src is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module + + value = getattr(import_module(src, __name__), name) + globals()[name] = value + return value + + +__all__ = [ + "CLAUDE_HOST_CREDENTIAL_TOKEN_REF", + "CODEX_HOST_CREDENTIAL_TOKEN_REF", + "EGRESS_HOSTNAME", + "EGRESS_ROUTES_FILENAME", + "EGRESS_ROUTES_IN_CONTAINER", + "Egress", + "EgressPlan", + "EgressRoute", + "egress_manifest_routes", + "egress_render_routes", + "egress_resolve_token_values", + "egress_routes_for_bottle", + "egress_agent_env_entries", + "egress_gateway_env_entries", + "egress_token_env_map", +] diff --git a/bot_bottle/egress/plan.py b/bot_bottle/egress/plan.py new file mode 100644 index 00000000..ecca0244 --- /dev/null +++ b/bot_bottle/egress/plan.py @@ -0,0 +1,48 @@ +"""Egress launch DTOs (PRD 0017). + +`EgressRoute` (the host-side extension of the addon's wire `Route`) and +`EgressPlan` (the launch plan the backend contract references). Pure value +types — the route-building / rendering logic and the `Egress` service live in +`egress.service`. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from ..gateway.egress.addon_core import Route + + +@dataclass(frozen=True) +class EgressRoute(Route): + """Host-side extension of the addon's `Route`. + + Inherits `host`, `matches`, `auth_scheme`, and `token_env` + from `egress_addon_core.Route` — those are the fields that cross the + YAML wire into the gateway. The fields below are host-only and + are never serialised to the addon. + + `token_ref` is the host env var the CLI reads at launch and forwards + into the container's environ under `token_env`. + + `roles` carries the manifest route's role tuple (reserved for + future use; always empty today).""" + + token_ref: str = "" + roles: tuple[str, ...] = () + + +@dataclass(frozen=True) +class EgressPlan: + slug: str + routes_path: Path + routes: tuple[EgressRoute, ...] + token_env_map: dict[str, str] + internal_network: str = "" + egress_network: str = "" + mitmproxy_ca_host_path: Path = Path() + mitmproxy_ca_cert_only_host_path: Path = Path() + log: int = 0 + canary: str = "" + canary_env: str = "" diff --git a/bot_bottle/egress.py b/bot_bottle/egress/service.py similarity index 86% rename from bot_bottle/egress.py rename to bot_bottle/egress/service.py index 1abaf6ae..76a88298 100644 --- a/bot_bottle/egress.py +++ b/bot_bottle/egress/service.py @@ -1,33 +1,33 @@ -"""Per-bottle egress proxy (PRD 0017, PRD 0053). +"""The `Egress` host-side service (PRD 0017). -This module defines the abstract proxy (`Egress`), its plan -dataclass (`EgressPlan`), and the resolved per-route shape -(`EgressRoute`). The gateway's start/stop lifecycle is backend- -specific and lives on concrete subclasses (see -`bot_bottle/backend/docker/egress.py`). +`Egress` builds a bottle's egress plan at launch: resolve the manifest's egress +routes, render the gateway's `routes.yaml`, assign per-route token slots, and +plant the exfil canary. The service also resolves the launch-time token values +and the agent/gateway env entries the backend injects. The runtime enforcement +(the mitmproxy addon) lives in `bot_bottle.gateway.egress.addon*`. """ from __future__ import annotations import dataclasses import secrets -from abc import ABC -from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING -from .egress_addon_core import ( +from ..gateway.egress.addon_core import ( ON_MATCH_REDACT, HeaderMatch as CoreHeaderMatch, MatchEntry as CoreMatchEntry, PathMatch as CorePathMatch, Route, ) -from .errors import MissingEnvVarError -from .log import die +from ..errors import MissingEnvVarError +from ..log import die +from .plan import EgressPlan, EgressRoute if TYPE_CHECKING: - from .manifest import ManifestBottle + from ..manifest import ManifestBottle + CODEX_HOST_CREDENTIAL_TOKEN_REF = "BOT_BOTTLE_CODEX_HOST_ACCESS_TOKEN" CLAUDE_HOST_CREDENTIAL_TOKEN_REF = "BOT_BOTTLE_CLAUDE_HOST_ACCESS_TOKEN" @@ -82,39 +82,6 @@ def egress_agent_env_entries(plan: "EgressPlan") -> tuple[str, ...]: return () -@dataclass(frozen=True) -class EgressRoute(Route): - """Host-side extension of the addon's `Route`. - - Inherits `host`, `matches`, `auth_scheme`, and `token_env` - from `egress_addon_core.Route` — those are the fields that cross the - YAML wire into the gateway. The fields below are host-only and - are never serialised to the addon. - - `token_ref` is the host env var the CLI reads at launch and forwards - into the container's environ under `token_env`. - - `roles` carries the manifest route's role tuple (reserved for - future use; always empty today).""" - - token_ref: str = "" - roles: tuple[str, ...] = () - - -@dataclass(frozen=True) -class EgressPlan: - slug: str - routes_path: Path - routes: tuple[EgressRoute, ...] - token_env_map: dict[str, str] - internal_network: str = "" - egress_network: str = "" - mitmproxy_ca_host_path: Path = Path() - mitmproxy_ca_cert_only_host_path: Path = Path() - log: int = 0 - canary: str = "" - canary_env: str = "" - def egress_manifest_routes( bottle: ManifestBottle, @@ -389,7 +356,11 @@ def egress_resolve_token_values( return out -class Egress(ABC): +class Egress: + """The host-side egress service. The backend drives `prepare` at launch, + then `resolve_token_values` to resolve the routes' upstream credentials and + `agent_env_entries` for the agent's egress env. Stateless.""" + def prepare( self, bottle: ManifestBottle, @@ -416,20 +387,13 @@ class Egress(ABC): canary_env=_random_canary_env(), ) -__all__ = [ - "CLAUDE_HOST_CREDENTIAL_TOKEN_REF", - "CODEX_HOST_CREDENTIAL_TOKEN_REF", - "EGRESS_HOSTNAME", - "EGRESS_ROUTES_FILENAME", - "EGRESS_ROUTES_IN_CONTAINER", - "Egress", - "EgressPlan", - "EgressRoute", - "egress_manifest_routes", - "egress_render_routes", - "egress_resolve_token_values", - "egress_routes_for_bottle", - "egress_agent_env_entries", - "egress_gateway_env_entries", - "egress_token_env_map", -] + def resolve_token_values( + self, token_env_map: dict[str, str], host_env: dict[str, str], + ) -> dict[str, str]: + """Resolve each route's upstream credential from the host env at launch. + Raises `MissingEnvVarError` for an unset/empty referenced host var.""" + return egress_resolve_token_values(token_env_map, host_env) + + def agent_env_entries(self, plan: EgressPlan) -> tuple[str, ...]: + """The agent-visible egress env entries (the exfil canary).""" + return egress_agent_env_entries(plan) diff --git a/bot_bottle/gateway/__init__.py b/bot_bottle/gateway/__init__.py new file mode 100644 index 00000000..a1ad85ab --- /dev/null +++ b/bot_bottle/gateway/__init__.py @@ -0,0 +1,119 @@ +"""The consolidated per-host gateway (PRD 0070). + +The core consolidation win: **one** persistent gateway per host, shared by +every bottle, instead of a gateway per bottle. It's safe to share +because the attribution invariant (source IP + identity token, see +`registry`) lets the gateway attribute each request to the right bottle — +so per-bottle policy lives in one long-lived process keyed on who's calling. + +`Gateway` is the backend-neutral lifecycle contract (mirrors `LaunchBroker`): +ensure the single instance is up, report it, tear it down. The docker +implementation (`DockerGateway`) lives in `backend/docker/gateway.py`; a +firecracker gateway VM slots in later. + +The defining behaviour is **idempotent singleton**: `ensure_running` starts +the instance if absent and is a no-op if it's already up, so N bottle +launches never spawn N gateways. +""" + +from __future__ import annotations + +import abc +import os +from pathlib import Path + +from ..paths import host_gateway_ca_dir + +# The gateway's mitmproxy writes its CA a beat after the container starts, so +# reads poll for it rather than assuming it's there on a fresh launch. +CA_POLL_SECONDS = 0.5 +DEFAULT_CA_TIMEOUT_SECONDS = 30.0 + +GATEWAY_NAME = "bot-bottle-orch-gateway" +GATEWAY_LABEL = "bot-bottle-orch-gateway=1" +# The single user-defined network the gateway and every agent bottle share. +# Agents attach here with a pinned IP and reach the gateway's egress / +# git-http / supervise ports by its address — no host port publishing, and +# the source IP the gateway attributes by is the address on this network. +GATEWAY_NETWORK = "bot-bottle-gateway" + +# mitmproxy's CA dir in the bundle. The host's gateway-CA dir (see +# `host_gateway_ca_dir`) is bind-mounted here so the gateway's self-generated +# CA stays STABLE across container recreation — every agent installs this one +# CA to trust the shared gateway's TLS interception, so it must not rotate when +# the gateway restarts. A host bind-mount rather than a named volume: a named +# volume is silently wiped by `docker volume prune`, minting a fresh CA that +# breaks every running bottle (issue #450). +MITMPROXY_HOME = "/home/mitmproxy/.mitmproxy" +GATEWAY_CA_CERT = f"{MITMPROXY_HOME}/mitmproxy-ca-cert.pem" + +# The CA material mitmproxy writes into its confdir. mitmproxy reuses these on +# startup when present and generates them only on first run, so persisting them +# is what makes the CA stable; deleting them (see `rotate_gateway_ca`) forces a +# fresh CA on the next start. `mitmproxy-ca.pem` (cert + private key) is the +# signing identity; the rest are derived encodings agents/clients consume. +GATEWAY_CA_GLOB = "mitmproxy-ca*" + +# The gateway data-plane image + its Dockerfile. Kept as a local constant +# rather than imported from the backend layer, which would drag +# the whole backend layer into the lean orchestrator (see #359); unify when +# that lands. Env override matches the backend's BOT_BOTTLE_GATEWAY_IMAGE. +GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest") +GATEWAY_DOCKERFILE = "Dockerfile.gateway" +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def rotate_gateway_ca(ca_dir: Path | None = None) -> list[Path]: + """Delete the persisted mitmproxy CA so the next gateway start mints a + fresh one — the explicit, deliberate CA-rollover path (issue #450). + + Persistence keeps the CA stable across restarts precisely because mitmproxy + reuses the on-disk CA; rotation is therefore just removing that material. + Returns the files removed (empty when there was no CA yet); idempotent. + + This only clears the on-disk CA. It does NOT stop the running gateway (whose + mitmproxy still holds the old CA in memory) or re-provision agents — the + caller recreates the gateway to mint the new CA and re-attaches bottles. + `rotate-ca` on the orchestrator CLI wires those steps together.""" + ca_dir = ca_dir if ca_dir is not None else host_gateway_ca_dir() + removed: list[Path] = [] + for path in sorted(ca_dir.glob(GATEWAY_CA_GLOB)): + path.unlink() + removed.append(path) + return removed + + +class GatewayError(Exception): + """The shared gateway failed to build/start/stop (non-zero `docker` exit).""" + + +class Gateway(abc.ABC): + """Lifecycle of the single per-host gateway. Backend-neutral.""" + + name: str + + def ensure_built(self) -> None: + """Ensure the gateway's image / rootfs exists, building it if needed. + Default: nothing to build (e.g. a stub or a pre-pulled image).""" + return + + @abc.abstractmethod + def ensure_running(self) -> None: + """Start the gateway if it isn't already up. Idempotent: a no-op + when it's already running (that's the whole point — one per host). + Assumes the image exists — call `ensure_built()` first.""" + + @abc.abstractmethod + def is_running(self) -> bool: + """True iff the gateway instance is currently up.""" + + @abc.abstractmethod + def stop(self) -> None: + """Remove the gateway. Idempotent — absent is success.""" + + +__all__ = [ + "Gateway", "GatewayError", "rotate_gateway_ca", + "GATEWAY_NAME", "GATEWAY_LABEL", "GATEWAY_IMAGE", "GATEWAY_NETWORK", + "GATEWAY_CA_CERT", "GATEWAY_CA_GLOB", +] diff --git a/bot_bottle/gateway_init.py b/bot_bottle/gateway/bootstrap.py similarity index 96% rename from bot_bottle/gateway_init.py rename to bot_bottle/gateway/bootstrap.py index 7e9df7e2..4e1c6d16 100644 --- a/bot_bottle/gateway_init.py +++ b/bot_bottle/gateway/bootstrap.py @@ -58,10 +58,10 @@ _READY_GATED_DAEMONS: tuple[str, ...] = ("git-gate", "git-http") # The data-plane daemons instead hold the pre-minted `gateway` JWT they present. # Scoping each to its process (even in the combined infra container) keeps a # compromised data-plane daemon from reading the key and minting a `cli` token -# (issue #469 review). Values match paths.CONTROL_PLANE_TOKEN_ENV / -# CONTROL_AUTH_JWT_ENV; hardcoded here so this supervisor stays import-light. -_SIGNING_KEY_ENV = "BOT_BOTTLE_CONTROL_PLANE_TOKEN" -_GATEWAY_JWT_ENV = "BOT_BOTTLE_CONTROL_AUTH_JWT" +# (issue #469 review). Values match paths.ORCHESTRATOR_TOKEN_ENV / +# ORCHESTRATOR_AUTH_JWT_ENV; hardcoded here so this supervisor stays import-light. +_SIGNING_KEY_ENV = "BOT_BOTTLE_ORCHESTRATOR_TOKEN" +_GATEWAY_JWT_ENV = "BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT" # Daemons that must be requested explicitly via BOT_BOTTLE_GATEWAY_DAEMONS # and are NOT started in the default (env-var-unset) case. The orchestrator @@ -100,8 +100,8 @@ _DAEMONS: tuple[_DaemonSpec, ...] = ( )), _DaemonSpec("egress", ("/bin/sh", "/app/egress-entrypoint.sh")), _DaemonSpec("git-gate", ("/bin/sh", "/git-gate-entrypoint.sh")), - _DaemonSpec("git-http", ("python3", "-m", "bot_bottle.git_http_backend")), - _DaemonSpec("supervise", ("python3", "-m", "bot_bottle.supervise_server")), + _DaemonSpec("git-http", ("python3", "-m", "bot_bottle.gateway.git_gate.http_backend")), + _DaemonSpec("supervise", ("python3", "-m", "bot_bottle.gateway.supervisor.server")), ) @@ -173,7 +173,7 @@ def _spawn(spec: _DaemonSpec) -> subprocess.Popen[bytes]: return proc -class _Supervisor: +class _DaemonManager: """Holds the running children + shutdown state. Pulled out so the test suite can drive it with fake commands.""" @@ -387,7 +387,7 @@ def main(argv: Sequence[str] | None = None) -> int: _log("no daemons selected; nothing to do") return 0 - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() signal.signal(signal.SIGTERM, lambda *_: sup.request_shutdown("SIGTERM")) # type: ignore diff --git a/bot_bottle/gateway/egress/__init__.py b/bot_bottle/gateway/egress/__init__.py new file mode 100644 index 00000000..e406e602 --- /dev/null +++ b/bot_bottle/gateway/egress/__init__.py @@ -0,0 +1,9 @@ +"""Gateway-side (data-plane) egress service: the mitmproxy addon and its +pure decision core, DLP detectors, and route/DLP config parsing. + +These are the long-running data-plane pieces (loaded by mitmdump inside the +`bot-bottle-gateway` image), distinct from the host-side `bot_bottle.egress` +service that renders routes and prepares env. Import the concrete modules +directly (`from bot_bottle.gateway.egress.addon_core import ...`) — this +package deliberately does no eager work so leaf imports stay cheap. +""" diff --git a/bot_bottle/egress_addon.py b/bot_bottle/gateway/egress/addon.py similarity index 99% rename from bot_bottle/egress_addon.py rename to bot_bottle/gateway/egress/addon.py index 8766623f..a5292f2c 100644 --- a/bot_bottle/egress_addon.py +++ b/bot_bottle/gateway/egress/addon.py @@ -16,8 +16,8 @@ import typing from mitmproxy import http # type: ignore[import-not-found] # pylint: disable=import-error from bot_bottle.constants import IDENTITY_HEADER -from bot_bottle.dlp_detectors import redact_tokens, strip_crlf -from bot_bottle.egress_addon_core import ( +from bot_bottle.gateway.egress.dlp_detectors import redact_tokens, strip_crlf +from bot_bottle.gateway.egress.addon_core import ( LOG_BLOCKS, LOG_FULL, DEFAULT_OUTBOUND_ON_MATCH, @@ -40,8 +40,8 @@ from bot_bottle.egress_addon_core import ( scan_inbound, scan_outbound, ) -from bot_bottle.policy_resolver import PolicyResolveError, PolicyResolver -from bot_bottle.supervise_types import ( +from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver +from bot_bottle.supervisor.types import ( STATUS_APPROVED, STATUS_MODIFIED, STATUSES, diff --git a/bot_bottle/egress_addon_core.py b/bot_bottle/gateway/egress/addon_core.py similarity index 99% rename from bot_bottle/egress_addon_core.py rename to bot_bottle/gateway/egress/addon_core.py index bf14fbb2..3d0bfca1 100644 --- a/bot_bottle/egress_addon_core.py +++ b/bot_bottle/gateway/egress/addon_core.py @@ -16,11 +16,11 @@ import re import typing from dataclasses import dataclass -from .yaml_subset import YamlSubsetError, parse_yaml_subset +from ...yaml_subset import YamlSubsetError, parse_yaml_subset # DLP detector-config parsing lives in a sibling module. Re-exported below # so existing `from egress_addon_core import ON_MATCH_*` callers keep working. -from .egress_dlp_config import ( +from .dlp_config import ( DEFAULT_OUTBOUND_ON_MATCH, INBOUND_DETECTOR_NAMES, ON_MATCH_BLOCK, diff --git a/bot_bottle/egress_dlp_config.py b/bot_bottle/gateway/egress/dlp_config.py similarity index 100% rename from bot_bottle/egress_dlp_config.py rename to bot_bottle/gateway/egress/dlp_config.py diff --git a/bot_bottle/dlp_detectors.py b/bot_bottle/gateway/egress/dlp_detectors.py similarity index 99% rename from bot_bottle/dlp_detectors.py rename to bot_bottle/gateway/egress/dlp_detectors.py index 2ca286a7..d0c8d1b9 100644 --- a/bot_bottle/dlp_detectors.py +++ b/bot_bottle/gateway/egress/dlp_detectors.py @@ -19,7 +19,7 @@ from math import log2 from collections import Counter from urllib.parse import quote as url_quote -from .egress_addon_core import ScanResult +from .addon_core import ScanResult # --------------------------------------------------------------------------- diff --git a/bot_bottle/egress_entrypoint.sh b/bot_bottle/gateway/egress/entrypoint.sh similarity index 100% rename from bot_bottle/egress_entrypoint.sh rename to bot_bottle/gateway/egress/entrypoint.sh diff --git a/bot_bottle/gateway/git_gate/__init__.py b/bot_bottle/gateway/git_gate/__init__.py new file mode 100644 index 00000000..67c9fe7d --- /dev/null +++ b/bot_bottle/gateway/git_gate/__init__.py @@ -0,0 +1,7 @@ +"""Gateway-side (data-plane) git-gate service: pure host-side hook rendering +(PRD 0008) and the smart-HTTP backend that fronts git-gate repos. + +The host-side git-gate service (provisioning, preflight) lives in +`bot_bottle.git_gate`; this is the data-plane counterpart. Import the concrete +modules directly (`from bot_bottle.gateway.git_gate.render import ...`). +""" diff --git a/bot_bottle/git_http_backend.py b/bot_bottle/gateway/git_gate/http_backend.py similarity index 99% rename from bot_bottle/git_http_backend.py rename to bot_bottle/gateway/git_gate/http_backend.py index 21f924fa..c4860e7e 100644 --- a/bot_bottle/git_http_backend.py +++ b/bot_bottle/gateway/git_gate/http_backend.py @@ -27,7 +27,7 @@ from pathlib import Path from urllib.parse import urlsplit from bot_bottle.constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER -from bot_bottle.policy_resolver import PolicyResolveError, PolicyResolver +from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver DEFAULT_PORT = 9420 diff --git a/bot_bottle/git_gate_render.py b/bot_bottle/gateway/git_gate/render.py similarity index 97% rename from bot_bottle/git_gate_render.py rename to bot_bottle/gateway/git_gate/render.py index 04c038c1..241ed9d8 100644 --- a/bot_bottle/git_gate_render.py +++ b/bot_bottle/gateway/git_gate/render.py @@ -3,9 +3,9 @@ Builds the agent's `.gitconfig` insteadOf rewrites, the known_hosts line, and the entrypoint / pre-receive / access-hook scripts the gateway runs. No docker or forge calls — exposed for tests and reuse across -backends. Split out of `git_gate.py` so the control surface (`GitGate`) -and the deploy-key lifecycle (`git_gate_provision`) each read on their -own; `git_gate` re-exports these names for API stability.""" +backends. The host-side service (`git_gate.GitGate`) and the deploy-key +lifecycle (`git_gate.provision`) each read on their own; `git_gate` +re-exports these names for API stability.""" from __future__ import annotations @@ -14,8 +14,8 @@ import shlex from dataclasses import dataclass from pathlib import Path -from .constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER -from .manifest import ManifestBottle, ManifestGitEntry +from ...constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER +from ...manifest import ManifestBottle, ManifestGitEntry # Short network alias for git-gate inside the gateway. The # agent's `.gitconfig` insteadOf rewrites resolve through this name. @@ -282,11 +282,11 @@ from pathlib import Path # identity_token), resolved server-side, so the proposal lands under the # calling bottle exactly as a direct write once did. try: - from bot_bottle.policy_resolver import PolicyResolver, PolicyResolveError - from bot_bottle.supervise_types import TOOL_GITLEAKS_ALLOW + from bot_bottle.gateway.policy_resolver import PolicyResolver, PolicyResolveError + from bot_bottle.supervisor.types import TOOL_GITLEAKS_ALLOW except ImportError: from policy_resolver import PolicyResolver, PolicyResolveError - from supervise_types import TOOL_GITLEAKS_ALLOW + from supervisor.types import TOOL_GITLEAKS_ALLOW report_path = Path(sys.argv[1]) source_ip = os.environ.get("SUPERVISE_SOURCE_IP", "") @@ -374,7 +374,7 @@ import sys # Non-blocking poll over the control plane. A decided proposal is archived # server-side on read, so no separate archive step is needed here. try: - from bot_bottle.policy_resolver import PolicyResolver, PolicyResolveError + from bot_bottle.gateway.policy_resolver import PolicyResolver, PolicyResolveError except ImportError: from policy_resolver import PolicyResolver, PolicyResolveError diff --git a/bot_bottle/policy_resolver.py b/bot_bottle/gateway/policy_resolver.py similarity index 96% rename from bot_bottle/policy_resolver.py rename to bot_bottle/gateway/policy_resolver.py index 9298a12b..a61f3daf 100644 --- a/bot_bottle/policy_resolver.py +++ b/bot_bottle/gateway/policy_resolver.py @@ -41,16 +41,16 @@ DEFAULT_TIMEOUT_SECONDS = 2.0 # rather than imported because this module is COPYed flat into the gateway image, # free of bot-bottle imports — same rationale as IDENTITY_HEADER in egress_addon # / git_http_backend. -CONTROL_AUTH_HEADER = "x-bot-bottle-control-auth" -CONTROL_AUTH_JWT_ENV = "BOT_BOTTLE_CONTROL_AUTH_JWT" +ORCHESTRATOR_AUTH_HEADER = "x-bot-bottle-orchestrator-auth" +ORCHESTRATOR_AUTH_JWT_ENV = "BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT" -def _control_auth_headers() -> dict[str, str]: +def _orchestrator_auth_headers() -> dict[str, str]: """The auth header to send, or {} when no token is configured (an open control plane, e.g. Firecracker behind its nft boundary — sending nothing is correct there and harmlessly ignored).""" - token = os.environ.get(CONTROL_AUTH_JWT_ENV, "").strip() - return {CONTROL_AUTH_HEADER: token} if token else {} + token = os.environ.get(ORCHESTRATOR_AUTH_JWT_ENV, "").strip() + return {ORCHESTRATOR_AUTH_HEADER: token} if token else {} class PolicyResolveError(RuntimeError): @@ -74,7 +74,7 @@ class PolicyResolver: body = json.dumps(payload).encode() req = urllib.request.Request( f"{self._base}{path}", data=body, method="POST", - headers={"Content-Type": "application/json", **_control_auth_headers()}, + headers={"Content-Type": "application/json", **_orchestrator_auth_headers()}, ) try: with urllib.request.urlopen(req, timeout=self._timeout) as resp: diff --git a/bot_bottle/gateway/supervisor/__init__.py b/bot_bottle/gateway/supervisor/__init__.py new file mode 100644 index 00000000..47b48b79 --- /dev/null +++ b/bot_bottle/gateway/supervisor/__init__.py @@ -0,0 +1,7 @@ +"""Gateway-side (data-plane) supervise service: the supervise daemon's HTTP +server (PRD 0013). + +The host-side supervise control lives in `bot_bottle.orchestrator.supervisor`; +this is the in-gateway daemon the agents reach. Import the concrete module +directly (`from bot_bottle.gateway.supervisor.server import ...`). +""" diff --git a/bot_bottle/supervise_server.py b/bot_bottle/gateway/supervisor/server.py similarity index 99% rename from bot_bottle/supervise_server.py rename to bot_bottle/gateway/supervisor/server.py index aad06d1a..35d53c53 100644 --- a/bot_bottle/supervise_server.py +++ b/bot_bottle/gateway/supervisor/server.py @@ -58,11 +58,11 @@ import typing from dataclasses import dataclass from bot_bottle.constants import IDENTITY_HEADER -from bot_bottle.egress_addon_core import ( +from bot_bottle.gateway.egress.addon_core import ( LOG_OFF, load_config, resolve_client_context, route_to_yaml_dict, ) -from bot_bottle.policy_resolver import PolicyResolveError, PolicyResolver -from bot_bottle import supervise as _sv +from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver +from bot_bottle.supervisor import types as _sv # --- JSON-RPC / MCP plumbing ---------------------------------------------- diff --git a/bot_bottle/git_gate.py b/bot_bottle/git_gate.py deleted file mode 100644 index d7c0f963..00000000 --- a/bot_bottle/git_gate.py +++ /dev/null @@ -1,169 +0,0 @@ -"""Per-agent git-gate (PRD 0008). - -A third per-agent daemon that fronts the bottle's declared git -upstreams as a transparent mirror. Each `bottle.git` entry maps to -a bare repo on the gate; `git daemon` serves the bare repos over -`git:///.git`. Two hooks make the mirror bidirectional: - -- **`pre-receive`** (push path) — gitleaks-scans incoming refs and, - on clean, forwards them to the real upstream with the - gate-resident credential. -- **`--access-hook`** (fetch path) — runs `git fetch origin --prune` - against the real upstream before every `upload-pack`, so an - agent fetch returns whatever the upstream has *now*. Fail-closed - if the upstream is unreachable. - -The agent never sees the upstream credential under either path. - -Why a separate daemon (not folded into egress or ssh-gate): the -gate is the only one of the three that holds upstream push -credentials. Mixing it with egress would put push creds in the -same blast radius as internet-facing TLS interception; mixing it -with ssh-gate would force ssh-gate above L4 and into git-protocol -land. See `docs/prds/0008-git-gate.md`. - -This module defines the abstract gate (`GitGate`) and its plan -dataclass (`GitGatePlan`). The gateway's start/stop lifecycle is -backend-specific and lives on concrete subclasses (see -`bot_bottle/backend/docker/git_gate.py`).""" - - -from __future__ import annotations - -from abc import ABC -from dataclasses import dataclass -from pathlib import Path - -from .manifest import ManifestBottle - -# Rendering and the deploy-key lifecycle live in sibling modules; the -# names are re-exported here (see __all__) so existing -# `from bot_bottle.git_gate import …` callers are unchanged. -from .git_gate_render import ( - GIT_GATE_HOSTNAME, - GIT_GATE_TIMEOUT_SECS, - GitGateUpstream, - git_gate_known_hosts_line, - git_gate_render_access_hook, - git_gate_render_entrypoint, - git_gate_render_provision, - git_gate_render_gitconfig, - git_gate_render_hook, - git_gate_upstreams_for_bottle, - _gitconfig_validate_value, -) -from .git_gate_provision import ( - provision_git_gate_dynamic_keys, - revoke_git_gate_provisioned_keys, - _provision_dynamic_key, - _resolve_identity_file, -) - -@dataclass(frozen=True) -class GitGatePlan: - """Output of GitGate.prepare; consumed by .start. - - The script + slug + upstream fields are filled at prepare time - (host-side, side-effect-free on docker). The network fields are - populated by the backend's launch step via `dataclasses.replace` - once those networks exist. Empty defaults are sentinels meaning - "not yet set"; `.start` validates that they are populated. - - `hook_script` is the shared `pre-receive` for push-time gating; - `access_hook_script` is `git daemon`'s `--access-hook` for the - fetch-time upstream refresh.""" - - slug: str - entrypoint_script: Path - hook_script: Path - access_hook_script: Path - upstreams: tuple[GitGateUpstream, ...] - internal_network: str = "" - egress_network: str = "" - - - - -class GitGate(ABC): - """The per-agent git-gate. Encapsulates the host-side prepare - (upstream lift + entrypoint/hook render); the gateway's - start/stop lifecycle is backend-specific and lives on concrete - subclasses.""" - - def prepare(self, bottle: ManifestBottle, slug: str, stage_dir: Path) -> GitGatePlan: - """Compute the upstream table from `bottle.git` and write the - entrypoint, pre-receive hook, and access-hook scripts (mode - 600) under `stage_dir`. Pure host-side, no docker subprocess. - - For `gitea` key entries, the returned upstream intentionally - has an empty identity file. Backend launch fills that in after - the operator confirms the preflight. - - Returned plan is incomplete: the launch step must fill - `internal_network` / `egress_network` via `dataclasses.replace` - before passing the plan to `.start`.""" - upstreams = git_gate_upstreams_for_bottle(bottle) - entrypoint = stage_dir / "git_gate_entrypoint.sh" - entrypoint.write_text(git_gate_render_entrypoint(upstreams)) - entrypoint.chmod(0o600) - hook = stage_dir / "git_gate_pre_receive.sh" - hook.write_text(git_gate_render_hook()) - hook.chmod(0o600) - access_hook = stage_dir / "git_gate_access_hook.sh" - access_hook.write_text(git_gate_render_access_hook()) - # 0o700 (not 0o600): git daemon execs --access-hook directly, - # not via `sh`, so the script needs the x bit. The gateway copy - # does not necessarily preserve this mode (`docker cp` does, the - # Apple `container cp` does not), so provision_git_gate re-applies - # +x on the gateway side — see backend/docker/gateway_provision.py. - access_hook.chmod(0o700) - upstreams_with_files: list[GitGateUpstream] = [] - for u in upstreams: - known_hosts_file = Path() - if u.known_host_key: - known_hosts_file = stage_dir / f"{u.name}-known_hosts" - known_hosts_file.write_text( - git_gate_known_hosts_line( - u.upstream_host, u.upstream_port, u.known_host_key, - ) - ) - known_hosts_file.chmod(0o600) - upstreams_with_files.append( - GitGateUpstream( - name=u.name, - upstream_url=u.upstream_url, - upstream_host=u.upstream_host, - upstream_port=u.upstream_port, - identity_file=u.identity_file, - known_host_key=u.known_host_key, - known_hosts_file=known_hosts_file, - ) - ) - return GitGatePlan( - slug=slug, - entrypoint_script=entrypoint, - hook_script=hook, - access_hook_script=access_hook, - upstreams=tuple(upstreams_with_files), - ) - - -__all__ = [ - "GIT_GATE_HOSTNAME", - "GIT_GATE_TIMEOUT_SECS", - "GitGateUpstream", - "GitGatePlan", - "GitGate", - "git_gate_upstreams_for_bottle", - "git_gate_render_gitconfig", - "git_gate_known_hosts_line", - "git_gate_render_entrypoint", - "git_gate_render_provision", - "git_gate_render_hook", - "git_gate_render_access_hook", - "provision_git_gate_dynamic_keys", - "revoke_git_gate_provisioned_keys", - "_gitconfig_validate_value", - "_provision_dynamic_key", - "_resolve_identity_file", -] diff --git a/bot_bottle/git_gate/__init__.py b/bot_bottle/git_gate/__init__.py new file mode 100644 index 00000000..f6b6256e --- /dev/null +++ b/bot_bottle/git_gate/__init__.py @@ -0,0 +1,98 @@ +"""Per-agent git-gate (PRD 0008). + +The git-gate fronts a bottle's declared git upstreams as a transparent mirror: +a `pre-receive` hook gitleaks-scans pushes and forwards clean refs to the real +upstream with a gate-resident credential; an `--access-hook` refreshes from the +upstream before fetches. The agent never sees the upstream credential. + +Layout: + + * `service` — the `GitGate` host-side service (prepare / provision / revoke / + preflight) the backend drives at launch. + * `plan` — `GitGatePlan`, the launch DTO (in the backend contract). + * `provision`, `host_key` — the deploy-key + host-key host-side helpers the + service delegates to. + +The rendering + the in-gateway hook execution live in +`bot_bottle.gateway.git_gate.render`. The public names are re-exported lazily +via `__getattr__`, so `from bot_bottle.git_gate import …` keeps working and +importing `git_gate.plan` (the contract's dependency) doesn't drag in the +provisioning / forge-API code. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .service import GitGate + from .plan import GitGatePlan + from .provision import ( + provision_git_gate_dynamic_keys, + revoke_git_gate_provisioned_keys, + ) + from ..gateway.git_gate.render import ( + GIT_GATE_HOSTNAME, + GIT_GATE_TIMEOUT_SECS, + GitGateUpstream, + git_gate_known_hosts_line, + git_gate_render_access_hook, + git_gate_render_entrypoint, + git_gate_render_gitconfig, + git_gate_render_hook, + git_gate_render_provision, + git_gate_upstreams_for_bottle, + ) + + +# Public name -> relative module that defines it. Render/hook names come from +# the gateway (where the in-gateway execution lives); the service, plan, and +# provisioning are this package's own submodules. +_LAZY: dict[str, str] = { + "GitGate": ".service", + "GitGatePlan": ".plan", + "provision_git_gate_dynamic_keys": ".provision", + "revoke_git_gate_provisioned_keys": ".provision", + "_provision_dynamic_key": ".provision", + "_resolve_identity_file": ".provision", + "GIT_GATE_HOSTNAME": "..gateway.git_gate.render", + "GIT_GATE_TIMEOUT_SECS": "..gateway.git_gate.render", + "GitGateUpstream": "..gateway.git_gate.render", + "git_gate_upstreams_for_bottle": "..gateway.git_gate.render", + "git_gate_render_gitconfig": "..gateway.git_gate.render", + "git_gate_known_hosts_line": "..gateway.git_gate.render", + "git_gate_render_entrypoint": "..gateway.git_gate.render", + "git_gate_render_provision": "..gateway.git_gate.render", + "git_gate_render_hook": "..gateway.git_gate.render", + "git_gate_render_access_hook": "..gateway.git_gate.render", + "_gitconfig_validate_value": "..gateway.git_gate.render", +} + + +def __getattr__(name: str) -> Any: + src = _LAZY.get(name) + if src is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module + + value = getattr(import_module(src, __name__), name) + globals()[name] = value + return value + + +__all__ = [ + "GitGate", + "GitGatePlan", + "GitGateUpstream", + "GIT_GATE_HOSTNAME", + "GIT_GATE_TIMEOUT_SECS", + "git_gate_upstreams_for_bottle", + "git_gate_render_gitconfig", + "git_gate_known_hosts_line", + "git_gate_render_entrypoint", + "git_gate_render_provision", + "git_gate_render_hook", + "git_gate_render_access_hook", + "provision_git_gate_dynamic_keys", + "revoke_git_gate_provisioned_keys", +] diff --git a/bot_bottle/git_gate_host_key.py b/bot_bottle/git_gate/host_key.py similarity index 98% rename from bot_bottle/git_gate_host_key.py rename to bot_bottle/git_gate/host_key.py index 2210e8e6..5c2ba788 100644 --- a/bot_bottle/git_gate_host_key.py +++ b/bot_bottle/git_gate/host_key.py @@ -16,9 +16,9 @@ import sys from pathlib import Path from typing import cast -from .log import die, info -from .manifest import Manifest -from .yaml_subset import YamlSubsetError, parse_frontmatter, serialize_yaml_subset +from ..log import die, info +from ..manifest import Manifest +from ..yaml_subset import YamlSubsetError, parse_frontmatter, serialize_yaml_subset # Preferred key types, most secure first. diff --git a/bot_bottle/git_gate/plan.py b/bot_bottle/git_gate/plan.py new file mode 100644 index 00000000..edd1bd21 --- /dev/null +++ b/bot_bottle/git_gate/plan.py @@ -0,0 +1,36 @@ +"""`GitGatePlan` — the git-gate launch plan (DTO). + +The output of `GitGate.prepare`, consumed by the backend launch step and the +`BottlePlan` contract. A pure value type (its `upstreams` element type +`GitGateUpstream` is the neutral render dataclass). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from ..gateway.git_gate.render import GitGateUpstream + + +@dataclass(frozen=True) +class GitGatePlan: + """Output of GitGate.prepare; consumed by .start. + + The script + slug + upstream fields are filled at prepare time (host-side, + side-effect-free on docker). The network fields are populated by the + backend's launch step via `dataclasses.replace` once those networks exist. + Empty defaults are sentinels meaning "not yet set"; `.start` validates that + they are populated. + + `hook_script` is the shared `pre-receive` for push-time gating; + `access_hook_script` is `git daemon`'s `--access-hook` for the fetch-time + upstream refresh.""" + + slug: str + entrypoint_script: Path + hook_script: Path + access_hook_script: Path + upstreams: tuple[GitGateUpstream, ...] + internal_network: str = "" + egress_network: str = "" diff --git a/bot_bottle/git_gate_provision.py b/bot_bottle/git_gate/provision.py similarity index 93% rename from bot_bottle/git_gate_provision.py rename to bot_bottle/git_gate/provision.py index 00f42fc6..bae03a72 100644 --- a/bot_bottle/git_gate_provision.py +++ b/bot_bottle/git_gate/provision.py @@ -13,14 +13,14 @@ import dataclasses from pathlib import Path from typing import TYPE_CHECKING -from .bottle_state import globalize_slug -from .errors import MissingEnvVarError -from .log import info -from .manifest import ManifestBottle, ManifestGitEntry -from .git_gate_render import GitGateUpstream +from ..bottle_state import globalize_slug +from ..errors import MissingEnvVarError +from ..log import info +from ..manifest import ManifestBottle, ManifestGitEntry +from ..gateway.git_gate.render import GitGateUpstream if TYPE_CHECKING: - from .git_gate import GitGatePlan + from .plan import GitGatePlan def _provision_dynamic_key( entry: ManifestGitEntry, @@ -32,7 +32,7 @@ def _provision_dynamic_key( Returns the host-side path to the private key file so the caller can inject it into the GitGateUpstream as `identity_file`.""" - from .deploy_key_provisioner import get_provisioner + from ..deploy_key_provisioner import get_provisioner pk = entry.Key token = os.environ.get(pk.forge_token_env) if token is None: @@ -70,7 +70,7 @@ def revoke_git_gate_provisioned_keys(bottle: ManifestBottle, stage_dir: Path) -> Called at teardown after containers stop. Raises if any revocation fails — a stranded key is a security concern that the operator must address manually.""" - from .deploy_key_provisioner import get_provisioner + from ..deploy_key_provisioner import get_provisioner for entry in bottle.git: if entry.Key.provider != "gitea": continue diff --git a/bot_bottle/git_gate/service.py b/bot_bottle/git_gate/service.py new file mode 100644 index 00000000..cfcc8aab --- /dev/null +++ b/bot_bottle/git_gate/service.py @@ -0,0 +1,114 @@ +"""The `GitGate` host-side service (PRD 0008). + +The git-gate fronts a bottle's declared git upstreams as a transparent mirror: +a `pre-receive` hook gitleaks-scans pushes and forwards clean refs to the real +upstream with a gate-resident credential; an `--access-hook` refreshes from the +upstream before fetches. The agent never sees the upstream credential. + +`GitGate` is the host-side service the backend drives at launch: build the plan +(`prepare`), provision / revoke the per-upstream deploy keys, and preflight the +upstream host keys. The rendering it emits + the in-gateway hook execution live +in `bot_bottle.gateway.git_gate.render`. +""" + +from __future__ import annotations + +from pathlib import Path + +from ..manifest import Manifest, ManifestBottle +from ..gateway.git_gate.render import ( + GitGateUpstream, + git_gate_known_hosts_line, + git_gate_render_access_hook, + git_gate_render_entrypoint, + git_gate_render_hook, + git_gate_upstreams_for_bottle, +) +from .plan import GitGatePlan +from . import provision as _provision +from . import host_key as _host_key + + +class GitGate: + """The per-agent git-gate host-side service. Stateless — the backend's + launch step drives `prepare` → `provision_dynamic_keys` and, at teardown, + `revoke_provisioned_keys`; `preflight_host_keys` gates a launch on the + upstream host keys being known.""" + + def prepare(self, bottle: ManifestBottle, slug: str, stage_dir: Path) -> GitGatePlan: + """Compute the upstream table from `bottle.git` and write the + entrypoint, pre-receive hook, and access-hook scripts (mode 600) under + `stage_dir`. Pure host-side, no docker subprocess. + + For `gitea` key entries, the returned upstream intentionally has an + empty identity file. Backend launch fills that in after the operator + confirms the preflight. + + Returned plan is incomplete: the launch step must fill + `internal_network` / `egress_network` via `dataclasses.replace` before + passing the plan to `.start`.""" + upstreams = git_gate_upstreams_for_bottle(bottle) + entrypoint = stage_dir / "git_gate_entrypoint.sh" + entrypoint.write_text(git_gate_render_entrypoint(upstreams)) + entrypoint.chmod(0o600) + hook = stage_dir / "git_gate_pre_receive.sh" + hook.write_text(git_gate_render_hook()) + hook.chmod(0o600) + access_hook = stage_dir / "git_gate_access_hook.sh" + access_hook.write_text(git_gate_render_access_hook()) + # 0o700 (not 0o600): git daemon execs --access-hook directly, not via + # `sh`, so the script needs the x bit. The gateway copy does not + # necessarily preserve this mode (`docker cp` does, the Apple `container + # cp` does not), so provisioning re-applies +x on the gateway side — see + # backend/docker/gateway_provision.py. + access_hook.chmod(0o700) + upstreams_with_files: list[GitGateUpstream] = [] + for u in upstreams: + known_hosts_file = Path() + if u.known_host_key: + known_hosts_file = stage_dir / f"{u.name}-known_hosts" + known_hosts_file.write_text( + git_gate_known_hosts_line( + u.upstream_host, u.upstream_port, u.known_host_key, + ) + ) + known_hosts_file.chmod(0o600) + upstreams_with_files.append( + GitGateUpstream( + name=u.name, + upstream_url=u.upstream_url, + upstream_host=u.upstream_host, + upstream_port=u.upstream_port, + identity_file=u.identity_file, + known_host_key=u.known_host_key, + known_hosts_file=known_hosts_file, + ) + ) + return GitGatePlan( + slug=slug, + entrypoint_script=entrypoint, + hook_script=hook, + access_hook_script=access_hook, + upstreams=tuple(upstreams_with_files), + ) + + def provision_dynamic_keys( + self, bottle: ManifestBottle, plan: GitGatePlan, stage_dir: Path, + ) -> GitGatePlan: + """Mint the `gitea` upstreams' ephemeral deploy keys via the forge API + and return the plan with their identity files filled in.""" + return _provision.provision_git_gate_dynamic_keys(bottle, plan, stage_dir) + + def revoke_provisioned_keys(self, bottle: ManifestBottle, stage_dir: Path) -> None: + """Revoke every deploy key provisioned for `bottle` (teardown).""" + _provision.revoke_git_gate_provisioned_keys(bottle, stage_dir) + + def preflight_host_keys( + self, manifest: Manifest, *, headless: bool, home_md: Path | None, + ) -> Manifest: + """Ensure every git-gate upstream has a known host key, populating + missing ones (or erroring in headless mode). Returns the manifest, + possibly updated with fetched keys.""" + return _host_key.preflight_host_keys( + manifest, headless=headless, home_md=home_md, + ) diff --git a/bot_bottle/image_cache.py b/bot_bottle/image_cache.py index b2f802a4..65538e39 100644 --- a/bot_bottle/image_cache.py +++ b/bot_bottle/image_cache.py @@ -5,10 +5,7 @@ from __future__ import annotations from datetime import datetime, timezone from pathlib import Path -try: - from .config_store import ConfigStore -except ImportError: - from config_store import ConfigStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module +from .store.config_store import ConfigStore class StaleImageError(Exception): diff --git a/bot_bottle/manifest/__init__.py b/bot_bottle/manifest/__init__.py new file mode 100644 index 00000000..1dde0285 --- /dev/null +++ b/bot_bottle/manifest/__init__.py @@ -0,0 +1,118 @@ +"""Manifest dataclasses (PRD 0011 layout). + +Reads the per-file manifest tree: + + $HOME/.bot-bottle/bottles/.md — one bottle per file + $HOME/.bot-bottle/agents/.md — home-resident agents + $CWD/.bot-bottle/agents/.md — cwd-supplied agents + +Each file is Markdown with YAML frontmatter. The frontmatter holds +the structured config (see schema below); for agents the body is +the system prompt, for bottles the body is human documentation +(ignored by the parser). + +Bottle schema (frontmatter): + extends: # optional (PRD 0025) + env: { : , ... } + git-gate: # optional (PRD 0047) + user: { name: , email: } # optional + repos: { : , ... } # optional + egress: { routes: [ , ... ] } + # route keys: host, matches, auth, role, dlp + supervise: # optional (default true) + nested_containers: # optional (default false) + +Agent schema (frontmatter): + bottle: # required + skills: [ , ... ] # optional + git-gate: + user: { name: , email: } # optional; overlays bottle + # Claude Code subagent passthrough fields — accepted, ignored: + name, description, model, color, memory + +The agent file's Markdown body is the system prompt (stripped). +Unknown top-level frontmatter keys raise ManifestError with a hint. + +Bottles can ONLY live under $HOME. A bottles/ dir under $CWD is a +warn at load time and contributes nothing. The trust boundary is +expressed as filesystem layout rather than resolver logic. + +Two types are exported: + + ManifestIndex — the multi-agent/bottle collection returned by + resolve() and from_json_obj(). Used for agent + selection (all_agent_names), validation + (require_agent), and lazy loading (load_for_agent). + This is the pre-preflight form. + + Manifest — a single-agent/bottle value type holding exactly + one agent: ManifestAgent and one bottle: + ManifestBottle (with the agent's git-gate.user + already overlaid). Returned by load_for_agent(). + This is the post-preflight form passed to backends. + +ManifestIndex.from_json_obj is preserved as a programmatic entry +point (used by tests) that takes a dict with the same field names — +useful for building manifests without on-disk files. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .index import Manifest, ManifestIndex + from .util import ManifestError + from .agent import ManifestAgent, ManifestAgentProvider + from .bottle import ManifestBottle + from .egress import EGRESS_AUTH_SCHEMES, ManifestEgressConfig, ManifestEgressRoute + from .git import ManifestGitEntry, ManifestGitUser, ManifestKeyConfig + + +# Facade name -> submodule that defines it. The aggregate model (`Manifest`, +# `ManifestIndex`) lives in `index`; the piece types in their own modules. +_LAZY_MODULES: dict[str, str] = { + "Manifest": "index", + "ManifestIndex": "index", + "ManifestError": "util", + "ManifestAgent": "agent", + "ManifestAgentProvider": "agent", + "ManifestBottle": "bottle", + "EGRESS_AUTH_SCHEMES": "egress", + "ManifestEgressRoute": "egress", + "ManifestEgressConfig": "egress", + "ManifestGitEntry": "git", + "ManifestGitUser": "git", + "ManifestKeyConfig": "git", +} + + +def __getattr__(name: str) -> Any: + """Lazily surface the manifest facade names from their submodules and cache + them at package level — so importing a leaf (e.g. `manifest.util`) doesn't + build the whole manifest model, while `from bot_bottle.manifest import + Manifest` keeps working.""" + mod = _LAZY_MODULES.get(name) + if mod is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module + + value = getattr(import_module(f"{__name__}.{mod}"), name) + globals()[name] = value + return value + + +__all__ = [ + "Manifest", + "ManifestIndex", + "ManifestError", + "ManifestGitEntry", + "ManifestGitUser", + "ManifestKeyConfig", + "ManifestAgentProvider", + "ManifestAgent", + "ManifestBottle", + "EGRESS_AUTH_SCHEMES", + "ManifestEgressRoute", + "ManifestEgressConfig", +] diff --git a/bot_bottle/manifest_agent.py b/bot_bottle/manifest/agent.py similarity index 98% rename from bot_bottle/manifest_agent.py rename to bot_bottle/manifest/agent.py index c6071ad4..f5d76ceb 100644 --- a/bot_bottle/manifest_agent.py +++ b/bot_bottle/manifest/agent.py @@ -5,10 +5,10 @@ from __future__ import annotations from dataclasses import dataclass, field from typing import cast -from .agent_provider import PROVIDER_TEMPLATES -from .manifest_util import ManifestError, as_json_object -from .manifest_git import ManifestGitUser -from .manifest_schema import AGENT_MODEL_KEYS, is_valid_entity_name +from ..agent_provider import PROVIDER_TEMPLATES +from .util import ManifestError, as_json_object +from .git import ManifestGitUser +from .schema import AGENT_MODEL_KEYS, is_valid_entity_name @dataclass(frozen=True) diff --git a/bot_bottle/manifest_bottle.py b/bot_bottle/manifest/bottle.py similarity index 87% rename from bot_bottle/manifest_bottle.py rename to bot_bottle/manifest/bottle.py index b4b2015e..ae76d7e3 100644 --- a/bot_bottle/manifest_bottle.py +++ b/bot_bottle/manifest/bottle.py @@ -1,13 +1,13 @@ """The `ManifestBottle` value type. -Split out of `manifest.py` so the `extends:`/loader resolvers can import it -without a circular dependency: `manifest.py` imports those resolvers, while -they only need this value type. Everything here depends on leaf modules -(`manifest_util`, `manifest_agent`, `manifest_egress`, `manifest_git`, -`manifest_schema`), so this module sits at the bottom of the manifest layer. +Split out of the package facade (`manifest/__init__.py`) so the `extends:`/ +loader resolvers can import it without a circular dependency: the facade +imports those resolvers, while they only need this value type. Everything here +depends on leaf modules (`util`, `agent`, `egress`, `git`, `schema`), so this +module sits at the bottom of the manifest layer. -`manifest.py` re-exports `ManifestBottle`, so existing -`from .manifest import ManifestBottle` callers are unaffected. +The facade re-exports `ManifestBottle`, so existing +`from bot_bottle.manifest import ManifestBottle` callers are unaffected. """ from __future__ import annotations @@ -15,11 +15,11 @@ from __future__ import annotations from dataclasses import dataclass, field from typing import Mapping -from .manifest_util import ManifestError, as_json_object -from .manifest_agent import ManifestAgentProvider -from .manifest_egress import ManifestEgressConfig -from .manifest_git import ManifestGitEntry, ManifestGitUser, parse_git_gate_config -from .manifest_schema import BOTTLE_KEYS +from .util import ManifestError, as_json_object +from .agent import ManifestAgentProvider +from .egress import ManifestEgressConfig +from .git import ManifestGitEntry, ManifestGitUser, parse_git_gate_config +from .schema import BOTTLE_KEYS __all__ = ["ManifestBottle"] diff --git a/bot_bottle/manifest_egress.py b/bot_bottle/manifest/egress.py similarity index 99% rename from bot_bottle/manifest_egress.py rename to bot_bottle/manifest/egress.py index c373f324..a9f22cdb 100644 --- a/bot_bottle/manifest_egress.py +++ b/bot_bottle/manifest/egress.py @@ -6,7 +6,7 @@ import re from dataclasses import dataclass from typing import cast -from .manifest_util import ManifestError, as_json_object +from .util import ManifestError, as_json_object EGRESS_AUTH_SCHEMES = ("Bearer", "token") diff --git a/bot_bottle/manifest_extends.py b/bot_bottle/manifest/extends.py similarity index 98% rename from bot_bottle/manifest_extends.py rename to bot_bottle/manifest/extends.py index 698d23f6..c96b032c 100644 --- a/bot_bottle/manifest_extends.py +++ b/bot_bottle/manifest/extends.py @@ -2,10 +2,10 @@ from __future__ import annotations -from .manifest_bottle import ManifestBottle -from .manifest_egress import ManifestEgressConfig, validate_egress_routes -from .manifest_git import ManifestGitUser, parse_git_gate_config -from .manifest_util import ManifestError, as_json_object +from .bottle import ManifestBottle +from .egress import ManifestEgressConfig, validate_egress_routes +from .git import ManifestGitUser, parse_git_gate_config +from .util import ManifestError, as_json_object def _overlay_declared_bool( diff --git a/bot_bottle/manifest_git.py b/bot_bottle/manifest/git.py similarity index 99% rename from bot_bottle/manifest_git.py rename to bot_bottle/manifest/git.py index f620c153..44c87b68 100644 --- a/bot_bottle/manifest_git.py +++ b/bot_bottle/manifest/git.py @@ -5,7 +5,7 @@ from __future__ import annotations import re from dataclasses import dataclass -from .manifest_util import ManifestError, as_json_object +from .util import ManifestError, as_json_object # Shell-safe characters for git-gate repo names. Names are embedded in # the generated entrypoint shell script (shlex.quote is the primary diff --git a/bot_bottle/manifest.py b/bot_bottle/manifest/index.py similarity index 82% rename from bot_bottle/manifest.py rename to bot_bottle/manifest/index.py index f8b63076..8996560d 100644 --- a/bot_bottle/manifest.py +++ b/bot_bottle/manifest/index.py @@ -1,59 +1,10 @@ -"""Manifest dataclasses (PRD 0011 layout). +"""The `Manifest` / `ManifestIndex` aggregate model. -Reads the per-file manifest tree: - - $HOME/.bot-bottle/bottles/.md — one bottle per file - $HOME/.bot-bottle/agents/.md — home-resident agents - $CWD/.bot-bottle/agents/.md — cwd-supplied agents - -Each file is Markdown with YAML frontmatter. The frontmatter holds -the structured config (see schema below); for agents the body is -the system prompt, for bottles the body is human documentation -(ignored by the parser). - -Bottle schema (frontmatter): - extends: # optional (PRD 0025) - env: { : , ... } - git-gate: # optional (PRD 0047) - user: { name: , email: } # optional - repos: { : , ... } # optional - egress: { routes: [ , ... ] } - # route keys: host, matches, auth, role, dlp - supervise: # optional (default true) - nested_containers: # optional (default false) - -Agent schema (frontmatter): - bottle: # required - skills: [ , ... ] # optional - git-gate: - user: { name: , email: } # optional; overlays bottle - # Claude Code subagent passthrough fields — accepted, ignored: - name, description, model, color, memory - -The agent file's Markdown body is the system prompt (stripped). -Unknown top-level frontmatter keys raise ManifestError with a hint. - -Bottles can ONLY live under $HOME. A bottles/ dir under $CWD is a -warn at load time and contributes nothing. The trust boundary is -expressed as filesystem layout rather than resolver logic. - -Two types are exported: - - ManifestIndex — the multi-agent/bottle collection returned by - resolve() and from_json_obj(). Used for agent - selection (all_agent_names), validation - (require_agent), and lazy loading (load_for_agent). - This is the pre-preflight form. - - Manifest — a single-agent/bottle value type holding exactly - one agent: ManifestAgent and one bottle: - ManifestBottle (with the agent's git-gate.user - already overlaid). Returned by load_for_agent(). - This is the post-preflight form passed to backends. - -ManifestIndex.from_json_obj is preserved as a programmatic entry -point (used by tests) that takes a dict with the same field names — -useful for building manifests without on-disk files. +The parsed manifest document (`Manifest`) and the multi-agent index over it +(`ManifestIndex`), plus the bottle-resolution helpers that stitch the pieces +(`agent`, `bottle`, `egress`, `git`, `extends`, `loader`, `schema`) into an +effective manifest. This is the heavy aggregate — the thin package `__init__` +re-exports `Manifest` / `ManifestIndex` lazily. """ from __future__ import annotations @@ -63,41 +14,20 @@ from dataclasses import dataclass, field, replace from pathlib import Path from typing import Mapping -from .log import warn -from .manifest_util import ManifestError, as_json_object -from .manifest_agent import ManifestAgent, ManifestAgentProvider -from .manifest_bottle import ManifestBottle -from .manifest_egress import ( - EGRESS_AUTH_SCHEMES, - ManifestEgressConfig, - ManifestEgressRoute, -) -from .manifest_extends import merge_bottles_runtime, resolve_bottles -from .manifest_git import ManifestGitEntry, ManifestGitUser, ManifestKeyConfig -from .manifest_loader import ( +from ..log import warn +from .util import ManifestError, as_json_object +from .agent import ManifestAgent +from .bottle import ManifestBottle +from .extends import merge_bottles_runtime, resolve_bottles +from .git import ManifestGitUser +from .loader import ( check_stale_json, load_bottle_chain_from_dir, scan_agent_names, scan_bottle_names, ) -from .manifest_schema import validate_agent_frontmatter_keys -from .yaml_subset import YamlSubsetError, parse_frontmatter - -# Re-export everything that callers currently import from this module. -__all__ = [ - "ManifestError", - "ManifestGitEntry", - "ManifestGitUser", - "ManifestKeyConfig", - "ManifestAgentProvider", - "EGRESS_AUTH_SCHEMES", - "ManifestEgressRoute", - "ManifestEgressConfig", - "ManifestAgent", - "ManifestBottle", - "ManifestIndex", - "Manifest", -] +from .schema import validate_agent_frontmatter_keys +from ..yaml_subset import YamlSubsetError, parse_frontmatter def _section_dict(value: object, label: str) -> dict[str, object]: diff --git a/bot_bottle/manifest_loader.py b/bot_bottle/manifest/loader.py similarity index 93% rename from bot_bottle/manifest_loader.py rename to bot_bottle/manifest/loader.py index c88af320..f19130a3 100644 --- a/bot_bottle/manifest_loader.py +++ b/bot_bottle/manifest/loader.py @@ -4,15 +4,15 @@ from __future__ import annotations from pathlib import Path -from .log import warn -from .manifest_bottle import ManifestBottle -from .manifest_extends import resolve_bottles -from .manifest_schema import ( +from ..log import warn +from .bottle import ManifestBottle +from .extends import resolve_bottles +from .schema import ( entity_name_from_path, validate_bottle_frontmatter_keys, ) -from .manifest_util import ManifestError -from .yaml_subset import YamlSubsetError, parse_frontmatter +from .util import ManifestError +from ..yaml_subset import YamlSubsetError, parse_frontmatter def check_stale_json(dir_path: Path, md_dir: Path, label: str) -> None: diff --git a/bot_bottle/manifest_schema.py b/bot_bottle/manifest/schema.py similarity index 98% rename from bot_bottle/manifest_schema.py rename to bot_bottle/manifest/schema.py index 165718a3..512231a6 100644 --- a/bot_bottle/manifest_schema.py +++ b/bot_bottle/manifest/schema.py @@ -68,7 +68,7 @@ def _validate_frontmatter_keys( keys: object, allowed_keys: frozenset[str], ) -> None: - from .manifest_util import ManifestError + from .util import ManifestError key_set = set(keys) # type: ignore unknown = key_set - allowed_keys # type: ignore diff --git a/bot_bottle/manifest_util.py b/bot_bottle/manifest/util.py similarity index 100% rename from bot_bottle/manifest_util.py rename to bot_bottle/manifest/util.py diff --git a/bot_bottle/orchestrator/__init__.py b/bot_bottle/orchestrator/__init__.py index 47ff2826..7597a69c 100644 --- a/bot_bottle/orchestrator/__init__.py +++ b/bot_bottle/orchestrator/__init__.py @@ -5,19 +5,19 @@ A single persistent per-host service that will run the gateway functions agent launches. This package is being built bottom-up, starting with the backend-neutral "consolidation core" that needs no VM packaging: - * `registry` — the SQLite runtime-state store + fail-closed + * `store.registry_store` — the SQLite runtime-state store + fail-closed attribution (source IP + per-bottle identity token). * `broker` — the signed, structured launch-request contract + a `LaunchBroker` (stub for the harness) that verifies provenance before acting. - * `gateway` — the consolidated per-host gateway: a `Gateway` - lifecycle contract (idempotent singleton) + a - `DockerGateway` impl. One gateway shared by all - bottles instead of one per bottle. + * `gateway` — the consolidated per-host gateway: the `Gateway` + lifecycle contract (idempotent singleton). One + gateway shared by all bottles instead of one per + bottle; backend impls live under `backend/*/gateway`. * `service` — the `Orchestrator`: owns the registry, brokers the launch lifecycle (launch/teardown), manages the shared gateway, attributes. - * `control_plane` — the HTTP control-plane RPC (launch / teardown / + * `server` — the HTTP control-plane RPC (launch / teardown / list / attribute / gateway / health). The actual backend-native launch (a real docker/firecracker broker) and @@ -28,19 +28,59 @@ orchestrator -> firecracker). from __future__ import annotations -from .registry import BottleRecord, RegistryStore, new_identity_token -from .broker import ( - BrokerAuthError, - LaunchBroker, - LaunchRequest, - StubBroker, - sign_request, - verify_request, -) -from .docker_broker import DockerBroker, DockerBrokerError -from .gateway import DockerGateway, Gateway, GatewayError -from .service import Orchestrator -from .control_plane import ControlPlaneServer, dispatch, make_server +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .store.registry_store import BottleRecord, RegistryStore, new_identity_token + from .broker import ( + BrokerAuthError, + LaunchBroker, + LaunchRequest, + StubBroker, + sign_request, + verify_request, + ) + from .docker_broker import DockerBroker, DockerBrokerError + from ..gateway import Gateway, GatewayError + from .service import Orchestrator + from .server import OrchestratorServer, dispatch, make_server + + +# Facade name -> submodule that defines it. Lazy so importing a leaf (or the +# parent package, e.g. via `orchestrator.store.store_manager`) doesn't drag the +# whole orchestrator — server, docker_broker + the docker backend, http — while +# `from bot_bottle.orchestrator import RegistryStore` keeps working. +_LAZY: dict[str, str] = { + "BottleRecord": ".store.registry_store", + "RegistryStore": ".store.registry_store", + "new_identity_token": ".store.registry_store", + "BrokerAuthError": ".broker", + "LaunchBroker": ".broker", + "LaunchRequest": ".broker", + "StubBroker": ".broker", + "sign_request": ".broker", + "verify_request": ".broker", + "DockerBroker": ".docker_broker", + "DockerBrokerError": ".docker_broker", + "Gateway": "..gateway", + "GatewayError": "..gateway", + "Orchestrator": ".service", + "OrchestratorServer": ".server", + "dispatch": ".server", + "make_server": ".server", +} + + +def __getattr__(name: str) -> Any: + src = _LAZY.get(name) + if src is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + from importlib import import_module + + value = getattr(import_module(src, __name__), name) + globals()[name] = value + return value + __all__ = [ "BottleRecord", @@ -50,15 +90,14 @@ __all__ = [ "LaunchBroker", "LaunchRequest", "StubBroker", + "sign_request", + "verify_request", "DockerBroker", "DockerBrokerError", "Gateway", - "DockerGateway", "GatewayError", - "sign_request", - "verify_request", "Orchestrator", - "ControlPlaneServer", + "OrchestratorServer", "dispatch", "make_server", ] diff --git a/bot_bottle/orchestrator/__main__.py b/bot_bottle/orchestrator/__main__.py index 1606e12e..6fad8364 100644 --- a/bot_bottle/orchestrator/__main__.py +++ b/bot_bottle/orchestrator/__main__.py @@ -16,13 +16,12 @@ import secrets from pathlib import Path from .. import log -from ..store_manager import StoreManager +from .store.store_manager import StoreManager from .broker import LaunchBroker, StubBroker -from .control_plane import make_server +from .server import make_server from .docker_broker import DockerBroker -from .registry import RegistryStore, default_db_path +from .store.registry_store import RegistryStore, default_db_path from .service import Orchestrator -from .gateway import DockerGateway, Gateway def main(argv: list[str] | None = None) -> int: @@ -38,10 +37,6 @@ def main(argv: list[str] | None = None) -> int: "--broker", choices=("stub", "docker"), default="stub", help="launch broker: 'stub' records requests; 'docker' runs containers", ) - parser.add_argument( - "--gateway", action="store_true", - help="run one consolidated per-host gateway (build-if-missing)", - ) args = parser.parse_args(argv) registry = RegistryStore(args.db) @@ -57,20 +52,7 @@ def main(argv: list[str] | None = None) -> int: # anything; 'docker' runs real containers (firecracker drops in later). secret = secrets.token_bytes(32) broker: LaunchBroker = DockerBroker(secret) if args.broker == "docker" else StubBroker(secret) - # Standalone `--gateway` (not the consolidated flow, where the host - # lifecycle runs the gateway). The gateway resolves against this same - # process; the URL is only reachable when they share a docker network. - gateway: Gateway | None = ( - DockerGateway(orchestrator_url=f"http://{args.host}:{args.port}") - if args.gateway else None - ) - orchestrator = Orchestrator(registry, broker, secret, gateway) - - # One persistent per-host gateway, shared by every bottle: build the - # bundle image if missing, then bring the singleton up (idempotent). - if gateway is not None: - orchestrator.ensure_gateway() - log.info("consolidated gateway ensured", context={"name": gateway.name}) + orchestrator = Orchestrator(registry, broker, secret) server = make_server(orchestrator, host=args.host, port=args.port) bound_host, bound_port = server.server_address[0], server.server_address[1] diff --git a/bot_bottle/orchestrator/client.py b/bot_bottle/orchestrator/client.py index a349c645..b6c95492 100644 --- a/bot_bottle/orchestrator/client.py +++ b/bot_bottle/orchestrator/client.py @@ -18,9 +18,9 @@ import urllib.request from collections.abc import Iterable from dataclasses import dataclass -from ..control_auth import ROLE_CLI, mint -from ..paths import host_control_plane_token -from .control_plane import CONTROL_AUTH_HEADER +from ..orchestrator_auth import ROLE_CLI, mint +from ..paths import host_orchestrator_token +from .server import ORCHESTRATOR_AUTH_HEADER DEFAULT_TIMEOUT_SECONDS = 5.0 @@ -32,7 +32,7 @@ def _host_auth_token() -> str: "" means 'send no auth header' — correct against an open (unconfigured) control plane, and harmlessly rejected by a secured one.""" try: - return mint(ROLE_CLI, host_control_plane_token()) + return mint(ROLE_CLI, host_orchestrator_token()) except (OSError, ValueError): return "" @@ -83,7 +83,7 @@ class OrchestratorClient: data = json.dumps(body).encode() if body is not None else None headers = {"Content-Type": "application/json"} if data is not None else {} if self._auth_token: - headers[CONTROL_AUTH_HEADER] = self._auth_token + headers[ORCHESTRATOR_AUTH_HEADER] = self._auth_token req = urllib.request.Request( f"{self._base}{path}", data=data, method=method, headers=headers, ) @@ -252,14 +252,14 @@ def discover_orchestrator_url(*, timeout: float = 2.0) -> str: candidates.append("http://127.0.0.1:8099") try: # firecracker: infra VM control plane on the orchestrator TAP from ..backend.firecracker import netpool - from ..backend.firecracker.infra_vm import CONTROL_PLANE_PORT + from ..backend.firecracker.infra_vm import ORCHESTRATOR_PORT candidates.append( - f"http://{netpool.orch_slot().guest_ip}:{CONTROL_PLANE_PORT}") + f"http://{netpool.orch_slot().guest_ip}:{ORCHESTRATOR_PORT}") except Exception: # noqa: BLE001 — backend optional / not firecracker pass try: # macOS: infra container control plane on its host-only address - from ..backend.macos_container.infra import probe_control_plane_url - url = probe_control_plane_url() + from ..backend.macos_container.infra import probe_orchestrator_url + url = probe_orchestrator_url() if url: candidates.append(url) except Exception: # noqa: BLE001 — backend optional / not macOS diff --git a/bot_bottle/orchestrator/docker_broker.py b/bot_bottle/orchestrator/docker_broker.py index e5af91c5..2e22845e 100644 --- a/bot_bottle/orchestrator/docker_broker.py +++ b/bot_bottle/orchestrator/docker_broker.py @@ -15,7 +15,7 @@ from __future__ import annotations import subprocess -from ..docker_cmd import run_docker +from ..backend.docker.util import run_docker from .broker import LaunchBroker, LaunchRequest CONTAINER_PREFIX = "bot-bottle-orch-" diff --git a/bot_bottle/orchestrator/lifecycle.py b/bot_bottle/orchestrator/lifecycle.py index b75e87d4..d57b14da 100644 --- a/bot_bottle/orchestrator/lifecycle.py +++ b/bot_bottle/orchestrator/lifecycle.py @@ -1,94 +1,29 @@ -"""Orchestrator + gateway lifecycle (PRD 0070, docker slice). +"""Shared orchestrator-process constants + helpers (PRD 0070). -Runs both the orchestrator control plane and the gateway data plane inside -a single `bot-bottle-infra` container on the shared gateway network — -matching the structure already used by the macOS and Firecracker backends. -`gateway_init` is PID 1 and supervises both; the infra container is an -idempotent per-host singleton. - -The combined container replaces the prior two-container split -(bot-bottle-orchestrator + bot-bottle-orch-gateway). The host CLI reaches -the control plane via a published loopback port; gateway daemons reach it -over 127.0.0.1 (same container). +Backend-neutral pieces the per-backend infra services build on: the +control-plane port, the startup timeout, the start-error, and the source hash +used to detect a code change. The per-backend infra lifecycle lives with each +backend — docker: `backend.docker.infra.DockerInfraService`; macOS: +`backend.macos_container.infra`; firecracker: `backend.firecracker.infra_vm`. """ from __future__ import annotations import hashlib -import os -import time -import urllib.error -import urllib.request from pathlib import Path -from .. import log -from ..control_auth import ROLE_GATEWAY, mint -from ..docker_cmd import run_docker -from ..paths import ( - CONTROL_AUTH_JWT_ENV, - CONTROL_PLANE_TOKEN_ENV, - bot_bottle_root, - host_control_plane_token, - host_gateway_ca_dir, -) -from .gateway import ( - GATEWAY_DOCKERFILE, - GATEWAY_IMAGE, - GATEWAY_NETWORK, - GatewayError, - MITMPROXY_HOME, -) - DEFAULT_PORT = 8099 DEFAULT_STARTUP_TIMEOUT_SECONDS = 45.0 -INFRA_NAME = "bot-bottle-infra" -INFRA_LABEL = "bot-bottle-infra=1" -# The combined infra image: gateway data plane + orchestrator content. -# Built from Dockerfile.infra (FROM gateway + COPY --from orchestrator). -INFRA_IMAGE = os.environ.get("BOT_BOTTLE_INFRA_IMAGE", "bot-bottle-infra:latest") -INFRA_DOCKERFILE = "Dockerfile.infra" -# Baked as a container label so `ensure_running` can detect whether the -# running container is executing the current bind-mounted source. -INFRA_SOURCE_HASH_LABEL = "bot-bottle-infra-source-hash" - -# Orchestrator image: the single canonical definition of the control-plane -# content (lean: python:3.12-slim + bot_bottle package, no mitmproxy/git). -# Used as a build intermediate: `Dockerfile.infra` COPY --from this image. -ORCHESTRATOR_IMAGE = os.environ.get( - "BOT_BOTTLE_ORCHESTRATOR_IMAGE", "bot-bottle-orchestrator:latest" -) -ORCHESTRATOR_DOCKERFILE = "Dockerfile.orchestrator" - -# The gateway daemons + orchestrator the infra container runs. -# BOT_BOTTLE_GATEWAY_DAEMONS listing `orchestrator` opts it in to -# gateway_init's supervise tree (see gateway_init._OPT_IN_DAEMONS). -_INFRA_DAEMONS = "egress,git-http,supervise,orchestrator" - -# The bind-mount path for the live control-plane source inside the -# container. Separate from /app so the gateway's baked scripts -# (egress_addon.py, egress-entrypoint.sh) are not overlaid. -_SRC_IN_CONTAINER = "/bot-bottle-src" -# Bot-bottle host-root bind-mount inside the container (DB + state). The -# orchestrator control plane opens bot-bottle.db under here (via BOT_BOTTLE_ROOT -# -> host_db_path()); it is the ONLY process in the container with a file -# handle on it (PRD 0070 / issue #469). -_ROOT_IN_CONTAINER = "/bot-bottle-root" - -_HEALTH_POLL_SECONDS = 0.25 -_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0 - -_REPO_ROOT = Path(__file__).resolve().parents[2] - class OrchestratorStartError(RuntimeError): - """The infra container did not become healthy within the timeout.""" + """The infra container/VM did not become healthy within the timeout.""" def source_hash(repo_root: Path) -> str: """Content hash of the orchestrator's bind-mounted Python source (the `bot_bottle` package the control-plane process imports). Changes only - when the code that would actually run changes — `ensure_running` + when the code that would actually run changes — a backend's `ensure_running` recreates the container on a mismatch so a code change takes effect, but leaves a healthy up-to-date container alone to preserve in-memory egress tokens.""" @@ -99,179 +34,9 @@ def source_hash(repo_root: Path) -> str: return h.hexdigest() -class OrchestratorService: - """Manages the single per-host infra container (control plane + gateway). - Callers only need `ensure_running()` + `url`. - - `infra_name` / `infra_label` let backends run independent infra containers - on the same host without name collisions (e.g. isolated integration tests - that can't share the production INFRA_NAME singleton).""" - - def __init__( - self, - *, - port: int = DEFAULT_PORT, - network: str = GATEWAY_NETWORK, - image: str = INFRA_IMAGE, - repo_root: Path = _REPO_ROOT, - host_root: Path | None = None, - infra_name: str = INFRA_NAME, - infra_label: str = INFRA_LABEL, - ) -> None: - self.port = port - self.network = network - self.image = image - self._repo_root = repo_root - self._host_root = host_root or bot_bottle_root() - self._infra_name = infra_name - self._infra_label = infra_label - - @property - def url(self) -> str: - """Host-side control-plane URL (published loopback port).""" - return f"http://127.0.0.1:{self.port}" - - def is_healthy(self, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS) -> bool: - try: - with urllib.request.urlopen(f"{self.url}/health", timeout=timeout) as resp: - return resp.status == 200 - except (urllib.error.URLError, TimeoutError, OSError): - return False - - def _container_running(self, name: str) -> bool: - proc = run_docker(["docker", "ps", "--filter", f"name=^/{name}$", "--format", "{{.Names}}"]) - return name in proc.stdout.split() - - def _infra_source_current(self, current_hash: str) -> bool: - """True iff the running infra container was started from the current - bind-mounted source. Mirrors the macOS backend's `_source_current`.""" - if not self._container_running(self._infra_name): - return False - proc = run_docker([ - "docker", "inspect", "--format", - "{{ index .Config.Labels \"" + INFRA_SOURCE_HASH_LABEL + "\" }}", - self._infra_name, - ]) - if proc.returncode != 0: - return True # can't compare → don't churn a working container - return proc.stdout.strip() == current_hash - - def _ensure_network(self) -> None: - if run_docker(["docker", "network", "inspect", self.network]).returncode == 0: - return - proc = run_docker(["docker", "network", "create", self.network]) - if proc.returncode != 0 and "already exists" not in proc.stderr: - raise GatewayError( - f"gateway network {self.network} failed to create: {proc.stderr.strip()}" - ) - - def _build_images(self) -> None: - """Build the gateway base, the orchestrator intermediate, then the - infra image. All are cache-aware: a no-op when nothing changed.""" - for tag, dockerfile in ( - (GATEWAY_IMAGE, GATEWAY_DOCKERFILE), - (ORCHESTRATOR_IMAGE, ORCHESTRATOR_DOCKERFILE), - (self.image, INFRA_DOCKERFILE), - ): - argv = ["docker", "build", "-t", tag, - "-f", str(self._repo_root / dockerfile), - str(self._repo_root)] - if os.environ.get("BOT_BOTTLE_NO_CACHE"): - argv.insert(2, "--no-cache") - proc = run_docker(argv) - if proc.returncode != 0: - raise GatewayError(f"{dockerfile} build failed: {proc.stderr.strip()}") - - def _run_infra_container(self, current_hash: str) -> None: - """Start the combined infra container (idempotent: clears a stale - fixed-name container first). Labels the container with `current_hash` - so a later `ensure_running` can detect a real code change.""" - self._ensure_network() - run_docker(["docker", "rm", "--force", self._infra_name]) - _signing_key = host_control_plane_token() - proc = run_docker([ - "docker", "run", "--detach", - "--name", self._infra_name, - "--label", self._infra_label, - "--label", f"{INFRA_SOURCE_HASH_LABEL}={current_hash}", - "--network", self.network, - # Host CLI reaches the control plane here (loopback only). - # gateway_init always starts the orchestrator on DEFAULT_PORT (8099) - # inside the container; self.port is the host-side published port. - "--publish", f"127.0.0.1:{self.port}:{DEFAULT_PORT}", - # Persist the mitmproxy CA on the host so it survives container - # recreation AND docker volume pruning (issue #450): every agent - # trusts this one CA, so a fresh one would break all running bottles. - "--volume", f"{host_gateway_ca_dir()}:{MITMPROXY_HOME}", - # Live control-plane source, mounted to a path that does not - # overlay the gateway's baked /app scripts. - "--volume", f"{self._repo_root}:{_SRC_IN_CONTAINER}:ro", - # PYTHONPATH lets the orchestrator (and other Python daemons) - # import the live source ahead of the installed package. - "--env", f"PYTHONPATH={_SRC_IN_CONTAINER}", - # Orchestrator registry DB on the host (sole writer: control plane). - "--volume", f"{self._host_root}:{_ROOT_IN_CONTAINER}", - "--env", f"BOT_BOTTLE_ROOT={_ROOT_IN_CONTAINER}", - # Control-plane signing key (orchestrator: verifies tokens) + the - # pre-minted `gateway` JWT (data-plane daemons: present it). gateway_init - # scopes each to its process, so a compromised data-plane daemon never - # sees the key and can't mint a `cli` token (issue #469 review). - "--env", CONTROL_PLANE_TOKEN_ENV, - "--env", CONTROL_AUTH_JWT_ENV, - # Gateway daemons reach the orchestrator over loopback at its - # fixed internal port (DEFAULT_PORT), independent of self.port. - "--env", f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{DEFAULT_PORT}", - # Opt the orchestrator into gateway_init's supervise tree. - "--env", f"BOT_BOTTLE_GATEWAY_DAEMONS={_INFRA_DAEMONS}", - self.image, - ], env={ - **os.environ, - CONTROL_PLANE_TOKEN_ENV: _signing_key, - CONTROL_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key), - }) - if proc.returncode != 0: - raise OrchestratorStartError( - f"infra container failed to start: {proc.stderr.strip()}" - ) - - def ensure_running( - self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS, - ) -> str: - """Ensure the infra container (control plane + gateway) is up; return - the host control-plane URL. Idempotent — a healthy container on current - source is left untouched. Raises `OrchestratorStartError` on timeout.""" - self._build_images() - - current_hash = source_hash(self._repo_root) - if self.is_healthy() and self._infra_source_current(current_hash): - return self.url - - log.info("starting infra container", context={"name": self._infra_name}) - self._run_infra_container(current_hash) - - deadline = time.monotonic() + startup_timeout - while time.monotonic() < deadline: - if self.is_healthy(): - log.info("infra container healthy", context={"url": self.url}) - return self.url - time.sleep(_HEALTH_POLL_SECONDS) - raise OrchestratorStartError( - f"infra container at {self.url} did not become healthy within {startup_timeout:g}s" - ) - - def stop(self) -> None: - """Remove the infra container (idempotent).""" - run_docker(["docker", "rm", "--force", self._infra_name]) - - __all__ = [ - "OrchestratorService", - "OrchestratorStartError", - "INFRA_NAME", - "INFRA_IMAGE", - "INFRA_SOURCE_HASH_LABEL", - "ORCHESTRATOR_IMAGE", "DEFAULT_PORT", "DEFAULT_STARTUP_TIMEOUT_SECONDS", + "OrchestratorStartError", "source_hash", ] diff --git a/bot_bottle/orchestrator/rotate_ca.py b/bot_bottle/orchestrator/rotate_ca.py index 5495c779..93030054 100644 --- a/bot_bottle/orchestrator/rotate_ca.py +++ b/bot_bottle/orchestrator/rotate_ca.py @@ -23,10 +23,10 @@ from __future__ import annotations import sys from pathlib import Path -from ..docker_cmd import run_docker +from ..backend.docker.util import run_docker from ..paths import host_gateway_ca_dir -from .gateway import GATEWAY_NAME, rotate_gateway_ca -from .lifecycle import INFRA_NAME +from ..gateway import GATEWAY_NAME, rotate_gateway_ca +from ..backend.docker.infra import INFRA_NAME # The containers whose mitmproxy would still be serving the old CA from memory: # the consolidated infra container and the standalone per-host gateway. diff --git a/bot_bottle/orchestrator/control_plane.py b/bot_bottle/orchestrator/server.py similarity index 95% rename from bot_bottle/orchestrator/control_plane.py rename to bot_bottle/orchestrator/server.py index 5760923d..a7ec7a4a 100644 --- a/bot_bottle/orchestrator/control_plane.py +++ b/bot_bottle/orchestrator/server.py @@ -48,7 +48,7 @@ via the orchestrator. Register/deregister without a launch are internal to `Orchestrator`, not exposed here. Routing/handling is the pure function `dispatch()` so it is unit-testable -without a socket; `Handler` / `ControlPlaneServer` / `make_server` are a +without a socket; `Handler` / `OrchestratorServer` / `make_server` are a thin stdlib adapter around it. Listing redacts identity tokens — they are returned only once, to the caller that launches the bottle. """ @@ -63,22 +63,22 @@ import sys import typing from urllib.parse import urlsplit -from ..control_auth import ROLE_CLI, ROLES, verify -from ..paths import CONTROL_PLANE_TOKEN_ENV -from ..supervise_types import TOOLS +from ..orchestrator_auth import ROLE_CLI, ROLES, verify +from ..paths import ORCHESTRATOR_TOKEN_ENV +from ..supervisor.types import TOOLS from .service import Orchestrator # JSON body payload type (parsed request / rendered response). Json = dict[str, object] # The request header carrying the caller's role-scoped control-plane token (a -# signed JWT naming the caller's role — see control_auth). The role gates which +# signed JWT naming the caller's role — see orchestrator_auth). The role gates which # routes the caller may reach: the data plane holds a `gateway` token good only # for the agent-facing lookups; the host CLI holds a `cli` token for the # operator/mutating routes. An agent that can merely *reach* the port holds no # token at all, and a compromised gateway holds only `gateway` — neither can # drive the operator routes (approve proposals, rewrite policy, read tokens). -CONTROL_AUTH_HEADER = "x-bot-bottle-control-auth" +ORCHESTRATOR_AUTH_HEADER = "x-bot-bottle-orchestrator-auth" # The routes the data plane (role `gateway`) is allowed to reach — exactly the # per-request lookups PolicyResolver makes. Every other authenticated route is @@ -116,7 +116,7 @@ def dispatch( # pylint: disable=too-many-return-statements,too-many-branches `role` is the caller's verified control-plane role (`gateway` or `cli`), or None for an unauthenticated request; an open-mode server (no signing key - configured — see `ControlPlaneServer`) passes `cli`. Every route except + configured — see `OrchestratorServer`) passes `cli`. Every route except `GET /health` requires a role: a missing role is 401, and a role that doesn't cover the route is 403 — so a `gateway` data-plane token can reach `/resolve` + `/supervise/{propose,poll}` but not the operator routes @@ -365,10 +365,10 @@ class Handler(http.server.BaseHTTPRequestHandler): crashing the connection, so one bad request can't take the control plane down for the caller.""" server = self.server - assert isinstance(server, ControlPlaneServer) + assert isinstance(server, OrchestratorServer) length = int(self.headers.get("Content-Length") or 0) body = self.rfile.read(length) if length > 0 else b"" - role = server.role_for(self.headers.get(CONTROL_AUTH_HEADER, "")) + role = server.role_for(self.headers.get(ORCHESTRATOR_AUTH_HEADER, "")) try: status, payload = dispatch( server.orchestrator, method, self.path, body, role=role) @@ -396,11 +396,11 @@ class Handler(http.server.BaseHTTPRequestHandler): self._serve("DELETE") -class ControlPlaneServer(socketserver.ThreadingMixIn, http.server.HTTPServer): +class OrchestratorServer(socketserver.ThreadingMixIn, http.server.HTTPServer): """Threading HTTP server that carries the orchestrator for its handlers. Holds the per-host control-plane *signing key* (from - `$BOT_BOTTLE_CONTROL_PLANE_TOKEN`, injected by the launcher into the + `$BOT_BOTTLE_ORCHESTRATOR_TOKEN`, injected by the launcher into the orchestrator process only) and verifies each request's role-scoped token against it. When a key is set, every route but `/health` requires a valid token whose role covers the route; when it is unset the server runs **open** @@ -413,11 +413,11 @@ class ControlPlaneServer(socketserver.ThreadingMixIn, http.server.HTTPServer): def __init__(self, address: tuple[str, int], orchestrator: Orchestrator) -> None: self.orchestrator = orchestrator - self._signing_key = os.environ.get(CONTROL_PLANE_TOKEN_ENV, "").strip() + self._signing_key = os.environ.get(ORCHESTRATOR_TOKEN_ENV, "").strip() if not self._signing_key: sys.stderr.write( "orchestrator: WARNING — no control-plane signing key " - f"(${CONTROL_PLANE_TOKEN_ENV}); running WITHOUT caller " + f"(${ORCHESTRATOR_TOKEN_ENV}); running WITHOUT caller " "authentication. Any client that can reach this port can drive " "it. Backends that put the control plane on an agent-reachable " "network MUST set this.\n" @@ -438,13 +438,13 @@ class ControlPlaneServer(socketserver.ThreadingMixIn, http.server.HTTPServer): def make_server( orchestrator: Orchestrator, host: str = "127.0.0.1", port: int = 0 -) -> ControlPlaneServer: +) -> OrchestratorServer: """Build (but do not start) a control-plane server. `port=0` binds an ephemeral port — read `server.server_address` for the actual one.""" - return ControlPlaneServer((host, port), orchestrator) + return OrchestratorServer((host, port), orchestrator) __all__ = [ - "dispatch", "Handler", "ControlPlaneServer", "make_server", "Json", - "CONTROL_AUTH_HEADER", + "dispatch", "Handler", "OrchestratorServer", "make_server", "Json", + "ORCHESTRATOR_AUTH_HEADER", ] diff --git a/bot_bottle/orchestrator/service.py b/bot_bottle/orchestrator/service.py index 629ab16c..5b7e1091 100644 --- a/bot_bottle/orchestrator/service.py +++ b/bot_bottle/orchestrator/service.py @@ -26,9 +26,8 @@ from collections.abc import Iterable from datetime import datetime, timezone from .broker import LaunchBroker, LaunchRequest, sign_request -from .registry import DEFAULT_REAP_GRACE_SECONDS, BottleRecord, RegistryStore -from .gateway import Gateway -from ..supervise import ( +from .store.registry_store import DEFAULT_REAP_GRACE_SECONDS, BottleRecord, RegistryStore +from .supervisor import ( AuditEntry, COMPONENT_FOR_TOOL, POLL_STATUS_PENDING, @@ -40,16 +39,9 @@ from ..supervise import ( STATUS_REJECTED, TOOL_EGRESS_ALLOW, TOOL_EGRESS_BLOCK, - archive_all_proposals, - list_all_pending_proposals, - read_proposal, - read_response, - render_diff, - sha256_hex, - write_audit_entry, - write_proposal, - write_response, + Supervisor, ) +from ..util import render_diff # Operator decision → Response.status. The apply half (egress tools) runs @@ -72,12 +64,14 @@ class Orchestrator: registry: RegistryStore, broker: LaunchBroker, sign_secret: bytes, - gateway: Gateway | None = None, + supervisor: Supervisor | None = None, ) -> None: self.registry = registry self._broker = broker self._secret = sign_secret - self._gateway = gateway + # The supervise service (queue + audit I/O). Injectable so tests can + # point it at a temp DB; defaults to the host DB. + self._supervisor = supervisor or Supervisor() # Per-bottle egress auth tokens (env_name -> value), keyed by bottle_id. # Held **in memory only** — never written to the registry DB — so the # gateway can inject each bottle's upstream credential without secrets @@ -107,7 +101,7 @@ class Orchestrator: if tokens: self._tokens[rec.bottle_id] = dict(tokens) if env_var_secret: - from .secret_store import encrypt_value + from .store.secret_store import encrypt_value encrypted = {k: encrypt_value(env_var_secret, v) for k, v in tokens.items()} self.registry.store_agent_secrets(rec.bottle_id, encrypted) req = LaunchRequest( @@ -137,7 +131,7 @@ class Orchestrator: self.registry.deregister(bottle_id) self._tokens.pop(bottle_id, None) # Reap any supervise proposals the gone bottle never acknowledged. - archive_all_proposals(bottle_id) + self._supervisor.archive_all_proposals(bottle_id) return True def reconcile( @@ -163,7 +157,7 @@ class Orchestrator: for rec in reaped: self._tokens.pop(rec.bottle_id, None) # Reap any supervise proposals the gone bottle never acknowledged. - archive_all_proposals(rec.bottle_id) + self._supervisor.archive_all_proposals(rec.bottle_id) return [rec.bottle_id for rec in reaped] def tokens_for(self, bottle_id: str) -> dict[str, str]: @@ -221,9 +215,8 @@ class Orchestrator: tool=tool, proposed_file=proposed_file, justification=justification, - current_file_hash=sha256_hex(proposed_file), ) - write_proposal(proposal) + self._supervisor.write_proposal(proposal) return proposal.id def supervise_poll_response(self, bottle_id: str, proposal_id: str) -> dict[str, object]: @@ -244,10 +237,10 @@ class Orchestrator: Reads are scoped to `bottle_id` (the queue key), so a caller can never read another bottle's proposal even with a guessed id.""" try: - response = read_response(bottle_id, proposal_id) + response = self._supervisor.read_response(bottle_id, proposal_id) except FileNotFoundError: try: - read_proposal(bottle_id, proposal_id) + self._supervisor.read_proposal(bottle_id, proposal_id) except FileNotFoundError: return {"status": POLL_STATUS_UNKNOWN} return {"status": POLL_STATUS_PENDING} @@ -266,7 +259,7 @@ class Orchestrator: orchestrator-assigned bottle_id, which is opaque to an operator). The CLI renders the label but still responds against `bottle_slug`.""" out: list[dict[str, object]] = [] - for p in list_all_pending_proposals(): + for p in self._supervisor.list_all_pending_proposals(): d = p.to_dict() d["bottle_label"] = self._label_for(p.bottle_slug) out.append(d) @@ -329,7 +322,7 @@ class Orchestrator: if status is None: return False, f"unknown decision {decision!r}" try: - proposal = read_proposal(bottle_slug, proposal_id) + proposal = self._supervisor.read_proposal(bottle_slug, proposal_id) except FileNotFoundError: return False, "no such proposal" @@ -345,19 +338,22 @@ class Orchestrator: diff_before, diff_after = rec.policy, new_policy self.set_policy(rec.bottle_id, new_policy) - write_response(bottle_slug, Response( + self._supervisor.write_response(bottle_slug, Response( proposal_id=proposal_id, status=status, notes=notes, final_file=final_file, )) component = COMPONENT_FOR_TOOL.get(proposal.tool) if component is not None: - write_audit_entry(AuditEntry( + self._supervisor.write_audit_entry(AuditEntry( timestamp=datetime.now(timezone.utc).isoformat(), bottle_slug=bottle_slug, component=component, operator_action=status, operator_notes=notes, justification=proposal.justification, - diff=render_diff(diff_before, diff_after, label=component), + diff=render_diff( + diff_before, diff_after, label=component, + before_title="current", after_label="proposed", + ), )) return True, "" @@ -370,7 +366,7 @@ class Orchestrator: value with *env_var_secret*, and restores ``_tokens[bottle_id]``. Returns True on success, False when no stored secrets exist for this bottle or decryption fails (wrong key / corrupt data).""" - from .secret_store import decrypt_value + from .store.secret_store import decrypt_value encrypted = self.registry.get_agent_secrets(bottle_id) if not encrypted: return False @@ -383,22 +379,15 @@ class Orchestrator: # --- consolidated gateway ---------------------------------------------- - def ensure_gateway(self) -> None: - """Ensure the single per-host gateway is built and up (idempotent). - No-op when no gateway is configured.""" - if self._gateway is not None: - self._gateway.ensure_built() - self._gateway.ensure_running() - def gateway_status(self) -> dict[str, object]: - """Report the shared gateway for the control plane / console.""" - if self._gateway is None: - return {"configured": False} - return { - "configured": True, - "name": self._gateway.name, - "running": self._gateway.is_running(), - } + """Report the shared gateway for the control plane / console. + + The orchestrator no longer owns a standalone gateway lifecycle — the + consolidated flow runs the gateway data plane inside the per-host infra + container/VM (see `backend/*/gateway`), so this reports `configured: + false`. Retained for the documented `GET /gateway` control-plane + contract.""" + return {"configured": False} __all__ = ["Orchestrator"] diff --git a/bot_bottle/orchestrator/store/__init__.py b/bot_bottle/orchestrator/store/__init__.py new file mode 100644 index 00000000..4b2b70fa --- /dev/null +++ b/bot_bottle/orchestrator/store/__init__.py @@ -0,0 +1,10 @@ +"""Orchestrator-owned SQLite stores. + +The stores whose tables only the orchestrator (control plane) opens: the +per-host bottle `registry_store`, the supervise proposal/response `queue_store`, +the agent-secret `secret_store`, and the orchestrator `config_store`. They build +on the shared `DbStore` / `migrations` base in `bot_bottle.store`. + +Callers import the concrete module directly, e.g. +`from bot_bottle.orchestrator.store.queue_store import QueueStore`. +""" diff --git a/bot_bottle/orchestrator/config_store.py b/bot_bottle/orchestrator/store/config_store.py similarity index 96% rename from bot_bottle/orchestrator/config_store.py rename to bot_bottle/orchestrator/store/config_store.py index 2cd6f854..251f0ef6 100644 --- a/bot_bottle/orchestrator/config_store.py +++ b/bot_bottle/orchestrator/store/config_store.py @@ -11,9 +11,9 @@ import os import sqlite3 from pathlib import Path -from ..db_store import DbStore -from ..migrations import TableMigrations -from ..paths import host_db_path +from ...store.db_store import DbStore +from ...store.migrations import TableMigrations +from ...paths import host_db_path TEARDOWN_TIMEOUT_ENV = "BOT_BOTTLE_ORCHESTRATOR_TEARDOWN_TIMEOUT_SECONDS" DEFAULT_TEARDOWN_TIMEOUT_SECONDS = 30.0 diff --git a/bot_bottle/queue_store.py b/bot_bottle/orchestrator/store/queue_store.py similarity index 85% rename from bot_bottle/queue_store.py rename to bot_bottle/orchestrator/store/queue_store.py index 2dae5c9e..b9d8b1a8 100644 --- a/bot_bottle/queue_store.py +++ b/bot_bottle/orchestrator/store/queue_store.py @@ -6,16 +6,10 @@ import os import sqlite3 from pathlib import Path -try: - from .supervise_types import Proposal, Response - from .paths import host_db_path - from .db_store import DbStore - from .migrations import TableMigrations -except ImportError: - from supervise_types import Proposal, Response # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from paths import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from db_store import DbStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module +from ...supervisor.types import Proposal, Response +from ...paths import host_db_path +from ...store.db_store import DbStore +from ...store.migrations import TableMigrations class QueueStore(DbStore): @@ -173,25 +167,6 @@ class QueueStore(DbStore): raise FileNotFoundError(proposal_id) return self._row_to_response(row) - def archive_proposal(self, proposal_id: str) -> None: - if not self.db_path.is_file(): - return - with self._connection() as conn: - conn.execute( - """ - UPDATE supervise_proposals SET archived = 1 - WHERE queue_key = ? AND id = ? - """, - (self.queue_key, proposal_id), - ) - conn.execute( - """ - UPDATE supervise_responses SET archived = 1 - WHERE queue_key = ? AND proposal_id = ? - """, - (self.queue_key, proposal_id), - ) - def archive_all(self) -> None: """Archive every proposal + response for this queue key. Used to reap a bottle's supervise records when it is torn down or reconciled away — diff --git a/bot_bottle/orchestrator/registry.py b/bot_bottle/orchestrator/store/registry_store.py similarity index 99% rename from bot_bottle/orchestrator/registry.py rename to bot_bottle/orchestrator/store/registry_store.py index a7f34b19..05a2eb66 100644 --- a/bot_bottle/orchestrator/registry.py +++ b/bot_bottle/orchestrator/store/registry_store.py @@ -36,9 +36,9 @@ from collections.abc import Iterable from dataclasses import dataclass from pathlib import Path -from ..db_store import DbStore -from ..migrations import TableMigrations -from ..paths import host_db_path +from ...store.db_store import DbStore +from ...store.migrations import TableMigrations +from ...paths import host_db_path # 256 bits of urandom, URL-safe — unguessable per-bottle identity token. IDENTITY_TOKEN_BYTES = 32 diff --git a/bot_bottle/orchestrator/secret_store.py b/bot_bottle/orchestrator/store/secret_store.py similarity index 100% rename from bot_bottle/orchestrator/secret_store.py rename to bot_bottle/orchestrator/store/secret_store.py diff --git a/bot_bottle/store_manager.py b/bot_bottle/orchestrator/store/store_manager.py similarity index 54% rename from bot_bottle/store_manager.py rename to bot_bottle/orchestrator/store/store_manager.py index f8807a40..2e40e0fe 100644 --- a/bot_bottle/store_manager.py +++ b/bot_bottle/orchestrator/store/store_manager.py @@ -1,34 +1,34 @@ -"""Singleton manager for all bot-bottle SQLite stores (PRD 0013).""" +"""The orchestrator's store manager (PRD 0013 / 0070). + +One store manager per service. Post-PRD-0070 the orchestrator is the sole +opener of `bot-bottle.db`, so it owns migrating every table in it: the supervise +`queue_store` (local to `orchestrator.store`) plus the `audit_store` and +`config_store` from the shared `bot_bottle.store` base. The data plane never +touches these — it reaches state over the control-plane RPC. +""" from __future__ import annotations from pathlib import Path -try: - from .audit_store import AuditStore - from .config_store import ConfigStore - from .queue_store import QueueStore -except ImportError: - from audit_store import AuditStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from config_store import ConfigStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from queue_store import QueueStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module +from .queue_store import QueueStore +from ...store.audit_store import AuditStore +from ...store.config_store import ConfigStore _instance: StoreManager | None = None class StoreManager: - """Owns db_path and delegates migrate/is_migrated across all stores. + """Owns db_path and delegates migrate/is_migrated across the orchestrator's + stores. - Use instance() for normal access. Call reset(db_path) in tests to swap - the singleton to a temp path, then reset() with no args to restore the + Use instance() for normal access. Call reset(db_path) in tests to swap the + singleton to a temp path, then reset() with no args to restore the default.""" def __init__(self, db_path: Path | None = None) -> None: if db_path is None: - try: - from .paths import host_db_path - except ImportError: - from paths import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module + from ...paths import host_db_path db_path = host_db_path() self.db_path = db_path diff --git a/bot_bottle/orchestrator/supervisor.py b/bot_bottle/orchestrator/supervisor.py new file mode 100644 index 00000000..7db78398 --- /dev/null +++ b/bot_bottle/orchestrator/supervisor.py @@ -0,0 +1,91 @@ +"""The orchestrator's supervise service. + +`Supervisor` is the service object the `Orchestrator` owns and calls: it wraps +the supervise queue + audit stores (queue proposals, record operator responses, +write audit entries, stage the DB at launch). Binding those operations to one +object — optionally scoped to a `db_path` — makes the orchestrator's supervise +dependency explicit and injectable in tests. + +For convenience this package also re-exports the neutral vocabulary +(`bot_bottle.supervisor`'s types + `SupervisePlan`), so orchestrator-side +callers import from one place. (Generic helpers like `render_diff` / +`sha256_hex` live in `bot_bottle.util`.) The data plane must NOT import this +package — it imports +`bot_bottle.supervisor` (neutral) directly and reaches the queue over the +control-plane RPC. +""" + +from __future__ import annotations + +from pathlib import Path + +from ..supervisor.types import * # noqa: F401,F403 — re-export the neutral vocabulary +from ..supervisor.plan import SupervisePlan +from .store.store_manager import StoreManager +from .store.queue_store import QueueStore +from ..store.audit_store import AuditStore +from ..supervisor.types import AuditEntry, Proposal, Response + + +class Supervisor: + """Host-side supervise service: the queue + audit I/O and launch-time DB + staging the orchestrator drives. Stateless apart from an optional `db_path` + (None → the host DB, resolved per store as before), so a test can point a + `Supervisor` at a temp database.""" + + def __init__(self, db_path: Path | None = None) -> None: + self._db_path = db_path + + def _queue(self, queue_key: str) -> QueueStore: + return QueueStore(queue_key, self._db_path) + + def _audit(self) -> AuditStore: + return AuditStore(self._db_path) + + # --- Queue I/O --------------------------------------------------------- + + def write_proposal(self, proposal: Proposal) -> Path: + """Persist `proposal` in the queue database, mode 0o600.""" + return self._queue(proposal.bottle_slug).write_proposal(proposal) + + def read_proposal(self, bottle_slug: str, proposal_id: str) -> Proposal: + return self._queue(bottle_slug).read_proposal(proposal_id) + + def list_pending_proposals(self, bottle_slug: str) -> list[Proposal]: + """A single bottle's undecided proposals, FIFO by arrival.""" + return self._queue(bottle_slug).list_pending_proposals() + + def list_all_pending_proposals(self) -> list[Proposal]: + """All pending proposals across bottles, sorted FIFO.""" + return self._queue("").list_all_pending_proposals() + + def write_response(self, bottle_slug: str, response: Response) -> Path: + return self._queue(bottle_slug).write_response(response) + + def read_response(self, bottle_slug: str, proposal_id: str) -> Response: + return self._queue(bottle_slug).read_response(proposal_id) + + def archive_all_proposals(self, bottle_slug: str) -> None: + """Archive every proposal + response for `bottle_slug` (bottle teardown + / reconcile cleanup). Idempotent.""" + self._queue(bottle_slug).archive_all() + + # --- Audit log --------------------------------------------------------- + + def write_audit_entry(self, entry: AuditEntry) -> Path: + """Append `entry` to the host supervise audit table.""" + return self._audit().write_audit_entry(entry) + + def read_audit_entries(self, component: str, slug: str) -> list[AuditEntry]: + """Load all audit entries for the given component+slug.""" + return self._audit().read_audit_entries(component, slug) + + # --- Launch-time staging ---------------------------------------------- + + def prepare(self, slug: str, stage_dir: Path) -> SupervisePlan: + """Stage the host database and return the plan. Called by the backend at + launch; `internal_network` is set by the launch step before `.start`.""" + del stage_dir + mgr = StoreManager(self._db_path) if self._db_path else StoreManager.instance() + mgr.migrate() + return SupervisePlan(slug=slug, db_path=mgr.db_path) diff --git a/bot_bottle/control_auth.py b/bot_bottle/orchestrator_auth.py similarity index 97% rename from bot_bottle/control_auth.py rename to bot_bottle/orchestrator_auth.py index 23cf3513..54f47620 100644 --- a/bot_bottle/control_auth.py +++ b/bot_bottle/orchestrator_auth.py @@ -15,8 +15,8 @@ Only the orchestrator (and the host CLI, which shares the host trust domain) holds the signing key; the gateway is handed a pre-minted ``gateway`` token it cannot rewrite into a ``cli`` token. So a compromised data-plane process can no longer approve its own supervise proposals or drive operator routes — it can -only present the ``gateway`` role it was issued (control_plane rejects it on -operator routes with 403). +only present the ``gateway`` role it was issued (the control plane rejects it +on operator routes with 403). Stdlib-only (HMAC-SHA256 over a JSON payload); no JWT dependency — the project carries no runtime pip deps. Tokens are **signed, not encrypted** (the role is diff --git a/bot_bottle/paths.py b/bot_bottle/paths.py index 629edb55..7b991cbf 100644 --- a/bot_bottle/paths.py +++ b/bot_bottle/paths.py @@ -33,20 +33,19 @@ HOST_DB_FILENAME = "bot-bottle.db" # The per-host control-plane secret file, and the env var the launchers inject # its value into. The control plane requires this secret on every mutating / -# reading route (see orchestrator/control_plane.py); it is held only by the +# reading route (see orchestrator/server.py); it is held only by the # trusted callers (control plane, gateway, host CLI) and never handed to an # agent, so an agent that can reach the control-plane port still can't drive it. -CONTROL_PLANE_TOKEN_FILENAME = "control-plane-token" -# The env var carrying the control-plane *signing key* — held only by the +ORCHESTRATOR_TOKEN_FILENAME = "orchestrator-token" +# The env var carrying the orchestrator's *signing key* — held only by the # orchestrator (to verify tokens) and the host CLI (to mint its own), never by -# the data plane. Same value as the host token file; the name is unchanged for -# backward compatibility with existing launchers. -CONTROL_PLANE_TOKEN_ENV = "BOT_BOTTLE_CONTROL_PLANE_TOKEN" +# the data plane. Same value as the host token file. +ORCHESTRATOR_TOKEN_ENV = "BOT_BOTTLE_ORCHESTRATOR_TOKEN" # The env var carrying the data plane's pre-minted `gateway`-role token (a # signed JWT the launcher mints from the signing key). The gateway presents this # on /resolve + /supervise/{propose,poll}; it never holds the signing key, so it # cannot forge a higher-privilege `cli` token (issue #469 review). -CONTROL_AUTH_JWT_ENV = "BOT_BOTTLE_CONTROL_AUTH_JWT" +ORCHESTRATOR_AUTH_JWT_ENV = "BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT" # The host directory holding the gateway's persistent mitmproxy CA. Bind-mounted # into the infra/gateway container at mitmproxy's confdir so the self-generated @@ -98,7 +97,7 @@ def host_gateway_ca_dir() -> Path: return ca_dir -def host_control_plane_token() -> str: +def host_orchestrator_token() -> str: """The per-host control-plane secret, minted (256-bit, url-safe) and persisted 0600 on first use, then reused. @@ -107,7 +106,7 @@ def host_control_plane_token() -> str: *host* artifact — the file lives under the root the agent never mounts, and the env var is set only on the trusted containers — so reading it here is safe on the host launch path but the value never reaches a bottle.""" - path = bot_bottle_root() / CONTROL_PLANE_TOKEN_FILENAME + path = bot_bottle_root() / ORCHESTRATOR_TOKEN_FILENAME try: existing = path.read_text().strip() if existing: @@ -131,13 +130,13 @@ def host_control_plane_token() -> str: __all__ = [ "HOST_DB_FILENAME", - "CONTROL_PLANE_TOKEN_FILENAME", - "CONTROL_PLANE_TOKEN_ENV", - "CONTROL_AUTH_JWT_ENV", + "ORCHESTRATOR_TOKEN_FILENAME", + "ORCHESTRATOR_TOKEN_ENV", + "ORCHESTRATOR_AUTH_JWT_ENV", "GATEWAY_CA_DIRNAME", "bot_bottle_root", "host_db_path", "host_db_dir", "host_gateway_ca_dir", - "host_control_plane_token", + "host_orchestrator_token", ] diff --git a/bot_bottle/store/__init__.py b/bot_bottle/store/__init__.py new file mode 100644 index 00000000..0a7f1f32 --- /dev/null +++ b/bot_bottle/store/__init__.py @@ -0,0 +1,13 @@ +"""SQLite-backed persistence base for bot-bottle. + +The shared store base: the `DbStore` class (versioned `migrations`) and the +concrete stores built on it that aren't owned by a single service — +`ConfigStore` and `AuditStore`. + +Service-owned stores live under their service: the supervise `queue_store`, the +`secret_store`, the orchestrator `config_store`, and the orchestrator +`store_manager` are under `bot_bottle.orchestrator.store`. + +Callers import the concrete module they need directly, e.g. +`from bot_bottle.store.db_store import DbStore`. +""" diff --git a/bot_bottle/audit_store.py b/bot_bottle/store/audit_store.py similarity index 80% rename from bot_bottle/audit_store.py rename to bot_bottle/store/audit_store.py index 2f48f7c8..d80451b1 100644 --- a/bot_bottle/audit_store.py +++ b/bot_bottle/store/audit_store.py @@ -5,16 +5,10 @@ from __future__ import annotations import sqlite3 from pathlib import Path -try: - from .supervise_types import AuditEntry - from .paths import host_db_path - from .db_store import DbStore - from .migrations import TableMigrations -except ImportError: - from supervise_types import AuditEntry # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from paths import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from db_store import DbStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module +from ..supervisor.types import AuditEntry +from ..paths import host_db_path +from .db_store import DbStore +from .migrations import TableMigrations class AuditStore(DbStore): diff --git a/bot_bottle/config_store.py b/bot_bottle/store/config_store.py similarity index 80% rename from bot_bottle/config_store.py rename to bot_bottle/store/config_store.py index bc2f607b..8d7811e7 100644 --- a/bot_bottle/config_store.py +++ b/bot_bottle/store/config_store.py @@ -4,14 +4,9 @@ from __future__ import annotations from pathlib import Path -try: - from .db_store import DbStore - from .migrations import TableMigrations - from .paths import host_db_path -except ImportError: - from db_store import DbStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from paths import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module +from .db_store import DbStore +from .migrations import TableMigrations +from ..paths import host_db_path DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS = 1 diff --git a/bot_bottle/db_store.py b/bot_bottle/store/db_store.py similarity index 90% rename from bot_bottle/db_store.py rename to bot_bottle/store/db_store.py index 256f29e0..49bcced0 100644 --- a/bot_bottle/db_store.py +++ b/bot_bottle/store/db_store.py @@ -6,10 +6,7 @@ import sqlite3 from contextlib import contextmanager from pathlib import Path -try: - from .migrations import TableMigrations -except ImportError: - from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module +from .migrations import TableMigrations class DbVersionError(Exception): diff --git a/bot_bottle/migrations.py b/bot_bottle/store/migrations.py similarity index 100% rename from bot_bottle/migrations.py rename to bot_bottle/store/migrations.py diff --git a/bot_bottle/supervise.py b/bot_bottle/supervise.py deleted file mode 100644 index 184500f9..00000000 --- a/bot_bottle/supervise.py +++ /dev/null @@ -1,298 +0,0 @@ -"""Per-bottle supervise plane (PRD 0013). - -The supervise plane is the per-bottle MCP daemon plus its host-side -queue/audit support. The daemon (bot_bottle.supervise_server) -sits on the bottle's internal network and exposes MCP tools the agent -calls when it needs an operator-reviewed egress change: - - * egress-block / allow — agent proposes a new routes.yaml - -Each tool call: the agent passes the full proposed file plus a -justification text. The gateway validates the proposal syntactically, -writes it to the host SQLite queue table, and holds the tool-call -connection open. The operator's supervise TUI -(bot_bottle.cli.supervise) sees the proposal, accepts -approve / modify / reject, and writes a response row. The gateway sees -the response and returns `{status, notes}` to the agent. - -This module defines the host-side library: dataclasses for the queue -record shapes, queue read/write helpers, the audit log writer, and the -diff renderer. The in-gateway daemon lives in -bot_bottle/supervise_server.py; the supervise daemon's container -lifecycle is owned by the gateway (PRD 0024). - -For 0013 the supervisor's approval handlers are deliberately no-ops: -on approval the audit log is written and the response file is -delivered to the agent, but no host-side config change happens. The -remediation engines that wire real config changes land in PRDs 0014, -0015, and 0016. -""" - -from __future__ import annotations - -import difflib -import hashlib -import time -from abc import ABC -from dataclasses import dataclass -from pathlib import Path - -from .supervise_types import ( - ACTION_OPERATOR_EDIT, - AuditEntry, - POLL_STATUS_PENDING, - POLL_STATUS_UNKNOWN, - Proposal, - Response, - STATUSES, - STATUS_APPROVED, - STATUS_MODIFIED, - STATUS_REJECTED, - TOOLS, - TOOL_CHECK_PROPOSAL, - TOOL_EGRESS_ALLOW, - TOOL_EGRESS_BLOCK, - TOOL_EGRESS_TOKEN_ALLOW, - TOOL_GITLEAKS_ALLOW, - TOOL_LIST_EGRESS_ROUTES, -) - - -try: - from .paths import bot_bottle_root -except ImportError: # flat imports inside the gateway - from paths import bot_bottle_root # type: ignore[import-not-found,no-redef] # pylint: disable=import-error,no-name-in-module - - -SUPERVISE_HOSTNAME = "supervise" -SUPERVISE_PORT = 9100 - -# The supervise daemon uses these to query egress's -# introspection endpoint for the `list-egress-routes` MCP -# tool. The hostname + port match egress's docker network -# listen port (see backend.docker.egress.EGRESS_PORT). The supervise -# daemon runs inside the gateway alongside egress, so loopback -# is the stable address across docker, firecracker, and Apple -# Container backends. -EGRESS_FORWARD_PROXY = "http://127.0.0.1:9099" -EGRESS_INTROSPECT_URL = "http://_egress.local/allowlist" - -COMPONENT_FOR_TOOL: dict[str, str] = { - TOOL_EGRESS_ALLOW: "egress", - TOOL_EGRESS_BLOCK: "egress", -} - -DB_PATH_IN_CONTAINER = "/run/supervise/bot-bottle.db" -DEFAULT_POLL_INTERVAL_SEC = 0.5 - - -# --- Paths ----------------------------------------------------------------- - - -def audit_dir() -> Path: - return bot_bottle_root() / "audit" - - -def audit_log_path(component: str, slug: str) -> Path: - return audit_dir() / f"{component}-{slug}.log" - - -try: - from .queue_store import QueueStore - from .audit_store import AuditStore - from .store_manager import StoreManager -except ImportError: - # Gateway: files are flat-copied under /app, not a package. - from queue_store import QueueStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from audit_store import AuditStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - from store_manager import StoreManager # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module - - -# --- Queue I/O ------------------------------------------------------------- - - -def write_proposal(proposal: Proposal) -> Path: - """Persist `proposal` in the queue database, mode 0o600. - Directory is created if missing.""" - return QueueStore(proposal.bottle_slug).write_proposal(proposal) - - -def read_proposal(bottle_slug: str, proposal_id: str) -> Proposal: - return QueueStore(bottle_slug).read_proposal(proposal_id) - - -def list_pending_proposals(bottle_slug: str) -> list[Proposal]: - """All proposals for `bottle_slug` that do not yet have a matching - response. Sorted by `arrival_timestamp` so the operator - sees the queue FIFO.""" - return QueueStore(bottle_slug).list_pending_proposals() - - -def list_all_pending_proposals() -> list[Proposal]: - """All pending proposals across bottles, sorted FIFO.""" - return QueueStore("").list_all_pending_proposals() - - -def write_response(bottle_slug: str, response: Response) -> Path: - return QueueStore(bottle_slug).write_response(response) - - -def read_response(bottle_slug: str, proposal_id: str) -> Response: - return QueueStore(bottle_slug).read_response(proposal_id) - - -def wait_for_response( - bottle_slug: str, - proposal_id: str, - *, - poll_interval: float = DEFAULT_POLL_INTERVAL_SEC, - deadline: float | None = None, -) -> Response: - """Block until a response file appears for `proposal_id`, then - return it. `deadline` is an absolute time.monotonic() value after - which the wait raises TimeoutError. None waits forever — the - natural shape, since the operator's response time is unbounded. - - Polls SQLite so the implementation stays portable and stdlib-only.""" - store = QueueStore(bottle_slug) - while True: - try: - return store.read_response(proposal_id) - except FileNotFoundError: - pass - if deadline is not None and time.monotonic() >= deadline: - raise TimeoutError(f"no response for proposal {proposal_id!r}") - time.sleep(poll_interval) - - -def archive_proposal(bottle_slug: str, proposal_id: str) -> None: - """Mark both proposal and response rows processed. - Idempotent — missing rows are silently skipped.""" - QueueStore(bottle_slug).archive_proposal(proposal_id) - - -def archive_all_proposals(bottle_slug: str) -> None: - """Archive every proposal + response for `bottle_slug` (bottle teardown / - reconcile cleanup). Idempotent.""" - QueueStore(bottle_slug).archive_all() - - -# --- Audit log ------------------------------------------------------------- - - -def write_audit_entry(entry: AuditEntry) -> Path: - """Append `entry` to the host supervise audit table.""" - return AuditStore().write_audit_entry(entry) - - -def read_audit_entries(component: str, slug: str) -> list[AuditEntry]: - """Load all audit entries for the given component+slug.""" - return AuditStore().read_audit_entries(component, slug) - - -# --- Diff rendering -------------------------------------------------------- - - -def render_diff(before: str, after: str, *, label: str = "config") -> str: - """Unified diff suitable for the audit log + TUI. Empty diff (no - changes) renders as the empty string.""" - diff = difflib.unified_diff( - before.splitlines(keepends=True), - after.splitlines(keepends=True), - fromfile=f"{label} (current)", - tofile=f"{label} (proposed)", - lineterm="", - ) - parts = list(diff) - if not parts: - return "" - return "".join(p if p.endswith("\n") else p + "\n" for p in parts).rstrip("\n") - - -def sha256_hex(content: str) -> str: - return hashlib.sha256(content.encode("utf-8")).hexdigest() - - -# --- Gateway plan + abstract lifecycle ------------------------------------- - - -@dataclass(frozen=True) -class SupervisePlan: - """Output of Supervise.prepare; consumed by .start. - - `db_path` is the host database bind-mounted into the gateway at - /run/supervise/bot-bottle.db. `internal_network` is empty at - prepare time; the backend's launch step fills it via - dataclasses.replace before calling .start.""" - - slug: str - db_path: Path - internal_network: str = "" - - -class Supervise(ABC): - """Per-bottle supervise daemon. Encapsulates host-side database - staging; the gateway's start/stop lifecycle is backend-specific.""" - - def prepare( - self, - slug: str, - stage_dir: Path, - ) -> SupervisePlan: - """Stage the host database. Returns the plan; `internal_network` - must be set by the launch step before .start runs.""" - del stage_dir - mgr = StoreManager.instance() - mgr.migrate() - return SupervisePlan( - slug=slug, - db_path=mgr.db_path, - ) - - -__all__ = [ - "ACTION_OPERATOR_EDIT", - "AuditEntry", - "AuditStore", - "COMPONENT_FOR_TOOL", - "DEFAULT_POLL_INTERVAL_SEC", - "DB_PATH_IN_CONTAINER", - "Proposal", - "QueueStore", - "Response", - "StoreManager", - "STATUSES", - "STATUS_APPROVED", - "STATUS_MODIFIED", - "STATUS_REJECTED", - "POLL_STATUS_PENDING", - "POLL_STATUS_UNKNOWN", - "SUPERVISE_HOSTNAME", - "SUPERVISE_PORT", - "Supervise", - "SupervisePlan", - "TOOLS", - "EGRESS_FORWARD_PROXY", - "EGRESS_INTROSPECT_URL", - "TOOL_CHECK_PROPOSAL", - "TOOL_EGRESS_ALLOW", - "TOOL_EGRESS_BLOCK", - "TOOL_GITLEAKS_ALLOW", - "TOOL_EGRESS_TOKEN_ALLOW", - "TOOL_LIST_EGRESS_ROUTES", - "archive_proposal", - "archive_all_proposals", - "audit_dir", - "audit_log_path", - "list_pending_proposals", - "list_all_pending_proposals", - "read_audit_entries", - "read_proposal", - "read_response", - "render_diff", - "sha256_hex", - "wait_for_response", - "write_audit_entry", - "write_proposal", - "write_response", -] diff --git a/bot_bottle/supervisor/__init__.py b/bot_bottle/supervisor/__init__.py new file mode 100644 index 00000000..7d002f9e --- /dev/null +++ b/bot_bottle/supervisor/__init__.py @@ -0,0 +1,16 @@ +"""Neutral supervise vocabulary — importable by every tier. + +The supervise plane spans the control plane (orchestrator) and the data plane +(gateway `supervise_server`): the gateway proposes/polls over RPC, the +orchestrator queues and applies operator decisions. Both sides must agree on +the wire types and constants, so those live here — a neutral package neither +tier's private code, importable by `bot_bottle.gateway` **and** +`bot_bottle.orchestrator` without either depending on the other. + + * `types` — the `Proposal`/`Response`/`AuditEntry` dataclasses, the tool / + status / poll-status constants, and the shared daemon constants. + * `plan` — `SupervisePlan`, the launch-time staging DTO. + +The orchestrator-only half (queue I/O, diff rendering, the `Supervisor` +lifecycle) lives under `bot_bottle.orchestrator.supervisor`. +""" diff --git a/bot_bottle/supervisor/plan.py b/bot_bottle/supervisor/plan.py new file mode 100644 index 00000000..e61cf915 --- /dev/null +++ b/bot_bottle/supervisor/plan.py @@ -0,0 +1,26 @@ +"""`SupervisePlan` — the launch-time supervise staging DTO. + +A pure value type (no store access), so it lives in the neutral package: the +backend builds it at launch and the orchestrator's `Supervisor.prepare` returns +it. The behaviour that fills it in — staging the host database — is the +orchestrator's, in `bot_bottle.orchestrator.supervisor.Supervisor`. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class SupervisePlan: + """Output of `Supervisor.prepare`; consumed by `.start`. + + `db_path` is the host database bind-mounted into the gateway at + /run/supervise/bot-bottle.db. `internal_network` is empty at prepare time; + the backend's launch step fills it via `dataclasses.replace` before calling + `.start`.""" + + slug: str + db_path: Path + internal_network: str = "" diff --git a/bot_bottle/supervise_types.py b/bot_bottle/supervisor/types.py similarity index 82% rename from bot_bottle/supervise_types.py rename to bot_bottle/supervisor/types.py index 6bb65e5d..cd6c3913 100644 --- a/bot_bottle/supervise_types.py +++ b/bot_bottle/supervisor/types.py @@ -15,6 +15,8 @@ import uuid from dataclasses import dataclass from datetime import datetime, timezone +from ..util import sha256_hex + TOOL_EGRESS_BLOCK = "egress-block" TOOL_EGRESS_ALLOW = "egress-allow" TOOL_GITLEAKS_ALLOW = "gitleaks-allow" @@ -48,6 +50,24 @@ POLL_STATUS_UNKNOWN = "unknown" ACTION_OPERATOR_EDIT = "operator-edit" +# Shared supervise-daemon constants. The gateway `supervise_server` and the +# backend read these, so they live with the wire types in the neutral package +# (never in the orchestrator's private code the data plane can't import). +SUPERVISE_HOSTNAME = "supervise" +SUPERVISE_PORT = 9100 +# The supervise daemon queries egress's introspection endpoint for the +# `list-egress-routes` tool over loopback (egress runs alongside it in the +# gateway), stable across docker / firecracker / Apple Container backends. +EGRESS_FORWARD_PROXY = "http://127.0.0.1:9099" +EGRESS_INTROSPECT_URL = "http://_egress.local/allowlist" +DB_PATH_IN_CONTAINER = "/run/supervise/bot-bottle.db" +DEFAULT_POLL_INTERVAL_SEC = 0.5 + +COMPONENT_FOR_TOOL: dict[str, str] = { + TOOL_EGRESS_ALLOW: "egress", + TOOL_EGRESS_BLOCK: "egress", +} + def _require_str(raw: dict[str, object], key: str) -> str: value = raw.get(key) @@ -76,7 +96,6 @@ class Proposal: tool: str, proposed_file: str, justification: str, - current_file_hash: str, now: datetime | None = None, ) -> "Proposal": ts = (now or datetime.now(timezone.utc)).isoformat() @@ -87,7 +106,7 @@ class Proposal: proposed_file=proposed_file, justification=justification, arrival_timestamp=ts, - current_file_hash=current_file_hash, + current_file_hash=sha256_hex(proposed_file), ) def to_dict(self) -> dict[str, object]: @@ -175,4 +194,11 @@ __all__ = [ "TOOL_EGRESS_TOKEN_ALLOW", "TOOL_GITLEAKS_ALLOW", "TOOL_LIST_EGRESS_ROUTES", + "SUPERVISE_HOSTNAME", + "SUPERVISE_PORT", + "EGRESS_FORWARD_PROXY", + "EGRESS_INTROSPECT_URL", + "DB_PATH_IN_CONTAINER", + "DEFAULT_POLL_INTERVAL_SEC", + "COMPONENT_FOR_TOOL", ] diff --git a/bot_bottle/util.py b/bot_bottle/util.py index 8cc09200..767e365f 100644 --- a/bot_bottle/util.py +++ b/bot_bottle/util.py @@ -5,11 +5,43 @@ level deeper, under their backend package.""" from __future__ import annotations +import difflib +import hashlib import ipaddress import os import sys +def sha256_hex(content: str) -> str: + """Hex SHA-256 of a UTF-8 string.""" + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + +def render_diff( + before: str, + after: str, + *, + label: str = "config", + before_title: str, + after_label: str, +) -> str: + """Unified diff of `before` vs `after`, with the two sides headed + `{label} ({before_title})` and `{label} ({after_label})`. Empty diff (no + changes) renders as the empty string. The side titles are the caller's — + e.g. "current"/"proposed" for a supervise proposal.""" + diff = difflib.unified_diff( + before.splitlines(keepends=True), + after.splitlines(keepends=True), + fromfile=f"{label} ({before_title})", + tofile=f"{label} ({after_label})", + lineterm="", + ) + parts = list(diff) + if not parts: + return "" + return "".join(p if p.endswith("\n") else p + "\n" for p in parts).rstrip("\n") + + def is_ip_literal(value: str) -> bool: try: ipaddress.ip_address(value) diff --git a/scripts/critical-modules.txt b/scripts/critical-modules.txt index fa542f22..845e4154 100644 --- a/scripts/critical-modules.txt +++ b/scripts/critical-modules.txt @@ -8,18 +8,18 @@ # # One module path per line, relative to the repo root. Blank lines and # `#` comments are ignored. -bot_bottle/egress_addon.py -bot_bottle/egress_addon_core.py -bot_bottle/dlp_detectors.py +bot_bottle/gateway/egress/addon.py +bot_bottle/gateway/egress/addon_core.py +bot_bottle/gateway/egress/dlp_detectors.py bot_bottle/egress.py bot_bottle/manifest.py bot_bottle/manifest_egress.py bot_bottle/manifest_agent.py bot_bottle/manifest_schema.py bot_bottle/git_gate.py -bot_bottle/git_gate_render.py +bot_bottle/gateway/git_gate/render.py bot_bottle/git_gate_provision.py -bot_bottle/git_http_backend.py +bot_bottle/gateway/git_gate/http_backend.py bot_bottle/supervise.py bot_bottle/yaml_subset.py bot_bottle/bottle_state.py diff --git a/tests/integration/test_gateway_image.py b/tests/integration/test_gateway_image.py index 87e5d971..7c02a28b 100644 --- a/tests/integration/test_gateway_image.py +++ b/tests/integration/test_gateway_image.py @@ -91,7 +91,7 @@ class TestGatewayImage(unittest.TestCase): # Probe that the package imports resolve inside the image. rc, out = self._run_in_image( "python3", "-c", - "from bot_bottle import supervise, supervise_server; print('ok')", + "from bot_bottle.supervisor import types; from bot_bottle.gateway.supervisor import server as supervise_server; print('ok')", ) self.assertEqual(0, rc, msg=out) self.assertIn("ok", out) diff --git a/tests/integration/test_multitenant_isolation.py b/tests/integration/test_multitenant_isolation.py index 7ed30314..e32f9e2f 100644 --- a/tests/integration/test_multitenant_isolation.py +++ b/tests/integration/test_multitenant_isolation.py @@ -29,8 +29,8 @@ from bot_bottle.backend.docker.consolidated_launch import ( from bot_bottle.backend.docker.egress import EGRESS_PORT from bot_bottle.backend.docker.gateway_net import next_free_ip from bot_bottle.orchestrator.client import OrchestratorClient -from bot_bottle.orchestrator.gateway import GATEWAY_IMAGE, GATEWAY_NAME, GATEWAY_NETWORK -from bot_bottle.orchestrator.lifecycle import OrchestratorService +from bot_bottle.gateway import GATEWAY_IMAGE, GATEWAY_NAME, GATEWAY_NETWORK +from bot_bottle.backend.docker.infra import DockerInfraService from tests._docker import skip_unless_docker # One upstream reachable under two names; reflects the Authorization header so @@ -85,7 +85,7 @@ class TestMultitenantIsolation(unittest.TestCase): self._tmp = tempfile.TemporaryDirectory() self.addCleanup(self._tmp.cleanup) # Throwaway root → a clean registry DB, independent of the host's. - self.svc = OrchestratorService(host_root=Path(self._tmp.name)) + self.svc = DockerInfraService(host_root=Path(self._tmp.name)) self.addCleanup(self._teardown_docker) # ensure_running builds the bundle image (slow on a cold cache) and # brings up the shared network + gateway + orchestrator. diff --git a/tests/integration/test_orchestrator_docker_control_plane_auth.py b/tests/integration/test_orchestrator_docker_auth.py similarity index 92% rename from tests/integration/test_orchestrator_docker_control_plane_auth.py rename to tests/integration/test_orchestrator_docker_auth.py index aedb8caa..50ecaa58 100644 --- a/tests/integration/test_orchestrator_docker_control_plane_auth.py +++ b/tests/integration/test_orchestrator_docker_auth.py @@ -25,10 +25,10 @@ import tempfile import unittest from pathlib import Path -from bot_bottle.control_auth import ROLE_CLI, ROLE_GATEWAY, mint +from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY, mint from bot_bottle.orchestrator.client import OrchestratorClient -from bot_bottle.orchestrator.lifecycle import OrchestratorService -from bot_bottle.paths import host_control_plane_token +from bot_bottle.backend.docker.infra import DockerInfraService +from bot_bottle.paths import host_orchestrator_token from tests._docker import skip_unless_docker # Fixed (not per-run-suffixed) so repeated runs reuse the same layer-cached @@ -46,20 +46,20 @@ _TEST_INFRA_IMAGE = "bot-bottle-infra:itest" "runner's /workspace — same host-bind-mount constraint as the other " "bottle-bringup integration tests", ) -class TestDockerControlPlaneAuthIntegration(unittest.TestCase): +class TestDockerOrchestratorAuthIntegration(unittest.TestCase): @classmethod def setUpClass(cls) -> None: suffix = secrets.token_hex(4) cls._tmp = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with cls.addClassCleanup(cls._tmp.cleanup) - # host_control_plane_token() — both the token read below and the one - # OrchestratorService injects into the container's env — resolves its + # host_orchestrator_token() — both the token read below and the one + # DockerInfraService injects into the container's env — resolves its # path via the *ambient* BOT_BOTTLE_ROOT env var, not the host_root # kwarg passed to the constructor (that kwarg only controls the DB # bind-mount destination). Without pointing the env var at the same # throwaway dir, this "isolated" test would read/write the developer's - # real ~/.bot-bottle/control-plane-token. + # real ~/.bot-bottle/orchestrator-token. previous_root = os.environ.get("BOT_BOTTLE_ROOT") def _restore_root() -> None: @@ -78,7 +78,7 @@ class TestDockerControlPlaneAuthIntegration(unittest.TestCase): cls._teardown_docker, infra_name, network, host_root ) - cls.svc = OrchestratorService( + cls.svc = DockerInfraService( infra_name=infra_name, network=network, image=_TEST_INFRA_IMAGE, @@ -88,7 +88,7 @@ class TestDockerControlPlaneAuthIntegration(unittest.TestCase): cls.svc.ensure_running() # The control plane now verifies role-scoped signed tokens, not the raw # key. Mint one of each role from the host signing key (issue #469 review). - signing_key = host_control_plane_token() + signing_key = host_orchestrator_token() cls.cli_token = mint(ROLE_CLI, signing_key) cls.gateway_token = mint(ROLE_GATEWAY, signing_key) diff --git a/tests/integration/test_orchestrator_docker_gateway.py b/tests/integration/test_orchestrator_docker_gateway.py index 38e4b762..453dccd1 100644 --- a/tests/integration/test_orchestrator_docker_gateway.py +++ b/tests/integration/test_orchestrator_docker_gateway.py @@ -10,7 +10,7 @@ import secrets import subprocess import unittest -from bot_bottle.orchestrator.gateway import DockerGateway +from bot_bottle.backend.docker.gateway import DockerGateway from tests._docker import skip_unless_docker IMAGE = "busybox" diff --git a/tests/integration/test_orchestrator_docker_gateway_build.py b/tests/integration/test_orchestrator_docker_gateway_build.py index 5de444f0..4eb59cb0 100644 --- a/tests/integration/test_orchestrator_docker_gateway_build.py +++ b/tests/integration/test_orchestrator_docker_gateway_build.py @@ -11,7 +11,7 @@ from __future__ import annotations import subprocess import unittest -from bot_bottle.orchestrator.gateway import DockerGateway +from bot_bottle.backend.docker.gateway import DockerGateway from tests._docker import skip_unless_docker IMAGE = "busybox" diff --git a/tests/unit/_docker_bottle_plan.py b/tests/unit/_docker_bottle_plan.py index c3a16c2f..78f4b19e 100644 --- a/tests/unit/_docker_bottle_plan.py +++ b/tests/unit/_docker_bottle_plan.py @@ -16,7 +16,7 @@ from bot_bottle.backend.docker.bottle_plan import DockerBottlePlan from bot_bottle.egress import EgressPlan, EgressRoute from bot_bottle.git_gate import GitGatePlan, GitGateUpstream from bot_bottle.manifest import ManifestIndex -from bot_bottle.supervise import SupervisePlan +from bot_bottle.supervisor.plan import SupervisePlan SLUG = "demo-abc12" diff --git a/tests/unit/test_backend_selection.py b/tests/unit/test_backend_selection.py index a5a4c187..25b221c4 100644 --- a/tests/unit/test_backend_selection.py +++ b/tests/unit/test_backend_selection.py @@ -13,6 +13,7 @@ import unittest from unittest.mock import patch from bot_bottle import backend as backend_mod +from bot_bottle.backend import selection as _sel from bot_bottle.backend import ( ActiveAgent, enumerate_active_agents, @@ -40,7 +41,7 @@ class TestGetBottleBackend(unittest.TestCase): return True with patch.dict(os.environ, {}, clear=True), \ - patch.object(backend_mod, "_backends", { + patch.object(_sel, "_backends", { "macos-container": _FakeBackend(), "docker": _FakeBackend(), }): @@ -62,11 +63,11 @@ class TestGetBottleBackend(unittest.TestCase): with patch.dict(os.environ, {}, clear=True), \ patch.object(backend_mod.FirecrackerBottleBackend, "is_host_capable", classmethod(lambda cls: False)), \ - patch.object(backend_mod, "_backends", { + patch.object(_sel, "_backends", { "macos-container": _FakeBackend("macos-container", False), "docker": _FakeBackend("docker", True), }), \ - patch.object(backend_mod, "read_tty_line", return_value="d"): + patch.object(_sel, "read_tty_line", return_value="d"): b = get_bottle_backend() self.assertEqual("docker", b.name) @@ -85,7 +86,7 @@ class TestGetBottleBackend(unittest.TestCase): with patch.dict(os.environ, {}, clear=True), \ patch.object(backend_mod.FirecrackerBottleBackend, "is_host_capable", classmethod(lambda cls: True)), \ - patch.object(backend_mod, "_backends", { + patch.object(_sel, "_backends", { "macos-container": _FakeBackend("macos-container", False), "firecracker": _FakeBackend("firecracker", False), "docker": _FakeBackend("docker", True), @@ -94,7 +95,7 @@ class TestGetBottleBackend(unittest.TestCase): self.assertEqual("firecracker", b.name) def test_unknown_dies(self): - with patch.object(backend_mod, "die", side_effect=SystemExit("die")): + with patch.object(_sel, "die", side_effect=SystemExit("die")): with self.assertRaises(SystemExit): get_bottle_backend("nonexistent") @@ -111,11 +112,11 @@ class TestGetBottleBackend(unittest.TestCase): with patch.dict(os.environ, {}, clear=True), \ patch.object(backend_mod.FirecrackerBottleBackend, "is_host_capable", classmethod(lambda cls: False)), \ - patch.object(backend_mod, "_backends", { + patch.object(_sel, "_backends", { "macos-container": _FakeBackend("macos-container", False), "docker": _FakeBackend("docker", False), }), \ - patch.object(backend_mod, "die", side_effect=SystemExit("die")): + patch.object(_sel, "die", side_effect=SystemExit("die")): with self.assertRaises(SystemExit): get_bottle_backend() @@ -132,12 +133,12 @@ class TestGetBottleBackend(unittest.TestCase): with patch.dict(os.environ, {}, clear=True), \ patch.object(backend_mod.FirecrackerBottleBackend, "is_host_capable", classmethod(lambda cls: False)), \ - patch.object(backend_mod, "_backends", { + patch.object(_sel, "_backends", { "macos-container": _FakeBackend("macos-container", False), "docker": _FakeBackend("docker", True), }), \ - patch.object(backend_mod, "read_tty_line", return_value="q"), \ - patch.object(backend_mod, "die", side_effect=SystemExit("die")): + patch.object(_sel, "read_tty_line", return_value="q"), \ + patch.object(_sel, "die", side_effect=SystemExit("die")): with self.assertRaises(SystemExit): get_bottle_backend() @@ -155,11 +156,11 @@ class TestGetBottleBackend(unittest.TestCase): with patch.dict(os.environ, {}, clear=True), \ patch.object(backend_mod.FirecrackerBottleBackend, "is_host_capable", classmethod(lambda cls: False)), \ - patch.object(backend_mod, "_backends", { + patch.object(_sel, "_backends", { "macos-container": _FakeBackend("macos-container", False), "docker": _FakeBackend("docker", True), }), \ - patch.object(backend_mod, "die", side_effect=SystemExit("die")): + patch.object(_sel, "die", side_effect=SystemExit("die")): with self.assertRaises(SystemExit): get_bottle_backend(prompt=False) @@ -176,13 +177,13 @@ class TestGetBottleBackend(unittest.TestCase): with patch.dict(os.environ, {}, clear=True), \ patch.object(backend_mod.FirecrackerBottleBackend, "is_host_capable", classmethod(lambda cls: False)), \ - patch.object(backend_mod, "_backends", { + patch.object(_sel, "_backends", { "macos-container": _FakeBackend("macos-container", False), "docker": _FakeBackend("docker", True), }), \ - patch.object(backend_mod, "read_tty_line", return_value="i"), \ - patch.object(backend_mod, "_print_vm_install_instructions") as mock_inst, \ - patch.object(backend_mod, "die", side_effect=SystemExit("die")): + patch.object(_sel, "read_tty_line", return_value="i"), \ + patch.object(_sel, "_print_vm_install_instructions") as mock_inst, \ + patch.object(_sel, "die", side_effect=SystemExit("die")): with self.assertRaises(SystemExit): get_bottle_backend() mock_inst.assert_called_once() @@ -218,9 +219,9 @@ class TestPrintVmInstallInstructions(unittest.TestCase): def test_linux_prints_firecracker_instructions(self): from bot_bottle.backend import _print_vm_install_instructions - with patch.object(backend_mod, "_platform_vm_suggestion", + with patch.object(_sel, "_platform_vm_suggestion", return_value="firecracker"), \ - patch.object(backend_mod, "info") as mock_info: + patch.object(_sel, "info") as mock_info: _print_vm_install_instructions() messages = [str(c[0][0]) for c in mock_info.call_args_list] @@ -229,9 +230,9 @@ class TestPrintVmInstallInstructions(unittest.TestCase): def test_macos_prints_apple_container_instructions(self): from bot_bottle.backend import _print_vm_install_instructions - with patch.object(backend_mod, "_platform_vm_suggestion", + with patch.object(_sel, "_platform_vm_suggestion", return_value="macos-container"), \ - patch.object(backend_mod, "info") as mock_info: + patch.object(_sel, "info") as mock_info: _print_vm_install_instructions() messages = [str(c[0][0]) for c in mock_info.call_args_list] @@ -274,7 +275,7 @@ class TestEnumerateActiveAgents(unittest.TestCase): return self._items with patch.object( - backend_mod, "_backends", + _sel, "_backends", {"docker": _FakeBackend([a]), "firecracker": _FakeBackend([b])}, ): self.assertEqual([a, b], enumerate_active_agents()) @@ -308,7 +309,7 @@ class TestEnumerateActiveAgents(unittest.TestCase): return self._items with patch.object( - backend_mod, "_backends", + _sel, "_backends", { "docker": _FakeBackend([newer, tie_b]), "firecracker": _FakeBackend([missing_metadata, tie_a]), @@ -328,7 +329,7 @@ class TestEnumerateActiveAgents(unittest.TestCase): return [] with patch.object( - backend_mod, "_backends", + _sel, "_backends", {"docker": _FakeBackend(), "firecracker": _FakeBackend()}, ): self.assertEqual([], enumerate_active_agents()) @@ -359,7 +360,7 @@ class TestEnumerateActiveAgents(unittest.TestCase): return self._items with patch.object( - backend_mod, "_backends", + _sel, "_backends", { "docker": _FakeBackend([present], available=True), "firecracker": _FakeBackend([hidden], available=False), @@ -375,7 +376,7 @@ class TestHasBackend(unittest.TestCase): return False with patch.object( - backend_mod, "_backends", {"docker": _FakeBackend()}, + _sel, "_backends", {"docker": _FakeBackend()}, ): from bot_bottle.backend import has_backend self.assertFalse(has_backend("docker")) @@ -390,10 +391,10 @@ class TestEnsureOrchestrator(unittest.TestCase): the orchestrator + gateway containers; firecracker boots the infra VM; macos-container starts the infra container.""" - def test_docker_delegates_to_orchestrator_service(self): + def test_docker_delegates_to_infra_service(self): b = get_bottle_backend("docker") with patch( - "bot_bottle.orchestrator.lifecycle.OrchestratorService" + "bot_bottle.backend.docker.infra.DockerInfraService" ) as service_cls: service_cls.return_value.ensure_running.return_value = ( "http://127.0.0.1:8099" @@ -407,7 +408,7 @@ class TestEnsureOrchestrator(unittest.TestCase): with patch( "bot_bottle.backend.firecracker.infra_vm.ensure_running" ) as ensure_running: - ensure_running.return_value.control_plane_url = ( + ensure_running.return_value.orchestrator_url = ( "http://10.243.255.1:8099" ) url = b.ensure_orchestrator() @@ -418,7 +419,7 @@ class TestEnsureOrchestrator(unittest.TestCase): with patch( "bot_bottle.backend.macos_container.infra.MacosInfraService" ) as service_cls: - service_cls.return_value.ensure_running.return_value.control_plane_url = ( + service_cls.return_value.ensure_running.return_value.orchestrator_url = ( "http://192.168.128.2:8099" ) url = b.ensure_orchestrator() diff --git a/tests/unit/test_cli_backend.py b/tests/unit/test_cli_backend.py index 16eea2c8..40f2890c 100644 --- a/tests/unit/test_cli_backend.py +++ b/tests/unit/test_cli_backend.py @@ -12,7 +12,7 @@ import subprocess import unittest from unittest.mock import MagicMock, patch -from bot_bottle.cli import backend as cmd +from bot_bottle.cli.commands import backend as cmd from bot_bottle.backend.docker import setup as docker_setup diff --git a/tests/unit/test_cli_cleanup_cross_backend.py b/tests/unit/test_cli_cleanup_cross_backend.py index e1f4be8e..47ab1334 100644 --- a/tests/unit/test_cli_cleanup_cross_backend.py +++ b/tests/unit/test_cli_cleanup_cross_backend.py @@ -11,7 +11,7 @@ from __future__ import annotations import unittest from unittest.mock import patch, MagicMock -from bot_bottle.cli import cleanup as cmd +from bot_bottle.cli.commands import cleanup as cmd def _make_backend(empty: bool = True): diff --git a/tests/unit/test_cli_commit.py b/tests/unit/test_cli_commit.py index 6a7e7fff..9eee5423 100644 --- a/tests/unit/test_cli_commit.py +++ b/tests/unit/test_cli_commit.py @@ -7,7 +7,7 @@ import unittest from pathlib import Path from unittest.mock import MagicMock, patch -from bot_bottle.cli.commit import cmd_commit +from bot_bottle.cli.commands.commit import cmd_commit from tests.unit import use_bottle_root from bot_bottle import bottle_state from bot_bottle.backend.freeze import CommitCancelled @@ -42,7 +42,7 @@ class TestCmdCommitSlugArg(_FakeHomeMixin, unittest.TestCase): slug = "dev-abc12" self._write_meta(slug, "docker") - with patch("bot_bottle.cli.commit.get_freezer") as mock_gf: + with patch("bot_bottle.cli.commands.commit.get_freezer") as mock_gf: mock_freezer = MagicMock() mock_gf.return_value = mock_freezer rc = cmd_commit([slug]) @@ -56,7 +56,7 @@ class TestCmdCommitSlugArg(_FakeHomeMixin, unittest.TestCase): slug = "dev-abc12" self._write_meta(slug, "") - with patch("bot_bottle.cli.commit.get_freezer") as mock_gf: + with patch("bot_bottle.cli.commands.commit.get_freezer") as mock_gf: mock_freezer = MagicMock() mock_gf.return_value = mock_freezer rc = cmd_commit([slug]) @@ -68,7 +68,7 @@ class TestCmdCommitSlugArg(_FakeHomeMixin, unittest.TestCase): slug = "dev-abc12" self._write_meta(slug, "macos-container") - with patch("bot_bottle.cli.commit.get_freezer") as mock_gf: + with patch("bot_bottle.cli.commands.commit.get_freezer") as mock_gf: mock_freezer = MagicMock() mock_gf.return_value = mock_freezer rc = cmd_commit([slug]) @@ -81,7 +81,7 @@ class TestCmdCommitSlugArg(_FakeHomeMixin, unittest.TestCase): slug = "dev-abc12" self._write_meta(slug, "firecracker") - with patch("bot_bottle.cli.commit.get_freezer") as mock_gf: + with patch("bot_bottle.cli.commands.commit.get_freezer") as mock_gf: mock_freezer = MagicMock() mock_gf.return_value = mock_freezer rc = cmd_commit([slug]) @@ -93,7 +93,7 @@ class TestCmdCommitSlugArg(_FakeHomeMixin, unittest.TestCase): slug = "dev-abc12" self._write_meta(slug, "macos-container") - with patch("bot_bottle.cli.commit.get_freezer") as mock_gf: + with patch("bot_bottle.cli.commands.commit.get_freezer") as mock_gf: mock_freezer = MagicMock() mock_freezer.commit_slug.side_effect = CommitCancelled mock_gf.return_value = mock_freezer @@ -111,9 +111,9 @@ class TestCmdCommitNoActiveBottles(_FakeHomeMixin, unittest.TestCase): def test_dies_when_no_active_bottles_and_no_slug(self): with patch( - "bot_bottle.cli.commit.enumerate_active_agents", return_value=[], + "bot_bottle.cli.commands.commit.enumerate_active_agents", return_value=[], ), patch( - "bot_bottle.cli.commit.die", side_effect=SystemExit("die"), + "bot_bottle.cli.commands.commit.die", side_effect=SystemExit("die"), ) as mock_die: with self.assertRaises(SystemExit): cmd_commit([]) @@ -124,9 +124,9 @@ class TestCmdCommitNoActiveBottles(_FakeHomeMixin, unittest.TestCase): active = MagicMock() active.slug = "dev-abc12" with patch( - "bot_bottle.cli.commit.enumerate_active_agents", return_value=[active], + "bot_bottle.cli.commands.commit.enumerate_active_agents", return_value=[active], ), patch( - "bot_bottle.cli.commit.tui.filter_select", return_value=None, + "bot_bottle.cli.commands.commit.tui.filter_select", return_value=None, ): rc = cmd_commit([]) diff --git a/tests/unit/test_cli_dispatch.py b/tests/unit/test_cli_dispatch.py index 2981f73f..bdc37bf9 100644 --- a/tests/unit/test_cli_dispatch.py +++ b/tests/unit/test_cli_dispatch.py @@ -12,10 +12,10 @@ from unittest.mock import patch import bot_bottle.cli as climod from bot_bottle.cli import main -from bot_bottle.db_store import DbStore +from bot_bottle.store.db_store import DbStore from bot_bottle.log import Die from bot_bottle.manifest import ManifestError -from bot_bottle.store_manager import StoreManager +from bot_bottle.orchestrator.store.store_manager import StoreManager class TestMainDispatch(unittest.TestCase): diff --git a/tests/unit/test_cli_login.py b/tests/unit/test_cli_login.py index 0d4c13c7..67c00e01 100644 --- a/tests/unit/test_cli_login.py +++ b/tests/unit/test_cli_login.py @@ -14,27 +14,27 @@ from unittest.mock import MagicMock, patch class TestFlagParsing(unittest.TestCase): def test_console_url_flag(self) -> None: - from bot_bottle.cli.login import _flag + from bot_bottle.cli.commands.login import _flag self.assertEqual(_flag(["--console-url", "http://x"], "--console-url"), "http://x") def test_console_url_equals_form(self) -> None: - from bot_bottle.cli.login import _flag + from bot_bottle.cli.commands.login import _flag self.assertEqual( _flag(["--console-url=http://x"], "--console-url"), "http://x" ) def test_label_flag(self) -> None: - from bot_bottle.cli.login import _flag + from bot_bottle.cli.commands.login import _flag self.assertEqual(_flag(["--label", "my-mac"], "--label"), "my-mac") def test_missing_flag_returns_none(self) -> None: - from bot_bottle.cli.login import _flag + from bot_bottle.cli.commands.login import _flag self.assertIsNone(_flag([], "--console-url")) class TestHttpHelpers(unittest.TestCase): def test_post_sends_json_and_decodes_response(self) -> None: - from bot_bottle.cli.login import _post + from bot_bottle.cli.commands.login import _post response = MagicMock() response.__enter__.return_value.read.return_value = b'{"ok": true}' @@ -48,7 +48,7 @@ class TestHttpHelpers(unittest.TestCase): self.assertEqual(request.get_header("Content-type"), "application/json") def test_get_decodes_success_response(self) -> None: - from bot_bottle.cli.login import _get + from bot_bottle.cli.commands.login import _get response = MagicMock() response.__enter__.return_value.status = 200 @@ -59,7 +59,7 @@ class TestHttpHelpers(unittest.TestCase): ) def test_get_returns_http_error_status(self) -> None: - from bot_bottle.cli.login import _get + from bot_bottle.cli.commands.login import _get error = urllib.error.HTTPError( "http://console/status", 410, "gone", Message(), None @@ -70,7 +70,7 @@ class TestHttpHelpers(unittest.TestCase): class TestSaveCredentials(unittest.TestCase): def test_writes_json_and_sets_perms(self) -> None: - from bot_bottle.cli.login import _save_credentials + from bot_bottle.cli.commands.login import _save_credentials with tempfile.TemporaryDirectory() as tmp: with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}): @@ -85,7 +85,7 @@ class TestSaveCredentials(unittest.TestCase): def test_temp_file_is_private_before_replace(self) -> None: """Temp file must be 0600 at the moment os.replace is called.""" - from bot_bottle.cli.login import _save_credentials + from bot_bottle.cli.commands.login import _save_credentials from pathlib import Path as _Path tmp_perms_at_replace: list[int] = [] @@ -107,7 +107,7 @@ class TestSaveCredentials(unittest.TestCase): def test_cleanup_on_write_failure(self) -> None: """Temp file is removed and no credentials remain if replace fails.""" - from bot_bottle.cli.login import _save_credentials + from bot_bottle.cli.commands.login import _save_credentials with tempfile.TemporaryDirectory() as tmp: with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}): @@ -120,12 +120,12 @@ class TestSaveCredentials(unittest.TestCase): class TestCmdLoginMissingUrl(unittest.TestCase): def test_help_returns_0(self) -> None: - from bot_bottle.cli.login import cmd_login + from bot_bottle.cli.commands.login import cmd_login self.assertEqual(cmd_login(["--help"]), 0) def test_returns_1_without_url(self) -> None: - from bot_bottle.cli.login import cmd_login + from bot_bottle.cli.commands.login import cmd_login with patch.dict(os.environ, {}, clear=True): os.environ.pop("BB_CONSOLE_URL", None) result = cmd_login([]) @@ -133,7 +133,7 @@ class TestCmdLoginMissingUrl(unittest.TestCase): def test_reads_env_var(self) -> None: """Exits 1 (network error) not because of missing URL when env var is set.""" - from bot_bottle.cli.login import cmd_login + from bot_bottle.cli.commands.login import cmd_login def _fail_post(_url: str, _payload: dict[str, Any]) -> dict[str, Any]: raise OSError("connection refused") @@ -146,7 +146,7 @@ class TestCmdLoginMissingUrl(unittest.TestCase): "BOT_BOTTLE_ROOT": tmp, }, ): - with patch("bot_bottle.cli.login._post", side_effect=_fail_post): + with patch("bot_bottle.cli.commands.login._post", side_effect=_fail_post): result = cmd_login([]) self.assertEqual(result, 1) @@ -155,7 +155,7 @@ class TestCmdLoginFlow(unittest.TestCase): def _run_with_mocks( self, poll_responses: list[dict[str, Any]], tmp: str ) -> int: - from bot_bottle.cli.login import cmd_login + from bot_bottle.cli.commands.login import cmd_login start_resp = { "device_code": "dc123", @@ -175,8 +175,8 @@ class TestCmdLoginFlow(unittest.TestCase): return 200, next(poll_iter, {"status": "pending"}) with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}): - with patch("bot_bottle.cli.login._post", side_effect=_fake_post): - with patch("bot_bottle.cli.login._get", side_effect=_fake_get): + with patch("bot_bottle.cli.commands.login._post", side_effect=_fake_post): + with patch("bot_bottle.cli.commands.login._get", side_effect=_fake_get): with patch("time.sleep"): return cmd_login(["--console-url", "http://console"]) @@ -202,7 +202,7 @@ class TestCmdLoginFlow(unittest.TestCase): self.assertEqual(result, 1) def test_timeout_returns_1(self) -> None: - from bot_bottle.cli.login import cmd_login + from bot_bottle.cli.commands.login import cmd_login start_resp = { "device_code": "dc", @@ -213,13 +213,13 @@ class TestCmdLoginFlow(unittest.TestCase): with tempfile.TemporaryDirectory() as tmp: with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}): - with patch("bot_bottle.cli.login._post", return_value=start_resp): + with patch("bot_bottle.cli.commands.login._post", return_value=start_resp): result = cmd_login(["--console-url", "http://console"]) self.assertEqual(result, 1) def test_poll_interval_from_server_is_used(self) -> None: """time.sleep must be called with the server-provided poll_interval.""" - from bot_bottle.cli.login import cmd_login + from bot_bottle.cli.commands.login import cmd_login server_interval = 7 start_resp = { @@ -241,9 +241,9 @@ class TestCmdLoginFlow(unittest.TestCase): with tempfile.TemporaryDirectory() as tmp: with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}): - with patch("bot_bottle.cli.login._post", return_value=start_resp): + with patch("bot_bottle.cli.commands.login._post", return_value=start_resp): with patch( - "bot_bottle.cli.login._get", side_effect=_fake_get + "bot_bottle.cli.commands.login._get", side_effect=_fake_get ): with patch("time.sleep") as mock_sleep: result = cmd_login(["--console-url", "http://console"]) diff --git a/tests/unit/test_cli_start_headless.py b/tests/unit/test_cli_start_headless.py index 092b26c8..e259acb0 100644 --- a/tests/unit/test_cli_start_headless.py +++ b/tests/unit/test_cli_start_headless.py @@ -14,7 +14,7 @@ import os import unittest from unittest.mock import MagicMock, patch -import bot_bottle.cli.start as start_mod +import bot_bottle.cli.commands.start as start_mod import bot_bottle.cli.tui as tui_mod from bot_bottle.backend import ActiveAgent from bot_bottle.log import Die @@ -53,15 +53,15 @@ class TestCmdStartHeadless(unittest.TestCase): ["researcher", "implementer"], ["claude", "dev"], agent_bottle="claude" ) patch( - "bot_bottle.cli.start.ManifestIndex.resolve", + "bot_bottle.cli.commands.start.ManifestIndex.resolve", return_value=self._manifest, ).start() self._launch_mock = patch( - "bot_bottle.cli.start._launch_bottle", return_value=0 + "bot_bottle.cli.commands.start._launch_bottle", return_value=0 ).start() # No bottles running by default → no label collision. patch( - "bot_bottle.cli.start.enumerate_active_agents", return_value=[] + "bot_bottle.cli.commands.start.enumerate_active_agents", return_value=[] ).start() # If any TUI picker fires in headless mode, that's a bug. self._agent_picker = patch.object(tui_mod, "filter_select").start() @@ -71,7 +71,7 @@ class TestCmdStartHeadless(unittest.TestCase): os.environ.pop("BOT_BOTTLE_BACKEND", None) # PTY check uses os.isatty(sys.stdin.fileno()); stub both so # headless unit tests aren't blocked on a real TTY. - patch("bot_bottle.cli.start.os.isatty", return_value=True).start() + patch("bot_bottle.cli.commands.start.os.isatty", return_value=True).start() self.addCleanup(patch.stopall) def _spec(self): @@ -128,7 +128,7 @@ class TestCmdStartHeadless(unittest.TestCase): def test_no_bottle_and_no_default_dies(self): manifest = _make_manifest(["researcher"], ["claude"], agent_bottle="") with patch( - "bot_bottle.cli.start.ManifestIndex.resolve", return_value=manifest + "bot_bottle.cli.commands.start.ManifestIndex.resolve", return_value=manifest ): with self.assertRaises(Die): start_mod.cmd_start( @@ -170,7 +170,7 @@ class TestCmdStartHeadless(unittest.TestCase): def test_label_collision_uniquifies(self): with patch( - "bot_bottle.cli.start.enumerate_active_agents", + "bot_bottle.cli.commands.start.enumerate_active_agents", return_value=[_active_agent("researcher")], ): start_mod.cmd_start( @@ -203,9 +203,9 @@ class TestPrepareWithPreflight(unittest.TestCase): mock_backend.name = "test-backend" render = MagicMock() - with patch("bot_bottle.cli.start.get_bottle_backend", return_value=mock_backend), \ - patch("bot_bottle.cli.start._identity_from_plan", return_value="id"), \ - patch("bot_bottle.cli.start.info"): + with patch("bot_bottle.cli.commands.start.get_bottle_backend", return_value=mock_backend), \ + patch("bot_bottle.cli.commands.start._identity_from_plan", return_value="id"), \ + patch("bot_bottle.cli.commands.start.info"): start_mod.prepare_with_preflight( MagicMock(), stage_dir=Path("/tmp"), @@ -223,7 +223,7 @@ class TestNoAgentsDefined(unittest.TestCase): def test_no_agents_defined_returns_1(self): manifest = _make_manifest([], []) with patch( - "bot_bottle.cli.start.ManifestIndex.resolve", return_value=manifest + "bot_bottle.cli.commands.start.ManifestIndex.resolve", return_value=manifest ), patch("sys.stderr", io.StringIO()) as err: rc = start_mod.cmd_start([]) self.assertEqual(1, rc) @@ -236,7 +236,7 @@ class TestTextRenderPreflight(unittest.TestCase): def test_backend_name_in_output(self): render = start_mod._text_render_preflight() plan = MagicMock() - with patch("bot_bottle.cli.start._manifest_to_yaml", return_value=""), \ + with patch("bot_bottle.cli.commands.start._manifest_to_yaml", return_value=""), \ patch("sys.stderr", io.StringIO()) as err: render(plan, "my-backend") self.assertIn("my-backend", err.getvalue()) diff --git a/tests/unit/test_cli_start_selector.py b/tests/unit/test_cli_start_selector.py index 6af13b44..5819dac8 100644 --- a/tests/unit/test_cli_start_selector.py +++ b/tests/unit/test_cli_start_selector.py @@ -14,7 +14,7 @@ import unittest from collections.abc import Mapping, Sequence from unittest.mock import MagicMock, patch -import bot_bottle.cli.start as start_mod +import bot_bottle.cli.commands.start as start_mod import bot_bottle.cli.tui as tui_mod from bot_bottle.backend import ActiveAgent @@ -38,13 +38,13 @@ class TestCmdStartSelector(unittest.TestCase): def setUp(self): self._manifest = _make_manifest(["researcher", "implementer"], ["claude", "dev"]) self._resolve_patch = patch( - "bot_bottle.cli.start.ManifestIndex.resolve", + "bot_bottle.cli.commands.start.ManifestIndex.resolve", return_value=self._manifest, ) self._resolve_patch.start() self._launch_patch = patch( - "bot_bottle.cli.start._launch_bottle", + "bot_bottle.cli.commands.start._launch_bottle", return_value=0, ) self._launch_mock = self._launch_patch.start() @@ -66,7 +66,7 @@ class TestCmdStartSelector(unittest.TestCase): self._modal_patch.start() self._image_policy_patch = patch( - "bot_bottle.cli.start._select_image_policy", + "bot_bottle.cli.commands.start._select_image_policy", return_value="fresh", ) self._image_policy_patch.start() @@ -141,14 +141,14 @@ class TestCmdStartSelector(unittest.TestCase): self.assertEqual(("claude", "dev"), spec.bottle_names) def test_image_policy_forwarded_to_spec(self): - with patch("bot_bottle.cli.start._select_image_policy", return_value="cached"): + with patch("bot_bottle.cli.commands.start._select_image_policy", return_value="cached"): start_mod.cmd_start(["researcher"]) self._launch_mock.assert_called_once() spec = self._launch_mock.call_args[0][0] self.assertEqual("cached", spec.image_policy) def test_image_policy_cancel_returns_0(self): - with patch("bot_bottle.cli.start._select_image_policy", return_value=None): + with patch("bot_bottle.cli.commands.start._select_image_policy", return_value=None): rc = start_mod.cmd_start(["researcher"]) self.assertEqual(0, rc) self._launch_mock.assert_not_called() @@ -169,7 +169,7 @@ class TestCmdStartSelector(unittest.TestCase): ["implementer"], ["claude", "dev"], agent_bottle="claude" ) with patch( - "bot_bottle.cli.start.ManifestIndex.resolve", return_value=manifest + "bot_bottle.cli.commands.start.ManifestIndex.resolve", return_value=manifest ): start_mod.cmd_start(["implementer"]) call_kwargs = self._bottle_picker_mock.call_args @@ -178,7 +178,7 @@ class TestCmdStartSelector(unittest.TestCase): def test_no_agent_bottle_empty_initial(self): manifest = _make_manifest(["researcher"], ["claude", "dev"], agent_bottle="") with patch( - "bot_bottle.cli.start.ManifestIndex.resolve", return_value=manifest + "bot_bottle.cli.commands.start.ManifestIndex.resolve", return_value=manifest ): start_mod.cmd_start(["researcher"]) call_kwargs = self._bottle_picker_mock.call_args @@ -228,19 +228,19 @@ class TestCmdStartLabelCollision(unittest.TestCase): def setUp(self): self._manifest = _make_manifest(["researcher"], ["claude"]) - patch("bot_bottle.cli.start.ManifestIndex.resolve", return_value=self._manifest).start() + patch("bot_bottle.cli.commands.start.ManifestIndex.resolve", return_value=self._manifest).start() self._launch_mock = patch( - "bot_bottle.cli.start._launch_bottle", return_value=0, + "bot_bottle.cli.commands.start._launch_bottle", return_value=0, ).start() # Stub the bottle picker to always return a selection. patch.object(tui_mod, "filter_multiselect", return_value=["claude"]).start() - patch("bot_bottle.cli.start._select_image_policy", return_value="fresh").start() + patch("bot_bottle.cli.commands.start._select_image_policy", return_value="fresh").start() self.addCleanup(patch.stopall) def test_no_collision_proceeds_without_reprompt(self): with ( patch.object(tui_mod, "name_color_modal", return_value=("researcher", "")) as modal, - patch("bot_bottle.cli.start.enumerate_active_agents", return_value=[]), + patch("bot_bottle.cli.commands.start.enumerate_active_agents", return_value=[]), ): rc = start_mod.cmd_start(["researcher"]) self.assertEqual(0, rc) @@ -261,7 +261,7 @@ class TestCmdStartLabelCollision(unittest.TestCase): with ( patch.object(tui_mod, "name_color_modal", side_effect=_modal) as modal, patch( - "bot_bottle.cli.start.enumerate_active_agents", + "bot_bottle.cli.commands.start.enumerate_active_agents", side_effect=[[collision_agent], []], ), ): @@ -336,7 +336,7 @@ class TestManifestToYaml(unittest.TestCase): agent_provider_template: str = "claude", ): from bot_bottle.manifest import Manifest, ManifestBottle - from bot_bottle.manifest_agent import ManifestAgent, ManifestAgentProvider + from bot_bottle.manifest.agent import ManifestAgent, ManifestAgentProvider agent = ManifestAgent(skills=tuple(skills)) bottle = ManifestBottle( diff --git a/tests/unit/test_cli_start_settle.py b/tests/unit/test_cli_start_settle.py index 4b1c405b..55666b8f 100644 --- a/tests/unit/test_cli_start_settle.py +++ b/tests/unit/test_cli_start_settle.py @@ -10,7 +10,7 @@ from pathlib import Path from tests.unit import use_bottle_root from bot_bottle import bottle_state -from bot_bottle.cli import start as start_mod +from bot_bottle.cli.commands import start as start_mod class _FakeHomeMixin: diff --git a/tests/unit/test_cli_start_stale.py b/tests/unit/test_cli_start_stale.py index 87625ae3..97651a17 100644 --- a/tests/unit/test_cli_start_stale.py +++ b/tests/unit/test_cli_start_stale.py @@ -56,7 +56,7 @@ class TestLaunchBottleStaleHandling(unittest.TestCase): ) def _run_launch(self, **patch_kwargs: Any) -> int: - import bot_bottle.cli.start as start_mod + import bot_bottle.cli.commands.start as start_mod spec = self._spec() with patch.object(start_mod, "prepare_with_preflight", return_value=(_fake_plan(), "dev-abc")), \ @@ -71,7 +71,7 @@ class TestLaunchBottleStaleHandling(unittest.TestCase): def test_headless_stale_calls_die(self) -> None: """In headless mode (assume_yes=True), a StaleImageError from prelaunch_checks must call die().""" - import bot_bottle.cli.start as start_mod + import bot_bottle.cli.commands.start as start_mod backend_mock = MagicMock() backend_mock.prelaunch_checks.side_effect = StaleImageError("image is 5 day(s) old") @@ -85,7 +85,7 @@ class TestLaunchBottleStaleHandling(unittest.TestCase): def test_interactive_user_declines_stops_before_launch(self) -> None: """Interactive user answering 'n' → launch is never called.""" - import bot_bottle.cli.start as start_mod + import bot_bottle.cli.commands.start as start_mod backend_mock = MagicMock() backend_mock.prelaunch_checks.side_effect = StaleImageError("image is 5 day(s) old") @@ -100,7 +100,7 @@ class TestLaunchBottleStaleHandling(unittest.TestCase): def test_interactive_user_confirms_launches_once(self) -> None: """Interactive user answering 'y' → prelaunch stale error is bypassed; launch called once.""" - import bot_bottle.cli.start as start_mod + import bot_bottle.cli.commands.start as start_mod bottle_mock = MagicMock() bottle_mock.name = "dev-abc" @@ -122,7 +122,7 @@ class TestLaunchBottleStaleHandling(unittest.TestCase): def test_interactive_yes_uppercase_also_accepted(self) -> None: """'Y' or 'YES' should also be accepted as confirmation.""" - import bot_bottle.cli.start as start_mod + import bot_bottle.cli.commands.start as start_mod bottle_mock = MagicMock() bottle_mock.name = "dev-abc" diff --git a/tests/unit/test_config_store.py b/tests/unit/test_config_store.py index 5987ae05..b47cb6dc 100644 --- a/tests/unit/test_config_store.py +++ b/tests/unit/test_config_store.py @@ -7,11 +7,11 @@ import tempfile import unittest from pathlib import Path -from bot_bottle.config_store import ( +from bot_bottle.store.config_store import ( DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS, ConfigStore, ) -from bot_bottle.store_manager import StoreManager +from bot_bottle.orchestrator.store.store_manager import StoreManager class TestConfigStore(unittest.TestCase): diff --git a/tests/unit/test_contrib_claude_provider.py b/tests/unit/test_contrib_claude_provider.py index 0e8589b2..1dc5db3c 100644 --- a/tests/unit/test_contrib_claude_provider.py +++ b/tests/unit/test_contrib_claude_provider.py @@ -25,7 +25,7 @@ from bot_bottle.contrib.claude.agent_provider import ClaudeAgentProvider from bot_bottle.egress import EgressPlan from bot_bottle.git_gate import GitGatePlan from bot_bottle.manifest import ManifestIndex -from bot_bottle.supervise import SupervisePlan +from bot_bottle.supervisor.plan import SupervisePlan _URL = "http://supervise:9100/" diff --git a/tests/unit/test_contrib_codex_provider.py b/tests/unit/test_contrib_codex_provider.py index c53d2fb3..4b2ca1f3 100644 --- a/tests/unit/test_contrib_codex_provider.py +++ b/tests/unit/test_contrib_codex_provider.py @@ -25,7 +25,7 @@ from bot_bottle.contrib.codex.agent_provider import CodexAgentProvider from bot_bottle.egress import EgressPlan from bot_bottle.git_gate import GitGatePlan from bot_bottle.manifest import ManifestIndex -from bot_bottle.supervise import SupervisePlan +from bot_bottle.supervisor.plan import SupervisePlan _URL = "http://supervise:9100/" diff --git a/tests/unit/test_db_store.py b/tests/unit/test_db_store.py index c447d1d3..04046eea 100644 --- a/tests/unit/test_db_store.py +++ b/tests/unit/test_db_store.py @@ -7,8 +7,8 @@ import tempfile import unittest from pathlib import Path -from bot_bottle.db_store import DbStore -from bot_bottle.migrations import TableMigrations +from bot_bottle.store.db_store import DbStore +from bot_bottle.store.migrations import TableMigrations def _store(tmp: Path) -> DbStore: diff --git a/tests/unit/test_dlp_detectors.py b/tests/unit/test_dlp_detectors.py index 0cbd54f9..5310252a 100644 --- a/tests/unit/test_dlp_detectors.py +++ b/tests/unit/test_dlp_detectors.py @@ -7,7 +7,7 @@ import base64 import gzip import unittest -from bot_bottle.dlp_detectors import ( +from bot_bottle.gateway.egress.dlp_detectors import ( ENTROPY_BLOCK_THRESHOLD, PARTIAL_MATCH_MIN_LEN, REDACT, @@ -465,17 +465,17 @@ class TestMatchedAndSafeTokens(unittest.TestCase): class TestStripCrlf(unittest.TestCase): def test_removes_url_encoded_crlf(self): - from bot_bottle.dlp_detectors import strip_crlf + from bot_bottle.gateway.egress.dlp_detectors import strip_crlf out = strip_crlf("next=%0d%0aX-Injected: evil") self.assertNotRegex(out, r"%0[dD]%0[aA]") def test_removes_literal_header_injection(self): - from bot_bottle.dlp_detectors import strip_crlf + from bot_bottle.gateway.egress.dlp_detectors import strip_crlf out = strip_crlf("value\r\nX-Injected: evil") self.assertIsNone(scan_crlf_injection(out)) def test_leaves_clean_text_unchanged(self): - from bot_bottle.dlp_detectors import strip_crlf + from bot_bottle.gateway.egress.dlp_detectors import strip_crlf self.assertEqual("/api/v1/data?q=hello", strip_crlf("/api/v1/data?q=hello")) class TestAlnumProjection(unittest.TestCase): diff --git a/tests/unit/test_orchestrator_lifecycle.py b/tests/unit/test_docker_infra.py similarity index 95% rename from tests/unit/test_orchestrator_lifecycle.py rename to tests/unit/test_docker_infra.py index 26728530..bf85eb41 100644 --- a/tests/unit/test_orchestrator_lifecycle.py +++ b/tests/unit/test_docker_infra.py @@ -8,21 +8,23 @@ import urllib.error from pathlib import Path from unittest.mock import MagicMock, Mock, patch -from bot_bottle.orchestrator.gateway import GatewayError -from bot_bottle.orchestrator.lifecycle import ( +from bot_bottle.gateway import GatewayError +from bot_bottle.backend.docker.infra import ( INFRA_NAME, INFRA_SOURCE_HASH_LABEL, - OrchestratorService, + DockerInfraService, +) +from bot_bottle.orchestrator.lifecycle import ( OrchestratorStartError, source_hash, ) from bot_bottle.paths import GATEWAY_CA_DIRNAME from tests.unit import use_bottle_root -_URLOPEN = "bot_bottle.orchestrator.lifecycle.urllib.request.urlopen" -_RUN = "bot_bottle.orchestrator.lifecycle.run_docker" -_SLEEP = "bot_bottle.orchestrator.lifecycle.time.sleep" -_MONOTONIC = "bot_bottle.orchestrator.lifecycle.time.monotonic" +_URLOPEN = "bot_bottle.backend.docker.infra.urllib.request.urlopen" +_RUN = "bot_bottle.backend.docker.infra.run_docker" +_SLEEP = "bot_bottle.backend.docker.infra.time.sleep" +_MONOTONIC = "bot_bottle.backend.docker.infra.time.monotonic" def _health(status: int) -> MagicMock: @@ -35,12 +37,12 @@ def _proc(returncode: int = 0, stdout: str = "", stderr: str = "") -> Mock: return Mock(returncode=returncode, stdout=stdout, stderr=stderr) -class TestOrchestratorService(unittest.TestCase): +class TestDockerInfraService(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() self.addCleanup(self._tmp.cleanup) self.addCleanup(use_bottle_root(Path(self._tmp.name))) - self.svc = OrchestratorService(port=8099) + self.svc = DockerInfraService(port=8099) def test_url(self) -> None: self.assertEqual("http://127.0.0.1:8099", self.svc.url) @@ -159,7 +161,7 @@ class TestOrchestratorService(unittest.TestCase): return _proc(stdout="") return _proc() - svc = OrchestratorService(port=20001) + svc = DockerInfraService(port=20001) with patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \ patch(_RUN, side_effect=fake), patch(_SLEEP): svc.ensure_running() diff --git a/tests/unit/test_egress.py b/tests/unit/test_egress.py index 6d3e1346..2ec776a4 100644 --- a/tests/unit/test_egress.py +++ b/tests/unit/test_egress.py @@ -10,7 +10,6 @@ from bot_bottle.egress import ( Egress, EgressPlan, EgressRoute, - _yaml_str_escape, egress_agent_env_entries, egress_manifest_routes, egress_render_routes, @@ -19,6 +18,7 @@ from bot_bottle.egress import ( egress_gateway_env_entries, egress_token_env_map, ) +from bot_bottle.egress.service import _yaml_str_escape from bot_bottle.errors import MissingEnvVarError from bot_bottle.manifest import ManifestIndex from bot_bottle.yaml_subset import parse_yaml_subset @@ -344,7 +344,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertEqual([], parse_yaml_subset(rendered)["routes"]) def test_round_trip_through_addon_core(self): - from bot_bottle.egress_addon_core import load_config + from bot_bottle.gateway.egress.addon_core import load_config b = _bottle([ {"host": "api.github.com", "auth": {"scheme": "Bearer", "token_ref": "GH_PAT"}, @@ -363,7 +363,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertEqual("", addon_routes[2].auth_scheme) def test_dlp_round_trips(self): - from bot_bottle.egress_addon_core import load_config + from bot_bottle.gateway.egress.addon_core import load_config b = _bottle([{"host": "x.example", "dlp": { "outbound_detectors": ["token_patterns"], "inbound_detectors": False, @@ -375,7 +375,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertEqual((), addon_routes[0].inbound_detectors) def test_outbound_on_match_round_trips(self): - from bot_bottle.egress_addon_core import load_config + from bot_bottle.gateway.egress.addon_core import load_config b = _bottle([{"host": "logs.example", "dlp": { "outbound_on_match": "redact", }}]) @@ -392,7 +392,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertNotIn("outbound_on_match", rendered) def test_git_fetch_policy_round_trips(self): - from bot_bottle.egress_addon_core import load_config + from bot_bottle.gateway.egress.addon_core import load_config b = _bottle([{"host": "github.com", "git": {"fetch": True}}]) routes = egress_routes_for_bottle(b) rendered = egress_render_routes(routes) @@ -405,7 +405,7 @@ class TestRenderRoutes(unittest.TestCase): it, but the renderer in between dropped it — so the flag never reached the proxy and registry pulls kept failing with "unauthorized" while the config looked correct everywhere it was inspected.""" - from bot_bottle.egress_addon_core import load_config + from bot_bottle.gateway.egress.addon_core import load_config b = _bottle([{"host": "registry-1.docker.io", "preserve_auth": True}]) routes = egress_routes_for_bottle(b) rendered = egress_render_routes(routes) @@ -416,7 +416,7 @@ class TestRenderRoutes(unittest.TestCase): b = _bottle([{"host": "x.example"}]) rendered = egress_render_routes(egress_routes_for_bottle(b)) self.assertNotIn("preserve_auth", rendered) - from bot_bottle.egress_addon_core import load_config + from bot_bottle.gateway.egress.addon_core import load_config self.assertFalse(load_config(rendered).routes[0].preserve_auth) def test_log_zero_omitted_from_render(self): @@ -434,7 +434,7 @@ class TestRenderRoutes(unittest.TestCase): self.assertTrue(rendered.startswith(f"log: {level}\n")) def test_log_level_round_trips_to_addon_core(self): - from bot_bottle.egress_addon_core import load_config, LOG_FULL + from bot_bottle.gateway.egress.addon_core import load_config, LOG_FULL b = _bottle([{"host": "x.example"}]) routes = egress_routes_for_bottle(b) rendered = egress_render_routes(routes, log=LOG_FULL) @@ -444,7 +444,7 @@ class TestRenderRoutes(unittest.TestCase): def test_log_via_manifest_flows_to_render(self): from bot_bottle.manifest import ManifestIndex - from bot_bottle.egress_addon_core import load_config, LOG_BLOCKS + from bot_bottle.gateway.egress.addon_core import load_config, LOG_BLOCKS m = ManifestIndex.from_json_obj({ "bottles": {"dev": {"egress": { "log": 1, @@ -512,7 +512,7 @@ class TestRenderRoutesEscaping(unittest.TestCase): self.assertEqual('Bear"er', parsed[0]["inspect"]["auth_scheme"]) def test_path_value_with_double_quote_round_trips(self): - from bot_bottle.egress_addon_core import PathMatch, MatchEntry + from bot_bottle.gateway.egress.addon_core import PathMatch, MatchEntry routes = (EgressRoute( host="api.example", matches=(MatchEntry(paths=(PathMatch(type="prefix", value='/v1/"quoted"/'),)),), @@ -521,7 +521,7 @@ class TestRenderRoutesEscaping(unittest.TestCase): self.assertEqual('/v1/"quoted"/', parsed[0]["inspect"]["matches"][0]["paths"][0]["value"]) def test_header_value_with_double_quote_round_trips(self): - from bot_bottle.egress_addon_core import HeaderMatch, MatchEntry + from bot_bottle.gateway.egress.addon_core import HeaderMatch, MatchEntry routes = (EgressRoute( host="api.example", matches=(MatchEntry(headers=(HeaderMatch(name="x-h", value='val"ue'),)),), @@ -598,7 +598,7 @@ class TestCanaryGeneration(unittest.TestCase): self.assertNotEqual(plan_a.canary, plan_b.canary) def test_canary_detected_by_scan_known_secrets(self): - from bot_bottle.dlp_detectors import scan_known_secrets + from bot_bottle.gateway.egress.dlp_detectors import scan_known_secrets plan = self._make_plan() env = {plan.canary_env: plan.canary} diff --git a/tests/unit/test_egress_addon_core.py b/tests/unit/test_egress_addon_core.py index a88d1ba0..a99e7831 100644 --- a/tests/unit/test_egress_addon_core.py +++ b/tests/unit/test_egress_addon_core.py @@ -12,7 +12,7 @@ import unittest from pathlib import Path from urllib.parse import urlsplit -from bot_bottle.egress_addon_core import ( +from bot_bottle.gateway.egress.addon_core import ( LOG_BLOCKS, LOG_FULL, LOG_OFF, @@ -1377,15 +1377,15 @@ class TestScanOutboundEnhanced(unittest.TestCase): class TestOutboundDetectorNames(unittest.TestCase): def test_entropy_in_outbound_detector_names(self): - from bot_bottle.egress_addon_core import OUTBOUND_DETECTOR_NAMES + from bot_bottle.gateway.egress.addon_core import OUTBOUND_DETECTOR_NAMES self.assertIn("entropy", OUTBOUND_DETECTOR_NAMES) def test_known_secrets_in_outbound_detector_names(self): - from bot_bottle.egress_addon_core import OUTBOUND_DETECTOR_NAMES + from bot_bottle.gateway.egress.addon_core import OUTBOUND_DETECTOR_NAMES self.assertIn("known_secrets", OUTBOUND_DETECTOR_NAMES) def test_token_patterns_in_outbound_detector_names(self): - from bot_bottle.egress_addon_core import OUTBOUND_DETECTOR_NAMES + from bot_bottle.gateway.egress.addon_core import OUTBOUND_DETECTOR_NAMES self.assertIn("token_patterns", OUTBOUND_DETECTOR_NAMES) diff --git a/tests/unit/test_egress_addon_log_redaction.py b/tests/unit/test_egress_addon_log_redaction.py index 8b64f677..c6196d4c 100644 --- a/tests/unit/test_egress_addon_log_redaction.py +++ b/tests/unit/test_egress_addon_log_redaction.py @@ -36,7 +36,7 @@ def _ensure_shims() -> None: _ensure_shims() -from bot_bottle.egress_addon import EgressAddon # noqa: E402 (import after shims) +from bot_bottle.gateway.egress.addon import EgressAddon # noqa: E402 (import after shims) # --------------------------------------------------------------------------- diff --git a/tests/unit/test_egress_addon_request_flow.py b/tests/unit/test_egress_addon_request_flow.py index 6896e9d0..415bb8d1 100644 --- a/tests/unit/test_egress_addon_request_flow.py +++ b/tests/unit/test_egress_addon_request_flow.py @@ -194,22 +194,22 @@ def _ensure_shims() -> None: _ensure_shims() -import bot_bottle.egress_addon as _ea_mod # noqa: E402 (after shims) -from bot_bottle.egress_addon import EgressAddon # noqa: E402 (after shims) -from bot_bottle.egress_addon import ( # noqa: E402 +import bot_bottle.gateway.egress.addon as _ea_mod # noqa: E402 (after shims) +from bot_bottle.gateway.egress.addon import EgressAddon # noqa: E402 (after shims) +from bot_bottle.gateway.egress.addon import ( # noqa: E402 DEFAULT_INBOUND_SCAN_LIMIT_BYTES, DEFAULT_TOKEN_ALLOW_TIMEOUT_SECONDS, _inbound_scan_limit_from_env, _token_allow_timeout_from_env, ) -from bot_bottle.egress_addon_core import ( # noqa: E402 +from bot_bottle.gateway.egress.addon_core import ( # noqa: E402 Config, LOG_BLOCKS, LOG_FULL, Route, route_to_yaml_dict, ) -from bot_bottle.policy_resolver import PolicyResolveError # noqa: E402 +from bot_bottle.gateway.policy_resolver import PolicyResolveError # noqa: E402 # --------------------------------------------------------------------------- diff --git a/tests/unit/test_egress_core_parsing.py b/tests/unit/test_egress_core_parsing.py index 7762a93d..161efb4b 100644 --- a/tests/unit/test_egress_core_parsing.py +++ b/tests/unit/test_egress_core_parsing.py @@ -8,7 +8,7 @@ from __future__ import annotations import unittest -from bot_bottle.egress_addon_core import ( +from bot_bottle.gateway.egress.addon_core import ( HeaderMatch, MatchEntry, PathMatch, diff --git a/tests/unit/test_egress_entrypoint.py b/tests/unit/test_egress_entrypoint.py index fc0bb120..0c846ac6 100644 --- a/tests/unit/test_egress_entrypoint.py +++ b/tests/unit/test_egress_entrypoint.py @@ -21,7 +21,7 @@ from pathlib import Path _SCRIPT = ( Path(__file__).resolve().parent.parent.parent - / "bot_bottle" / "egress_entrypoint.sh" + / "bot_bottle" / "gateway" / "egress" / "entrypoint.sh" ) diff --git a/tests/unit/test_egress_multitenant.py b/tests/unit/test_egress_multitenant.py index 6d99e0d3..2aa5c869 100644 --- a/tests/unit/test_egress_multitenant.py +++ b/tests/unit/test_egress_multitenant.py @@ -4,7 +4,7 @@ from __future__ import annotations import unittest -from bot_bottle.egress_addon_core import ( +from bot_bottle.gateway.egress.addon_core import ( DENY_RESOLVER_ERROR, DENY_UNATTRIBUTED, DENY_UNPARSEABLE, @@ -12,7 +12,7 @@ from bot_bottle.egress_addon_core import ( resolve_client_config, resolve_client_context, ) -from bot_bottle.policy_resolver import PolicyResolveError +from bot_bottle.gateway.policy_resolver import PolicyResolveError class _FakeResolver: diff --git a/tests/unit/test_firecracker_infra_vm.py b/tests/unit/test_firecracker_infra_vm.py index a809344b..157a5c5b 100644 --- a/tests/unit/test_firecracker_infra_vm.py +++ b/tests/unit/test_firecracker_infra_vm.py @@ -31,7 +31,7 @@ class TestSigningKeySync(unittest.TestCase): patch.object(infra_vm, "bot_bottle_root", return_value=root): out = infra_vm._with_signing_key(self._infra()) self.assertIsInstance(out, infra_vm.InfraVm) # returned for chaining - token = root / infra_vm.CONTROL_PLANE_TOKEN_FILENAME + token = root / infra_vm.ORCHESTRATOR_TOKEN_FILENAME self.assertEqual("the-signing-key", token.read_text()) self.assertEqual(0o600, token.stat().st_mode & 0o777) # It cat'd the guest volume path over SSH. @@ -44,16 +44,16 @@ class TestSigningKeySync(unittest.TestCase): with patch.object(infra_vm.subprocess, "run", return_value=proc), \ patch.object(infra_vm, "bot_bottle_root", return_value=root): infra_vm._with_signing_key(self._infra()) # no raise - self.assertFalse((root / infra_vm.CONTROL_PLANE_TOKEN_FILENAME).exists()) + self.assertFalse((root / infra_vm.ORCHESTRATOR_TOKEN_FILENAME).exists()) -class TestControlPlaneUrl(unittest.TestCase): +class TestOrchestratorUrl(unittest.TestCase): def test_url_uses_guest_ip_and_port(self): infra = infra_vm.InfraVm( vm=MagicMock(), guest_ip="10.243.255.1", private_key=Path("/k")) self.assertEqual( - f"http://10.243.255.1:{infra_vm.CONTROL_PLANE_PORT}", - infra.control_plane_url, + f"http://10.243.255.1:{infra_vm.ORCHESTRATOR_PORT}", + infra.orchestrator_url, ) @@ -72,7 +72,7 @@ class TestBuildInfraRootfs(unittest.TestCase): self.assertIn("bot_bottle.orchestrator", init) # Gateway launches via the installed package (there is no # /app/gateway_init.py file since the daemons moved into bot_bottle). - self.assertIn("bot_bottle.gateway_init", init) + self.assertIn("bot_bottle.gateway.bootstrap", init) self.assertIn("export PATH=", init) # Persistent registry volume mounted at the DB dir before the CP starts. self.assertIn("/dev/vdb", init) @@ -81,11 +81,11 @@ class TestBuildInfraRootfs(unittest.TestCase): # Role-scoped control-plane auth (issue #469 review): the orchestrator # gets the signing key, the gateway daemons get a pre-minted `gateway` # JWT — never open mode in the infra VM. - self.assertIn("host_control_plane_token", init) # key generated on the volume + self.assertIn("host_orchestrator_token", init) # key generated on the volume self.assertIn("mint, ROLE_GATEWAY", init) # gateway JWT minted from it - self.assertIn('BOT_BOTTLE_CONTROL_PLANE_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator', + self.assertIn('BOT_BOTTLE_ORCHESTRATOR_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator', init) # key -> orchestrator only - self.assertIn('BOT_BOTTLE_CONTROL_AUTH_JWT="$GW_JWT"', init) # JWT -> gateway daemons + self.assertIn('BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT="$GW_JWT"', init) # JWT -> gateway daemons class TestSshGatewayTransport(unittest.TestCase): diff --git a/tests/unit/test_gateway_init.py b/tests/unit/test_gateway_init.py index 17b5213b..1fb471f5 100644 --- a/tests/unit/test_gateway_init.py +++ b/tests/unit/test_gateway_init.py @@ -1,6 +1,6 @@ """Unit: gateway data-plane init supervisor (PRD 0070; PRD 0024 bundle shape). -Tests both the helper functions in `bot_bottle.gateway_init` +Tests both the helper functions in `bot_bottle.gateway.bootstrap` and the supervisor's end-to-end signal / exit-code behavior. The end-to-end tests use real subprocesses (`sleep`, `/bin/sh -c '...'`) — short-lived, no docker required — so they run under `tests/unit/` @@ -18,9 +18,9 @@ import warnings from pathlib import Path from unittest.mock import patch -from bot_bottle.gateway_init import ( +from bot_bottle.gateway.bootstrap import ( _DaemonSpec, - _Supervisor, + _DaemonManager, _argv_for_daemon, _env_for_daemon, _selected_daemons, @@ -64,7 +64,7 @@ class TestEnvForDaemon(unittest.TestCase): self.assertNotIn("X", self._BASE) -class TestControlPlaneEnvScoping(unittest.TestCase): +class TestOrchestratorEnvScoping(unittest.TestCase): """The control-plane signing key stays with the orchestrator; the pre-minted `gateway` JWT goes to the data-plane daemons (issue #469 review). Scoping them per-process keeps a compromised data-plane daemon from reading the key @@ -72,20 +72,20 @@ class TestControlPlaneEnvScoping(unittest.TestCase): _BASE = { "PATH": "/usr/bin", - "BOT_BOTTLE_CONTROL_PLANE_TOKEN": "sk-x", - "BOT_BOTTLE_CONTROL_AUTH_JWT": "gw-jwt", + "BOT_BOTTLE_ORCHESTRATOR_TOKEN": "sk-x", + "BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT": "gw-jwt", } def test_orchestrator_gets_key_not_jwt(self): env = _env_for_daemon("orchestrator", self._BASE) - self.assertEqual("sk-x", env["BOT_BOTTLE_CONTROL_PLANE_TOKEN"]) - self.assertNotIn("BOT_BOTTLE_CONTROL_AUTH_JWT", env) + self.assertEqual("sk-x", env["BOT_BOTTLE_ORCHESTRATOR_TOKEN"]) + self.assertNotIn("BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT", env) def test_data_plane_daemons_get_jwt_not_key(self): for name in ("egress", "git-gate", "git-http", "supervise"): env = _env_for_daemon(name, self._BASE) - self.assertNotIn("BOT_BOTTLE_CONTROL_PLANE_TOKEN", env, name) - self.assertEqual("gw-jwt", env["BOT_BOTTLE_CONTROL_AUTH_JWT"], name) + self.assertNotIn("BOT_BOTTLE_ORCHESTRATOR_TOKEN", env, name) + self.assertEqual("gw-jwt", env["BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT"], name) class TestSelectedDaemons(unittest.TestCase): @@ -169,7 +169,7 @@ class TestDaemonArgv(unittest.TestCase): class TestSupervisor(unittest.TestCase): - """End-to-end: drive `_Supervisor` directly with fake commands. + """End-to-end: drive `_DaemonManager` directly with fake commands. We don't go through `main()` because main installs signal handlers process-wide, which collides with the test runner.""" @@ -177,7 +177,7 @@ class TestSupervisor(unittest.TestCase): warnings.simplefilter("error", ResourceWarning) self.addCleanup(warnings.resetwarnings) - def _drive(self, sup: _Supervisor, max_wait_s: float = 6.0) -> int: + def _drive(self, sup: _DaemonManager, max_wait_s: float = 6.0) -> int: deadline = time.monotonic() + max_wait_s while not sup.tick(): if time.monotonic() > deadline: @@ -193,7 +193,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("a", ("/bin/sh", "-c", ":")), _DaemonSpec("b", ("/bin/sh", "-c", ":")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.1) sup.request_shutdown(reason="test") @@ -209,7 +209,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("crasher", ("/bin/sh", "-c", "exit 1")), _DaemonSpec("longrun", (SLEEP, "30")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() original_pid = sup.procs[0][1].pid @@ -235,7 +235,7 @@ class TestSupervisor(unittest.TestCase): def test_single_daemon_crash_is_restarted_before_tick_completes(self): specs = [_DaemonSpec("crasher", ("/bin/sh", "-c", "exit 1"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() original_pid = sup.procs[0][1].pid time.sleep(0.1) @@ -259,7 +259,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("crasher", ("/bin/sh", "-c", "exit 1")), _DaemonSpec("longrun", (SLEEP, "30")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.3) # let crasher die sup.request_shutdown(reason="test") @@ -271,7 +271,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("a", ("/bin/sh", "-c", "exit 0")), _DaemonSpec("b", ("/bin/sh", "-c", "exit 2")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() original_pids = [p.pid for _, p in sup.procs] time.sleep(0.1) @@ -309,7 +309,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("egress", sighup_marker), _DaemonSpec("other", (SLEEP, "30")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.3) # let Python install the handler @@ -331,7 +331,7 @@ class TestSupervisor(unittest.TestCase): def test_forward_signal_unknown_daemon_no_op(self): specs = [_DaemonSpec("a", (SLEEP, "30"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() delivered = sup.forward_signal(signal.SIGHUP, "ghost") self.assertFalse(delivered) @@ -345,7 +345,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("git-gate", (SLEEP, "30")), _DaemonSpec("supervise", (SLEEP, "30")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.1) old_git_gate_pid = sup.procs[0][1].pid @@ -370,7 +370,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("git-gate", (SLEEP, "30")), _DaemonSpec("supervise", (SLEEP, "30")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.1) old_git_gate_pid = sup.procs[0][1].pid @@ -392,7 +392,7 @@ class TestSupervisor(unittest.TestCase): def test_repeated_restart_requests_coalesce(self): specs = [_DaemonSpec("git-gate", (SLEEP, "30"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.1) @@ -415,7 +415,7 @@ class TestSupervisor(unittest.TestCase): def test_request_restart_unknown_daemon_no_op(self): specs = [_DaemonSpec("a", (SLEEP, "30"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() ok = sup.request_restart("ghost") self.assertFalse(ok) @@ -425,7 +425,7 @@ class TestSupervisor(unittest.TestCase): def test_restart_unknown_daemon_no_op(self): specs = [_DaemonSpec("a", (SLEEP, "30"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() ok = sup.restart_daemon("ghost") self.assertFalse(ok) @@ -434,7 +434,7 @@ class TestSupervisor(unittest.TestCase): def test_restart_during_shutdown_is_no_op(self): specs = [_DaemonSpec("git-gate", (SLEEP, "30"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() sup.request_shutdown(reason="test") ok = sup.restart_daemon("git-gate") @@ -444,7 +444,7 @@ class TestSupervisor(unittest.TestCase): def test_pending_restart_dropped_during_shutdown(self): specs = [_DaemonSpec("git-gate", (SLEEP, "30"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.1) old_pid = sup.procs[0][1].pid @@ -464,7 +464,7 @@ class TestSupervisor(unittest.TestCase): _DaemonSpec("a", (SLEEP, "60")), _DaemonSpec("b", (SLEEP, "60")), ] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.2) # let them actually start sup.request_shutdown(reason="test") @@ -484,12 +484,12 @@ class TestSupervisor(unittest.TestCase): "trap '' TERM; sleep 30", ) specs = [_DaemonSpec("stubborn", ignore_term)] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.3) # let `trap` register sup.request_shutdown(reason="test") - with patch("bot_bottle.gateway_init._GRACE_SECONDS", 0.3): + with patch("bot_bottle.gateway.bootstrap._GRACE_SECONDS", 0.3): rc = self._drive(sup, max_wait_s=4.0) # Process was SIGKILL'd → returncode -9 on POSIX. @@ -498,7 +498,7 @@ class TestSupervisor(unittest.TestCase): def test_idempotent_shutdown_requests(self): specs = [_DaemonSpec("a", (SLEEP, "60"))] - sup = _Supervisor(specs) + sup = _DaemonManager(specs) sup.start_all() time.sleep(0.1) first_at = None @@ -531,7 +531,7 @@ class TestMainEndToEnd(unittest.TestCase): helper = ( "import os, runpy, sys\n" - "from bot_bottle import gateway_init as si\n" + "from bot_bottle.gateway import bootstrap as si\n" "si._DAEMONS = (\n" f" si._DaemonSpec('alpha', ({SLEEP!r},'30')),\n" f" si._DaemonSpec('beta', ({SLEEP!r},'30')),\n" diff --git a/tests/unit/test_git_gate.py b/tests/unit/test_git_gate.py index fd899f0a..bf7a891d 100644 --- a/tests/unit/test_git_gate.py +++ b/tests/unit/test_git_gate.py @@ -230,7 +230,7 @@ class TestHookRender(unittest.TestCase): # execute from the bare repo directory, so the embedded Python must # include /app and support both import layouts. self.assertIn('PYTHONPATH="/app${PYTHONPATH:+:$PYTHONPATH}"', hook) - self.assertIn("from bot_bottle.policy_resolver import PolicyResolver", hook) + self.assertIn("from bot_bottle.gateway.policy_resolver import PolicyResolver", hook) self.assertIn("from policy_resolver import PolicyResolver", hook) def test_inline_gitleaks_allow_fails_closed_without_supervisor(self): @@ -385,13 +385,13 @@ class TestDynamicKeyProvisioning(unittest.TestCase): def test_resolve_identity_file_gitea_provisions_key(self): entry = self._gitea_manifest().bottles["dev"].git[0] - with patch("bot_bottle.git_gate_provision._provision_dynamic_key", return_value="/tmp/provisioned-key") as mock_provision: + with patch("bot_bottle.git_gate.provision._provision_dynamic_key", return_value="/tmp/provisioned-key") as mock_provision: self.assertEqual("/tmp/provisioned-key", _resolve_identity_file(entry, "demo", self.stage)) mock_provision.assert_called_once() def test_prepare_defers_gitea_key_provisioning(self): bottle = self._gitea_manifest().bottles["dev"] - with patch("bot_bottle.git_gate_provision._provision_dynamic_key") as mock_provision: + with patch("bot_bottle.git_gate.provision._provision_dynamic_key") as mock_provision: plan = _StubGate().prepare(bottle, "demo", self.stage) mock_provision.assert_not_called() @@ -402,7 +402,7 @@ class TestDynamicKeyProvisioning(unittest.TestCase): plan = _StubGate().prepare(bottle, "demo", self.stage) with patch( - "bot_bottle.git_gate_provision._provision_dynamic_key", + "bot_bottle.git_gate.provision._provision_dynamic_key", return_value="/tmp/provisioned-key", ) as mock_provision: updated = provision_git_gate_dynamic_keys(bottle, plan, self.stage) diff --git a/tests/unit/test_git_gate_host_key.py b/tests/unit/test_git_gate_host_key.py index 3835d140..4a6541cd 100644 --- a/tests/unit/test_git_gate_host_key.py +++ b/tests/unit/test_git_gate_host_key.py @@ -7,7 +7,7 @@ import unittest from pathlib import Path from unittest.mock import MagicMock, patch -from bot_bottle.git_gate_host_key import ( +from bot_bottle.git_gate.host_key import ( add_host_key_to_frontmatter, find_and_update_bottle_file, find_repo_bottle_file, @@ -358,7 +358,7 @@ git-gate: with tempfile.TemporaryDirectory() as d: path = self._write_bottle(d, "dev", content) with patch( - "bot_bottle.git_gate_host_key.find_repo_bottle_file", + "bot_bottle.git_gate.host_key.find_repo_bottle_file", return_value=path, ), patch.object(Path, "read_text", side_effect=OSError("Permission denied")): result = find_and_update_bottle_file(Path(d), "myrepo", "ssh-ed25519 AAAA") @@ -382,7 +382,7 @@ git-gate: with tempfile.TemporaryDirectory() as d: path = self._write_bottle(d, "dev", content) with patch( - "bot_bottle.git_gate_host_key.add_host_key_to_frontmatter", + "bot_bottle.git_gate.host_key.add_host_key_to_frontmatter", return_value=content, # no change ): result = find_and_update_bottle_file(Path(d), "myrepo", "ssh-ed25519 AAAA") @@ -467,10 +467,10 @@ git-gate: mock_prompt = MagicMock(side_effect=["y", "n"]) # confirm key, skip save with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", return_value="ssh-ed25519 FETCHED", ), patch( - "bot_bottle.git_gate_host_key.prompt_tty", mock_prompt, + "bot_bottle.git_gate.host_key.prompt_tty", mock_prompt, ), patch("sys.stderr"): preflight_host_keys(manifest, headless=False, home_md=Path(home)) @@ -520,10 +520,10 @@ class TestPreflightHostKeys(unittest.TestCase): manifest = _manifest_with_git() with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", return_value="ssh-ed25519 FETCHED", ), patch( - "bot_bottle.git_gate_host_key.prompt_tty", + "bot_bottle.git_gate.host_key.prompt_tty", side_effect=["y", "n"], # confirm key, don't persist ), patch("sys.stderr"): result = preflight_host_keys(manifest, headless=False, home_md=None) @@ -534,10 +534,10 @@ class TestPreflightHostKeys(unittest.TestCase): manifest = _manifest_with_git() with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", return_value="ssh-ed25519 FETCHED", ), patch( - "bot_bottle.git_gate_host_key.prompt_tty", + "bot_bottle.git_gate.host_key.prompt_tty", return_value="n", ), patch("sys.stderr"), self.assertRaises(SystemExit): preflight_host_keys(manifest, headless=False, home_md=None) @@ -557,10 +557,10 @@ class TestPreflightHostKeys(unittest.TestCase): ) with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", return_value="ssh-ed25519 FETCHED", ), patch( - "bot_bottle.git_gate_host_key.prompt_tty", + "bot_bottle.git_gate.host_key.prompt_tty", side_effect=["y", "y"], # confirm key, yes persist ), patch("sys.stderr"): result = preflight_host_keys( @@ -586,10 +586,10 @@ class TestPreflightHostKeys(unittest.TestCase): ) with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", return_value="ssh-ed25519 FETCHED", ), patch( - "bot_bottle.git_gate_host_key.prompt_tty", + "bot_bottle.git_gate.host_key.prompt_tty", side_effect=["y", "n"], # confirm key, no persist ), patch("sys.stderr"): result = preflight_host_keys( @@ -604,7 +604,7 @@ class TestPreflightHostKeys(unittest.TestCase): manifest = _manifest_with_git() with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", side_effect=RuntimeError("connection refused"), ), patch("sys.stderr"), self.assertRaises(SystemExit): preflight_host_keys(manifest, headless=False, home_md=None) @@ -620,10 +620,10 @@ class TestPreflightHostKeys(unittest.TestCase): import io buf = io.StringIO() with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", return_value="ssh-ed25519 FETCHED", ), patch( - "bot_bottle.git_gate_host_key.prompt_tty", + "bot_bottle.git_gate.host_key.prompt_tty", side_effect=["y"], # confirm key only; no save prompt when no file found ), patch("sys.stderr", buf): result = preflight_host_keys( @@ -651,13 +651,13 @@ class TestPreflightHostKeys(unittest.TestCase): import io buf = io.StringIO() with patch( - "bot_bottle.git_gate_host_key.fetch_host_key", + "bot_bottle.git_gate.host_key.fetch_host_key", return_value="ssh-ed25519 FETCHED", ), patch( - "bot_bottle.git_gate_host_key.prompt_tty", + "bot_bottle.git_gate.host_key.prompt_tty", side_effect=["y", "y"], # confirm key, yes persist ), patch( - "bot_bottle.git_gate_host_key.find_and_update_bottle_file", + "bot_bottle.git_gate.host_key.find_and_update_bottle_file", return_value=False, ), patch("sys.stderr", buf): result = preflight_host_keys( diff --git a/tests/unit/test_git_gate_provision_render.py b/tests/unit/test_git_gate_provision_render.py index 3fc1a076..d2ba9a7f 100644 --- a/tests/unit/test_git_gate_provision_render.py +++ b/tests/unit/test_git_gate_provision_render.py @@ -4,7 +4,7 @@ from __future__ import annotations import unittest -from bot_bottle.git_gate_render import ( +from bot_bottle.gateway.git_gate.render import ( GitGateUpstream, git_gate_render_entrypoint, git_gate_render_provision, diff --git a/tests/unit/test_git_gate_render_provision.py b/tests/unit/test_git_gate_render_provision.py index 481a63ca..81263e47 100644 --- a/tests/unit/test_git_gate_render_provision.py +++ b/tests/unit/test_git_gate_render_provision.py @@ -21,7 +21,7 @@ from bot_bottle.git_gate import ( git_gate_render_gitconfig, revoke_git_gate_provisioned_keys, ) -from bot_bottle.manifest_git import ManifestGitEntry, ManifestKeyConfig +from bot_bottle.manifest.git import ManifestGitEntry, ManifestKeyConfig def _entry(**kw: Any) -> ManifestGitEntry: diff --git a/tests/unit/test_git_http_backend.py b/tests/unit/test_git_http_backend.py index 405f0b04..89f162cb 100644 --- a/tests/unit/test_git_http_backend.py +++ b/tests/unit/test_git_http_backend.py @@ -10,7 +10,7 @@ from pathlib import Path from unittest import mock from bot_bottle.git_gate import GIT_GATE_TIMEOUT_SECS -from bot_bottle.git_http_backend import GitHttpHandler, MAX_BODY_BYTES +from bot_bottle.gateway.git_gate.http_backend import GitHttpHandler, MAX_BODY_BYTES # The git-http backend is resolver-only: every request is attributed to a @@ -199,7 +199,7 @@ class TestGitHttpBackend(unittest.TestCase): subprocess.CompletedProcess(["git"], 0, backend_response, b""), ] with mock.patch( - "bot_bottle.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_gate.http_backend.subprocess.run", side_effect=calls, ) as run: request = urllib.request.Request( @@ -265,7 +265,7 @@ class TestGitHttpBackend(unittest.TestCase): subprocess.CompletedProcess(["git"], 0, backend_response, b""), ] with mock.patch( - "bot_bottle.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_gate.http_backend.subprocess.run", side_effect=calls, ) as run: req = urllib.request.Request( @@ -309,7 +309,7 @@ class TestGitHttpBackend(unittest.TestCase): denial = b"git-gate: upstream fetch failed; refusing to serve stale data\n" with mock.patch( - "bot_bottle.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_gate.http_backend.subprocess.run", return_value=subprocess.CompletedProcess( ["hook"], 1, b"", denial, ), @@ -355,7 +355,7 @@ class TestGitHttpBackend(unittest.TestCase): self.addCleanup(server.server_close) with mock.patch( - "bot_bottle.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_gate.http_backend.subprocess.run", return_value=subprocess.CompletedProcess( ["hook"], 2, b"", b"", ), @@ -402,7 +402,7 @@ class TestGitHttpBackend(unittest.TestCase): self.addCleanup(server.server_close) with mock.patch( - "bot_bottle.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_gate.http_backend.subprocess.run", side_effect=PermissionError(13, "Permission denied"), ): buf = io.StringIO() @@ -461,7 +461,7 @@ class TestMalformedStatusHeader(unittest.TestCase): def _get_with_backend_response(self, cgi_response: bytes) -> int: with mock.patch( - "bot_bottle.git_http_backend.subprocess.run", + "bot_bottle.gateway.git_gate.http_backend.subprocess.run", return_value=mock.Mock(returncode=0, stdout=cgi_response), ): req = urllib.request.Request( @@ -545,7 +545,7 @@ class TestContentLengthBounds(unittest.TestCase): # With a valid Content-Length the handler proceeds into # git http-backend; that will fail (no real git repo) but the # status won't be 400 or 413. - with mock.patch("bot_bottle.git_http_backend.subprocess.run") as run: + with mock.patch("bot_bottle.gateway.git_gate.http_backend.subprocess.run") as run: run.return_value = mock.Mock( returncode=0, stdout=( diff --git a/tests/unit/test_git_http_multitenant.py b/tests/unit/test_git_http_multitenant.py index b425f5f9..229e7dfe 100644 --- a/tests/unit/test_git_http_multitenant.py +++ b/tests/unit/test_git_http_multitenant.py @@ -10,8 +10,8 @@ from __future__ import annotations import unittest from pathlib import Path -from bot_bottle.git_http_backend import resolve_sandbox_root -from bot_bottle.policy_resolver import PolicyResolveError +from bot_bottle.gateway.git_gate.http_backend import resolve_sandbox_root +from bot_bottle.gateway.policy_resolver import PolicyResolveError _BASE = Path("/git") diff --git a/tests/unit/test_macos_consolidated_launch.py b/tests/unit/test_macos_consolidated_launch.py index b0f0b840..812e14e6 100644 --- a/tests/unit/test_macos_consolidated_launch.py +++ b/tests/unit/test_macos_consolidated_launch.py @@ -58,7 +58,7 @@ class TestEnsureGateway(unittest.TestCase): from bot_bottle.backend.macos_container.infra import InfraEndpoint service = MagicMock() service.ensure_running.return_value = InfraEndpoint( - control_plane_url="http://192.168.128.2:8099", + orchestrator_url="http://192.168.128.2:8099", gateway_ip="192.168.128.2", ) service.network = "bot-bottle-mac-gateway" @@ -72,7 +72,7 @@ class TestEnsureGateway(unittest.TestCase): self.assertEqual("PEM", endpoint.gateway_ca_pem) self.assertEqual("bot-bottle-mac-gateway", endpoint.network) - def test_control_plane_and_gateway_share_one_address(self) -> None: + def test_orchestrator_and_gateway_share_one_address(self) -> None: """One infra container hosts both, so the gateway IP and the control-plane host are the same.""" endpoint = self._run(self._service()) diff --git a/tests/unit/test_macos_infra.py b/tests/unit/test_macos_infra.py index cf746dca..f1341ecd 100644 --- a/tests/unit/test_macos_infra.py +++ b/tests/unit/test_macos_infra.py @@ -10,7 +10,7 @@ from bot_bottle.backend.macos_container.infra import ( INFRA_DB_VOLUME, MacosInfraService, OrchestratorStartError, - probe_control_plane_url, + probe_orchestrator_url, ) _INFRA = "bot_bottle.backend.macos_container.infra" @@ -48,7 +48,7 @@ class TestInfraRun(unittest.TestCase): self.assertIn("bot_bottle.orchestrator", script) # Gateway launches via the installed package (there is no # /app/gateway_init.py file since the daemons moved into bot_bottle). - self.assertIn("bot_bottle.gateway_init", script) + self.assertIn("bot_bottle.gateway.bootstrap", script) self.assertIn("127.0.0.1", script) # they reach each other on loopback def test_db_is_a_container_only_volume(self) -> None: @@ -110,7 +110,7 @@ class TestInfraEnsureRunning(unittest.TestCase): mod.try_container_ipv4_on_network.return_value = "192.168.128.2" endpoint = svc.ensure_running() run.assert_not_called() - self.assertEqual("http://192.168.128.2:8099", endpoint.control_plane_url) + self.assertEqual("http://192.168.128.2:8099", endpoint.orchestrator_url) self.assertEqual("192.168.128.2", endpoint.gateway_ip) def test_changed_source_recreates(self) -> None: @@ -176,16 +176,16 @@ class TestCaCertPem(unittest.TestCase): svc.ca_cert_pem(timeout=0) -class TestProbeControlPlane(unittest.TestCase): +class TestProbeOrchestrator(unittest.TestCase): def test_returns_url_when_running(self) -> None: with patch(f"{_INFRA}.container_mod") as mod: mod.try_container_ipv4_on_network.return_value = "192.168.128.2" - self.assertEqual("http://192.168.128.2:8099", probe_control_plane_url()) + self.assertEqual("http://192.168.128.2:8099", probe_orchestrator_url()) def test_empty_when_absent(self) -> None: with patch(f"{_INFRA}.container_mod") as mod: mod.try_container_ipv4_on_network.return_value = "" - self.assertEqual("", probe_control_plane_url()) + self.assertEqual("", probe_orchestrator_url()) if __name__ == "__main__": diff --git a/tests/unit/test_manifest_bottle_merge.py b/tests/unit/test_manifest_bottle_merge.py index 03fc7dea..256680d1 100644 --- a/tests/unit/test_manifest_bottle_merge.py +++ b/tests/unit/test_manifest_bottle_merge.py @@ -14,7 +14,7 @@ import unittest from pathlib import Path from bot_bottle.manifest import ManifestBottle, ManifestError, ManifestIndex -from bot_bottle.manifest_extends import merge_bottles_runtime +from bot_bottle.manifest.extends import merge_bottles_runtime def _index(bottles: dict[str, object], agents: dict[str, object]) -> ManifestIndex: @@ -44,7 +44,7 @@ class TestMergeBottlesRuntime(unittest.TestCase): self.assertEqual("y", result.env["ONLY_OVERRIDE"]) def test_egress_routes_concatenated(self): - from bot_bottle.manifest_egress import ManifestEgressConfig, ManifestEgressRoute + from bot_bottle.manifest.egress import ManifestEgressConfig, ManifestEgressRoute r1 = ManifestEgressRoute(Host="api.a.com") r2 = ManifestEgressRoute(Host="api.b.com") base = ManifestBottle(egress=ManifestEgressConfig(routes=(r1,))) diff --git a/tests/unit/test_manifest_validation.py b/tests/unit/test_manifest_validation.py index 618717d4..b762c2a0 100644 --- a/tests/unit/test_manifest_validation.py +++ b/tests/unit/test_manifest_validation.py @@ -14,12 +14,12 @@ from unittest.mock import patch from bot_bottle.env import resolve_env from bot_bottle.errors import MissingEnvVarError from bot_bottle.manifest import Manifest, ManifestBottle, ManifestIndex -from bot_bottle.manifest_agent import ( +from bot_bottle.manifest.agent import ( ManifestAgent, ManifestAgentProvider, _parse_provider_settings, ) -from bot_bottle.manifest_util import ManifestError +from bot_bottle.manifest.util import ManifestError def _idx(obj: dict[str, object]) -> ManifestIndex: diff --git a/tests/unit/test_control_auth.py b/tests/unit/test_orchestrator_auth.py similarity index 91% rename from tests/unit/test_control_auth.py rename to tests/unit/test_orchestrator_auth.py index bfcb7cc7..78e25596 100644 --- a/tests/unit/test_control_auth.py +++ b/tests/unit/test_orchestrator_auth.py @@ -6,7 +6,7 @@ import base64 import json import unittest -from bot_bottle.control_auth import ROLE_CLI, ROLE_GATEWAY, ROLES, mint, verify +from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY, ROLES, mint, verify _KEY = "test-key" @@ -45,14 +45,14 @@ class TestMintVerify(unittest.TestCase): def test_validly_signed_but_wrong_alg_rejected(self) -> None: # Alg-confusion: even a *correctly signed* token whose header claims a # non-HS256 alg must be rejected. - from bot_bottle.control_auth import _sign # noqa: PLC0415 + from bot_bottle.orchestrator_auth import _sign # noqa: PLC0415 signing_input = f"{_b64({'alg': 'none', 'typ': 'JWT'})}.{_b64({'role': 'cli'})}" self.assertIsNone(verify(f"{signing_input}.{_sign(_KEY, signing_input)}", _KEY)) def test_validly_signed_but_undecodable_rejected(self) -> None: # A correct signature over a header that isn't valid base64/JSON still # fails closed rather than raising. - from bot_bottle.control_auth import _sign # noqa: PLC0415 + from bot_bottle.orchestrator_auth import _sign # noqa: PLC0415 signing_input = f"!!!not-base64!!!.{_b64({'role': 'cli'})}" self.assertIsNone(verify(f"{signing_input}.{_sign(_KEY, signing_input)}", _KEY)) @@ -67,7 +67,7 @@ class TestMintVerify(unittest.TestCase): header, _p, _s = mint(ROLE_CLI, _KEY).split(".") # Re-sign a token carrying an unknown role — a valid signature but a # role the control plane doesn't recognise must still be rejected. - from bot_bottle.control_auth import _sign # noqa: PLC0415 + from bot_bottle.orchestrator_auth import _sign # noqa: PLC0415 payload = _b64({"role": "root"}) signing_input = f"{header}.{payload}" forged = f"{signing_input}.{_sign(_KEY, signing_input)}" diff --git a/tests/unit/test_orchestrator_client.py b/tests/unit/test_orchestrator_client.py index 4086adec..a9f77da2 100644 --- a/tests/unit/test_orchestrator_client.py +++ b/tests/unit/test_orchestrator_client.py @@ -7,7 +7,7 @@ import unittest import urllib.error from unittest.mock import MagicMock, patch -from bot_bottle.control_auth import ROLE_CLI, verify +from bot_bottle.orchestrator_auth import ROLE_CLI, verify from bot_bottle.orchestrator.client import ( OrchestratorClient, OrchestratorClientError, @@ -20,13 +20,13 @@ _URLOPEN = "bot_bottle.orchestrator.client.urllib.request.urlopen" class TestHostAuthToken(unittest.TestCase): def test_mints_a_cli_token_from_the_host_key(self) -> None: - with patch("bot_bottle.orchestrator.client.host_control_plane_token", + with patch("bot_bottle.orchestrator.client.host_orchestrator_token", return_value="signing-key"): tok = _host_auth_token() self.assertEqual(ROLE_CLI, verify(tok, "signing-key")) def test_returns_empty_when_key_unreadable(self) -> None: - with patch("bot_bottle.orchestrator.client.host_control_plane_token", + with patch("bot_bottle.orchestrator.client.host_orchestrator_token", side_effect=OSError("no host root")): self.assertEqual("", _host_auth_token()) diff --git a/tests/unit/test_orchestrator_config_store.py b/tests/unit/test_orchestrator_config_store.py index ff439877..538a475d 100644 --- a/tests/unit/test_orchestrator_config_store.py +++ b/tests/unit/test_orchestrator_config_store.py @@ -15,7 +15,7 @@ import unittest from pathlib import Path from types import ModuleType -from bot_bottle.orchestrator.config_store import ( +from bot_bottle.orchestrator.store.config_store import ( DEFAULT_TEARDOWN_TIMEOUT_SECONDS, TEARDOWN_TIMEOUT_ENV, OrchestratorConfigStore, diff --git a/tests/unit/test_orchestrator_gateway.py b/tests/unit/test_orchestrator_gateway.py index 40a14c84..b23162e6 100644 --- a/tests/unit/test_orchestrator_gateway.py +++ b/tests/unit/test_orchestrator_gateway.py @@ -7,10 +7,10 @@ import unittest from pathlib import Path from unittest.mock import Mock, patch -from bot_bottle.orchestrator.gateway import ( +from bot_bottle.backend.docker.gateway import DockerGateway +from bot_bottle.gateway import ( GATEWAY_CA_CERT, GATEWAY_NAME, - DockerGateway, GatewayError, rotate_gateway_ca, ) @@ -20,7 +20,7 @@ from tests.unit import use_bottle_root _CA_PEM = "-----BEGIN CERTIFICATE-----\nMII...\n-----END CERTIFICATE-----\n" -_RUN_DOCKER = "bot_bottle.orchestrator.gateway.run_docker" +_RUN_DOCKER = "bot_bottle.backend.docker.gateway.run_docker" def _proc(returncode: int = 0, stdout: str = "", stderr: str = "") -> Mock: @@ -162,7 +162,7 @@ class TestDockerGateway(unittest.TestCase): # First read: CA not there yet; second read: present. seq = [_proc(returncode=1, stderr="No such file"), _proc(stdout=_CA_PEM)] with patch(_RUN_DOCKER, side_effect=seq), \ - patch("bot_bottle.orchestrator.gateway.time.sleep"): + patch("bot_bottle.backend.docker.gateway.time.sleep"): self.assertEqual(_CA_PEM, self.sc.ca_cert_pem(timeout=5)) def test_ensure_running_reuses_existing_network(self) -> None: diff --git a/tests/unit/test_orchestrator_registration.py b/tests/unit/test_orchestrator_registration.py index 846da4ea..738ecf84 100644 --- a/tests/unit/test_orchestrator_registration.py +++ b/tests/unit/test_orchestrator_registration.py @@ -7,7 +7,7 @@ import unittest from pathlib import Path from bot_bottle.egress import EgressPlan, EgressRoute -from bot_bottle.egress_addon_core import LOG_BLOCKS, load_config +from bot_bottle.gateway.egress.addon_core import LOG_BLOCKS, load_config from bot_bottle.orchestrator.registration import ( RegistrationInputs, egress_policy, diff --git a/tests/unit/test_orchestrator_registry.py b/tests/unit/test_orchestrator_registry_store.py similarity index 99% rename from tests/unit/test_orchestrator_registry.py rename to tests/unit/test_orchestrator_registry_store.py index 90d0ff43..8b46397a 100644 --- a/tests/unit/test_orchestrator_registry.py +++ b/tests/unit/test_orchestrator_registry_store.py @@ -9,7 +9,7 @@ import unittest from contextlib import closing from pathlib import Path -from bot_bottle.orchestrator.registry import ( +from bot_bottle.orchestrator.store.registry_store import ( BottleRecord, RegistryStore, new_identity_token, diff --git a/tests/unit/test_orchestrator_rotate_ca.py b/tests/unit/test_orchestrator_rotate_ca.py index 22199b8a..e3ee60b7 100644 --- a/tests/unit/test_orchestrator_rotate_ca.py +++ b/tests/unit/test_orchestrator_rotate_ca.py @@ -8,8 +8,8 @@ from pathlib import Path from unittest.mock import Mock, patch from bot_bottle.orchestrator import rotate_ca -from bot_bottle.orchestrator.gateway import GATEWAY_NAME -from bot_bottle.orchestrator.lifecycle import INFRA_NAME +from bot_bottle.gateway import GATEWAY_NAME +from bot_bottle.backend.docker.infra import INFRA_NAME from bot_bottle.paths import host_gateway_ca_dir from tests.unit import use_bottle_root diff --git a/tests/unit/test_orchestrator_secret_store.py b/tests/unit/test_orchestrator_secret_store.py index 64dd453d..2b0e0d51 100644 --- a/tests/unit/test_orchestrator_secret_store.py +++ b/tests/unit/test_orchestrator_secret_store.py @@ -4,7 +4,7 @@ from __future__ import annotations import unittest -from bot_bottle.orchestrator.secret_store import ( +from bot_bottle.orchestrator.store.secret_store import ( ENV_VAR_SECRET_NAME, decrypt_value, encrypt_value, diff --git a/tests/unit/test_orchestrator_control_plane.py b/tests/unit/test_orchestrator_server.py similarity index 97% rename from tests/unit/test_orchestrator_control_plane.py rename to tests/unit/test_orchestrator_server.py index b400bc5c..e3dfb1be 100644 --- a/tests/unit/test_orchestrator_control_plane.py +++ b/tests/unit/test_orchestrator_server.py @@ -19,17 +19,16 @@ from contextlib import closing from pathlib import Path from unittest.mock import patch -from bot_bottle.control_auth import ROLE_CLI, ROLE_GATEWAY, mint +from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY, mint from bot_bottle.orchestrator.broker import StubBroker -from bot_bottle.orchestrator.control_plane import dispatch, make_server -from bot_bottle.orchestrator.registry import BottleRecord, RegistryStore +from bot_bottle.orchestrator.server import dispatch, make_server +from bot_bottle.orchestrator.store.registry_store import BottleRecord, RegistryStore from bot_bottle.orchestrator.service import Orchestrator -from bot_bottle.store_manager import StoreManager -from bot_bottle.supervise import ( +from bot_bottle.orchestrator.store.store_manager import StoreManager +from bot_bottle.orchestrator.supervisor import ( Proposal, TOOL_EGRESS_ALLOW, - sha256_hex, - write_proposal, + Supervisor, ) @@ -285,7 +284,7 @@ class TestServerRoundTrip(unittest.TestCase): self.assertEqual(reg["bottle_id"], attr["bottle_id"]) -class TestControlPlaneAuth(unittest.TestCase): +class TestOrchestratorAuth(unittest.TestCase): """Role-scoped control-plane tokens (issue #400 / #469 review): every route but /health needs a valid token, and the token's role gates which routes it reaches — a `gateway` data-plane token can't drive the operator routes.""" @@ -350,7 +349,7 @@ class TestControlPlaneAuth(unittest.TestCase): self.assertIsNotNone(self.orch.registry.get(rec.bottle_id)) def _server_with_key(self, signing_key: str): - with patch.dict("os.environ", {"BOT_BOTTLE_CONTROL_PLANE_TOKEN": signing_key}): + with patch.dict("os.environ", {"BOT_BOTTLE_ORCHESTRATOR_TOKEN": signing_key}): server = make_server(self.orch, "127.0.0.1", 0) self.addCleanup(server.server_close) threading.Thread(target=server.serve_forever, daemon=True).start() @@ -361,7 +360,7 @@ class TestControlPlaneAuth(unittest.TestCase): def _status(self, url: str, *, header: str | None = None) -> int: req = urllib.request.Request(url) if header is not None: - req.add_header("x-bot-bottle-control-auth", header) + req.add_header("x-bot-bottle-orchestrator-auth", header) try: return urllib.request.urlopen(req, timeout=5).status except urllib.error.HTTPError as e: @@ -385,7 +384,7 @@ class TestControlPlaneAuth(unittest.TestCase): grants full cli access, so existing round-trip behavior is unchanged.""" with patch.dict("os.environ", {}, clear=False): import os - os.environ.pop("BOT_BOTTLE_CONTROL_PLANE_TOKEN", None) + os.environ.pop("BOT_BOTTLE_ORCHESTRATOR_TOKEN", None) server = make_server(self.orch, "127.0.0.1", 0) self.addCleanup(server.server_close) self.assertEqual(ROLE_CLI, server.role_for("")) @@ -420,8 +419,8 @@ class TestDispatchSupervise(unittest.TestCase): "10.243.0.1", metadata=json.dumps({"slug": slug}), policy="routes: []\n") p = Proposal.new( bottle_slug=slug, tool=TOOL_EGRESS_ALLOW, proposed_file=proposed, - justification="need it", current_file_hash=sha256_hex(proposed)) - write_proposal(p) + justification="need it") + Supervisor().write_proposal(p) return p.id def test_list_pending(self) -> None: diff --git a/tests/unit/test_orchestrator_service.py b/tests/unit/test_orchestrator_service.py index 06cf6114..8ba25891 100644 --- a/tests/unit/test_orchestrator_service.py +++ b/tests/unit/test_orchestrator_service.py @@ -12,18 +12,15 @@ from pathlib import Path from unittest.mock import patch from bot_bottle.orchestrator.broker import LaunchBroker, LaunchRequest, StubBroker -from bot_bottle.orchestrator.registry import RegistryStore +from bot_bottle.orchestrator.store.registry_store import RegistryStore from bot_bottle.orchestrator.service import Orchestrator -from bot_bottle.orchestrator.gateway import Gateway -from bot_bottle.orchestrator.secret_store import new_env_var_secret -from bot_bottle.store_manager import StoreManager -from bot_bottle.supervise import ( +from bot_bottle.orchestrator.store.secret_store import new_env_var_secret +from bot_bottle.orchestrator.store.store_manager import StoreManager +from bot_bottle.orchestrator.supervisor import ( Proposal, STATUS_APPROVED, TOOL_EGRESS_ALLOW, - read_response, - sha256_hex, - write_proposal, + Supervisor, ) @@ -38,29 +35,6 @@ class _FailingBroker(LaunchBroker): pass -class _FakeGateway(Gateway): - """In-memory gateway for wiring tests.""" - - def __init__(self) -> None: - self.name = "fake-gateway" - self.ensured = 0 - self.built = 0 - self._running = False - - def ensure_built(self) -> None: - self.built += 1 - - def ensure_running(self) -> None: - self.ensured += 1 - self._running = True - - def is_running(self) -> bool: - return self._running - - def stop(self) -> None: - self._running = False - - class TestOrchestrator(unittest.TestCase): def setUp(self) -> None: self._tmp = tempfile.TemporaryDirectory() @@ -168,24 +142,11 @@ class TestOrchestrator(unittest.TestCase): orch.launch_bottle("10.243.0.9") self.assertEqual([], self.store.all()) # no orphan - def test_gateway_unconfigured_by_default(self) -> None: + def test_gateway_status_reports_unconfigured(self) -> None: + # The orchestrator no longer owns a standalone gateway lifecycle; the + # consolidated flow runs the gateway data plane in the per-host infra + # container/VM. `GET /gateway` therefore always reports unconfigured. self.assertEqual({"configured": False}, self.orch.gateway_status()) - self.orch.ensure_gateway() # no-op, must not raise - - def test_gateway_wired_and_ensured(self) -> None: - sc = _FakeGateway() - orch = Orchestrator(self.store, self.broker, self.secret, gateway=sc) - self.assertEqual( - {"configured": True, "name": "fake-gateway", "running": False}, - orch.gateway_status(), - ) - orch.ensure_gateway() - self.assertEqual(1, sc.built) # ensure_gateway builds first, - self.assertEqual(1, sc.ensured) # then runs - self.assertEqual( - {"configured": True, "name": "fake-gateway", "running": True}, - orch.gateway_status(), - ) class TestOrchestratorSupervise(unittest.TestCase): @@ -221,8 +182,8 @@ class TestOrchestratorSupervise(unittest.TestCase): def _queue(self, slug: str, proposed: str) -> str: p = Proposal.new( bottle_slug=slug, tool=TOOL_EGRESS_ALLOW, proposed_file=proposed, - justification="need it", current_file_hash=sha256_hex(proposed)) - write_proposal(p) + justification="need it") + Supervisor().write_proposal(p) return p.id def test_pending_lists_queued_proposal(self) -> None: @@ -245,7 +206,7 @@ class TestOrchestratorSupervise(unittest.TestCase): assert rec is not None self.assertEqual(new_routes, rec.policy) # response written -> agent unblocks, proposal no longer pending - self.assertEqual(STATUS_APPROVED, read_response("demo", pid).status) + self.assertEqual(STATUS_APPROVED, Supervisor().read_response("demo", pid).status) self.assertEqual([], self.orch.supervise_pending()) def test_pending_carries_human_label(self) -> None: @@ -298,7 +259,7 @@ class TestOrchestratorSupervise(unittest.TestCase): rec = self.store.get(bottle_id) assert rec is not None self.assertEqual("routes:\n - host: existing.com\n", rec.policy) - self.assertEqual("rejected", read_response("demo", pid).status) + self.assertEqual("rejected", Supervisor().read_response("demo", pid).status) def test_unknown_proposal_is_error(self) -> None: ok, err = self.orch.supervise_respond( diff --git a/tests/unit/test_policy_resolver.py b/tests/unit/test_policy_resolver.py index 83544ff1..e034d238 100644 --- a/tests/unit/test_policy_resolver.py +++ b/tests/unit/test_policy_resolver.py @@ -7,15 +7,15 @@ import unittest import urllib.error from unittest.mock import MagicMock, patch -from bot_bottle.policy_resolver import ( - CONTROL_AUTH_HEADER, - CONTROL_AUTH_JWT_ENV, +from bot_bottle.gateway.policy_resolver import ( + ORCHESTRATOR_AUTH_HEADER, + ORCHESTRATOR_AUTH_JWT_ENV, PolicyResolveError, PolicyResolver, - _control_auth_headers, + _orchestrator_auth_headers, ) -_URLOPEN = "bot_bottle.policy_resolver.urllib.request.urlopen" +_URLOPEN = "bot_bottle.gateway.policy_resolver.urllib.request.urlopen" def _resp(payload: object) -> MagicMock: @@ -29,16 +29,16 @@ def _http_error(code: int) -> urllib.error.HTTPError: return urllib.error.HTTPError("http://x/resolve", code, "err", {}, None) # type: ignore[arg-type] -class TestControlAuthHeaders(unittest.TestCase): +class TestOrchestratorAuthHeaders(unittest.TestCase): def test_sends_the_gateway_jwt_when_configured(self) -> None: - with patch.dict("os.environ", {CONTROL_AUTH_JWT_ENV: "gateway.jwt.tok"}): - self.assertEqual({CONTROL_AUTH_HEADER: "gateway.jwt.tok"}, _control_auth_headers()) + with patch.dict("os.environ", {ORCHESTRATOR_AUTH_JWT_ENV: "gateway.jwt.tok"}): + self.assertEqual({ORCHESTRATOR_AUTH_HEADER: "gateway.jwt.tok"}, _orchestrator_auth_headers()) def test_sends_nothing_when_unset(self) -> None: import os with patch.dict("os.environ", {}, clear=False): - os.environ.pop(CONTROL_AUTH_JWT_ENV, None) - self.assertEqual({}, _control_auth_headers()) + os.environ.pop(ORCHESTRATOR_AUTH_JWT_ENV, None) + self.assertEqual({}, _orchestrator_auth_headers()) class TestPolicyResolver(unittest.TestCase): diff --git a/tests/unit/test_supervise.py b/tests/unit/test_supervise.py index 6381af09..13957058 100644 --- a/tests/unit/test_supervise.py +++ b/tests/unit/test_supervise.py @@ -1,39 +1,30 @@ """Unit: supervise queue + audit log + diff helpers (PRD 0013).""" import tempfile -import threading -import time import unittest from datetime import datetime, timezone from pathlib import Path -from bot_bottle import supervise +from bot_bottle.orchestrator import supervisor as supervise from bot_bottle.paths import host_db_path +from bot_bottle.util import render_diff, sha256_hex from tests.unit import use_bottle_root -from bot_bottle.audit_store import AuditStore -from bot_bottle.queue_store import QueueStore -from bot_bottle.supervise import ( +from bot_bottle.store.audit_store import AuditStore +from bot_bottle.orchestrator.store.queue_store import QueueStore +from bot_bottle.orchestrator.supervisor import ( AuditEntry, Proposal, Response, STATUS_APPROVED, STATUS_MODIFIED, STATUS_REJECTED, + Supervisor, TOOL_EGRESS_ALLOW, TOOL_GITLEAKS_ALLOW, - archive_proposal, - list_pending_proposals, - read_audit_entries, - read_proposal, - read_response, - render_diff, - sha256_hex, - wait_for_response, - write_audit_entry, - write_proposal, - write_response, ) +_SV = Supervisor() + FIXED_TS = datetime(2026, 5, 25, 12, 0, 0, tzinfo=timezone.utc) @@ -48,7 +39,6 @@ def _proposal( tool=tool, proposed_file=proposed, justification=justification, - current_file_hash=sha256_hex(proposed), now=FIXED_TS, ) @@ -127,93 +117,49 @@ class TestQueueIO(unittest.TestCase): def test_write_and_read_proposal(self): p = _proposal() - path = write_proposal(p) + path = _SV.write_proposal(p) self.assertTrue(path.exists()) self.assertEqual(host_db_path(), path) self.assertEqual(0o600, path.stat().st_mode & 0o777) - loaded = read_proposal(self.slug, p.id) + loaded = _SV.read_proposal(self.slug, p.id) self.assertEqual(p, loaded) def test_list_pending_excludes_responded(self): a = _proposal(justification="first") b = _proposal(justification="second") - write_proposal(a) - write_proposal(b) - write_response(self.slug, Response( + _SV.write_proposal(a) + _SV.write_proposal(b) + _SV.write_response(self.slug, Response( proposal_id=a.id, status=STATUS_APPROVED, notes="", )) - pending = list_pending_proposals(self.slug) + pending = _SV.list_pending_proposals(self.slug) self.assertEqual([b.id], [p.id for p in pending]) def test_list_pending_returns_empty_for_missing_slug(self): - self.assertEqual([], list_pending_proposals("nope")) + self.assertEqual([], _SV.list_pending_proposals("nope")) def test_list_pending_sorted_by_arrival(self): # Fabricate two with explicit timestamps. a = Proposal.new( bottle_slug="dev", tool=TOOL_EGRESS_ALLOW, proposed_file="routes:\n - host: early.example.com\n", justification="early", - current_file_hash="x", now=datetime(2026, 5, 25, 10, 0, 0, tzinfo=timezone.utc), ) b = Proposal.new( bottle_slug="dev", tool=TOOL_EGRESS_ALLOW, proposed_file="routes:\n - host: late.example.com\n", justification="late", - current_file_hash="x", now=datetime(2026, 5, 25, 14, 0, 0, tzinfo=timezone.utc), ) # Write in reverse order. - write_proposal(b) - write_proposal(a) - ordered = list_pending_proposals(self.slug) + _SV.write_proposal(b) + _SV.write_proposal(a) + ordered = _SV.list_pending_proposals(self.slug) self.assertEqual([a.id, b.id], [p.id for p in ordered]) def test_write_and_read_response(self): r = Response(proposal_id="xyz", status=STATUS_REJECTED, notes="no") - write_response(self.slug, r) - self.assertEqual(r, read_response(self.slug, "xyz")) - - def test_wait_for_response_returns_when_file_appears(self): - p = _proposal() - write_proposal(p) - - def write_after_delay(): - time.sleep(0.05) - write_response(self.slug, Response( - proposal_id=p.id, status=STATUS_APPROVED, notes="ok", - )) - - t = threading.Thread(target=write_after_delay) - t.start() - try: - r = wait_for_response(self.slug, p.id, poll_interval=0.01) - finally: - t.join() - self.assertEqual(STATUS_APPROVED, r.status) - self.assertEqual("ok", r.notes) - - def test_wait_for_response_times_out(self): - deadline = time.monotonic() + 0.05 - with self.assertRaises(TimeoutError): - wait_for_response( - self.slug, "never", - poll_interval=0.01, deadline=deadline, - ) - - def test_archive_proposal_hides_rows(self): - p = _proposal() - write_proposal(p) - write_response(self.slug, Response( - proposal_id=p.id, status=STATUS_APPROVED, notes="", - )) - archive_proposal(self.slug, p.id) - self.assertEqual([], list_pending_proposals(self.slug)) - with self.assertRaises(FileNotFoundError): - read_response(self.slug, p.id) - - def test_archive_is_idempotent_on_missing_files(self): - # Should not raise. - archive_proposal(self.slug, "nope") + _SV.write_response(self.slug, r) + self.assertEqual(r, _SV.read_response(self.slug, "xyz")) class TestAuditLog(unittest.TestCase): @@ -239,15 +185,15 @@ class TestAuditLog(unittest.TestCase): justification="agent needed gh-api token", diff="--- before\n+++ after\n", ) - path = write_audit_entry(e) + path = _SV.write_audit_entry(e) self.assertEqual(host_db_path(), path) self.assertEqual(0o600, path.stat().st_mode & 0o777) - loaded = read_audit_entries("cred-proxy", "dev") + loaded = _SV.read_audit_entries("cred-proxy", "dev") self.assertEqual([e], loaded) def test_appends_one_line_per_entry(self): for i in range(3): - write_audit_entry(AuditEntry( + _SV.write_audit_entry(AuditEntry( timestamp=f"2026-05-25T12:00:0{i}+00:00", bottle_slug="dev", component="egress", @@ -256,7 +202,7 @@ class TestAuditLog(unittest.TestCase): justification="", diff="", )) - entries = read_audit_entries("egress", "dev") + entries = _SV.read_audit_entries("egress", "dev") self.assertEqual(3, len(entries)) self.assertEqual( ["2026-05-25T12:00:00+00:00", "2026-05-25T12:00:01+00:00", @@ -265,7 +211,7 @@ class TestAuditLog(unittest.TestCase): ) def test_separate_logs_per_component_slug(self): - write_audit_entry(AuditEntry( + _SV.write_audit_entry(AuditEntry( timestamp="t", bottle_slug="dev", component="cred-proxy", @@ -274,7 +220,7 @@ class TestAuditLog(unittest.TestCase): justification="", diff="", )) - write_audit_entry(AuditEntry( + _SV.write_audit_entry(AuditEntry( timestamp="t", bottle_slug="dev", component="egress", @@ -283,7 +229,7 @@ class TestAuditLog(unittest.TestCase): justification="", diff="", )) - write_audit_entry(AuditEntry( + _SV.write_audit_entry(AuditEntry( timestamp="t", bottle_slug="other", component="cred-proxy", @@ -292,20 +238,20 @@ class TestAuditLog(unittest.TestCase): justification="", diff="", )) - self.assertEqual(1, len(read_audit_entries("cred-proxy", "dev"))) - self.assertEqual(1, len(read_audit_entries("egress", "dev"))) - self.assertEqual(1, len(read_audit_entries("cred-proxy", "other"))) + self.assertEqual(1, len(_SV.read_audit_entries("cred-proxy", "dev"))) + self.assertEqual(1, len(_SV.read_audit_entries("egress", "dev"))) + self.assertEqual(1, len(_SV.read_audit_entries("cred-proxy", "other"))) def test_read_audit_entries_missing_log_returns_empty(self): - self.assertEqual([], read_audit_entries("cred-proxy", "no-such-bottle")) + self.assertEqual([], _SV.read_audit_entries("cred-proxy", "no-such-bottle")) class TestDiffAndHash(unittest.TestCase): def test_render_diff_returns_empty_when_unchanged(self): - self.assertEqual("", render_diff("a\nb\n", "a\nb\n")) + self.assertEqual("", render_diff("a\nb\n", "a\nb\n", before_title="current", after_label="proposed")) def test_render_diff_shows_changes(self): - diff = render_diff("a\nb\nc\n", "a\nB\nc\n", label="routes.yaml") + diff = render_diff("a\nb\nc\n", "a\nB\nc\n", label="routes.yaml", before_title="current", after_label="proposed") self.assertIn("routes.yaml (current)", diff) self.assertIn("routes.yaml (proposed)", diff) self.assertIn("-b", diff) @@ -338,7 +284,6 @@ class TestToolConstants(unittest.TestCase): tool=supervise.TOOL_EGRESS_TOKEN_ALLOW, proposed_file="host: api.example.com\n", justification="false positive", - current_file_hash="h", ) self.assertEqual(p, Proposal.from_dict(p.to_dict())) @@ -352,7 +297,7 @@ class TestToolConstants(unittest.TestCase): ) -class _StubSupervise(supervise.Supervise): +class _StubSupervise(supervise.Supervisor): """Concrete Supervise subclass for testing the prepare template.""" def start(self, plan): # type: ignore diff --git a/tests/unit/test_supervise_cli.py b/tests/unit/test_supervise_cli.py index b785a43f..1d6679ba 100644 --- a/tests/unit/test_supervise_cli.py +++ b/tests/unit/test_supervise_cli.py @@ -12,14 +12,13 @@ import unittest from datetime import datetime, timezone from unittest.mock import MagicMock, patch -from bot_bottle.cli import supervise as supervise_cli -from bot_bottle.supervise import ( +from bot_bottle.cli.commands import supervise as supervise_cli +from bot_bottle.orchestrator.supervisor import ( Proposal, TOOL_EGRESS_ALLOW, TOOL_EGRESS_BLOCK, TOOL_GITLEAKS_ALLOW, TOOL_EGRESS_TOKEN_ALLOW, - sha256_hex, ) @@ -37,7 +36,7 @@ def _proposal(slug: str = "dev", tool: str = TOOL_EGRESS_ALLOW, payload = payloads.get(tool, "") return Proposal.new( bottle_slug=slug, tool=tool, proposed_file=payload, - justification=f"needed for {slug}", current_file_hash=sha256_hex(payload), + justification=f"needed for {slug}", now=now, ) diff --git a/tests/unit/test_supervise_cli_crash_logging.py b/tests/unit/test_supervise_cli_crash_logging.py index 0f07f08d..e36c0c34 100644 --- a/tests/unit/test_supervise_cli_crash_logging.py +++ b/tests/unit/test_supervise_cli_crash_logging.py @@ -18,7 +18,7 @@ from pathlib import Path from unittest import mock from tests.unit import use_bottle_root -from bot_bottle.cli import supervise as supervise_cli +from bot_bottle.cli.commands import supervise as supervise_cli from bot_bottle.log import Die, die diff --git a/tests/unit/test_supervise_edge.py b/tests/unit/test_supervise_edge.py index 4cedbfa3..82929e45 100644 --- a/tests/unit/test_supervise_edge.py +++ b/tests/unit/test_supervise_edge.py @@ -5,28 +5,24 @@ fallback paths.""" from __future__ import annotations import tempfile -import time import unittest from pathlib import Path from unittest.mock import patch -from bot_bottle import supervise -from bot_bottle.audit_store import AuditStore +from bot_bottle.orchestrator import supervisor as supervise +from bot_bottle.store.audit_store import AuditStore from bot_bottle.paths import bot_bottle_root -from bot_bottle.queue_store import QueueStore -from bot_bottle.supervise import ( +from bot_bottle.orchestrator.store.queue_store import QueueStore +from bot_bottle.orchestrator.supervisor import ( AuditEntry, Proposal, STATUS_APPROVED, + Supervisor, TOOL_EGRESS_ALLOW, - list_pending_proposals, - read_audit_entries, - read_proposal, - read_response, - wait_for_response, - write_audit_entry, ) +_SV = Supervisor() + def _proposal() -> Proposal: return Proposal.new( @@ -34,7 +30,6 @@ def _proposal() -> Proposal: tool=TOOL_EGRESS_ALLOW, proposed_file="x", justification="j", - current_file_hash="h", ) @@ -49,21 +44,21 @@ class TestReadMalformed(unittest.TestCase): with patch.dict("os.environ", {"HOME": d}): QueueStore("slug").migrate() with self.assertRaises(FileNotFoundError): - read_proposal("slug", "p") + _SV.read_proposal("slug", "p") def test_read_response_missing_row(self) -> None: with tempfile.TemporaryDirectory() as d: with patch.dict("os.environ", {"HOME": d}): QueueStore("slug").migrate() with self.assertRaises(FileNotFoundError): - read_response("slug", "p") + _SV.read_response("slug", "p") def test_list_pending_reads_db_only(self) -> None: with tempfile.TemporaryDirectory() as d: with patch.dict("os.environ", {"HOME": d}): QueueStore("slug").migrate() - supervise.write_proposal(_proposal()) - pending = list_pending_proposals("slug") + _SV.write_proposal(_proposal()) + pending = _SV.list_pending_proposals("slug") self.assertEqual(1, len(pending)) self.assertEqual("slug", pending[0].bottle_slug) @@ -72,42 +67,26 @@ class TestReadMalformed(unittest.TestCase): with patch.dict("os.environ", {"HOME": d}): QueueStore("slug").migrate() p = _proposal() - supervise.write_proposal(p) - supervise.write_response("slug", supervise.Response( + _SV.write_proposal(p) + _SV.write_response("slug", supervise.Response( proposal_id=p.id, status=STATUS_APPROVED, notes="", )) - self.assertEqual([], list_pending_proposals("slug")) - - -class TestWaitForResponse(unittest.TestCase): - def test_missing_response_times_out(self) -> None: - with tempfile.TemporaryDirectory() as d: - with patch.dict("os.environ", {"HOME": d}): - QueueStore("slug").migrate() - with self.assertRaises(TimeoutError): - wait_for_response("slug", "p", deadline=time.monotonic()) - - def test_empty_db_response_does_not_count(self) -> None: - with tempfile.TemporaryDirectory() as d: - with patch.dict("os.environ", {"HOME": d}): - QueueStore("slug").migrate() - with self.assertRaises(TimeoutError): - wait_for_response("slug", "p", deadline=time.monotonic()) + self.assertEqual([], _SV.list_pending_proposals("slug")) class TestReadAuditEntries(unittest.TestCase): def test_missing_log_returns_empty(self) -> None: with tempfile.TemporaryDirectory() as home, \ patch.dict("os.environ", {"HOME": home}): - self.assertEqual([], read_audit_entries("egress", "nope")) + self.assertEqual([], _SV.read_audit_entries("egress", "nope")) def test_reads_entries_from_db(self) -> None: with tempfile.TemporaryDirectory() as home, \ patch.dict("os.environ", {"HOME": home}): AuditStore().migrate() - write_audit_entry(AuditEntry( + _SV.write_audit_entry(AuditEntry( timestamp="t", bottle_slug="slug", component="egress", @@ -116,7 +95,7 @@ class TestReadAuditEntries(unittest.TestCase): justification="", diff="", )) - write_audit_entry(AuditEntry( + _SV.write_audit_entry(AuditEntry( timestamp="t", bottle_slug="other", component="egress", @@ -125,23 +104,10 @@ class TestReadAuditEntries(unittest.TestCase): justification="", diff="", )) - entries = read_audit_entries("egress", "slug") + entries = _SV.read_audit_entries("egress", "slug") self.assertEqual(1, len(entries)) self.assertEqual("approve", entries[0].operator_action) - def test_legacy_audit_log_file_does_not_count(self) -> None: - with tempfile.TemporaryDirectory() as home, \ - patch.dict("os.environ", {"HOME": home}): - path = supervise.audit_log_path("egress", "slug") - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text( - '{"timestamp": "t", "bottle_slug": "slug", "component": "egress",' - ' "operator_action": "approve", "operator_notes": "",' - ' "justification": "", "diff": ""}\n' - ) - entries = read_audit_entries("egress", "slug") - self.assertEqual([], entries) - class TestStoreGuardBranches(unittest.TestCase): """Direct QueueStore / AuditStore construction and early-return guard branches.""" @@ -170,14 +136,6 @@ class TestStoreGuardBranches(unittest.TestCase): db.unlink() self.assertEqual([], store.list_all_pending_proposals()) - def test_queue_store_missing_db_archive_is_noop(self): - with tempfile.TemporaryDirectory() as d: - db = Path(d) / "q.db" - store = QueueStore("key", db_path=db) - store.migrate() - db.unlink() - store.archive_proposal("anything") # must not raise - def test_queue_store_chmod_oserror_is_swallowed(self): with tempfile.TemporaryDirectory() as d: db = Path(d) / "q.db" diff --git a/tests/unit/test_supervise_server.py b/tests/unit/test_supervise_server.py index c666c556..52c575cb 100644 --- a/tests/unit/test_supervise_server.py +++ b/tests/unit/test_supervise_server.py @@ -18,12 +18,12 @@ from pathlib import Path from tests.unit import use_bottle_root -from bot_bottle import supervise as _sv -from bot_bottle import queue_store as _qs -from bot_bottle import audit_store as _as +from bot_bottle.orchestrator import supervisor as _sv +from bot_bottle.orchestrator.store import queue_store as _qs +from bot_bottle.store import audit_store as _as -from bot_bottle import supervise_server # noqa: E402 -from bot_bottle.supervise_server import ( +from bot_bottle.gateway.supervisor import server as supervise_server # noqa: E402 +from bot_bottle.gateway.supervisor.server import ( ERR_INTERNAL, ERR_INVALID_PARAMS, ERR_INVALID_REQUEST, @@ -50,6 +50,11 @@ from bot_bottle.supervise_server import ( validate_proposed_file, ) +# The orchestrator's supervise service, backing the fake resolver's queue reads +# and writes (the fake stands in for the RPC; state still goes through the real +# queue store). +_SV = _sv.Supervisor() + # Fixed caller identity for the handler tests. The control plane attributes by # (source_ip, identity_token); the fake resolver ignores them and answers for a # fixed bottle, since attribution itself is covered by the orchestrator tests. @@ -87,9 +92,9 @@ class _FakeSuperviseResolver: return None proposal = _sv.Proposal.new( bottle_slug=self.bottle_id, tool=tool, proposed_file=proposed_file, - justification=justification, current_file_hash=_sv.sha256_hex(proposed_file), + justification=justification, ) - _sv.write_proposal(proposal) + _SV.write_proposal(proposal) return proposal.id def poll_supervise( @@ -101,10 +106,10 @@ class _FakeSuperviseResolver: if self.bottle_id is None or self.poll_none: return None try: - response = _sv.read_response(self.bottle_id, proposal_id) + response = _SV.read_response(self.bottle_id, proposal_id) except FileNotFoundError: try: - _sv.read_proposal(self.bottle_id, proposal_id) + _SV.read_proposal(self.bottle_id, proposal_id) except FileNotFoundError: return {"status": _sv.POLL_STATUS_UNKNOWN} return {"status": _sv.POLL_STATUS_PENDING} @@ -377,10 +382,10 @@ class TestHandleToolsCall(unittest.TestCase): matching response — the operator half, out of band.""" def runner(): for _ in range(200): - pending = _sv.list_pending_proposals("dev") + pending = _SV.list_pending_proposals("dev") if pending: p = pending[0] - _sv.write_response("dev", _sv.Response( + _SV.write_response("dev", _sv.Response( proposal_id=p.id, status=status, notes=notes, )) return @@ -487,7 +492,7 @@ class TestHandleToolsCall(unittest.TestCase): responder.join() # A decided proposal drops off the operator's pending list (a response # row exists) — poll itself no longer archives (issue #469 review). - self.assertEqual([], _sv.list_pending_proposals("dev")) + self.assertEqual([], _SV.list_pending_proposals("dev")) def test_pending_response_times_out_without_archive(self): result = _tools_call( @@ -505,7 +510,7 @@ class TestHandleToolsCall(unittest.TestCase): text = result["content"][0]["text"] # type: ignore[index] self.assertIn("status: pending", text) self.assertIn("proposal remains queued", text) - self.assertEqual(1, len(_sv.list_pending_proposals("dev"))) + self.assertEqual(1, len(_SV.list_pending_proposals("dev"))) _ALLOW: dict[str, object] = { "name": _sv.TOOL_EGRESS_ALLOW, @@ -745,9 +750,8 @@ class TestNonBlockingSupervise(unittest.TestCase): tool=_sv.TOOL_EGRESS_ALLOW, proposed_file=self._ROUTES, justification="need example.com", - current_file_hash=_sv.sha256_hex(self._ROUTES), ) - _sv.write_proposal(p) + _SV.write_proposal(p) return p # --- pending response carries the id --- @@ -771,7 +775,7 @@ class TestNonBlockingSupervise(unittest.TestCase): self.assertFalse(result["isError"]) # type: ignore[index] text = result["content"][0]["text"] # type: ignore[index] self.assertIn("status: pending", text) - pending = _sv.list_pending_proposals("dev") + pending = _SV.list_pending_proposals("dev") self.assertEqual(1, len(pending)) # still queued, not archived self.assertIn(pending[0].id, text) # agent got the id to poll @@ -779,7 +783,7 @@ class TestNonBlockingSupervise(unittest.TestCase): def test_check_returns_approved_idempotently(self): p = self._seed_proposal() - _sv.write_response("dev", _sv.Response(proposal_id=p.id, status=_sv.STATUS_APPROVED, notes="ok")) + _SV.write_response("dev", _sv.Response(proposal_id=p.id, status=_sv.STATUS_APPROVED, notes="ok")) result = _check(self.resolver, {"arguments": {"proposal_id": p.id}}) self.assertFalse(result["isError"]) text = result["content"][0]["text"] # type: ignore[index] @@ -792,7 +796,7 @@ class TestNonBlockingSupervise(unittest.TestCase): def test_check_rejected_sets_isError(self): p = self._seed_proposal() - _sv.write_response("dev", _sv.Response(proposal_id=p.id, status=_sv.STATUS_REJECTED, notes="no")) + _SV.write_response("dev", _sv.Response(proposal_id=p.id, status=_sv.STATUS_REJECTED, notes="no")) result = _check(self.resolver, {"arguments": {"proposal_id": p.id}}) self.assertTrue(result["isError"]) self.assertIn("status: rejected", result["content"][0]["text"]) # type: ignore[index] @@ -804,7 +808,7 @@ class TestNonBlockingSupervise(unittest.TestCase): text = result["content"][0]["text"] # type: ignore[index] self.assertIn("status: pending", text) self.assertIn(p.id, text) - self.assertEqual(1, len(_sv.list_pending_proposals("dev"))) # not archived + self.assertEqual(1, len(_SV.list_pending_proposals("dev"))) # not archived def test_check_unknown_id_is_error(self): result = _check(self.resolver, {"arguments": {"proposal_id": "no-such-proposal"}}) @@ -848,15 +852,15 @@ class TestNonBlockingSupervise(unittest.TestCase): }, ServerConfig(response_timeout_seconds=0.05), ) - pid = _sv.list_pending_proposals("dev")[0].id + pid = _SV.list_pending_proposals("dev")[0].id self.assertIn(pid, result["content"][0]["text"]) # type: ignore[index] # 2. operator decides out-of-band - _sv.write_response("dev", _sv.Response(proposal_id=pid, status=_sv.STATUS_APPROVED, notes="ok")) + _SV.write_response("dev", _sv.Response(proposal_id=pid, status=_sv.STATUS_APPROVED, notes="ok")) # 3. agent resumes by polling — no re-proposing poll = _check(self.resolver, {"arguments": {"proposal_id": pid}}) self.assertFalse(poll["isError"]) self.assertIn("status: approved", poll["content"][0]["text"]) # type: ignore[index] - self.assertEqual([], _sv.list_pending_proposals("dev")) # resolved (response exists) + self.assertEqual([], _SV.list_pending_proposals("dev")) # resolved (response exists) if __name__ == "__main__":