fix(supervise): get bot-bottle.db off the data plane (supervise + egress proposals over RPC) #471

Open
didericis-claude wants to merge 44 commits from fix/db-off-data-plane-469 into main
52 changed files with 380 additions and 385 deletions
Showing only changes of commit 44e2b5a897 - Show all commits
+1 -1
View File
@@ -48,7 +48,7 @@ from ..git_gate import GitGatePlan
from ..log import die, info, warn
from ..util import read_tty_line
from ..manifest import Manifest, ManifestIndex
from ..supervise import SupervisePlan
from ..supervisor.plan import SupervisePlan
from ..util import expand_tilde
from ..env import resolve_env, ResolvedEnv
from ..workspace import WorkspacePlan, workspace_plan
+2 -2
View File
@@ -13,7 +13,7 @@ from ..egress import EgressPlan
from ..git_gate import GitGatePlan
from ..orchestrator.client import OrchestratorClient, RegisteredBottle
from ..orchestrator.registration import registration_inputs
from ..orchestrator.secret_store import new_env_var_secret
from ..orchestrator.store.secret_store import new_env_var_secret
from .docker.gateway_provision import GatewayTransport, deprovision_git_gate, provision_git_gate
@@ -58,7 +58,7 @@ def teardown_consolidated(
) -> None:
"""Deregister the bottle and remove its git-gate state. Both steps are
idempotent so this is safe from a cleanup trap."""
from ..orchestrator.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS
from ..orchestrator.store.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS
OrchestratorClient(
orchestrator_url,
timeout=timeout if timeout is not None else DEFAULT_TEARDOWN_TIMEOUT_SECONDS,
+2 -2
View File
@@ -24,12 +24,12 @@ from contextlib import contextmanager
from pathlib import Path
from typing import Generator, Sequence
from ...supervise import SUPERVISE_HOSTNAME, SUPERVISE_PORT
from ...supervisor.types import SUPERVISE_HOSTNAME, SUPERVISE_PORT
from ...agent_provider import AgentProvisionPlan
from ...egress import EgressPlan
from ...env import ResolvedEnv
from ...git_gate import GitGatePlan
from ...supervise import SupervisePlan
from ...supervisor.plan import SupervisePlan
from ...manifest import Manifest
from .. import ActiveAgent, BottleBackend, BottleImages, BottleSpec
from . import cleanup as _cleanup
@@ -17,7 +17,7 @@ from __future__ import annotations
from typing import Any
from ...egress import egress_agent_env_entries
from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME
from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
from .bottle_plan import DockerBottlePlan
from .egress import EGRESS_PORT
@@ -22,7 +22,7 @@ from ...git_gate import GitGatePlan
from ...orchestrator.client import OrchestratorClient
from ...gateway import GATEWAY_NETWORK
from ...orchestrator.lifecycle import INFRA_NAME, OrchestratorService
from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME
from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
from ...orchestrator.reprovision import reprovision_bottles
from ..consolidated_util import provision_bottle
from ..consolidated_util import teardown_consolidated as _teardown_util
+2 -2
View File
@@ -64,7 +64,7 @@ from .compose import (
write_compose_file,
)
from .consolidated_compose import consolidated_agent_compose
from ...orchestrator.config_store import resolve_teardown_timeout
from ...orchestrator.store.config_store import resolve_teardown_timeout
from .consolidated_launch import launch_consolidated, teardown_consolidated
from ...orchestrator.lifecycle import INFRA_NAME
from .gateway import DockerGateway
@@ -207,7 +207,7 @@ def launch(
# spec, value only in the subprocess env so it is never written to disk.
compose_env: dict[str, str] = {**os.environ, **plan.forwarded_env}
if plan.env_var_secret:
from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME
from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
compose_env[ENV_VAR_SECRET_NAME] = plan.env_var_secret
info(
f"docker compose up -d (project {project}, agent on shared "
+1 -1
View File
@@ -19,7 +19,7 @@ from ...env import ResolvedEnv
from ...agent_provider import AgentProvisionPlan
from ...egress import EgressPlan
from ...manifest import Manifest
from ...supervise import SupervisePlan
from ...supervisor.plan import SupervisePlan
from ...git_gate import GitGatePlan
def preflight() -> None:
+1 -1
View File
@@ -17,7 +17,7 @@ from ...egress import EgressPlan
from ...env import ResolvedEnv
from ...git_gate import GitGatePlan
from ...manifest import Manifest
from ...supervise import SupervisePlan
from ...supervisor.plan import SupervisePlan
from .. import ActiveAgent, BottleBackend, BottleImages, BottleSpec
from . import cleanup as _cleanup
from . import enumerate as _enumerate
@@ -38,7 +38,7 @@ from ...orchestrator.lifecycle import (
OrchestratorStartError, # re-exported so callers can catch it
)
from ...orchestrator.reprovision import reprovision_bottles
from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME
from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
from ..consolidated_util import (
provision_bottle,
teardown_consolidated as _teardown_util,
+3 -3
View File
@@ -48,14 +48,14 @@ from ...git_gate import (
)
from ...image_cache import check_stale_path
from ...log import die, info, warn
from ...supervise import SUPERVISE_PORT
from ...supervisor.types import SUPERVISE_PORT
from ..docker.egress import EGRESS_PORT
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
from . import firecracker_vm, image_builder, isolation_probe, netpool, util
from .bottle import FirecrackerBottle
from .bottle_plan import FirecrackerBottlePlan
from ...orchestrator.config_store import resolve_teardown_timeout
from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME
from ...orchestrator.store.config_store import resolve_teardown_timeout
from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
from .consolidated_launch import (
launch_consolidated,
persist_env_var_secret,
@@ -9,7 +9,7 @@ from ...egress import EgressPlan
from ...env import ResolvedEnv
from ...git_gate import GitGatePlan
from ...manifest import Manifest
from ...supervise import SupervisePlan
from ...supervisor.plan import SupervisePlan
from .. import BottleSpec
from . import util
from .bottle_plan import FirecrackerBottlePlan
@@ -10,7 +10,7 @@ from ...agent_provider import AgentProvisionPlan
from ...egress import EgressPlan
from ...env import ResolvedEnv
from ...git_gate import GitGatePlan
from ...supervise import SupervisePlan
from ...supervisor.plan import SupervisePlan
from ...manifest import Manifest
from .. import ActiveAgent, BottleBackend, BottleImages, BottleSpec
from . import cleanup as _cleanup
@@ -39,7 +39,7 @@ from ...git_gate import GitGatePlan
from ...log import info
from ...orchestrator.client import OrchestratorClient, OrchestratorClientError
from ...orchestrator.reprovision import reprovision_bottles
from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME
from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
from ..consolidated_util import (
provision_bottle,
teardown_consolidated as _teardown_util,
+3 -3
View File
@@ -56,7 +56,7 @@ from ...gateway.git_http_backend import DEFAULT_PORT as _GIT_HTTP_PORT
from ...image_cache import check_stale
from ...log import die, info, warn
from .. import BottleImages
from ...supervise import SUPERVISE_PORT
from ...supervisor.types import SUPERVISE_PORT
from ..docker.egress import EGRESS_PORT
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
from . import util as container_mod
@@ -68,8 +68,8 @@ from .gateway_hosts import (
)
from . import nested_containers as nested_containers_mod
from .bottle_plan import MacosContainerBottlePlan
from ...orchestrator.config_store import resolve_teardown_timeout
from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME, new_env_var_secret
from ...orchestrator.store.config_store import resolve_teardown_timeout
from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME, new_env_var_secret
from .consolidated_launch import (
GatewayEndpoint,
ensure_gateway,
@@ -8,7 +8,7 @@ from ...agent_provider import AgentProvisionPlan
from ...egress import EgressPlan
from ...env import ResolvedEnv
from ...git_gate import GitGatePlan
from ...supervise import SupervisePlan
from ...supervisor.plan import SupervisePlan
from ...manifest import Manifest
from .. import BottleSpec
from . import util as container_mod
+2 -1
View File
@@ -28,7 +28,8 @@ from ..egress import Egress, EgressPlan
from ..git_gate import GitGate, GitGatePlan
from ..log import die
from ..manifest import Manifest, ManifestBottle
from ..supervise import Supervise, SupervisePlan
from ..supervisor.plan import SupervisePlan
from ..orchestrator.supervisor.supervise import Supervise
from . import BottleSpec
+1 -1
View File
@@ -10,7 +10,7 @@ import sys
from ..errors import MissingEnvVarError
from ..log import Die, die, error
from ..manifest import ManifestError
from ..store.store_manager import StoreManager
from ..orchestrator.store.store_manager import StoreManager
from ._common import PROG
from . import list as _list_mod
from .backend import cmd_backend
+1 -1
View File
@@ -27,7 +27,7 @@ from ..orchestrator.client import (
discover_orchestrator_url,
)
from ..supervise import (
from ..supervisor.types import (
Proposal,
TOOL_EGRESS_ALLOW,
TOOL_EGRESS_BLOCK,
+1 -1
View File
@@ -41,7 +41,7 @@ from bot_bottle.gateway.egress_addon_core import (
scan_outbound,
)
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
from bot_bottle.supervise_types import (
from bot_bottle.supervisor.types import (
STATUS_APPROVED,
STATUS_MODIFIED,
STATUSES,
+2 -2
View File
@@ -283,10 +283,10 @@ from pathlib import Path
# calling bottle exactly as a direct write once did.
try:
from bot_bottle.gateway.policy_resolver import PolicyResolver, PolicyResolveError
from bot_bottle.supervise_types import TOOL_GITLEAKS_ALLOW
from bot_bottle.supervisor.types import TOOL_GITLEAKS_ALLOW
except ImportError:
from policy_resolver import PolicyResolver, PolicyResolveError
from supervise_types import TOOL_GITLEAKS_ALLOW
from supervisor.types import TOOL_GITLEAKS_ALLOW
report_path = Path(sys.argv[1])
source_ip = os.environ.get("SUPERVISE_SOURCE_IP", "")
+1 -1
View File
@@ -62,7 +62,7 @@ from bot_bottle.gateway.egress_addon_core import (
LOG_OFF, load_config, resolve_client_context, route_to_yaml_dict,
)
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
from bot_bottle import supervise as _sv
from bot_bottle.supervisor import types as _sv
# --- JSON-RPC / MCP plumbing ----------------------------------------------
+1 -1
View File
@@ -16,7 +16,7 @@ import secrets
from pathlib import Path
from .. import log
from ..store.store_manager import StoreManager
from .store.store_manager import StoreManager
from .broker import LaunchBroker, StubBroker
from .control_plane import make_server
from .docker_broker import DockerBroker
+1 -1
View File
@@ -65,7 +65,7 @@ from urllib.parse import urlsplit
from ..control_auth import ROLE_CLI, ROLES, verify
from ..paths import CONTROL_PLANE_TOKEN_ENV
from ..supervise_types import TOOLS
from ..supervisor.types import TOOLS
from .service import Orchestrator
# JSON body payload type (parsed request / rendered response).
+3 -3
View File
@@ -27,7 +27,7 @@ from datetime import datetime, timezone
from .broker import LaunchBroker, LaunchRequest, sign_request
from .registry import DEFAULT_REAP_GRACE_SECONDS, BottleRecord, RegistryStore
from ..supervise import (
from .supervisor import (
AuditEntry,
COMPONENT_FOR_TOOL,
POLL_STATUS_PENDING,
@@ -104,7 +104,7 @@ class Orchestrator:
if tokens:
self._tokens[rec.bottle_id] = dict(tokens)
if env_var_secret:
from .secret_store import encrypt_value
from .store.secret_store import encrypt_value
encrypted = {k: encrypt_value(env_var_secret, v) for k, v in tokens.items()}
self.registry.store_agent_secrets(rec.bottle_id, encrypted)
req = LaunchRequest(
@@ -367,7 +367,7 @@ class Orchestrator:
value with *env_var_secret*, and restores ``_tokens[bottle_id]``.
Returns True on success, False when no stored secrets exist for this
bottle or decryption fails (wrong key / corrupt data)."""
from .secret_store import decrypt_value
from .store.secret_store import decrypt_value
encrypted = self.registry.get_agent_secrets(bottle_id)
if not encrypted:
return False
+10
View File
@@ -0,0 +1,10 @@
"""Orchestrator-owned SQLite stores.
The stores whose tables only the orchestrator (control plane) opens: the
supervise proposal/response `queue_store`, the agent-secret `secret_store`, and
the orchestrator `config_store`. They build on the shared `DbStore` /
`migrations` base in `bot_bottle.store`.
Callers import the concrete module directly, e.g.
`from bot_bottle.orchestrator.store.queue_store import QueueStore`.
"""
@@ -11,9 +11,9 @@ import os
import sqlite3
from pathlib import Path
from ..store.db_store import DbStore
from ..store.migrations import TableMigrations
from ..paths import host_db_path
from ...store.db_store import DbStore
from ...store.migrations import TableMigrations
from ...paths import host_db_path
TEARDOWN_TIMEOUT_ENV = "BOT_BOTTLE_ORCHESTRATOR_TEARDOWN_TIMEOUT_SECONDS"
DEFAULT_TEARDOWN_TIMEOUT_SECONDS = 30.0
@@ -7,12 +7,12 @@ import sqlite3
from pathlib import Path
try:
from ..supervise_types import Proposal, Response
from ..paths import host_db_path
from .db_store import DbStore
from .migrations import TableMigrations
from ...supervisor.types import Proposal, Response
from ...paths import host_db_path
from ...store.db_store import DbStore
from ...store.migrations import TableMigrations
except ImportError:
from supervise_types import Proposal, Response # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
from supervisor.types import Proposal, Response # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
from paths import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
from db_store import DbStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
@@ -1,34 +1,34 @@
"""Singleton manager for all bot-bottle SQLite stores (PRD 0013)."""
"""The orchestrator's store manager (PRD 0013 / 0070).
One store manager per service. Post-PRD-0070 the orchestrator is the sole
opener of `bot-bottle.db`, so it owns migrating every table in it: the supervise
`queue_store` (local to `orchestrator.store`) plus the `audit_store` and
`config_store` from the shared `bot_bottle.store` base. The data plane never
touches these it reaches state over the control-plane RPC.
"""
from __future__ import annotations
from pathlib import Path
try:
from .audit_store import AuditStore
from .config_store import ConfigStore
from .queue_store import QueueStore
except ImportError:
from audit_store import AuditStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
from config_store import ConfigStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
from queue_store import QueueStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
from .queue_store import QueueStore
from ...store.audit_store import AuditStore
from ...store.config_store import ConfigStore
_instance: StoreManager | None = None
class StoreManager:
"""Owns db_path and delegates migrate/is_migrated across all stores.
"""Owns db_path and delegates migrate/is_migrated across the orchestrator's
stores.
Use instance() for normal access. Call reset(db_path) in tests to swap
the singleton to a temp path, then reset() with no args to restore the
Use instance() for normal access. Call reset(db_path) in tests to swap the
singleton to a temp path, then reset() with no args to restore the
default."""
def __init__(self, db_path: Path | None = None) -> None:
if db_path is None:
try:
from ..paths import host_db_path
except ImportError:
from paths import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
from ...paths import host_db_path
db_path = host_db_path()
self.db_path = db_path
@@ -0,0 +1,34 @@
"""The orchestrator's view of the supervise plane.
Bundles the orchestrator-owned supervise surface: the queue/audit I/O and diff
rendering (`queue`) and the `Supervise` lifecycle (`supervise`). For
convenience it re-exports the neutral vocabulary (`bot_bottle.supervisor`'s
types + `SupervisePlan`) so orchestrator-side callers service, CLI, tests
import from one place.
The data plane must NOT import this package; it imports `bot_bottle.supervisor`
(neutral) directly and reaches the queue over the control-plane RPC.
"""
from __future__ import annotations
from ...supervisor.types import * # noqa: F401,F403 — re-export the neutral vocabulary
from ...supervisor.plan import SupervisePlan
from .queue import (
archive_all_proposals,
archive_proposal,
audit_dir,
audit_log_path,
list_all_pending_proposals,
list_pending_proposals,
read_audit_entries,
read_proposal,
read_response,
render_diff,
sha256_hex,
wait_for_response,
write_audit_entry,
write_proposal,
write_response,
)
from .supervise import Supervise
+141
View File
@@ -0,0 +1,141 @@
"""Orchestrator-side supervise queue + audit I/O and diff rendering.
The host-side library the orchestrator uses to drive the supervise flow: read /
write proposals and responses against the queue store, append audit entries,
and render operator diffs. Only the orchestrator opens `bot-bottle.db` (PRD
0070), so this lives under `orchestrator/` the data plane reaches it over the
control-plane RPC, never by importing this module.
"""
from __future__ import annotations
import difflib
import hashlib
import time
from pathlib import Path
from ...supervisor.types import (
AuditEntry,
DEFAULT_POLL_INTERVAL_SEC,
Proposal,
Response,
)
from ...paths import bot_bottle_root
from ..store.queue_store import QueueStore
from ...store.audit_store import AuditStore
# --- Paths -----------------------------------------------------------------
def audit_dir() -> Path:
return bot_bottle_root() / "audit"
def audit_log_path(component: str, slug: str) -> Path:
return audit_dir() / f"{component}-{slug}.log"
# --- Queue I/O -------------------------------------------------------------
def write_proposal(proposal: Proposal) -> Path:
"""Persist `proposal` in the queue database, mode 0o600.
Directory is created if missing."""
return QueueStore(proposal.bottle_slug).write_proposal(proposal)
def read_proposal(bottle_slug: str, proposal_id: str) -> Proposal:
return QueueStore(bottle_slug).read_proposal(proposal_id)
def list_pending_proposals(bottle_slug: str) -> list[Proposal]:
"""All proposals for `bottle_slug` that do not yet have a matching
response. Sorted by `arrival_timestamp` so the operator sees the queue
FIFO."""
return QueueStore(bottle_slug).list_pending_proposals()
def list_all_pending_proposals() -> list[Proposal]:
"""All pending proposals across bottles, sorted FIFO."""
return QueueStore("").list_all_pending_proposals()
def write_response(bottle_slug: str, response: Response) -> Path:
return QueueStore(bottle_slug).write_response(response)
def read_response(bottle_slug: str, proposal_id: str) -> Response:
return QueueStore(bottle_slug).read_response(proposal_id)
def wait_for_response(
bottle_slug: str,
proposal_id: str,
*,
poll_interval: float = DEFAULT_POLL_INTERVAL_SEC,
deadline: float | None = None,
) -> Response:
"""Block until a response file appears for `proposal_id`, then return it.
`deadline` is an absolute time.monotonic() value after which the wait raises
TimeoutError. None waits forever the natural shape, since the operator's
response time is unbounded.
Polls SQLite so the implementation stays portable and stdlib-only."""
store = QueueStore(bottle_slug)
while True:
try:
return store.read_response(proposal_id)
except FileNotFoundError:
pass
if deadline is not None and time.monotonic() >= deadline:
raise TimeoutError(f"no response for proposal {proposal_id!r}")
time.sleep(poll_interval)
def archive_proposal(bottle_slug: str, proposal_id: str) -> None:
"""Mark both proposal and response rows processed.
Idempotent missing rows are silently skipped."""
QueueStore(bottle_slug).archive_proposal(proposal_id)
def archive_all_proposals(bottle_slug: str) -> None:
"""Archive every proposal + response for `bottle_slug` (bottle teardown /
reconcile cleanup). Idempotent."""
QueueStore(bottle_slug).archive_all()
# --- Audit log -------------------------------------------------------------
def write_audit_entry(entry: AuditEntry) -> Path:
"""Append `entry` to the host supervise audit table."""
return AuditStore().write_audit_entry(entry)
def read_audit_entries(component: str, slug: str) -> list[AuditEntry]:
"""Load all audit entries for the given component+slug."""
return AuditStore().read_audit_entries(component, slug)
# --- Diff rendering --------------------------------------------------------
def render_diff(before: str, after: str, *, label: str = "config") -> str:
"""Unified diff suitable for the audit log + TUI. Empty diff (no changes)
renders as the empty string."""
diff = difflib.unified_diff(
before.splitlines(keepends=True),
after.splitlines(keepends=True),
fromfile=f"{label} (current)",
tofile=f"{label} (proposed)",
lineterm="",
)
parts = list(diff)
if not parts:
return ""
return "".join(p if p.endswith("\n") else p + "\n" for p in parts).rstrip("\n")
def sha256_hex(content: str) -> str:
return hashlib.sha256(content.encode("utf-8")).hexdigest()
@@ -0,0 +1,36 @@
"""The `Supervise` lifecycle — host-side database staging (PRD 0013 / 0070).
`Supervise.prepare` migrates the orchestrator's `bot-bottle.db` and returns the
neutral `SupervisePlan`. It touches the orchestrator store manager, so it lives
here rather than in the neutral `bot_bottle.supervisor` package; the backend
(which drives launch) may import it backend orchestrator is an allowed
direction, unlike gateway orchestrator.
"""
from __future__ import annotations
from abc import ABC
from pathlib import Path
from ..store.store_manager import StoreManager
from ...supervisor.plan import SupervisePlan
class Supervise(ABC):
"""Per-bottle supervise daemon. Encapsulates host-side database staging;
the gateway's start/stop lifecycle is backend-specific."""
def prepare(
self,
slug: str,
stage_dir: Path,
) -> SupervisePlan:
"""Stage the host database. Returns the plan; `internal_network` must be
set by the launch step before .start runs."""
del stage_dir
mgr = StoreManager.instance()
mgr.migrate()
return SupervisePlan(
slug=slug,
db_path=mgr.db_path,
)
+9 -5
View File
@@ -1,9 +1,13 @@
"""SQLite-backed persistence for bot-bottle.
"""SQLite-backed persistence base for bot-bottle.
The store family: a shared `DbStore` base (versioned `migrations`) and the
concrete stores built on it `ConfigStore`, `AuditStore`, `QueueStore` plus
`StoreManager`, which migrates them together against the one per-host DB.
The shared store base: the `DbStore` class (versioned `migrations`) and the
concrete stores built on it that aren't owned by a single service —
`ConfigStore` and `AuditStore`.
Service-owned stores live under their service: the supervise `queue_store`, the
`secret_store`, the orchestrator `config_store`, and the orchestrator
`store_manager` are under `bot_bottle.orchestrator.store`.
Callers import the concrete module they need directly, e.g.
`from bot_bottle.store.store_manager import StoreManager`.
`from bot_bottle.store.db_store import DbStore`.
"""
+2 -2
View File
@@ -6,12 +6,12 @@ import sqlite3
from pathlib import Path
try:
from ..supervise_types import AuditEntry
from ..supervisor.types import AuditEntry
from ..paths import host_db_path
from .db_store import DbStore
from .migrations import TableMigrations
except ImportError:
from supervise_types import AuditEntry # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
from supervisor.types import AuditEntry # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
from paths import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
from db_store import DbStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
-298
View File
@@ -1,298 +0,0 @@
"""Per-bottle supervise plane (PRD 0013).
The supervise plane is the per-bottle MCP daemon plus its host-side
queue/audit support. The daemon (bot_bottle.gateway.supervise_server)
sits on the bottle's internal network and exposes MCP tools the agent
calls when it needs an operator-reviewed egress change:
* egress-block / allow agent proposes a new routes.yaml
Each tool call: the agent passes the full proposed file plus a
justification text. The gateway validates the proposal syntactically,
writes it to the host SQLite queue table, and holds the tool-call
connection open. The operator's supervise TUI
(bot_bottle.cli.supervise) sees the proposal, accepts
approve / modify / reject, and writes a response row. The gateway sees
the response and returns `{status, notes}` to the agent.
This module defines the host-side library: dataclasses for the queue
record shapes, queue read/write helpers, the audit log writer, and the
diff renderer. The in-gateway daemon lives in
bot_bottle/gateway/supervise_server.py; the supervise daemon's container
lifecycle is owned by the gateway (PRD 0024).
For 0013 the supervisor's approval handlers are deliberately no-ops:
on approval the audit log is written and the response file is
delivered to the agent, but no host-side config change happens. The
remediation engines that wire real config changes land in PRDs 0014,
0015, and 0016.
"""
from __future__ import annotations
import difflib
import hashlib
import time
from abc import ABC
from dataclasses import dataclass
from pathlib import Path
from .supervise_types import (
ACTION_OPERATOR_EDIT,
AuditEntry,
POLL_STATUS_PENDING,
POLL_STATUS_UNKNOWN,
Proposal,
Response,
STATUSES,
STATUS_APPROVED,
STATUS_MODIFIED,
STATUS_REJECTED,
TOOLS,
TOOL_CHECK_PROPOSAL,
TOOL_EGRESS_ALLOW,
TOOL_EGRESS_BLOCK,
TOOL_EGRESS_TOKEN_ALLOW,
TOOL_GITLEAKS_ALLOW,
TOOL_LIST_EGRESS_ROUTES,
)
try:
from .paths import bot_bottle_root
except ImportError: # flat imports inside the gateway
from paths import bot_bottle_root # type: ignore[import-not-found,no-redef] # pylint: disable=import-error,no-name-in-module
SUPERVISE_HOSTNAME = "supervise"
SUPERVISE_PORT = 9100
# The supervise daemon uses these to query egress's
# introspection endpoint for the `list-egress-routes` MCP
# tool. The hostname + port match egress's docker network
# listen port (see backend.docker.egress.EGRESS_PORT). The supervise
# daemon runs inside the gateway alongside egress, so loopback
# is the stable address across docker, firecracker, and Apple
# Container backends.
EGRESS_FORWARD_PROXY = "http://127.0.0.1:9099"
EGRESS_INTROSPECT_URL = "http://_egress.local/allowlist"
COMPONENT_FOR_TOOL: dict[str, str] = {
TOOL_EGRESS_ALLOW: "egress",
TOOL_EGRESS_BLOCK: "egress",
}
DB_PATH_IN_CONTAINER = "/run/supervise/bot-bottle.db"
DEFAULT_POLL_INTERVAL_SEC = 0.5
# --- Paths -----------------------------------------------------------------
def audit_dir() -> Path:
return bot_bottle_root() / "audit"
def audit_log_path(component: str, slug: str) -> Path:
return audit_dir() / f"{component}-{slug}.log"
try:
from .store.queue_store import QueueStore
from .store.audit_store import AuditStore
from .store.store_manager import StoreManager
except ImportError:
# Gateway: files are flat-copied under /app, not a package.
from queue_store import QueueStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
from audit_store import AuditStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
from store_manager import StoreManager # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
# --- Queue I/O -------------------------------------------------------------
def write_proposal(proposal: Proposal) -> Path:
"""Persist `proposal` in the queue database, mode 0o600.
Directory is created if missing."""
return QueueStore(proposal.bottle_slug).write_proposal(proposal)
def read_proposal(bottle_slug: str, proposal_id: str) -> Proposal:
return QueueStore(bottle_slug).read_proposal(proposal_id)
def list_pending_proposals(bottle_slug: str) -> list[Proposal]:
"""All proposals for `bottle_slug` that do not yet have a matching
response. Sorted by `arrival_timestamp` so the operator
sees the queue FIFO."""
return QueueStore(bottle_slug).list_pending_proposals()
def list_all_pending_proposals() -> list[Proposal]:
"""All pending proposals across bottles, sorted FIFO."""
return QueueStore("").list_all_pending_proposals()
def write_response(bottle_slug: str, response: Response) -> Path:
return QueueStore(bottle_slug).write_response(response)
def read_response(bottle_slug: str, proposal_id: str) -> Response:
return QueueStore(bottle_slug).read_response(proposal_id)
def wait_for_response(
bottle_slug: str,
proposal_id: str,
*,
poll_interval: float = DEFAULT_POLL_INTERVAL_SEC,
deadline: float | None = None,
) -> Response:
"""Block until a response file appears for `proposal_id`, then
return it. `deadline` is an absolute time.monotonic() value after
which the wait raises TimeoutError. None waits forever the
natural shape, since the operator's response time is unbounded.
Polls SQLite so the implementation stays portable and stdlib-only."""
store = QueueStore(bottle_slug)
while True:
try:
return store.read_response(proposal_id)
except FileNotFoundError:
pass
if deadline is not None and time.monotonic() >= deadline:
raise TimeoutError(f"no response for proposal {proposal_id!r}")
time.sleep(poll_interval)
def archive_proposal(bottle_slug: str, proposal_id: str) -> None:
"""Mark both proposal and response rows processed.
Idempotent missing rows are silently skipped."""
QueueStore(bottle_slug).archive_proposal(proposal_id)
def archive_all_proposals(bottle_slug: str) -> None:
"""Archive every proposal + response for `bottle_slug` (bottle teardown /
reconcile cleanup). Idempotent."""
QueueStore(bottle_slug).archive_all()
# --- Audit log -------------------------------------------------------------
def write_audit_entry(entry: AuditEntry) -> Path:
"""Append `entry` to the host supervise audit table."""
return AuditStore().write_audit_entry(entry)
def read_audit_entries(component: str, slug: str) -> list[AuditEntry]:
"""Load all audit entries for the given component+slug."""
return AuditStore().read_audit_entries(component, slug)
# --- Diff rendering --------------------------------------------------------
def render_diff(before: str, after: str, *, label: str = "config") -> str:
"""Unified diff suitable for the audit log + TUI. Empty diff (no
changes) renders as the empty string."""
diff = difflib.unified_diff(
before.splitlines(keepends=True),
after.splitlines(keepends=True),
fromfile=f"{label} (current)",
tofile=f"{label} (proposed)",
lineterm="",
)
parts = list(diff)
if not parts:
return ""
return "".join(p if p.endswith("\n") else p + "\n" for p in parts).rstrip("\n")
def sha256_hex(content: str) -> str:
return hashlib.sha256(content.encode("utf-8")).hexdigest()
# --- Gateway plan + abstract lifecycle -------------------------------------
@dataclass(frozen=True)
class SupervisePlan:
"""Output of Supervise.prepare; consumed by .start.
`db_path` is the host database bind-mounted into the gateway at
/run/supervise/bot-bottle.db. `internal_network` is empty at
prepare time; the backend's launch step fills it via
dataclasses.replace before calling .start."""
slug: str
db_path: Path
internal_network: str = ""
class Supervise(ABC):
"""Per-bottle supervise daemon. Encapsulates host-side database
staging; the gateway's start/stop lifecycle is backend-specific."""
def prepare(
self,
slug: str,
stage_dir: Path,
) -> SupervisePlan:
"""Stage the host database. Returns the plan; `internal_network`
must be set by the launch step before .start runs."""
del stage_dir
mgr = StoreManager.instance()
mgr.migrate()
return SupervisePlan(
slug=slug,
db_path=mgr.db_path,
)
__all__ = [
"ACTION_OPERATOR_EDIT",
"AuditEntry",
"AuditStore",
"COMPONENT_FOR_TOOL",
"DEFAULT_POLL_INTERVAL_SEC",
"DB_PATH_IN_CONTAINER",
"Proposal",
"QueueStore",
"Response",
"StoreManager",
"STATUSES",
"STATUS_APPROVED",
"STATUS_MODIFIED",
"STATUS_REJECTED",
"POLL_STATUS_PENDING",
"POLL_STATUS_UNKNOWN",
"SUPERVISE_HOSTNAME",
"SUPERVISE_PORT",
"Supervise",
"SupervisePlan",
"TOOLS",
"EGRESS_FORWARD_PROXY",
"EGRESS_INTROSPECT_URL",
"TOOL_CHECK_PROPOSAL",
"TOOL_EGRESS_ALLOW",
"TOOL_EGRESS_BLOCK",
"TOOL_GITLEAKS_ALLOW",
"TOOL_EGRESS_TOKEN_ALLOW",
"TOOL_LIST_EGRESS_ROUTES",
"archive_proposal",
"archive_all_proposals",
"audit_dir",
"audit_log_path",
"list_pending_proposals",
"list_all_pending_proposals",
"read_audit_entries",
"read_proposal",
"read_response",
"render_diff",
"sha256_hex",
"wait_for_response",
"write_audit_entry",
"write_proposal",
"write_response",
]
+16
View File
@@ -0,0 +1,16 @@
"""Neutral supervise vocabulary — importable by every tier.
The supervise plane spans the control plane (orchestrator) and the data plane
(gateway `supervise_server`): the gateway proposes/polls over RPC, the
orchestrator queues and applies operator decisions. Both sides must agree on
the wire types and constants, so those live here a neutral package neither
tier's private code, importable by `bot_bottle.gateway` **and**
`bot_bottle.orchestrator` without either depending on the other.
* `types` the `Proposal`/`Response`/`AuditEntry` dataclasses, the tool /
status / poll-status constants, and the shared daemon constants.
* `plan` `SupervisePlan`, the launch-time staging DTO.
The orchestrator-only half (queue I/O, diff rendering, the `Supervise`
lifecycle) lives under `bot_bottle.orchestrator.supervisor`.
"""
+26
View File
@@ -0,0 +1,26 @@
"""`SupervisePlan` — the launch-time supervise staging DTO.
A pure value type (no store access), so it lives in the neutral package: the
backend builds it at launch and the orchestrator's `Supervise.prepare` returns
it. The behaviour that fills it in staging the host database is the
orchestrator's, in `bot_bottle.orchestrator.supervisor.supervise`.
"""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
@dataclass(frozen=True)
class SupervisePlan:
"""Output of `Supervise.prepare`; consumed by `.start`.
`db_path` is the host database bind-mounted into the gateway at
/run/supervise/bot-bottle.db. `internal_network` is empty at prepare time;
the backend's launch step fills it via `dataclasses.replace` before calling
`.start`."""
slug: str
db_path: Path
internal_network: str = ""
@@ -48,6 +48,24 @@ POLL_STATUS_UNKNOWN = "unknown"
ACTION_OPERATOR_EDIT = "operator-edit"
# Shared supervise-daemon constants. The gateway `supervise_server` and the
# backend read these, so they live with the wire types in the neutral package
# (never in the orchestrator's private code the data plane can't import).
SUPERVISE_HOSTNAME = "supervise"
SUPERVISE_PORT = 9100
# The supervise daemon queries egress's introspection endpoint for the
# `list-egress-routes` tool over loopback (egress runs alongside it in the
# gateway), stable across docker / firecracker / Apple Container backends.
EGRESS_FORWARD_PROXY = "http://127.0.0.1:9099"
EGRESS_INTROSPECT_URL = "http://_egress.local/allowlist"
DB_PATH_IN_CONTAINER = "/run/supervise/bot-bottle.db"
DEFAULT_POLL_INTERVAL_SEC = 0.5
COMPONENT_FOR_TOOL: dict[str, str] = {
TOOL_EGRESS_ALLOW: "egress",
TOOL_EGRESS_BLOCK: "egress",
}
def _require_str(raw: dict[str, object], key: str) -> str:
value = raw.get(key)
@@ -175,4 +193,11 @@ __all__ = [
"TOOL_EGRESS_TOKEN_ALLOW",
"TOOL_GITLEAKS_ALLOW",
"TOOL_LIST_EGRESS_ROUTES",
"SUPERVISE_HOSTNAME",
"SUPERVISE_PORT",
"EGRESS_FORWARD_PROXY",
"EGRESS_INTROSPECT_URL",
"DB_PATH_IN_CONTAINER",
"DEFAULT_POLL_INTERVAL_SEC",
"COMPONENT_FOR_TOOL",
]
+1 -1
View File
@@ -91,7 +91,7 @@ class TestGatewayImage(unittest.TestCase):
# Probe that the package imports resolve inside the image.
rc, out = self._run_in_image(
"python3", "-c",
"from bot_bottle import supervise; from bot_bottle.gateway import supervise_server; print('ok')",
"from bot_bottle.supervisor import types; from bot_bottle.gateway import supervise_server; print('ok')",
)
self.assertEqual(0, rc, msg=out)
self.assertIn("ok", out)
+1 -1
View File
@@ -16,7 +16,7 @@ from bot_bottle.backend.docker.bottle_plan import DockerBottlePlan
from bot_bottle.egress import EgressPlan, EgressRoute
from bot_bottle.git_gate import GitGatePlan, GitGateUpstream
from bot_bottle.manifest import ManifestIndex
from bot_bottle.supervise import SupervisePlan
from bot_bottle.supervisor.plan import SupervisePlan
SLUG = "demo-abc12"
+1 -1
View File
@@ -15,7 +15,7 @@ from bot_bottle.cli import main
from bot_bottle.store.db_store import DbStore
from bot_bottle.log import Die
from bot_bottle.manifest import ManifestError
from bot_bottle.store.store_manager import StoreManager
from bot_bottle.orchestrator.store.store_manager import StoreManager
class TestMainDispatch(unittest.TestCase):
+1 -1
View File
@@ -11,7 +11,7 @@ from bot_bottle.store.config_store import (
DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS,
ConfigStore,
)
from bot_bottle.store.store_manager import StoreManager
from bot_bottle.orchestrator.store.store_manager import StoreManager
class TestConfigStore(unittest.TestCase):
+1 -1
View File
@@ -25,7 +25,7 @@ from bot_bottle.contrib.claude.agent_provider import ClaudeAgentProvider
from bot_bottle.egress import EgressPlan
from bot_bottle.git_gate import GitGatePlan
from bot_bottle.manifest import ManifestIndex
from bot_bottle.supervise import SupervisePlan
from bot_bottle.supervisor.plan import SupervisePlan
_URL = "http://supervise:9100/"
+1 -1
View File
@@ -25,7 +25,7 @@ from bot_bottle.contrib.codex.agent_provider import CodexAgentProvider
from bot_bottle.egress import EgressPlan
from bot_bottle.git_gate import GitGatePlan
from bot_bottle.manifest import ManifestIndex
from bot_bottle.supervise import SupervisePlan
from bot_bottle.supervisor.plan import SupervisePlan
_URL = "http://supervise:9100/"
+1 -1
View File
@@ -15,7 +15,7 @@ import unittest
from pathlib import Path
from types import ModuleType
from bot_bottle.orchestrator.config_store import (
from bot_bottle.orchestrator.store.config_store import (
DEFAULT_TEARDOWN_TIMEOUT_SECONDS,
TEARDOWN_TIMEOUT_ENV,
OrchestratorConfigStore,
@@ -24,8 +24,8 @@ from bot_bottle.orchestrator.broker import StubBroker
from bot_bottle.orchestrator.control_plane import dispatch, make_server
from bot_bottle.orchestrator.registry import BottleRecord, RegistryStore
from bot_bottle.orchestrator.service import Orchestrator
from bot_bottle.store.store_manager import StoreManager
from bot_bottle.supervise import (
from bot_bottle.orchestrator.store.store_manager import StoreManager
from bot_bottle.orchestrator.supervisor import (
Proposal,
TOOL_EGRESS_ALLOW,
sha256_hex,
+1 -1
View File
@@ -4,7 +4,7 @@ from __future__ import annotations
import unittest
from bot_bottle.orchestrator.secret_store import (
from bot_bottle.orchestrator.store.secret_store import (
ENV_VAR_SECRET_NAME,
decrypt_value,
encrypt_value,
+3 -3
View File
@@ -14,9 +14,9 @@ from unittest.mock import patch
from bot_bottle.orchestrator.broker import LaunchBroker, LaunchRequest, StubBroker
from bot_bottle.orchestrator.registry import RegistryStore
from bot_bottle.orchestrator.service import Orchestrator
from bot_bottle.orchestrator.secret_store import new_env_var_secret
from bot_bottle.store.store_manager import StoreManager
from bot_bottle.supervise import (
from bot_bottle.orchestrator.store.secret_store import new_env_var_secret
from bot_bottle.orchestrator.store.store_manager import StoreManager
from bot_bottle.orchestrator.supervisor import (
Proposal,
STATUS_APPROVED,
TOOL_EGRESS_ALLOW,
+3 -3
View File
@@ -7,12 +7,12 @@ import unittest
from datetime import datetime, timezone
from pathlib import Path
from bot_bottle import supervise
from bot_bottle.orchestrator import supervisor as supervise
from bot_bottle.paths import host_db_path
from tests.unit import use_bottle_root
from bot_bottle.store.audit_store import AuditStore
from bot_bottle.store.queue_store import QueueStore
from bot_bottle.supervise import (
from bot_bottle.orchestrator.store.queue_store import QueueStore
from bot_bottle.orchestrator.supervisor import (
AuditEntry,
Proposal,
Response,
+1 -1
View File
@@ -13,7 +13,7 @@ from datetime import datetime, timezone
from unittest.mock import MagicMock, patch
from bot_bottle.cli import supervise as supervise_cli
from bot_bottle.supervise import (
from bot_bottle.orchestrator.supervisor import (
Proposal,
TOOL_EGRESS_ALLOW,
TOOL_EGRESS_BLOCK,
+3 -3
View File
@@ -10,11 +10,11 @@ import unittest
from pathlib import Path
from unittest.mock import patch
from bot_bottle import supervise
from bot_bottle.orchestrator import supervisor as supervise
from bot_bottle.store.audit_store import AuditStore
from bot_bottle.paths import bot_bottle_root
from bot_bottle.store.queue_store import QueueStore
from bot_bottle.supervise import (
from bot_bottle.orchestrator.store.queue_store import QueueStore
from bot_bottle.orchestrator.supervisor import (
AuditEntry,
Proposal,
STATUS_APPROVED,
+2 -2
View File
@@ -18,8 +18,8 @@ from pathlib import Path
from tests.unit import use_bottle_root
from bot_bottle import supervise as _sv
from bot_bottle.store import queue_store as _qs
from bot_bottle.orchestrator import supervisor as _sv
from bot_bottle.orchestrator.store import queue_store as _qs
from bot_bottle.store import audit_store as _as
from bot_bottle.gateway import supervise_server # noqa: E402