refactor(supervise): split the supervise plane by tier; per-service store managers

Give each service its own store package + manager, and cut the supervise module
along the control/data-plane boundary so nothing in the shared layer reaches up
into the orchestrator.

Stores, by owner:
  - bot_bottle/store/ keeps only the shared base (DbStore, migrations) and the
    concrete stores that aren't service-owned (audit_store, config_store).
  - bot_bottle/orchestrator/store/ now houses the orchestrator-owned stores —
    queue_store (supervise queue), secret_store, config_store — plus a new
    orchestrator store_manager that migrates them (composing audit/config
    downward from the base). The old shared store_manager is gone.

Supervise plane, by tier:
  - bot_bottle/supervisor/ (NEUTRAL, importable by every tier including the
    gateway): types.py (the Proposal/Response/AuditEntry wire types + the tool/
    status/poll constants + the shared daemon constants moved out of
    supervise.py) and plan.py (SupervisePlan, a pure DTO).
  - bot_bottle/orchestrator/supervisor/ (orchestrator-only): queue.py (the
    queue/audit I/O wrappers + render_diff + sha256_hex) and supervise.py (the
    Supervise lifecycle that stages the DB via the store manager). Its __init__
    re-exports the neutral vocabulary so orchestrator-side callers import from
    one place.

The gateway now imports only bot_bottle.supervisor.types (never
bot_bottle.supervise), so the data plane holds no code dependency on the
orchestrator — it reaches the queue over the control-plane RPC. This removes
the circular import that moving queue_store under orchestrator introduced
(supervise -> orchestrator -> service -> supervise).

supervise_types.py -> supervisor/types.py; supervise.py deleted (split). Full
unit suite green (2251).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

bot_bottle/orchestrator/__main__.py

@@ -16,7 +16,7 @@ import secrets
 from pathlib import Path
 
 from .. import log
-from ..store.store_manager import StoreManager
+from .store.store_manager import StoreManager
 from .broker import LaunchBroker, StubBroker
 from .control_plane import make_server
 from .docker_broker import DockerBroker

bot_bottle/orchestrator/control_plane.py

@@ -65,7 +65,7 @@ from urllib.parse import urlsplit
 
 from ..control_auth import ROLE_CLI, ROLES, verify
 from ..paths import CONTROL_PLANE_TOKEN_ENV
-from ..supervise_types import TOOLS
+from ..supervisor.types import TOOLS
 from .service import Orchestrator
 
 # JSON body payload type (parsed request / rendered response).

bot_bottle/orchestrator/service.py

@@ -27,7 +27,7 @@ from datetime import datetime, timezone
 
 from .broker import LaunchBroker, LaunchRequest, sign_request
 from .registry import DEFAULT_REAP_GRACE_SECONDS, BottleRecord, RegistryStore
-from ..supervise import (
+from .supervisor import (
     AuditEntry,
     COMPONENT_FOR_TOOL,
     POLL_STATUS_PENDING,
@@ -104,7 +104,7 @@ class Orchestrator:
         if tokens:
             self._tokens[rec.bottle_id] = dict(tokens)
             if env_var_secret:
-                from .secret_store import encrypt_value
+                from .store.secret_store import encrypt_value
                 encrypted = {k: encrypt_value(env_var_secret, v) for k, v in tokens.items()}
                 self.registry.store_agent_secrets(rec.bottle_id, encrypted)
         req = LaunchRequest(
@@ -367,7 +367,7 @@ class Orchestrator:
         value with *env_var_secret*, and restores ``_tokens[bottle_id]``.
         Returns True on success, False when no stored secrets exist for this
         bottle or decryption fails (wrong key / corrupt data)."""
-        from .secret_store import decrypt_value
+        from .store.secret_store import decrypt_value
         encrypted = self.registry.get_agent_secrets(bottle_id)
         if not encrypted:
             return False

bot_bottle/orchestrator/store/__init__.py

@@ -0,0 +1,10 @@
+"""Orchestrator-owned SQLite stores.
+
+The stores whose tables only the orchestrator (control plane) opens: the
+supervise proposal/response `queue_store`, the agent-secret `secret_store`, and
+the orchestrator `config_store`. They build on the shared `DbStore` /
+`migrations` base in `bot_bottle.store`.
+
+Callers import the concrete module directly, e.g.
+`from bot_bottle.orchestrator.store.queue_store import QueueStore`.
+"""

bot_bottle/orchestrator/store/config_store.py

@@ -11,9 +11,9 @@ import os
 import sqlite3
 from pathlib import Path
 
-from ..store.db_store import DbStore
-from ..store.migrations import TableMigrations
-from ..paths import host_db_path
+from ...store.db_store import DbStore
+from ...store.migrations import TableMigrations
+from ...paths import host_db_path
 
 TEARDOWN_TIMEOUT_ENV = "BOT_BOTTLE_ORCHESTRATOR_TEARDOWN_TIMEOUT_SECONDS"
 DEFAULT_TEARDOWN_TIMEOUT_SECONDS = 30.0

bot_bottle/orchestrator/store/queue_store.py

@@ -0,0 +1,234 @@
+"""SQLite-backed queue store for supervise proposals and responses (PRD 0013)."""
+
+from __future__ import annotations
+
+import os
+import sqlite3
+from pathlib import Path
+
+try:
+    from ...supervisor.types import Proposal, Response
+    from ...paths import host_db_path
+    from ...store.db_store import DbStore
+    from ...store.migrations import TableMigrations
+except ImportError:
+    from supervisor.types import Proposal, Response  # type: ignore[import-not-found]  # pylint: disable=import-error,no-name-in-module
+    from paths import host_db_path  # type: ignore[import-not-found]  # pylint: disable=import-error,no-name-in-module
+    from db_store import DbStore  # type: ignore[import-not-found]  # pylint: disable=import-error,no-name-in-module
+    from migrations import TableMigrations  # type: ignore[import-not-found]  # pylint: disable=import-error,no-name-in-module
+
+
+class QueueStore(DbStore):
+    """SQLite-backed persistent store for supervise proposals and responses."""
+
+    def __init__(self, queue_key: str, db_path: Path | None = None) -> None:
+        self.queue_key = queue_key
+        if db_path is not None:
+            resolved = db_path
+        else:
+            # In the gateway container SUPERVISE_DB_PATH points at the
+            # bind-mounted host DB. On the host this env var is never set,
+            # so we always fall through to host_db_path().
+            env_path = os.environ.get("SUPERVISE_DB_PATH", "").strip()
+            resolved = Path(env_path) if env_path else host_db_path()
+        # One entry per schema version: migrations[0] brings a fresh DB to
+        # version 1, [1] to version 2, etc.  Add new entries at the end; never
+        # edit existing ones.
+        migrations = TableMigrations("queue_store", [
+            # v1 — proposals table
+            """
+            CREATE TABLE IF NOT EXISTS supervise_proposals (
+                queue_key TEXT NOT NULL,
+                id TEXT NOT NULL,
+                bottle_slug TEXT NOT NULL,
+                tool TEXT NOT NULL,
+                proposed_file TEXT NOT NULL,
+                justification TEXT NOT NULL,
+                arrival_timestamp TEXT NOT NULL,
+                current_file_hash TEXT NOT NULL,
+                archived INTEGER NOT NULL DEFAULT 0,
+                PRIMARY KEY (queue_key, id)
+            )
+            """,
+            # v2 — responses table
+            """
+            CREATE TABLE IF NOT EXISTS supervise_responses (
+                queue_key TEXT NOT NULL,
+                proposal_id TEXT NOT NULL,
+                status TEXT NOT NULL,
+                notes TEXT NOT NULL,
+                final_file TEXT,
+                archived INTEGER NOT NULL DEFAULT 0,
+                PRIMARY KEY (queue_key, proposal_id)
+            )
+            """,
+        ])
+        super().__init__(resolved, migrations)
+
+    def write_proposal(self, proposal: Proposal) -> Path:
+        with self._connection() as conn:
+            conn.execute(
+                """
+                INSERT OR REPLACE INTO supervise_proposals (
+                    queue_key, id, bottle_slug, tool, proposed_file, justification,
+                    arrival_timestamp, current_file_hash, archived
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0)
+                """,
+                (
+                    self.queue_key,
+                    proposal.id,
+                    proposal.bottle_slug,
+                    proposal.tool,
+                    proposal.proposed_file,
+                    proposal.justification,
+                    proposal.arrival_timestamp,
+                    proposal.current_file_hash,
+                ),
+            )
+        self._chmod()
+        return self.db_path
+
+    def read_proposal(self, proposal_id: str) -> Proposal:
+        with self._connection() as conn:
+            row = conn.execute(
+                """
+                SELECT * FROM supervise_proposals
+                WHERE queue_key = ? AND id = ? AND archived = 0
+                """,
+                (self.queue_key, proposal_id),
+            ).fetchone()
+        if row is None:
+            raise FileNotFoundError(proposal_id)
+        return self._row_to_proposal(row)
+
+    def list_pending_proposals(self) -> list[Proposal]:
+        if not self.db_path.is_file():
+            return []
+        with self._connection() as conn:
+            rows = conn.execute(
+                """
+                SELECT p.* FROM supervise_proposals p
+                WHERE p.archived = 0
+                  AND p.queue_key = ?
+                  AND NOT EXISTS (
+                    SELECT 1 FROM supervise_responses r
+                    WHERE r.queue_key = p.queue_key
+                      AND r.proposal_id = p.id
+                      AND r.archived = 0
+                  )
+                ORDER BY p.arrival_timestamp, p.id
+                """,
+                (self.queue_key,),
+            ).fetchall()
+        return [self._row_to_proposal(row) for row in rows]
+
+    def list_all_pending_proposals(self) -> list[Proposal]:
+        if not self.db_path.is_file():
+            return []
+        with self._connection() as conn:
+            rows = conn.execute(
+                """
+                SELECT p.* FROM supervise_proposals p
+                WHERE p.archived = 0
+                  AND NOT EXISTS (
+                    SELECT 1 FROM supervise_responses r
+                    WHERE r.queue_key = p.queue_key
+                      AND r.proposal_id = p.id
+                      AND r.archived = 0
+                  )
+                ORDER BY p.arrival_timestamp, p.id
+                """
+            ).fetchall()
+        return [self._row_to_proposal(row) for row in rows]
+
+    def write_response(self, response: Response) -> Path:
+        with self._connection() as conn:
+            conn.execute(
+                """
+                INSERT OR REPLACE INTO supervise_responses (
+                    queue_key, proposal_id, status, notes, final_file, archived
+                ) VALUES (?, ?, ?, ?, ?, 0)
+                """,
+                (
+                    self.queue_key,
+                    response.proposal_id,
+                    response.status,
+                    response.notes,
+                    response.final_file,
+                ),
+            )
+        self._chmod()
+        return self.db_path
+
+    def read_response(self, proposal_id: str) -> Response:
+        with self._connection() as conn:
+            row = conn.execute(
+                """
+                SELECT * FROM supervise_responses
+                WHERE queue_key = ? AND proposal_id = ? AND archived = 0
+                """,
+                (self.queue_key, proposal_id),
+            ).fetchone()
+        if row is None:
+            raise FileNotFoundError(proposal_id)
+        return self._row_to_response(row)
+
+    def archive_proposal(self, proposal_id: str) -> None:
+        if not self.db_path.is_file():
+            return
+        with self._connection() as conn:
+            conn.execute(
+                """
+                UPDATE supervise_proposals SET archived = 1
+                WHERE queue_key = ? AND id = ?
+                """,
+                (self.queue_key, proposal_id),
+            )
+            conn.execute(
+                """
+                UPDATE supervise_responses SET archived = 1
+                WHERE queue_key = ? AND proposal_id = ?
+                """,
+                (self.queue_key, proposal_id),
+            )
+
+    def archive_all(self) -> None:
+        """Archive every proposal + response for this queue key. Used to reap a
+        bottle's supervise records when it is torn down or reconciled away —
+        the retention path for a client that received a decision but died before
+        acknowledging it. Idempotent."""
+        if not self.db_path.is_file():
+            return
+        with self._connection() as conn:
+            conn.execute(
+                "UPDATE supervise_proposals SET archived = 1 WHERE queue_key = ?",
+                (self.queue_key,),
+            )
+            conn.execute(
+                "UPDATE supervise_responses SET archived = 1 WHERE queue_key = ?",
+                (self.queue_key,),
+            )
+
+    @staticmethod
+    def _row_to_proposal(row: sqlite3.Row) -> Proposal:
+        return Proposal(
+            id=row["id"],
+            bottle_slug=row["bottle_slug"],
+            tool=row["tool"],
+            proposed_file=row["proposed_file"],
+            justification=row["justification"],
+            arrival_timestamp=row["arrival_timestamp"],
+            current_file_hash=row["current_file_hash"],
+        )
+
+    @staticmethod
+    def _row_to_response(row: sqlite3.Row) -> Response:
+        return Response(
+            proposal_id=row["proposal_id"],
+            status=row["status"],
+            notes=row["notes"],
+            final_file=row["final_file"],
+        )
+
+
+__all__ = ["QueueStore"]

bot_bottle/orchestrator/store/store_manager.py

@@ -0,0 +1,61 @@
+"""The orchestrator's store manager (PRD 0013 / 0070).
+
+One store manager per service. Post-PRD-0070 the orchestrator is the sole
+opener of `bot-bottle.db`, so it owns migrating every table in it: the supervise
+`queue_store` (local to `orchestrator.store`) plus the `audit_store` and
+`config_store` from the shared `bot_bottle.store` base. The data plane never
+touches these — it reaches state over the control-plane RPC.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from .queue_store import QueueStore
+from ...store.audit_store import AuditStore
+from ...store.config_store import ConfigStore
+
+_instance: StoreManager | None = None
+
+
+class StoreManager:
+    """Owns db_path and delegates migrate/is_migrated across the orchestrator's
+    stores.
+
+    Use instance() for normal access. Call reset(db_path) in tests to swap the
+    singleton to a temp path, then reset() with no args to restore the
+    default."""
+
+    def __init__(self, db_path: Path | None = None) -> None:
+        if db_path is None:
+            from ...paths import host_db_path
+            db_path = host_db_path()
+        self.db_path = db_path
+
+    @classmethod
+    def instance(cls) -> StoreManager:
+        global _instance
+        if _instance is None:
+            _instance = cls()
+        return _instance
+
+    @classmethod
+    def reset(cls, db_path: Path | None = None) -> None:
+        """Replace the singleton. Pass db_path for test isolation; omit to restore default."""
+        global _instance
+        _instance = cls(db_path)
+
+    def is_migrated(self) -> bool:
+        return (
+            QueueStore("", self.db_path).is_migrated()
+            and AuditStore(self.db_path).is_migrated()
+            and ConfigStore(self.db_path).is_migrated()
+        )
+
+    def migrate(self) -> None:
+        QueueStore("", self.db_path).migrate()
+        AuditStore(self.db_path).migrate()
+        ConfigStore(self.db_path).migrate()
+
+
+__all__ = ["StoreManager"]

bot_bottle/orchestrator/supervisor/__init__.py

@@ -0,0 +1,34 @@
+"""The orchestrator's view of the supervise plane.
+
+Bundles the orchestrator-owned supervise surface: the queue/audit I/O and diff
+rendering (`queue`) and the `Supervise` lifecycle (`supervise`). For
+convenience it re-exports the neutral vocabulary (`bot_bottle.supervisor`'s
+types + `SupervisePlan`) so orchestrator-side callers — service, CLI, tests —
+import from one place.
+
+The data plane must NOT import this package; it imports `bot_bottle.supervisor`
+(neutral) directly and reaches the queue over the control-plane RPC.
+"""
+
+from __future__ import annotations
+
+from ...supervisor.types import *  # noqa: F401,F403 — re-export the neutral vocabulary
+from ...supervisor.plan import SupervisePlan
+from .queue import (
+    archive_all_proposals,
+    archive_proposal,
+    audit_dir,
+    audit_log_path,
+    list_all_pending_proposals,
+    list_pending_proposals,
+    read_audit_entries,
+    read_proposal,
+    read_response,
+    render_diff,
+    sha256_hex,
+    wait_for_response,
+    write_audit_entry,
+    write_proposal,
+    write_response,
+)
+from .supervise import Supervise

bot_bottle/orchestrator/supervisor/queue.py

@@ -0,0 +1,141 @@
+"""Orchestrator-side supervise queue + audit I/O and diff rendering.
+
+The host-side library the orchestrator uses to drive the supervise flow: read /
+write proposals and responses against the queue store, append audit entries,
+and render operator diffs. Only the orchestrator opens `bot-bottle.db` (PRD
+0070), so this lives under `orchestrator/` — the data plane reaches it over the
+control-plane RPC, never by importing this module.
+"""
+
+from __future__ import annotations
+
+import difflib
+import hashlib
+import time
+from pathlib import Path
+
+from ...supervisor.types import (
+    AuditEntry,
+    DEFAULT_POLL_INTERVAL_SEC,
+    Proposal,
+    Response,
+)
+from ...paths import bot_bottle_root
+from ..store.queue_store import QueueStore
+from ...store.audit_store import AuditStore
+
+
+# --- Paths -----------------------------------------------------------------
+
+
+def audit_dir() -> Path:
+    return bot_bottle_root() / "audit"
+
+
+def audit_log_path(component: str, slug: str) -> Path:
+    return audit_dir() / f"{component}-{slug}.log"
+
+
+# --- Queue I/O -------------------------------------------------------------
+
+
+def write_proposal(proposal: Proposal) -> Path:
+    """Persist `proposal` in the queue database, mode 0o600.
+    Directory is created if missing."""
+    return QueueStore(proposal.bottle_slug).write_proposal(proposal)
+
+
+def read_proposal(bottle_slug: str, proposal_id: str) -> Proposal:
+    return QueueStore(bottle_slug).read_proposal(proposal_id)
+
+
+def list_pending_proposals(bottle_slug: str) -> list[Proposal]:
+    """All proposals for `bottle_slug` that do not yet have a matching
+    response. Sorted by `arrival_timestamp` so the operator sees the queue
+    FIFO."""
+    return QueueStore(bottle_slug).list_pending_proposals()
+
+
+def list_all_pending_proposals() -> list[Proposal]:
+    """All pending proposals across bottles, sorted FIFO."""
+    return QueueStore("").list_all_pending_proposals()
+
+
+def write_response(bottle_slug: str, response: Response) -> Path:
+    return QueueStore(bottle_slug).write_response(response)
+
+
+def read_response(bottle_slug: str, proposal_id: str) -> Response:
+    return QueueStore(bottle_slug).read_response(proposal_id)
+
+
+def wait_for_response(
+    bottle_slug: str,
+    proposal_id: str,
+    *,
+    poll_interval: float = DEFAULT_POLL_INTERVAL_SEC,
+    deadline: float | None = None,
+) -> Response:
+    """Block until a response file appears for `proposal_id`, then return it.
+    `deadline` is an absolute time.monotonic() value after which the wait raises
+    TimeoutError. None waits forever — the natural shape, since the operator's
+    response time is unbounded.
+
+    Polls SQLite so the implementation stays portable and stdlib-only."""
+    store = QueueStore(bottle_slug)
+    while True:
+        try:
+            return store.read_response(proposal_id)
+        except FileNotFoundError:
+            pass
+        if deadline is not None and time.monotonic() >= deadline:
+            raise TimeoutError(f"no response for proposal {proposal_id!r}")
+        time.sleep(poll_interval)
+
+
+def archive_proposal(bottle_slug: str, proposal_id: str) -> None:
+    """Mark both proposal and response rows processed.
+    Idempotent — missing rows are silently skipped."""
+    QueueStore(bottle_slug).archive_proposal(proposal_id)
+
+
+def archive_all_proposals(bottle_slug: str) -> None:
+    """Archive every proposal + response for `bottle_slug` (bottle teardown /
+    reconcile cleanup). Idempotent."""
+    QueueStore(bottle_slug).archive_all()
+
+
+# --- Audit log -------------------------------------------------------------
+
+
+def write_audit_entry(entry: AuditEntry) -> Path:
+    """Append `entry` to the host supervise audit table."""
+    return AuditStore().write_audit_entry(entry)
+
+
+def read_audit_entries(component: str, slug: str) -> list[AuditEntry]:
+    """Load all audit entries for the given component+slug."""
+    return AuditStore().read_audit_entries(component, slug)
+
+
+# --- Diff rendering --------------------------------------------------------
+
+
+def render_diff(before: str, after: str, *, label: str = "config") -> str:
+    """Unified diff suitable for the audit log + TUI. Empty diff (no changes)
+    renders as the empty string."""
+    diff = difflib.unified_diff(
+        before.splitlines(keepends=True),
+        after.splitlines(keepends=True),
+        fromfile=f"{label} (current)",
+        tofile=f"{label} (proposed)",
+        lineterm="",
+    )
+    parts = list(diff)
+    if not parts:
+        return ""
+    return "".join(p if p.endswith("\n") else p + "\n" for p in parts).rstrip("\n")
+
+
+def sha256_hex(content: str) -> str:
+    return hashlib.sha256(content.encode("utf-8")).hexdigest()

bot_bottle/orchestrator/supervisor/supervise.py

@@ -0,0 +1,36 @@
+"""The `Supervise` lifecycle — host-side database staging (PRD 0013 / 0070).
+
+`Supervise.prepare` migrates the orchestrator's `bot-bottle.db` and returns the
+neutral `SupervisePlan`. It touches the orchestrator store manager, so it lives
+here rather than in the neutral `bot_bottle.supervisor` package; the backend
+(which drives launch) may import it — backend → orchestrator is an allowed
+direction, unlike gateway → orchestrator.
+"""
+
+from __future__ import annotations
+
+from abc import ABC
+from pathlib import Path
+
+from ..store.store_manager import StoreManager
+from ...supervisor.plan import SupervisePlan
+
+
+class Supervise(ABC):
+    """Per-bottle supervise daemon. Encapsulates host-side database staging;
+    the gateway's start/stop lifecycle is backend-specific."""
+
+    def prepare(
+        self,
+        slug: str,
+        stage_dir: Path,
+    ) -> SupervisePlan:
+        """Stage the host database. Returns the plan; `internal_network` must be
+        set by the launch step before .start runs."""
+        del stage_dir
+        mgr = StoreManager.instance()
+        mgr.migrate()
+        return SupervisePlan(
+            slug=slug,
+            db_path=mgr.db_path,
+        )