From 9af02831ea681542e793aeb3fa973a3aa5bfcbd8 Mon Sep 17 00:00:00 2001 From: codex Date: Wed, 1 Jul 2026 16:53:23 +0000 Subject: [PATCH 01/10] docs(prd): add sqlite local storage plan --- docs/prds/prd-new-sqlite-local-storage.md | 140 ++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 docs/prds/prd-new-sqlite-local-storage.md diff --git a/docs/prds/prd-new-sqlite-local-storage.md b/docs/prds/prd-new-sqlite-local-storage.md new file mode 100644 index 0000000..1a24c69 --- /dev/null +++ b/docs/prds/prd-new-sqlite-local-storage.md @@ -0,0 +1,140 @@ +# PRD prd-new: SQLite local storage + +- **Status:** Draft +- **Author:** codex +- **Created:** 2026-07-01 +- **Issue:** #319 + +## Summary + +Add a small stdlib SQLite storage layer for bot-bottle host runtime state, +starting with the supervise queue and audit log. This replaces scattered JSON +queue files and JSONL audit logs with structured tables while preserving the +existing public supervise helper functions and sidecar queue mount contract. + +## Problem + +Bot-bottle currently stores supervise proposals and responses as individual JSON +files under `~/.bot-bottle/queue//`, and audit entries as JSONL files +under `~/.bot-bottle/audit/`. That worked for the original interactive TUI, but +new forge-native orchestration needs durable, queryable local state for queues, +audit trails, watchdogs, and lifecycle records. PR #318 started introducing +SQLite-shaped boilerplate for forge state; the storage foundation should live in +its own PR so forge work can build on the shared runtime store instead of adding +one-off persistence. + +## Goals / Success Criteria + +1. Supervise proposals and responses are persisted through SQLite. +2. Audit entries are persisted through SQLite. +3. Existing public supervise helpers keep their current call shape where + practical: `write_proposal`, `read_proposal`, `list_pending_proposals`, + `write_response`, `read_response`, `wait_for_response`, + `archive_proposal`, `write_audit_entry`, and `read_audit_entries`. +4. The sidecar queue mount still works across docker, smolmachines, and + macOS-container backends. +5. The implementation stays stdlib-only. +6. Unit tests cover queue round-trips, pending discovery, response waits, + archive semantics, audit round-trips, and path creation. + +## Non-goals + +- Migrating old JSON queue files or JSONL audit logs. +- Adding forge orchestration state tables. +- Adding egress metering or budget tables. +- Changing the supervise TUI workflow or remediation behavior. +- Introducing a third-party ORM or migration framework. + +## Design + +### Database locations + +Queue state remains tied to the mounted per-bottle queue directory: + +```text +~/.bot-bottle/queue//supervise.db +``` + +The supervise sidecar already receives that directory at +`/run/supervise/queue`, so both the sidecar and host TUI can read and write the +same SQLite file without changing backend mounts. + +Audit state uses the host-level local database: + +```text +~/.bot-bottle/bot-bottle.db +``` + +This creates the shared host database that later forge/native lifecycle work can +extend in separate PRDs. + +### Tables + +`supervise_proposals` lives in the per-queue database: + +```sql +CREATE TABLE supervise_proposals ( + id TEXT PRIMARY KEY, + bottle_slug TEXT NOT NULL, + tool TEXT NOT NULL, + proposed_file TEXT NOT NULL, + justification TEXT NOT NULL, + arrival_timestamp TEXT NOT NULL, + current_file_hash TEXT NOT NULL, + archived INTEGER NOT NULL DEFAULT 0 +); +``` + +`supervise_responses` lives in the same per-queue database: + +```sql +CREATE TABLE supervise_responses ( + proposal_id TEXT PRIMARY KEY, + status TEXT NOT NULL, + notes TEXT NOT NULL, + final_file TEXT, + archived INTEGER NOT NULL DEFAULT 0 +); +``` + +`supervise_audit_entries` lives in the host database: + +```sql +CREATE TABLE supervise_audit_entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + bottle_slug TEXT NOT NULL, + component TEXT NOT NULL, + operator_action TEXT NOT NULL, + operator_notes TEXT NOT NULL, + justification TEXT NOT NULL, + diff TEXT NOT NULL +); +``` + +### Compatibility + +The existing helper functions keep accepting `Path` arguments for queue +directories. Internally, they map the queue directory to `supervise.db` and +perform equivalent operations: + +- `list_pending_proposals` returns non-archived proposals without a non-archived + response, sorted by arrival time. +- `archive_proposal` marks matching proposal/response rows archived instead of + moving files into `processed/`. +- `wait_for_response` keeps the current polling behavior but polls SQLite. + +The old path helpers (`queue_dir_for_slug`, `audit_dir`, `audit_log_path`) stay +available for compatibility. `audit_log_path` no longer describes the active +storage location; callers should use `read_audit_entries`. + +## Implementation chunks + +1. Add SQLite store helpers for supervise queue and audit state. +2. Rewire `bot_bottle.supervise` queue/audit functions to the store. +3. Update supervise CLI discovery tests and queue/audit unit tests. +4. Run unit tests, pyright, and pylint for touched modules. + +## Open questions + +None. -- 2.52.0 From 08918f9a8a88230d0d23f7987823ba00a7d4a210 Mon Sep 17 00:00:00 2001 From: codex Date: Wed, 1 Jul 2026 16:56:23 +0000 Subject: [PATCH 02/10] feat(supervise): store queue and audit data in sqlite --- bot_bottle/supervise.py | 452 ++++++++++++++++++---------- tests/unit/test_supervise.py | 26 +- tests/unit/test_supervise_server.py | 4 +- 3 files changed, 302 insertions(+), 180 deletions(-) diff --git a/bot_bottle/supervise.py b/bot_bottle/supervise.py index 12d4b6c..ddd6c4f 100644 --- a/bot_bottle/supervise.py +++ b/bot_bottle/supervise.py @@ -34,8 +34,7 @@ from __future__ import annotations import dataclasses import difflib import hashlib -import json -import os +import sqlite3 import time import uuid from abc import ABC @@ -88,6 +87,8 @@ ACTION_OPERATOR_EDIT = "operator-edit" QUEUE_DIR_IN_CONTAINER = "/run/supervise/queue" DEFAULT_POLL_INTERVAL_SEC = 0.5 +HOST_DB_FILENAME = "bot-bottle.db" +QUEUE_DB_FILENAME = "supervise.db" # --- Paths ----------------------------------------------------------------- @@ -109,6 +110,14 @@ def audit_log_path(component: str, slug: str) -> Path: return audit_dir() / f"{component}-{slug}.log" +def host_db_path() -> Path: + return bot_bottle_root() / HOST_DB_FILENAME + + +def queue_db_path(queue_dir: Path) -> Path: + return queue_dir / QUEUE_DB_FILENAME + + # --- Dataclasses ----------------------------------------------------------- @@ -226,83 +235,29 @@ class AuditEntry: # --- Queue I/O ------------------------------------------------------------- -def _proposal_filename(proposal_id: str) -> str: - return f"{proposal_id}.proposal.json" - - -def _response_filename(proposal_id: str) -> str: - return f"{proposal_id}.response.json" - - -def _id_from_proposal_filename(path: Path) -> str | None: - name = path.name - if not name.endswith(".proposal.json"): - return None - return name[: -len(".proposal.json")] - - def write_proposal(queue_dir: Path, proposal: Proposal) -> Path: - """Persist `proposal` as JSON in the queue dir, mode 0o600. + """Persist `proposal` in the queue database, mode 0o600. Directory is created if missing.""" - queue_dir.mkdir(parents=True, exist_ok=True) - path = queue_dir / _proposal_filename(proposal.id) - payload = json.dumps(proposal.to_dict(), indent=2) + "\n" - _atomic_write(path, payload, mode=0o600) - return path + return _QueueStore(queue_dir).write_proposal(proposal) def read_proposal(queue_dir: Path, proposal_id: str) -> Proposal: - path = queue_dir / _proposal_filename(proposal_id) - with path.open() as f: - raw = json.load(f) - if not isinstance(raw, dict): - raise ValueError(f"{path}: top-level must be an object") - return Proposal.from_dict(raw) + return _QueueStore(queue_dir).read_proposal(proposal_id) def list_pending_proposals(queue_dir: Path) -> list[Proposal]: """All proposals in `queue_dir` that do not yet have a matching - response file. Sorted by `arrival_timestamp` so the operator + response. Sorted by `arrival_timestamp` so the operator sees the queue FIFO.""" - if not queue_dir.is_dir(): - return [] - out: list[Proposal] = [] - for path in sorted(queue_dir.glob("*.proposal.json")): - proposal_id = _id_from_proposal_filename(path) - if proposal_id is None: - continue - if (queue_dir / _response_filename(proposal_id)).exists(): - continue - try: - with path.open() as f: - raw = json.load(f) - except (OSError, json.JSONDecodeError): - continue - if not isinstance(raw, dict): - continue - try: - out.append(Proposal.from_dict(raw)) - except (KeyError, ValueError): - continue - out.sort(key=lambda p: p.arrival_timestamp) - return out + return _QueueStore(queue_dir).list_pending_proposals() def write_response(queue_dir: Path, response: Response) -> Path: - queue_dir.mkdir(parents=True, exist_ok=True) - path = queue_dir / _response_filename(response.proposal_id) - payload = json.dumps(response.to_dict(), indent=2) + "\n" - _atomic_write(path, payload, mode=0o600) - return path + return _QueueStore(queue_dir).write_response(response) def read_response(queue_dir: Path, proposal_id: str) -> Response: - path = queue_dir / _response_filename(proposal_id) - with path.open() as f: - raw = json.load(f) - if not isinstance(raw, dict): - raise ValueError(f"{path}: top-level must be an object") - return Response.from_dict(raw) + return _QueueStore(queue_dir).read_response(proposal_id) def wait_for_response( @@ -317,90 +272,35 @@ def wait_for_response( which the wait raises TimeoutError. None waits forever — the natural shape, since the operator's response time is unbounded. - Polls the filesystem so the implementation stays portable and - stdlib-only.""" - path = queue_dir / _response_filename(proposal_id) + Polls SQLite so the implementation stays portable and stdlib-only.""" + store = _QueueStore(queue_dir) while True: - if path.exists(): - try: - with path.open() as f: - raw = json.load(f) - except (OSError, json.JSONDecodeError): - raw = None - if isinstance(raw, dict): - try: - return Response.from_dict(raw) - except (KeyError, ValueError): - pass + try: + return store.read_response(proposal_id) + except FileNotFoundError: + pass if deadline is not None and time.monotonic() >= deadline: raise TimeoutError(f"no response for proposal {proposal_id!r}") time.sleep(poll_interval) def archive_proposal(queue_dir: Path, proposal_id: str) -> None: - """Move both proposal and response files to `/processed/`. - Idempotent — missing files are silently skipped.""" - processed = queue_dir / "processed" - processed.mkdir(parents=True, exist_ok=True) - for name in (_proposal_filename(proposal_id), _response_filename(proposal_id)): - src = queue_dir / name - if src.exists(): - src.rename(processed / name) + """Mark both proposal and response rows processed. + Idempotent — missing rows are silently skipped.""" + _QueueStore(queue_dir).archive_proposal(proposal_id) # --- Audit log ------------------------------------------------------------- def write_audit_entry(entry: AuditEntry) -> Path: - """Append `entry` as one JSON-Lines record to the per-bottle - audit log. Acquires an advisory exclusive lock so concurrent - writers don't interleave bytes.""" - path = audit_log_path(entry.component, entry.bottle_slug) - path.parent.mkdir(parents=True, exist_ok=True) - line = json.dumps(entry.to_dict(), sort_keys=False) + "\n" - fd = os.open(path, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600) - try: - _try_flock(fd) - try: - os.write(fd, line.encode("utf-8")) - finally: - _try_funlock(fd) - finally: - os.close(fd) - return path + """Append `entry` to the host supervise audit table.""" + return _AuditStore().write_audit_entry(entry) def read_audit_entries(component: str, slug: str) -> list[AuditEntry]: - """Load all audit entries for the given component+slug. Empty - list if the log doesn't exist.""" - path = audit_log_path(component, slug) - if not path.is_file(): - return [] - out: list[AuditEntry] = [] - with path.open() as f: - for raw_line in f: - raw_line = raw_line.strip() - if not raw_line: - continue - try: - raw = json.loads(raw_line) - except json.JSONDecodeError: - continue - if not isinstance(raw, dict): - continue - try: - out.append(AuditEntry( - timestamp=_require_str(raw, "timestamp"), - bottle_slug=_require_str(raw, "bottle_slug"), - component=_require_str(raw, "component"), - operator_action=_require_str(raw, "operator_action"), - operator_notes=_require_str(raw, "operator_notes"), - justification=_require_str(raw, "justification"), - diff=_require_str(raw, "diff"), - )) - except ValueError: - continue - return out + """Load all audit entries for the given component+slug.""" + return _AuditStore().read_audit_entries(component, slug) # --- Diff rendering -------------------------------------------------------- @@ -426,6 +326,260 @@ def sha256_hex(content: str) -> str: return hashlib.sha256(content.encode("utf-8")).hexdigest() +# --- SQLite storage -------------------------------------------------------- + + +class _QueueStore: + def __init__(self, queue_dir: Path) -> None: + self.db_path = queue_db_path(queue_dir) + self.db_path.parent.mkdir(parents=True, exist_ok=True) + self._init() + + def write_proposal(self, proposal: Proposal) -> Path: + with self._connect() as conn: + conn.execute( + """ + INSERT OR REPLACE INTO supervise_proposals ( + id, bottle_slug, tool, proposed_file, justification, + arrival_timestamp, current_file_hash, archived + ) VALUES (?, ?, ?, ?, ?, ?, ?, 0) + """, + ( + proposal.id, + proposal.bottle_slug, + proposal.tool, + proposal.proposed_file, + proposal.justification, + proposal.arrival_timestamp, + proposal.current_file_hash, + ), + ) + self._chmod() + return self.db_path + + def read_proposal(self, proposal_id: str) -> Proposal: + with self._connect() as conn: + row = conn.execute( + """ + SELECT * FROM supervise_proposals + WHERE id = ? AND archived = 0 + """, + (proposal_id,), + ).fetchone() + if row is None: + raise FileNotFoundError(proposal_id) + return _proposal_from_row(row) + + def list_pending_proposals(self) -> list[Proposal]: + if not self.db_path.is_file(): + return [] + with self._connect() as conn: + rows = conn.execute( + """ + SELECT p.* FROM supervise_proposals p + WHERE p.archived = 0 + AND NOT EXISTS ( + SELECT 1 FROM supervise_responses r + WHERE r.proposal_id = p.id AND r.archived = 0 + ) + ORDER BY p.arrival_timestamp, p.id + """ + ).fetchall() + return [_proposal_from_row(row) for row in rows] + + def write_response(self, response: Response) -> Path: + with self._connect() as conn: + conn.execute( + """ + INSERT OR REPLACE INTO supervise_responses ( + proposal_id, status, notes, final_file, archived + ) VALUES (?, ?, ?, ?, 0) + """, + ( + response.proposal_id, + response.status, + response.notes, + response.final_file, + ), + ) + self._chmod() + return self.db_path + + def read_response(self, proposal_id: str) -> Response: + with self._connect() as conn: + row = conn.execute( + """ + SELECT * FROM supervise_responses + WHERE proposal_id = ? AND archived = 0 + """, + (proposal_id,), + ).fetchone() + if row is None: + raise FileNotFoundError(proposal_id) + return _response_from_row(row) + + def archive_proposal(self, proposal_id: str) -> None: + if not self.db_path.is_file(): + return + with self._connect() as conn: + conn.execute( + "UPDATE supervise_proposals SET archived = 1 WHERE id = ?", + (proposal_id,), + ) + conn.execute( + """ + UPDATE supervise_responses SET archived = 1 + WHERE proposal_id = ? + """, + (proposal_id,), + ) + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + return conn + + def _init(self) -> None: + with self._connect() as conn: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS supervise_proposals ( + id TEXT PRIMARY KEY, + bottle_slug TEXT NOT NULL, + tool TEXT NOT NULL, + proposed_file TEXT NOT NULL, + justification TEXT NOT NULL, + arrival_timestamp TEXT NOT NULL, + current_file_hash TEXT NOT NULL, + archived INTEGER NOT NULL DEFAULT 0 + ) + """ + ) + conn.execute( + """ + CREATE TABLE IF NOT EXISTS supervise_responses ( + proposal_id TEXT PRIMARY KEY, + status TEXT NOT NULL, + notes TEXT NOT NULL, + final_file TEXT, + archived INTEGER NOT NULL DEFAULT 0 + ) + """ + ) + self._chmod() + + def _chmod(self) -> None: + try: + self.db_path.chmod(0o600) + except OSError: + pass + + +class _AuditStore: + def __init__(self, db_path: Path | None = None) -> None: + self.db_path = db_path or host_db_path() + self.db_path.parent.mkdir(parents=True, exist_ok=True) + self._init() + + def write_audit_entry(self, entry: AuditEntry) -> Path: + with self._connect() as conn: + conn.execute( + """ + INSERT INTO supervise_audit_entries ( + timestamp, bottle_slug, component, operator_action, + operator_notes, justification, diff + ) VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ( + entry.timestamp, + entry.bottle_slug, + entry.component, + entry.operator_action, + entry.operator_notes, + entry.justification, + entry.diff, + ), + ) + self._chmod() + return self.db_path + + def read_audit_entries(self, component: str, slug: str) -> list[AuditEntry]: + if not self.db_path.is_file(): + return [] + with self._connect() as conn: + rows = conn.execute( + """ + SELECT * FROM supervise_audit_entries + WHERE component = ? AND bottle_slug = ? + ORDER BY id + """, + (component, slug), + ).fetchall() + return [_audit_entry_from_row(row) for row in rows] + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + return conn + + def _init(self) -> None: + with self._connect() as conn: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS supervise_audit_entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + bottle_slug TEXT NOT NULL, + component TEXT NOT NULL, + operator_action TEXT NOT NULL, + operator_notes TEXT NOT NULL, + justification TEXT NOT NULL, + diff TEXT NOT NULL + ) + """ + ) + self._chmod() + + def _chmod(self) -> None: + try: + self.db_path.chmod(0o600) + except OSError: + pass + + +def _proposal_from_row(row: sqlite3.Row) -> Proposal: + return Proposal( + id=row["id"], + bottle_slug=row["bottle_slug"], + tool=row["tool"], + proposed_file=row["proposed_file"], + justification=row["justification"], + arrival_timestamp=row["arrival_timestamp"], + current_file_hash=row["current_file_hash"], + ) + + +def _response_from_row(row: sqlite3.Row) -> Response: + return Response( + proposal_id=row["proposal_id"], + status=row["status"], + notes=row["notes"], + final_file=row["final_file"], + ) + + +def _audit_entry_from_row(row: sqlite3.Row) -> AuditEntry: + return AuditEntry( + timestamp=row["timestamp"], + bottle_slug=row["bottle_slug"], + component=row["component"], + operator_action=row["operator_action"], + operator_notes=row["operator_notes"], + justification=row["justification"], + diff=row["diff"], + ) + + # --- Sidecar plan + abstract lifecycle ------------------------------------- @@ -474,40 +628,6 @@ def _require_str(raw: dict[str, object], key: str) -> str: return value -def _atomic_write(path: Path, content: str, *, mode: int) -> None: - """Atomic: write to a sibling tmp file, fsync, rename.""" - tmp = path.with_suffix(path.suffix + ".tmp") - fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode) - try: - os.write(fd, content.encode("utf-8")) - os.fsync(fd) - finally: - os.close(fd) - os.replace(tmp, path) - - -try: - import fcntl as _fcntl - - def _try_flock(fd: int) -> None: # type: ignore[reportRedeclaration] - try: - _fcntl.flock(fd, _fcntl.LOCK_EX) - except OSError: - pass - - def _try_funlock(fd: int) -> None: # type: ignore[reportRedeclaration] - try: - _fcntl.flock(fd, _fcntl.LOCK_UN) - except OSError: - pass -except ImportError: # pragma: no cover — Windows path - def _try_flock(fd: int) -> None: # noqa: F841 — Windows fallback - return None - - def _try_funlock(fd: int) -> None: # noqa: F841 — Windows fallback - return None - - __all__ = [ "ACTION_OPERATOR_EDIT", "AuditEntry", @@ -536,7 +656,9 @@ __all__ = [ "audit_dir", "audit_log_path", "bot_bottle_root", + "host_db_path", "list_pending_proposals", + "queue_db_path", "queue_dir_for_slug", "read_audit_entries", "read_proposal", diff --git a/tests/unit/test_supervise.py b/tests/unit/test_supervise.py index dfe8bd4..0e97d32 100644 --- a/tests/unit/test_supervise.py +++ b/tests/unit/test_supervise.py @@ -1,6 +1,5 @@ """Unit: supervise queue + audit log + diff helpers (PRD 0013).""" -import json import tempfile import threading import time @@ -19,8 +18,9 @@ from bot_bottle.supervise import ( TOOL_EGRESS_ALLOW, TOOL_GITLEAKS_ALLOW, archive_proposal, - audit_log_path, + host_db_path, list_pending_proposals, + queue_db_path, read_audit_entries, read_proposal, read_response, @@ -121,6 +121,7 @@ class TestQueueIO(unittest.TestCase): p = _proposal() path = write_proposal(self.queue_dir, p) self.assertTrue(path.exists()) + self.assertEqual(queue_db_path(self.queue_dir), path) self.assertEqual(0o600, path.stat().st_mode & 0o777) loaded = read_proposal(self.queue_dir, p.id) self.assertEqual(p, loaded) @@ -198,10 +199,9 @@ class TestQueueIO(unittest.TestCase): proposal_id=p.id, status=STATUS_APPROVED, notes="", )) archive_proposal(self.queue_dir, p.id) - self.assertFalse((self.queue_dir / f"{p.id}.proposal.json").exists()) - self.assertFalse((self.queue_dir / f"{p.id}.response.json").exists()) - self.assertTrue((self.queue_dir / "processed" / f"{p.id}.proposal.json").exists()) - self.assertTrue((self.queue_dir / "processed" / f"{p.id}.response.json").exists()) + self.assertEqual([], list_pending_proposals(self.queue_dir)) + with self.assertRaises(FileNotFoundError): + read_response(self.queue_dir, p.id) def test_archive_is_idempotent_on_missing_files(self): # Should not raise. @@ -237,6 +237,7 @@ class TestAuditLog(unittest.TestCase): diff="--- before\n+++ after\n", ) path = write_audit_entry(e) + self.assertEqual(host_db_path(), path) self.assertEqual(0o600, path.stat().st_mode & 0o777) loaded = read_audit_entries("cred-proxy", "dev") self.assertEqual([e], loaded) @@ -252,12 +253,13 @@ class TestAuditLog(unittest.TestCase): justification="", diff="", )) - path = audit_log_path("egress", "dev") - with path.open() as f: - lines = [line for line in f if line.strip()] - self.assertEqual(3, len(lines)) - for line in lines: - self.assertTrue(json.loads(line)) # each line is valid JSON + entries = read_audit_entries("egress", "dev") + self.assertEqual(3, len(entries)) + self.assertEqual( + ["2026-05-25T12:00:00+00:00", "2026-05-25T12:00:01+00:00", + "2026-05-25T12:00:02+00:00"], + [entry.timestamp for entry in entries], + ) def test_separate_logs_per_component_slug(self): write_audit_entry(AuditEntry( diff --git a/tests/unit/test_supervise_server.py b/tests/unit/test_supervise_server.py index 0eb11da..088dd39 100644 --- a/tests/unit/test_supervise_server.py +++ b/tests/unit/test_supervise_server.py @@ -413,9 +413,7 @@ class TestHandleToolsCall(unittest.TestCase): responder.join() # No pending proposals left after archive. self.assertEqual([], _sv.list_pending_proposals(self.queue_dir)) - # Both files moved to processed/. - processed = list((self.queue_dir / "processed").glob("*.json")) - self.assertEqual(2, len(processed)) + self.assertFalse((self.queue_dir / "processed").exists()) def test_pending_response_times_out_without_archive(self): config = ServerConfig( -- 2.52.0 From f1b8bbdfa158c1e7968c38d4ac7b440615928611 Mon Sep 17 00:00:00 2001 From: codex Date: Wed, 1 Jul 2026 16:57:45 +0000 Subject: [PATCH 03/10] test(supervise): update edge cases for sqlite storage --- tests/unit/test_supervise_edge.py | 89 ++++++++++++++++++------------- 1 file changed, 51 insertions(+), 38 deletions(-) diff --git a/tests/unit/test_supervise_edge.py b/tests/unit/test_supervise_edge.py index 7f05450..069d91d 100644 --- a/tests/unit/test_supervise_edge.py +++ b/tests/unit/test_supervise_edge.py @@ -4,7 +4,6 @@ fallback paths.""" from __future__ import annotations -import os import tempfile import time import unittest @@ -13,13 +12,16 @@ from unittest.mock import patch from bot_bottle import supervise from bot_bottle.supervise import ( + AuditEntry, Proposal, + STATUS_APPROVED, TOOL_EGRESS_ALLOW, list_pending_proposals, read_audit_entries, read_proposal, read_response, wait_for_response, + write_audit_entry, ) @@ -40,29 +42,29 @@ class TestPathHelpers(unittest.TestCase): def test_queue_dir_for_slug(self) -> None: self.assertIn("slug", str(supervise.queue_dir_for_slug("slug"))) - def test_id_from_non_proposal_filename(self) -> None: - self.assertIsNone(supervise._id_from_proposal_filename(Path("x.response.json"))) + def test_queue_db_path_for_slug_dir(self) -> None: + self.assertEqual( + Path("/tmp/queue/supervise.db"), + supervise.queue_db_path(Path("/tmp/queue")), + ) class TestReadMalformed(unittest.TestCase): - def test_read_proposal_non_dict(self) -> None: + def test_read_proposal_missing_row(self) -> None: with tempfile.TemporaryDirectory() as d: - (Path(d) / "p.proposal.json").write_text("[]") - with self.assertRaises(ValueError): + with self.assertRaises(FileNotFoundError): read_proposal(Path(d), "p") - def test_read_response_non_dict(self) -> None: + def test_read_response_missing_row(self) -> None: with tempfile.TemporaryDirectory() as d: - (Path(d) / "p.response.json").write_text("[]") - with self.assertRaises(ValueError): + with self.assertRaises(FileNotFoundError): read_response(Path(d), "p") - def test_list_pending_skips_malformed(self) -> None: + def test_list_pending_ignores_legacy_json_files(self) -> None: with tempfile.TemporaryDirectory() as d: qd = Path(d) (qd / "bad.proposal.json").write_text("{ not json") (qd / "arr.proposal.json").write_text("[]") - (qd / "incomplete.proposal.json").write_text("{}") # from_dict raises supervise.write_proposal(qd, _proposal()) # one valid pending = list_pending_proposals(qd) self.assertEqual(1, len(pending)) @@ -73,18 +75,21 @@ class TestReadMalformed(unittest.TestCase): qd = Path(d) p = _proposal() supervise.write_proposal(qd, p) - (qd / f"{p.id}.response.json").write_text("{}") # response exists -> skipped + supervise.write_response(qd, supervise.Response( + proposal_id=p.id, + status=STATUS_APPROVED, + notes="", + )) self.assertEqual([], list_pending_proposals(qd)) class TestWaitForResponse(unittest.TestCase): - def test_malformed_response_then_timeout(self) -> None: + def test_missing_response_times_out(self) -> None: with tempfile.TemporaryDirectory() as d: - (Path(d) / "p.response.json").write_text("{ not json") with self.assertRaises(TimeoutError): wait_for_response(Path(d), "p", deadline=time.monotonic()) - def test_incomplete_response_then_timeout(self) -> None: + def test_legacy_response_file_does_not_count(self) -> None: with tempfile.TemporaryDirectory() as d: (Path(d) / "p.response.json").write_text("{}") # dict but from_dict raises with self.assertRaises(TimeoutError): @@ -97,35 +102,43 @@ class TestReadAuditEntries(unittest.TestCase): patch.dict("os.environ", {"HOME": home}): self.assertEqual([], read_audit_entries("egress", "nope")) - def test_skips_malformed_lines(self) -> None: + def test_reads_entries_from_db(self) -> None: with tempfile.TemporaryDirectory() as home, \ patch.dict("os.environ", {"HOME": home}): - path = supervise.audit_log_path("egress", "slug") - path.parent.mkdir(parents=True, exist_ok=True) - valid = ( - '{"timestamp": "t", "bottle_slug": "slug", "component": "egress",' - ' "operator_action": "approve", "operator_notes": "",' - ' "justification": "", "diff": ""}' - ) - path.write_text( - "\n" # blank line skipped - "{ not json\n" # JSONDecodeError skipped - "[]\n" # not a dict skipped - "{}\n" # missing fields -> ValueError skipped - + valid + "\n" - ) + write_audit_entry(AuditEntry( + timestamp="t", + bottle_slug="slug", + component="egress", + operator_action="approve", + operator_notes="", + justification="", + diff="", + )) + write_audit_entry(AuditEntry( + timestamp="t", + bottle_slug="other", + component="egress", + operator_action="reject", + operator_notes="", + justification="", + diff="", + )) entries = read_audit_entries("egress", "slug") self.assertEqual(1, len(entries)) self.assertEqual("approve", entries[0].operator_action) - -class TestFlockFallback(unittest.TestCase): - def test_flock_on_closed_fd_is_swallowed(self) -> None: - # flock on a closed fd raises OSError(EBADF), which the helpers swallow. - fd = os.open(os.devnull, os.O_RDONLY) - os.close(fd) - supervise._try_flock(fd) - supervise._try_funlock(fd) + def test_legacy_audit_log_file_does_not_count(self) -> None: + with tempfile.TemporaryDirectory() as home, \ + patch.dict("os.environ", {"HOME": home}): + path = supervise.audit_log_path("egress", "slug") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + '{"timestamp": "t", "bottle_slug": "slug", "component": "egress",' + ' "operator_action": "approve", "operator_notes": "",' + ' "justification": "", "diff": ""}\n' + ) + entries = read_audit_entries("egress", "slug") + self.assertEqual([], entries) if __name__ == "__main__": -- 2.52.0 From 212551df9a3d803dd7e36a7c6a9e12846b6ac0dd Mon Sep 17 00:00:00 2001 From: codex Date: Wed, 1 Jul 2026 16:57:54 +0000 Subject: [PATCH 04/10] docs(prd): activate sqlite local storage --- docs/prds/prd-new-sqlite-local-storage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/prds/prd-new-sqlite-local-storage.md b/docs/prds/prd-new-sqlite-local-storage.md index 1a24c69..909721f 100644 --- a/docs/prds/prd-new-sqlite-local-storage.md +++ b/docs/prds/prd-new-sqlite-local-storage.md @@ -1,6 +1,6 @@ # PRD prd-new: SQLite local storage -- **Status:** Draft +- **Status:** Active - **Author:** codex - **Created:** 2026-07-01 - **Issue:** #319 -- 2.52.0 From 3067b067d2514ba808cd379c2f1b8c7b37e12c4d Mon Sep 17 00:00:00 2001 From: codex Date: Wed, 1 Jul 2026 19:33:43 +0000 Subject: [PATCH 05/10] fix(supervise): store queue rows in host sqlite db --- bot_bottle/backend/docker/compose.py | 8 ++ bot_bottle/backend/macos_container/launch.py | 4 +- bot_bottle/backend/smolmachines/launch.py | 4 +- bot_bottle/git_gate_render.py | 77 ++++++++++---------- bot_bottle/supervise.py | 70 +++++++++++++----- docs/prds/prd-new-sqlite-local-storage.md | 39 +++++----- tests/unit/test_compose.py | 3 + tests/unit/test_contrib_claude_provider.py | 1 + tests/unit/test_contrib_codex_provider.py | 1 + tests/unit/test_git_gate.py | 4 +- tests/unit/test_macos_container_launch.py | 9 ++- tests/unit/test_smolmachines_provision.py | 1 + tests/unit/test_supervise.py | 1 + tests/unit/test_supervise_edge.py | 2 +- tests/unit/test_supervise_server.py | 5 +- 15 files changed, 142 insertions(+), 87 deletions(-) diff --git a/bot_bottle/backend/docker/compose.py b/bot_bottle/backend/docker/compose.py index 11ebc2c..78112fc 100644 --- a/bot_bottle/backend/docker/compose.py +++ b/bot_bottle/backend/docker/compose.py @@ -34,6 +34,7 @@ from ...egress import ( from ...git_gate import GIT_GATE_HOSTNAME from ...log import die, warn from ...supervise import ( + DB_PATH_IN_CONTAINER, QUEUE_DIR_IN_CONTAINER, SUPERVISE_HOSTNAME, SUPERVISE_PORT, @@ -163,9 +164,16 @@ def _sidecar_bundle_service(plan: DockerBottlePlan) -> dict[str, Any]: if sp is not None: env += [ f"SUPERVISE_BOTTLE_SLUG={plan.slug}", + f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}", f"SUPERVISE_QUEUE_DIR={QUEUE_DIR_IN_CONTAINER}", f"SUPERVISE_PORT={SUPERVISE_PORT}", ] + volumes.append({ + "type": "bind", + "source": str(sp.db_path), + "target": DB_PATH_IN_CONTAINER, + "read_only": False, + }) volumes.append({ "type": "bind", "source": str(sp.queue_dir), diff --git a/bot_bottle/backend/macos_container/launch.py b/bot_bottle/backend/macos_container/launch.py index e0fe254..518644a 100644 --- a/bot_bottle/backend/macos_container/launch.py +++ b/bot_bottle/backend/macos_container/launch.py @@ -33,7 +33,7 @@ from ...git_gate import ( revoke_git_gate_provisioned_keys, ) from ...log import die, info, warn -from ...supervise import QUEUE_DIR_IN_CONTAINER, SUPERVISE_PORT +from ...supervise import DB_PATH_IN_CONTAINER, QUEUE_DIR_IN_CONTAINER, SUPERVISE_PORT from ...util import expand_tilde from ..docker.egress import EGRESS_CA_IN_CONTAINER, EGRESS_PORT from ..docker.git_gate import ( @@ -379,6 +379,7 @@ def _sidecar_env_entries(plan: MacosContainerBottlePlan) -> tuple[str, ...]: if plan.supervise_plan is not None: env += [ f"SUPERVISE_BOTTLE_SLUG={plan.slug}", + f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}", f"SUPERVISE_QUEUE_DIR={QUEUE_DIR_IN_CONTAINER}", f"SUPERVISE_PORT={SUPERVISE_PORT}", ] @@ -405,6 +406,7 @@ def _sidecar_mounts( sp = plan.supervise_plan if sp is not None: + mounts.append((str(sp.db_path), DB_PATH_IN_CONTAINER, False)) mounts.append((str(sp.queue_dir), QUEUE_DIR_IN_CONTAINER, False)) return tuple(mounts) diff --git a/bot_bottle/backend/smolmachines/launch.py b/bot_bottle/backend/smolmachines/launch.py index 483cf85..57de70b 100644 --- a/bot_bottle/backend/smolmachines/launch.py +++ b/bot_bottle/backend/smolmachines/launch.py @@ -27,7 +27,7 @@ from ...egress import ( egress_resolve_token_values, egress_sidecar_env_entries, ) -from ...supervise import QUEUE_DIR_IN_CONTAINER, SUPERVISE_PORT +from ...supervise import DB_PATH_IN_CONTAINER, QUEUE_DIR_IN_CONTAINER, SUPERVISE_PORT from ...util import expand_tilde from ..docker import util as docker_mod from ..docker.egress import ( @@ -369,9 +369,11 @@ def _bundle_launch_spec( daemons.append("supervise") env += [ f"SUPERVISE_BOTTLE_SLUG={plan.slug}", + f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}", f"SUPERVISE_QUEUE_DIR={QUEUE_DIR_IN_CONTAINER}", f"SUPERVISE_PORT={SUPERVISE_PORT}", ] + volumes.append((str(sp.db_path), DB_PATH_IN_CONTAINER, False)) volumes.append((str(sp.queue_dir), QUEUE_DIR_IN_CONTAINER, False)) # Container ports the agent reaches from the smolvm guest — diff --git a/bot_bottle/git_gate_render.py b/bot_bottle/git_gate_render.py index 8a442b8..656f0e9 100644 --- a/bot_bottle/git_gate_render.py +++ b/bot_bottle/git_gate_render.py @@ -234,9 +234,10 @@ import hashlib import json import os import sys -import uuid from pathlib import Path +from bot_bottle import supervise as _sv + report_path = Path(sys.argv[1]) queue_dir = os.environ.get("SUPERVISE_QUEUE_DIR", "") slug = os.environ.get("SUPERVISE_BOTTLE_SLUG", "") @@ -277,31 +278,19 @@ for i, finding in enumerate(raw, 1): ]) payload = "\n".join(lines).rstrip() + "\n" -proposal_id = str(uuid.uuid4()) -proposal = { - "id": proposal_id, - "bottle_slug": slug, - "tool": "gitleaks-allow", - "proposed_file": payload, - "justification": ( +proposal = _sv.Proposal.new( + bottle_slug=slug, + tool=_sv.TOOL_GITLEAKS_ALLOW, + proposed_file=payload, + justification=( "git-gate found gitleaks findings hidden by # gitleaks:allow; " "approve only for dummy test fixtures or confirmed false positives" ), - "arrival_timestamp": datetime.datetime.now( - datetime.timezone.utc - ).isoformat(), - "current_file_hash": hashlib.sha256(payload.encode("utf-8")).hexdigest(), -} -queue = Path(queue_dir) -queue.mkdir(parents=True, exist_ok=True) -path = queue / f"{proposal_id}.proposal.json" -tmp = path.with_suffix(path.suffix + ".tmp") -with tmp.open("w", encoding="utf-8") as f: - json.dump(proposal, f, indent=2) - f.write("\n") -os.chmod(tmp, 0o600) -os.replace(tmp, path) -print(proposal_id) + current_file_hash=hashlib.sha256(payload.encode("utf-8")).hexdigest(), + now=datetime.datetime.now(datetime.timezone.utc), +) +_sv.write_proposal(Path(queue_dir), proposal) +print(proposal.id) PY ) rc=$? @@ -315,7 +304,6 @@ PY fi queue_dir=${SUPERVISE_QUEUE_DIR:-} - response_file="$queue_dir/${proposal_id}.response.json" timeout=${SUPERVISE_GITLEAKS_ALLOW_TIMEOUT_SECONDS:-300} case "$timeout" in ''|*[!0-9]*) @@ -327,26 +315,36 @@ PY echo "git-gate: approve with './cli.py supervise' to continue this push" >&2 waited=0 while [ "$waited" -lt "$timeout" ]; do - if [ -f "$response_file" ]; then - status=$(python3 - "$response_file" <<'PY' -import json + status=$(python3 - "$queue_dir" "$proposal_id" <<'PY' import sys +from pathlib import Path + +from bot_bottle import supervise as _sv + try: - with open(sys.argv[1], encoding="utf-8") as f: - raw = json.load(f) -except (OSError, json.JSONDecodeError): - sys.exit(1) -status = raw.get("status") -if not isinstance(status, str): - sys.exit(1) -print(status) + response = _sv.read_response(Path(sys.argv[1]), sys.argv[2]) +except FileNotFoundError: + sys.exit(2) +print(response.status) PY - ) || status="" + ) + rc=$? + if [ "$rc" -eq 2 ]; then + status="" + elif [ "$rc" -ne 0 ]; then + status="invalid" + fi + if [ -n "$status" ]; then case "$status" in approved|modified) - mkdir -p "$queue_dir/processed" - mv -f "$queue_dir/${proposal_id}.proposal.json" "$queue_dir/processed/" 2>/dev/null || true - mv -f "$queue_dir/${proposal_id}.response.json" "$queue_dir/processed/" 2>/dev/null || true + python3 - "$queue_dir" "$proposal_id" <<'PY' || true +import sys +from pathlib import Path + +from bot_bottle import supervise as _sv + +_sv.archive_proposal(Path(sys.argv[1]), sys.argv[2]) +PY echo "git-gate: supervisor approved # gitleaks:allow for $ref" >&2 return 0 ;; @@ -499,4 +497,3 @@ if ! git -C "$repo_dir" rev-parse --verify HEAD >/dev/null 2>&1; then fi exit 0 """ - diff --git a/bot_bottle/supervise.py b/bot_bottle/supervise.py index ddd6c4f..3ec3679 100644 --- a/bot_bottle/supervise.py +++ b/bot_bottle/supervise.py @@ -34,6 +34,7 @@ from __future__ import annotations import dataclasses import difflib import hashlib +import os import sqlite3 import time import uuid @@ -86,9 +87,9 @@ STATUSES: tuple[str, ...] = (STATUS_APPROVED, STATUS_MODIFIED, STATUS_REJECTED) ACTION_OPERATOR_EDIT = "operator-edit" QUEUE_DIR_IN_CONTAINER = "/run/supervise/queue" +DB_PATH_IN_CONTAINER = "/run/supervise/bot-bottle.db" DEFAULT_POLL_INTERVAL_SEC = 0.5 HOST_DB_FILENAME = "bot-bottle.db" -QUEUE_DB_FILENAME = "supervise.db" # --- Paths ----------------------------------------------------------------- @@ -115,7 +116,9 @@ def host_db_path() -> Path: def queue_db_path(queue_dir: Path) -> Path: - return queue_dir / QUEUE_DB_FILENAME + del queue_dir + env_path = os.environ.get("SUPERVISE_DB_PATH", "").strip() + return Path(env_path) if env_path else host_db_path() # --- Dataclasses ----------------------------------------------------------- @@ -331,6 +334,7 @@ def sha256_hex(content: str) -> str: class _QueueStore: def __init__(self, queue_dir: Path) -> None: + self.queue_key = _queue_key(queue_dir) self.db_path = queue_db_path(queue_dir) self.db_path.parent.mkdir(parents=True, exist_ok=True) self._init() @@ -340,11 +344,12 @@ class _QueueStore: conn.execute( """ INSERT OR REPLACE INTO supervise_proposals ( - id, bottle_slug, tool, proposed_file, justification, + queue_key, id, bottle_slug, tool, proposed_file, justification, arrival_timestamp, current_file_hash, archived - ) VALUES (?, ?, ?, ?, ?, ?, ?, 0) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0) """, ( + self.queue_key, proposal.id, proposal.bottle_slug, proposal.tool, @@ -362,9 +367,9 @@ class _QueueStore: row = conn.execute( """ SELECT * FROM supervise_proposals - WHERE id = ? AND archived = 0 + WHERE queue_key = ? AND id = ? AND archived = 0 """, - (proposal_id,), + (self.queue_key, proposal_id), ).fetchone() if row is None: raise FileNotFoundError(proposal_id) @@ -378,12 +383,16 @@ class _QueueStore: """ SELECT p.* FROM supervise_proposals p WHERE p.archived = 0 + AND p.queue_key = ? AND NOT EXISTS ( SELECT 1 FROM supervise_responses r - WHERE r.proposal_id = p.id AND r.archived = 0 + WHERE r.queue_key = p.queue_key + AND r.proposal_id = p.id + AND r.archived = 0 ) ORDER BY p.arrival_timestamp, p.id - """ + """, + (self.queue_key,), ).fetchall() return [_proposal_from_row(row) for row in rows] @@ -392,10 +401,11 @@ class _QueueStore: conn.execute( """ INSERT OR REPLACE INTO supervise_responses ( - proposal_id, status, notes, final_file, archived - ) VALUES (?, ?, ?, ?, 0) + queue_key, proposal_id, status, notes, final_file, archived + ) VALUES (?, ?, ?, ?, ?, 0) """, ( + self.queue_key, response.proposal_id, response.status, response.notes, @@ -410,9 +420,9 @@ class _QueueStore: row = conn.execute( """ SELECT * FROM supervise_responses - WHERE proposal_id = ? AND archived = 0 + WHERE queue_key = ? AND proposal_id = ? AND archived = 0 """, - (proposal_id,), + (self.queue_key, proposal_id), ).fetchone() if row is None: raise FileNotFoundError(proposal_id) @@ -423,15 +433,18 @@ class _QueueStore: return with self._connect() as conn: conn.execute( - "UPDATE supervise_proposals SET archived = 1 WHERE id = ?", - (proposal_id,), + """ + UPDATE supervise_proposals SET archived = 1 + WHERE queue_key = ? AND id = ? + """, + (self.queue_key, proposal_id), ) conn.execute( """ UPDATE supervise_responses SET archived = 1 - WHERE proposal_id = ? + WHERE queue_key = ? AND proposal_id = ? """, - (proposal_id,), + (self.queue_key, proposal_id), ) def _connect(self) -> sqlite3.Connection: @@ -444,25 +457,29 @@ class _QueueStore: conn.execute( """ CREATE TABLE IF NOT EXISTS supervise_proposals ( - id TEXT PRIMARY KEY, + queue_key TEXT NOT NULL, + id TEXT NOT NULL, bottle_slug TEXT NOT NULL, tool TEXT NOT NULL, proposed_file TEXT NOT NULL, justification TEXT NOT NULL, arrival_timestamp TEXT NOT NULL, current_file_hash TEXT NOT NULL, - archived INTEGER NOT NULL DEFAULT 0 + archived INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (queue_key, id) ) """ ) conn.execute( """ CREATE TABLE IF NOT EXISTS supervise_responses ( - proposal_id TEXT PRIMARY KEY, + queue_key TEXT NOT NULL, + proposal_id TEXT NOT NULL, status TEXT NOT NULL, notes TEXT NOT NULL, final_file TEXT, - archived INTEGER NOT NULL DEFAULT 0 + archived INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (queue_key, proposal_id) ) """ ) @@ -580,6 +597,13 @@ def _audit_entry_from_row(row: sqlite3.Row) -> AuditEntry: ) +def _queue_key(queue_dir: Path) -> str: + env_slug = os.environ.get("SUPERVISE_BOTTLE_SLUG", "").strip() + if env_slug: + return env_slug + return queue_dir.name + + # --- Sidecar plan + abstract lifecycle ------------------------------------- @@ -594,6 +618,7 @@ class SupervisePlan: slug: str queue_dir: Path + db_path: Path internal_network: str = "" @@ -613,9 +638,13 @@ class Supervise(ABC): del stage_dir queue_dir = queue_dir_for_slug(slug) queue_dir.mkdir(parents=True, exist_ok=True) + db_path = host_db_path() + _QueueStore(queue_dir) + _AuditStore(db_path) return SupervisePlan( slug=slug, queue_dir=queue_dir, + db_path=db_path, ) # --- Helpers --------------------------------------------------------------- @@ -633,6 +662,7 @@ __all__ = [ "AuditEntry", "COMPONENT_FOR_TOOL", "DEFAULT_POLL_INTERVAL_SEC", + "DB_PATH_IN_CONTAINER", "Proposal", "QUEUE_DIR_IN_CONTAINER", "Response", diff --git a/docs/prds/prd-new-sqlite-local-storage.md b/docs/prds/prd-new-sqlite-local-storage.md index 909721f..bde2f6a 100644 --- a/docs/prds/prd-new-sqlite-local-storage.md +++ b/docs/prds/prd-new-sqlite-local-storage.md @@ -49,51 +49,50 @@ one-off persistence. ### Database locations -Queue state remains tied to the mounted per-bottle queue directory: - -```text -~/.bot-bottle/queue//supervise.db -``` - -The supervise sidecar already receives that directory at -`/run/supervise/queue`, so both the sidecar and host TUI can read and write the -same SQLite file without changing backend mounts. - -Audit state uses the host-level local database: +Queue and audit state use the host-level local database: ```text ~/.bot-bottle/bot-bottle.db ``` -This creates the shared host database that later forge/native lifecycle work can +The supervise sidecar receives that database as a writable bind mount at +`/run/supervise/bot-bottle.db` and gets the path through `SUPERVISE_DB_PATH`. +The existing per-slug queue directory mount remains in place for compatibility +with the supervise sidecar contract and any adjacent tooling that still expects a +queue directory, but the active queue records live in the host database. This +creates the shared host database that later forge/native lifecycle work can extend in separate PRDs. ### Tables -`supervise_proposals` lives in the per-queue database: +`supervise_proposals` lives in the host database: ```sql CREATE TABLE supervise_proposals ( - id TEXT PRIMARY KEY, + queue_key TEXT NOT NULL, + id TEXT NOT NULL, bottle_slug TEXT NOT NULL, tool TEXT NOT NULL, proposed_file TEXT NOT NULL, justification TEXT NOT NULL, arrival_timestamp TEXT NOT NULL, current_file_hash TEXT NOT NULL, - archived INTEGER NOT NULL DEFAULT 0 + archived INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (queue_key, id) ); ``` -`supervise_responses` lives in the same per-queue database: +`supervise_responses` lives in the host database: ```sql CREATE TABLE supervise_responses ( - proposal_id TEXT PRIMARY KEY, + queue_key TEXT NOT NULL, + proposal_id TEXT NOT NULL, status TEXT NOT NULL, notes TEXT NOT NULL, final_file TEXT, - archived INTEGER NOT NULL DEFAULT 0 + archived INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (queue_key, proposal_id) ); ``` @@ -115,8 +114,8 @@ CREATE TABLE supervise_audit_entries ( ### Compatibility The existing helper functions keep accepting `Path` arguments for queue -directories. Internally, they map the queue directory to `supervise.db` and -perform equivalent operations: +directories. Internally, they map the queue directory to a queue key and perform +equivalent operations against `~/.bot-bottle/bot-bottle.db`: - `list_pending_proposals` returns non-archived proposals without a non-archived response, sorted by arrival time. diff --git a/tests/unit/test_compose.py b/tests/unit/test_compose.py index 9f75be5..e874423 100644 --- a/tests/unit/test_compose.py +++ b/tests/unit/test_compose.py @@ -108,6 +108,7 @@ def _supervise_plan() -> SupervisePlan: return SupervisePlan( slug=SLUG, queue_dir=STATE / "supervise" / "queue", + db_path=STATE / "bot-bottle.db", internal_network=f"bot-bottle-net-{SLUG}", ) @@ -392,6 +393,7 @@ class TestSidecarBundleShape(unittest.TestCase): sc = self._render(supervise=True)["services"]["sidecars"] env_strings = sc["environment"] self.assertIn(f"SUPERVISE_BOTTLE_SLUG={SLUG}", env_strings) + self.assertIn("SUPERVISE_DB_PATH=/run/supervise/bot-bottle.db", env_strings) self.assertTrue(any(e.startswith("SUPERVISE_QUEUE_DIR=") for e in env_strings)) self.assertTrue(any(e.startswith("SUPERVISE_PORT=") for e in env_strings)) @@ -408,6 +410,7 @@ class TestSidecarBundleShape(unittest.TestCase): self.assertIn("/etc/egress", targets) self.assertIn("/git-gate-entrypoint.sh", targets) self.assertIn("/git-gate/creds/upstream-known_hosts", targets) + self.assertIn("/run/supervise/bot-bottle.db", targets) self.assertTrue(any("supervise/queue" in t or t.startswith("/run/supervise") for t in targets)) diff --git a/tests/unit/test_contrib_claude_provider.py b/tests/unit/test_contrib_claude_provider.py index 91bf73e..86d5231 100644 --- a/tests/unit/test_contrib_claude_provider.py +++ b/tests/unit/test_contrib_claude_provider.py @@ -75,6 +75,7 @@ def _plan( supervise_plan = SupervisePlan( slug="demo-abc12", queue_dir=Path("/tmp/queue"), + db_path=Path("/tmp/bot-bottle.db"), ) return DockerBottlePlan( spec=spec, diff --git a/tests/unit/test_contrib_codex_provider.py b/tests/unit/test_contrib_codex_provider.py index 3d3cf1c..97c45c5 100644 --- a/tests/unit/test_contrib_codex_provider.py +++ b/tests/unit/test_contrib_codex_provider.py @@ -78,6 +78,7 @@ def _plan( supervise_plan = SupervisePlan( slug="demo-abc12", queue_dir=Path("/tmp/queue"), + db_path=Path("/tmp/bot-bottle.db"), ) return DockerBottlePlan( spec=spec, diff --git a/tests/unit/test_git_gate.py b/tests/unit/test_git_gate.py index be6fc32..2f4e2f0 100644 --- a/tests/unit/test_git_gate.py +++ b/tests/unit/test_git_gate.py @@ -210,7 +210,9 @@ class TestHookRender(unittest.TestCase): # the suppressed findings for human approval. self.assertIn("--ignore-gitleaks-allow", hook) self.assertIn("--report-format=json", hook) - self.assertIn('"tool": "gitleaks-allow"', hook) + self.assertIn("tool=_sv.TOOL_GITLEAKS_ALLOW", hook) + self.assertIn("_sv.write_proposal", hook) + self.assertIn("_sv.read_response", hook) self.assertIn("SUPERVISE_QUEUE_DIR", hook) self.assertIn("SUPERVISE_BOTTLE_SLUG", hook) self.assertIn("supervisor approved # gitleaks:allow", hook) diff --git a/tests/unit/test_macos_container_launch.py b/tests/unit/test_macos_container_launch.py index 3e1038e..d96a055 100644 --- a/tests/unit/test_macos_container_launch.py +++ b/tests/unit/test_macos_container_launch.py @@ -71,7 +71,10 @@ def _plan( else: git_gate_plan = SimpleNamespace(upstreams=()) supervise_plan = ( - SimpleNamespace(queue_dir=Path("/state/supervise/queue")) + SimpleNamespace( + queue_dir=Path("/state/supervise/queue"), + db_path=Path("/state/bot-bottle.db"), + ) if supervise else None ) agent_provision = SimpleNamespace( @@ -136,6 +139,10 @@ class TestMacosContainerLaunchArgv(unittest.TestCase): f"type=bind,source={self.stage_dir},target=/etc/egress,readonly", argv, ) + self.assertIn( + "type=bind,source=/state/bot-bottle.db,target=/run/supervise/bot-bottle.db", + argv, + ) self.assertIn( "type=bind,source=/state/supervise/queue,target=/run/supervise/queue", argv, diff --git a/tests/unit/test_smolmachines_provision.py b/tests/unit/test_smolmachines_provision.py index d6b28c5..df92867 100644 --- a/tests/unit/test_smolmachines_provision.py +++ b/tests/unit/test_smolmachines_provision.py @@ -131,6 +131,7 @@ def _plan( supervise_plan = SupervisePlan( slug="demo-abc12", queue_dir=Path("/tmp/queue"), + db_path=Path("/tmp/bot-bottle.db"), ) return SmolmachinesBottlePlan( spec=spec, diff --git a/tests/unit/test_supervise.py b/tests/unit/test_supervise.py index 0e97d32..3d7419e 100644 --- a/tests/unit/test_supervise.py +++ b/tests/unit/test_supervise.py @@ -382,6 +382,7 @@ class TestSupervisePrepare(unittest.TestCase): def test_prepare_creates_queue(self): plan = _StubSupervise().prepare("dev", self.stage_dir) self.assertTrue(plan.queue_dir.is_dir()) + self.assertTrue(plan.db_path.is_file()) self.assertEqual("dev", plan.slug) self.assertEqual("", plan.internal_network) diff --git a/tests/unit/test_supervise_edge.py b/tests/unit/test_supervise_edge.py index 069d91d..d709a3f 100644 --- a/tests/unit/test_supervise_edge.py +++ b/tests/unit/test_supervise_edge.py @@ -44,7 +44,7 @@ class TestPathHelpers(unittest.TestCase): def test_queue_db_path_for_slug_dir(self) -> None: self.assertEqual( - Path("/tmp/queue/supervise.db"), + supervise.host_db_path(), supervise.queue_db_path(Path("/tmp/queue")), ) diff --git a/tests/unit/test_supervise_server.py b/tests/unit/test_supervise_server.py index 088dd39..8a95aa6 100644 --- a/tests/unit/test_supervise_server.py +++ b/tests/unit/test_supervise_server.py @@ -122,9 +122,10 @@ class TestRpcInternalErrorOnIoFailure(unittest.TestCase): def test_write_proposal_os_error_raises_internal(self): config = ServerConfig( bottle_slug="dev", - queue_dir=Path("/dev/null/cannot-exist"), + queue_dir=Path("/unused"), ) - with self.assertRaises(_RpcInternalError) as cm: + with patch.object(_sv, "write_proposal", side_effect=OSError("disk full")), \ + self.assertRaises(_RpcInternalError) as cm: handle_tools_call( { "name": _sv.TOOL_EGRESS_ALLOW, -- 2.52.0 From 29904609da4f829b14706789dd819fc98d6e0c71 Mon Sep 17 00:00:00 2001 From: codex Date: Wed, 1 Jul 2026 19:50:38 +0000 Subject: [PATCH 06/10] fix(supervise): remove queue directory from db-backed flow --- Dockerfile.sidecars | 4 +- bot_bottle/backend/docker/compose.py | 9 -- bot_bottle/backend/macos_container/launch.py | 4 +- bot_bottle/backend/smolmachines/launch.py | 4 +- bot_bottle/bottle_state.py | 5 +- bot_bottle/cli/supervise.py | 25 ++-- bot_bottle/egress_addon.py | 15 +-- bot_bottle/git_gate_render.py | 18 ++- bot_bottle/supervise.py | 117 +++++++++--------- bot_bottle/supervise_server.py | 26 ++-- docs/prds/prd-new-sqlite-local-storage.md | 29 ++--- tests/unit/test_compose.py | 4 - tests/unit/test_contrib_claude_provider.py | 1 - tests/unit/test_contrib_codex_provider.py | 1 - tests/unit/test_egress_addon_log_redaction.py | 1 - tests/unit/test_egress_addon_request_flow.py | 9 +- tests/unit/test_git_gate.py | 1 - tests/unit/test_macos_container_launch.py | 5 - tests/unit/test_smolmachines_provision.py | 1 - tests/unit/test_supervise.py | 64 ++++++---- tests/unit/test_supervise_cli.py | 42 +++---- tests/unit/test_supervise_edge.py | 62 +++++----- tests/unit/test_supervise_server.py | 35 +++--- 23 files changed, 212 insertions(+), 270 deletions(-) diff --git a/Dockerfile.sidecars b/Dockerfile.sidecars index a94960e..ba13551 100644 --- a/Dockerfile.sidecars +++ b/Dockerfile.sidecars @@ -18,7 +18,7 @@ # /git-gate-entrypoint.sh docker-cp'd at start time # /git-gate/creds/* docker-cp'd at start time # /git/* bare repos, populated at runtime -# /run/supervise/queue/ bind-mounted at run time +# /run/supervise/bot-bottle.db bind-mounted at run time # /home/mitmproxy/.mitmproxy/ mitmproxy CA dir # # Exposed ports inside the container: @@ -81,7 +81,7 @@ RUN mkdir -p \ /etc/git-gate \ /git-gate/creds \ /git \ - /run/supervise/queue \ + /run/supervise \ /home/mitmproxy/.mitmproxy # Documentation only — the compose renderer publishes whichever diff --git a/bot_bottle/backend/docker/compose.py b/bot_bottle/backend/docker/compose.py index 78112fc..9929610 100644 --- a/bot_bottle/backend/docker/compose.py +++ b/bot_bottle/backend/docker/compose.py @@ -35,7 +35,6 @@ from ...git_gate import GIT_GATE_HOSTNAME from ...log import die, warn from ...supervise import ( DB_PATH_IN_CONTAINER, - QUEUE_DIR_IN_CONTAINER, SUPERVISE_HOSTNAME, SUPERVISE_PORT, ) @@ -165,7 +164,6 @@ def _sidecar_bundle_service(plan: DockerBottlePlan) -> dict[str, Any]: env += [ f"SUPERVISE_BOTTLE_SLUG={plan.slug}", f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}", - f"SUPERVISE_QUEUE_DIR={QUEUE_DIR_IN_CONTAINER}", f"SUPERVISE_PORT={SUPERVISE_PORT}", ] volumes.append({ @@ -174,13 +172,6 @@ def _sidecar_bundle_service(plan: DockerBottlePlan) -> dict[str, Any]: "target": DB_PATH_IN_CONTAINER, "read_only": False, }) - volumes.append({ - "type": "bind", - "source": str(sp.queue_dir), - "target": QUEUE_DIR_IN_CONTAINER, - "read_only": False, - }) - internal_aliases = [EGRESS_HOSTNAME] if gp.upstreams: internal_aliases.append(GIT_GATE_HOSTNAME) diff --git a/bot_bottle/backend/macos_container/launch.py b/bot_bottle/backend/macos_container/launch.py index 518644a..10976d9 100644 --- a/bot_bottle/backend/macos_container/launch.py +++ b/bot_bottle/backend/macos_container/launch.py @@ -33,7 +33,7 @@ from ...git_gate import ( revoke_git_gate_provisioned_keys, ) from ...log import die, info, warn -from ...supervise import DB_PATH_IN_CONTAINER, QUEUE_DIR_IN_CONTAINER, SUPERVISE_PORT +from ...supervise import DB_PATH_IN_CONTAINER, SUPERVISE_PORT from ...util import expand_tilde from ..docker.egress import EGRESS_CA_IN_CONTAINER, EGRESS_PORT from ..docker.git_gate import ( @@ -380,7 +380,6 @@ def _sidecar_env_entries(plan: MacosContainerBottlePlan) -> tuple[str, ...]: env += [ f"SUPERVISE_BOTTLE_SLUG={plan.slug}", f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}", - f"SUPERVISE_QUEUE_DIR={QUEUE_DIR_IN_CONTAINER}", f"SUPERVISE_PORT={SUPERVISE_PORT}", ] return tuple(env) @@ -407,7 +406,6 @@ def _sidecar_mounts( sp = plan.supervise_plan if sp is not None: mounts.append((str(sp.db_path), DB_PATH_IN_CONTAINER, False)) - mounts.append((str(sp.queue_dir), QUEUE_DIR_IN_CONTAINER, False)) return tuple(mounts) diff --git a/bot_bottle/backend/smolmachines/launch.py b/bot_bottle/backend/smolmachines/launch.py index 57de70b..8736567 100644 --- a/bot_bottle/backend/smolmachines/launch.py +++ b/bot_bottle/backend/smolmachines/launch.py @@ -27,7 +27,7 @@ from ...egress import ( egress_resolve_token_values, egress_sidecar_env_entries, ) -from ...supervise import DB_PATH_IN_CONTAINER, QUEUE_DIR_IN_CONTAINER, SUPERVISE_PORT +from ...supervise import DB_PATH_IN_CONTAINER, SUPERVISE_PORT from ...util import expand_tilde from ..docker import util as docker_mod from ..docker.egress import ( @@ -370,11 +370,9 @@ def _bundle_launch_spec( env += [ f"SUPERVISE_BOTTLE_SLUG={plan.slug}", f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}", - f"SUPERVISE_QUEUE_DIR={QUEUE_DIR_IN_CONTAINER}", f"SUPERVISE_PORT={SUPERVISE_PORT}", ] volumes.append((str(sp.db_path), DB_PATH_IN_CONTAINER, False)) - volumes.append((str(sp.queue_dir), QUEUE_DIR_IN_CONTAINER, False)) # Container ports the agent reaches from the smolvm guest — # published on host loopback so the guest can dial via TSI + diff --git a/bot_bottle/bottle_state.py b/bot_bottle/bottle_state.py index 02fe3b1..7eb3316 100644 --- a/bot_bottle/bottle_state.py +++ b/bot_bottle/bottle_state.py @@ -284,9 +284,8 @@ def git_gate_state_dir(identity: str) -> Path: def supervise_state_dir(identity: str) -> Path: """State subdir reserved for supervise sidecar bind-mount sources. - The queue dir is intentionally NOT under here — it lives at - ~/.bot-bottle/queue// alongside the audit logs, so it - survives state-dir cleanup.""" + Runtime queue/audit rows live in the host-level bot-bottle SQLite + database, so they survive state-dir cleanup.""" return bottle_state_dir(identity) / _SUPERVISE_SUBDIR diff --git a/bot_bottle/cli/supervise.py b/bot_bottle/cli/supervise.py index 3eeaeb6..6f341bd 100644 --- a/bot_bottle/cli/supervise.py +++ b/bot_bottle/cli/supervise.py @@ -45,7 +45,7 @@ from ..supervise import ( TOOL_EGRESS_BLOCK, TOOL_GITLEAKS_ALLOW, TOOL_EGRESS_TOKEN_ALLOW, - list_pending_proposals, + list_all_pending_proposals, render_diff, write_audit_entry, write_response, @@ -63,10 +63,9 @@ _REPORT_ONLY_TOOLS: tuple[str, ...] = (TOOL_GITLEAKS_ALLOW, TOOL_EGRESS_TOKEN_AL @dataclass(frozen=True) class QueuedProposal: - """A pending proposal plus the queue dir it was found in.""" + """A pending proposal from the supervise queue.""" proposal: Proposal - queue_dir: Path # Errors any remediation engine may raise. Caught by the TUI key @@ -86,16 +85,11 @@ def apply_routes_change(slug: str, content: str) -> tuple[str, str]: def discover_pending() -> list[QueuedProposal]: - """Walk ~/.bot-bottle/queue/* and collect pending proposals.""" - queue_root = _supervise.bot_bottle_root() / "queue" - if not queue_root.is_dir(): - return [] - out: list[QueuedProposal] = [] - for slug_dir in sorted(queue_root.iterdir()): - if not slug_dir.is_dir(): - continue - for proposal in list_pending_proposals(slug_dir): - out.append(QueuedProposal(proposal=proposal, queue_dir=slug_dir)) + """Collect pending proposals across bottles.""" + out = [ + QueuedProposal(proposal=proposal) + for proposal in list_all_pending_proposals() + ] out.sort(key=lambda q: q.proposal.arrival_timestamp) return out @@ -118,7 +112,6 @@ def _detail_lines( (f"tool: {p.tool}", 0), (f"id: {p.id}", 0), (f"arrived: {p.arrival_timestamp}", 0), - (f"queue: {qp.queue_dir}", 0), ("", 0), ("justification:", 0), ] @@ -165,7 +158,7 @@ def approve( notes=notes, final_file=final_file, ) - write_response(qp.queue_dir, response) + write_response(qp.proposal.bottle_slug, response) _write_audit( qp, action=status, notes=notes, diff_before=diff_before, diff_after=diff_after, @@ -179,7 +172,7 @@ def reject(qp: QueuedProposal, *, reason: str) -> None: notes=reason, final_file=None, ) - write_response(qp.queue_dir, response) + write_response(qp.proposal.bottle_slug, response) _write_audit(qp, action=STATUS_REJECTED, notes=reason, diff_before="", diff_after="") diff --git a/bot_bottle/egress_addon.py b/bot_bottle/egress_addon.py index a28094e..d0cf0c7 100644 --- a/bot_bottle/egress_addon.py +++ b/bot_bottle/egress_addon.py @@ -79,14 +79,13 @@ class EgressAddon: # only — a restart re-prompts. Mutated only from the asyncio loop that # runs the addon hooks, so no lock is needed. self.safe_tokens: set[str] = set() - self._supervise_queue_dir = os.environ.get("SUPERVISE_QUEUE_DIR", "").strip() self._supervise_slug = os.environ.get("SUPERVISE_BOTTLE_SLUG", "").strip() self._token_allow_timeout = _token_allow_timeout_from_env(os.environ) self._reload(initial=True) self._install_sighup() def _supervise_available(self) -> bool: - return bool(self._supervise_queue_dir and self._supervise_slug) + return bool(self._supervise_slug) def _reload(self, *, initial: bool = False) -> None: try: @@ -393,9 +392,8 @@ class EgressAddon: justification=_TOKEN_ALLOW_JUSTIFICATION, current_file_hash=_sv.sha256_hex(payload), ) - queue_dir = Path(self._supervise_queue_dir) try: - _sv.write_proposal(queue_dir, proposal) + _sv.write_proposal(proposal) except OSError as e: sys.stderr.write( f"egress: could not queue token-allow proposal: {e}; " @@ -411,8 +409,8 @@ class EgressAddon: **self._req_ctx(flow), }) + "\n") - response = await self._await_token_response(queue_dir, proposal.id) - _sv.archive_proposal(queue_dir, proposal.id) + response = await self._await_token_response(proposal.id) + _sv.archive_proposal(self._supervise_slug, proposal.id) if response is not None and response.status in ( _sv.STATUS_APPROVED, _sv.STATUS_MODIFIED, @@ -439,16 +437,15 @@ class EgressAddon: async def _await_token_response( self, - queue_dir: Path, proposal_id: str, ) -> "_sv.Response | None": - """Poll the queue dir for the operator's response without blocking the + """Poll the DB for the operator's response without blocking the proxy event loop. Returns the Response, or None on timeout.""" loop = asyncio.get_running_loop() deadline = loop.time() + self._token_allow_timeout while True: try: - return _sv.read_response(queue_dir, proposal_id) + return _sv.read_response(self._supervise_slug, proposal_id) except (OSError, ValueError, KeyError): # Not written yet, or a partial/malformed write — retry until # the deadline, then fail closed. diff --git a/bot_bottle/git_gate_render.py b/bot_bottle/git_gate_render.py index 656f0e9..ba11926 100644 --- a/bot_bottle/git_gate_render.py +++ b/bot_bottle/git_gate_render.py @@ -239,9 +239,8 @@ from pathlib import Path from bot_bottle import supervise as _sv report_path = Path(sys.argv[1]) -queue_dir = os.environ.get("SUPERVISE_QUEUE_DIR", "") slug = os.environ.get("SUPERVISE_BOTTLE_SLUG", "") -if not queue_dir or not slug: +if not slug: sys.exit(2) try: @@ -289,7 +288,7 @@ proposal = _sv.Proposal.new( current_file_hash=hashlib.sha256(payload.encode("utf-8")).hexdigest(), now=datetime.datetime.now(datetime.timezone.utc), ) -_sv.write_proposal(Path(queue_dir), proposal) +_sv.write_proposal(proposal) print(proposal.id) PY ) @@ -303,7 +302,7 @@ PY return 1 fi - queue_dir=${SUPERVISE_QUEUE_DIR:-} + slug=${SUPERVISE_BOTTLE_SLUG:-} timeout=${SUPERVISE_GITLEAKS_ALLOW_TIMEOUT_SECONDS:-300} case "$timeout" in ''|*[!0-9]*) @@ -315,14 +314,14 @@ PY echo "git-gate: approve with './cli.py supervise' to continue this push" >&2 waited=0 while [ "$waited" -lt "$timeout" ]; do - status=$(python3 - "$queue_dir" "$proposal_id" <<'PY' + status=$(python3 - "$slug" "$proposal_id" <<'PY' import sys -from pathlib import Path from bot_bottle import supervise as _sv +slug = sys.argv[1] try: - response = _sv.read_response(Path(sys.argv[1]), sys.argv[2]) + response = _sv.read_response(slug, sys.argv[2]) except FileNotFoundError: sys.exit(2) print(response.status) @@ -337,13 +336,12 @@ PY if [ -n "$status" ]; then case "$status" in approved|modified) - python3 - "$queue_dir" "$proposal_id" <<'PY' || true + python3 - "$slug" "$proposal_id" <<'PY' || true import sys -from pathlib import Path from bot_bottle import supervise as _sv -_sv.archive_proposal(Path(sys.argv[1]), sys.argv[2]) +_sv.archive_proposal(sys.argv[1], sys.argv[2]) PY echo "git-gate: supervisor approved # gitleaks:allow for $ref" >&2 return 0 diff --git a/bot_bottle/supervise.py b/bot_bottle/supervise.py index 3ec3679..6605bb3 100644 --- a/bot_bottle/supervise.py +++ b/bot_bottle/supervise.py @@ -9,15 +9,14 @@ calls when it needs an operator-reviewed egress change: Each tool call: the agent passes the full proposed file plus a justification text. The sidecar validates the proposal syntactically, -writes it to the host's per-bottle queue dir, and holds the tool-call +writes it to the host SQLite queue table, and holds the tool-call connection open. The operator's supervise TUI (bot_bottle.cli.supervise) sees the proposal, accepts -approve / modify / reject, and writes a response file alongside the -proposal. The sidecar sees the response and returns `{status, notes}` -to the agent. +approve / modify / reject, and writes a response row. The sidecar sees +the response and returns `{status, notes}` to the agent. This module defines the host-side library: dataclasses for the queue -file shapes, queue read/write helpers, the audit log writer, and the +record shapes, queue read/write helpers, the audit log writer, and the diff renderer. The in-container sidecar lives in bot_bottle/supervise_server.py; the supervise daemon's container lifecycle is owned by the sidecar bundle (PRD 0024). @@ -86,7 +85,6 @@ STATUSES: tuple[str, ...] = (STATUS_APPROVED, STATUS_MODIFIED, STATUS_REJECTED) # `routes edit ` verb writes entries with this action. ACTION_OPERATOR_EDIT = "operator-edit" -QUEUE_DIR_IN_CONTAINER = "/run/supervise/queue" DB_PATH_IN_CONTAINER = "/run/supervise/bot-bottle.db" DEFAULT_POLL_INTERVAL_SEC = 0.5 HOST_DB_FILENAME = "bot-bottle.db" @@ -99,10 +97,6 @@ def bot_bottle_root() -> Path: return Path.home() / ".bot-bottle" -def queue_dir_for_slug(slug: str) -> Path: - return bot_bottle_root() / "queue" / slug - - def audit_dir() -> Path: return bot_bottle_root() / "audit" @@ -115,8 +109,7 @@ def host_db_path() -> Path: return bot_bottle_root() / HOST_DB_FILENAME -def queue_db_path(queue_dir: Path) -> Path: - del queue_dir +def queue_db_path() -> Path: env_path = os.environ.get("SUPERVISE_DB_PATH", "").strip() return Path(env_path) if env_path else host_db_path() @@ -126,9 +119,7 @@ def queue_db_path(queue_dir: Path) -> Path: @dataclass(frozen=True) class Proposal: - """One pending tool-call from the agent. The sidecar writes one - of these to the queue dir on a tool call; the operator's TUI - reads them; the sidecar polls for a matching Response.""" + """One pending tool-call from the agent.""" id: str bottle_slug: str @@ -182,7 +173,7 @@ class Proposal: @dataclass(frozen=True) class Response: """The operator's decision on a proposal. The TUI writes one of - these to the queue dir; the sidecar reads it and returns the + these to the queue table; the sidecar reads it and returns the `{status, notes}` pair to the agent's tool call. `final_file` carries the file content the supervisor will @@ -238,33 +229,38 @@ class AuditEntry: # --- Queue I/O ------------------------------------------------------------- -def write_proposal(queue_dir: Path, proposal: Proposal) -> Path: +def write_proposal(proposal: Proposal) -> Path: """Persist `proposal` in the queue database, mode 0o600. Directory is created if missing.""" - return _QueueStore(queue_dir).write_proposal(proposal) + return _QueueStore(proposal.bottle_slug).write_proposal(proposal) -def read_proposal(queue_dir: Path, proposal_id: str) -> Proposal: - return _QueueStore(queue_dir).read_proposal(proposal_id) +def read_proposal(bottle_slug: str, proposal_id: str) -> Proposal: + return _QueueStore(bottle_slug).read_proposal(proposal_id) -def list_pending_proposals(queue_dir: Path) -> list[Proposal]: - """All proposals in `queue_dir` that do not yet have a matching +def list_pending_proposals(bottle_slug: str) -> list[Proposal]: + """All proposals for `bottle_slug` that do not yet have a matching response. Sorted by `arrival_timestamp` so the operator sees the queue FIFO.""" - return _QueueStore(queue_dir).list_pending_proposals() + return _QueueStore(bottle_slug).list_pending_proposals() -def write_response(queue_dir: Path, response: Response) -> Path: - return _QueueStore(queue_dir).write_response(response) +def list_all_pending_proposals() -> list[Proposal]: + """All pending proposals across bottles, sorted FIFO.""" + return _QueueStore("").list_all_pending_proposals() -def read_response(queue_dir: Path, proposal_id: str) -> Response: - return _QueueStore(queue_dir).read_response(proposal_id) +def write_response(bottle_slug: str, response: Response) -> Path: + return _QueueStore(bottle_slug).write_response(response) + + +def read_response(bottle_slug: str, proposal_id: str) -> Response: + return _QueueStore(bottle_slug).read_response(proposal_id) def wait_for_response( - queue_dir: Path, + bottle_slug: str, proposal_id: str, *, poll_interval: float = DEFAULT_POLL_INTERVAL_SEC, @@ -276,7 +272,7 @@ def wait_for_response( natural shape, since the operator's response time is unbounded. Polls SQLite so the implementation stays portable and stdlib-only.""" - store = _QueueStore(queue_dir) + store = _QueueStore(bottle_slug) while True: try: return store.read_response(proposal_id) @@ -287,10 +283,10 @@ def wait_for_response( time.sleep(poll_interval) -def archive_proposal(queue_dir: Path, proposal_id: str) -> None: +def archive_proposal(bottle_slug: str, proposal_id: str) -> None: """Mark both proposal and response rows processed. Idempotent — missing rows are silently skipped.""" - _QueueStore(queue_dir).archive_proposal(proposal_id) + _QueueStore(bottle_slug).archive_proposal(proposal_id) # --- Audit log ------------------------------------------------------------- @@ -333,9 +329,9 @@ def sha256_hex(content: str) -> str: class _QueueStore: - def __init__(self, queue_dir: Path) -> None: - self.queue_key = _queue_key(queue_dir) - self.db_path = queue_db_path(queue_dir) + def __init__(self, queue_key: str) -> None: + self.queue_key = queue_key + self.db_path = queue_db_path() self.db_path.parent.mkdir(parents=True, exist_ok=True) self._init() @@ -396,6 +392,25 @@ class _QueueStore: ).fetchall() return [_proposal_from_row(row) for row in rows] + def list_all_pending_proposals(self) -> list[Proposal]: + if not self.db_path.is_file(): + return [] + with self._connect() as conn: + rows = conn.execute( + """ + SELECT p.* FROM supervise_proposals p + WHERE p.archived = 0 + AND NOT EXISTS ( + SELECT 1 FROM supervise_responses r + WHERE r.queue_key = p.queue_key + AND r.proposal_id = p.id + AND r.archived = 0 + ) + ORDER BY p.arrival_timestamp, p.id + """ + ).fetchall() + return [_proposal_from_row(row) for row in rows] + def write_response(self, response: Response) -> Path: with self._connect() as conn: conn.execute( @@ -597,13 +612,6 @@ def _audit_entry_from_row(row: sqlite3.Row) -> AuditEntry: ) -def _queue_key(queue_dir: Path) -> str: - env_slug = os.environ.get("SUPERVISE_BOTTLE_SLUG", "").strip() - if env_slug: - return env_slug - return queue_dir.name - - # --- Sidecar plan + abstract lifecycle ------------------------------------- @@ -611,39 +619,33 @@ def _queue_key(queue_dir: Path) -> str: class SupervisePlan: """Output of Supervise.prepare; consumed by .start. - `queue_dir` is the host directory bind-mounted into the sidecar - at /run/supervise/queue. `internal_network` is empty at prepare - time; the backend's launch step fills it via dataclasses.replace - before calling .start.""" + `db_path` is the host database bind-mounted into the sidecar at + /run/supervise/bot-bottle.db. `internal_network` is empty at + prepare time; the backend's launch step fills it via + dataclasses.replace before calling .start.""" slug: str - queue_dir: Path db_path: Path internal_network: str = "" class Supervise(ABC): - """Per-bottle supervise sidecar. Encapsulates the host-side - prepare (queue dir staging); the sidecar's start/stop lifecycle - is backend-specific.""" + """Per-bottle supervise sidecar. Encapsulates host-side database + staging; the sidecar's start/stop lifecycle is backend-specific.""" def prepare( self, slug: str, stage_dir: Path, ) -> SupervisePlan: - """Stage the per-bottle queue dir on the host. Returns the - plan; `internal_network` must be set by the launch step before - .start runs.""" + """Stage the host database. Returns the plan; `internal_network` + must be set by the launch step before .start runs.""" del stage_dir - queue_dir = queue_dir_for_slug(slug) - queue_dir.mkdir(parents=True, exist_ok=True) db_path = host_db_path() - _QueueStore(queue_dir) + _QueueStore(slug) _AuditStore(db_path) return SupervisePlan( slug=slug, - queue_dir=queue_dir, db_path=db_path, ) @@ -664,7 +666,6 @@ __all__ = [ "DEFAULT_POLL_INTERVAL_SEC", "DB_PATH_IN_CONTAINER", "Proposal", - "QUEUE_DIR_IN_CONTAINER", "Response", "STATUSES", "STATUS_APPROVED", @@ -688,8 +689,8 @@ __all__ = [ "bot_bottle_root", "host_db_path", "list_pending_proposals", + "list_all_pending_proposals", "queue_db_path", - "queue_dir_for_slug", "read_audit_entries", "read_proposal", "read_response", diff --git a/bot_bottle/supervise_server.py b/bot_bottle/supervise_server.py index 5fca484..089e106 100644 --- a/bot_bottle/supervise_server.py +++ b/bot_bottle/supervise_server.py @@ -7,14 +7,13 @@ config changes when stuck. The tools are `egress-allow`, Each queued tool call: 1. Validates the proposed file syntactically. - 2. Writes a Proposal to /run/supervise/queue/ (bind-mounted from - the host's ~/.bot-bottle/queue//). - 3. Blocks polling for a matching Response file. + 2. Writes a Proposal to the host SQLite database. + 3. Blocks polling for a matching Response row. 4. Returns the operator's `{status, notes}` to the agent. The bottle slug arrives via SUPERVISE_BOTTLE_SLUG env (stamped at -container creation by the backend's start step). The queue dir comes -from SUPERVISE_QUEUE_DIR (default `/run/supervise/queue`). +container creation by the backend's start step). SUPERVISE_DB_PATH +points at the bind-mounted host database. Speaks MCP over HTTP+JSON-RPC. Methods handled: @@ -42,7 +41,6 @@ import typing import urllib.error import urllib.request from dataclasses import dataclass -from pathlib import Path try: # Same-directory imports inside the bundle container; these files are @@ -277,7 +275,6 @@ def validate_proposed_file(tool: str, content: str) -> None: @dataclass(frozen=True) class ServerConfig: bottle_slug: str - queue_dir: Path response_timeout_seconds: float = DEFAULT_RESPONSE_TIMEOUT_SECONDS @@ -376,7 +373,7 @@ def handle_tools_call( current_file_hash=_sv.sha256_hex(proposed_file), ) try: - _sv.write_proposal(config.queue_dir, proposal) + _sv.write_proposal(proposal) except OSError as e: raise _RpcInternalError(f"failed to write proposal to queue: {e}") from e sys.stderr.write( @@ -387,7 +384,7 @@ def handle_tools_call( deadline = time.monotonic() + config.response_timeout_seconds try: response = _sv.wait_for_response( - config.queue_dir, + config.bottle_slug, proposal.id, poll_interval=MIN_RESPONSE_POLL_INTERVAL_SECONDS, deadline=deadline, @@ -399,7 +396,7 @@ def handle_tools_call( "isError": False, } try: - _sv.archive_proposal(config.queue_dir, proposal.id) + _sv.archive_proposal(config.bottle_slug, proposal.id) except OSError as e: raise _RpcInternalError(f"failed to archive proposal: {e}") from e @@ -539,7 +536,7 @@ class MCPHandler(http.server.BaseHTTPRequestHandler): class MCPServer(socketserver.ThreadingMixIn, http.server.HTTPServer): allow_reuse_address = True daemon_threads = True - config: ServerConfig = ServerConfig(bottle_slug="", queue_dir=Path()) + config: ServerConfig = ServerConfig(bottle_slug="") # --- Entry point ----------------------------------------------------------- @@ -548,21 +545,18 @@ class MCPServer(socketserver.ThreadingMixIn, http.server.HTTPServer): def serve( *, bottle_slug: str, - queue_dir: Path, port: int = _sv.SUPERVISE_PORT, bind: str = "0.0.0.0", response_timeout_seconds: float = DEFAULT_RESPONSE_TIMEOUT_SECONDS, ) -> typing.NoReturn: - queue_dir.mkdir(parents=True, exist_ok=True) server = MCPServer((bind, port), MCPHandler) server.config = ServerConfig( bottle_slug=bottle_slug, - queue_dir=queue_dir, response_timeout_seconds=response_timeout_seconds, ) sys.stderr.write( f"supervise listening on {bind}:{port}; " - f"slug={bottle_slug!r}; queue={queue_dir}; " + f"slug={bottle_slug!r}; " f"tools: {', '.join(t['name'] for t in TOOL_DEFINITIONS)}\n" # type: ignore[arg-type] ) sys.stderr.flush() @@ -581,7 +575,6 @@ def main(argv: list[str]) -> int: if not bottle_slug: sys.stderr.write("supervise: SUPERVISE_BOTTLE_SLUG env is unset\n") return 2 - queue_dir = Path(os.environ.get("SUPERVISE_QUEUE_DIR", _sv.QUEUE_DIR_IN_CONTAINER)) port = int(os.environ.get("SUPERVISE_PORT", str(_sv.SUPERVISE_PORT))) bind = os.environ.get("SUPERVISE_BIND", "0.0.0.0") try: @@ -591,7 +584,6 @@ def main(argv: list[str]) -> int: return 2 serve( bottle_slug=bottle_slug, - queue_dir=queue_dir, port=port, bind=bind, response_timeout_seconds=response_timeout_seconds, diff --git a/docs/prds/prd-new-sqlite-local-storage.md b/docs/prds/prd-new-sqlite-local-storage.md index bde2f6a..df5981e 100644 --- a/docs/prds/prd-new-sqlite-local-storage.md +++ b/docs/prds/prd-new-sqlite-local-storage.md @@ -27,12 +27,10 @@ one-off persistence. 1. Supervise proposals and responses are persisted through SQLite. 2. Audit entries are persisted through SQLite. -3. Existing public supervise helpers keep their current call shape where - practical: `write_proposal`, `read_proposal`, `list_pending_proposals`, - `write_response`, `read_response`, `wait_for_response`, - `archive_proposal`, `write_audit_entry`, and `read_audit_entries`. -4. The sidecar queue mount still works across docker, smolmachines, and - macOS-container backends. +3. Supervise queue helpers use the bottle slug / queue key instead of a queue + directory path. +4. The sidecar receives the host database mount across docker, smolmachines, + and macOS-container backends. 5. The implementation stays stdlib-only. 6. Unit tests cover queue round-trips, pending discovery, response waits, archive semantics, audit round-trips, and path creation. @@ -57,11 +55,9 @@ Queue and audit state use the host-level local database: The supervise sidecar receives that database as a writable bind mount at `/run/supervise/bot-bottle.db` and gets the path through `SUPERVISE_DB_PATH`. -The existing per-slug queue directory mount remains in place for compatibility -with the supervise sidecar contract and any adjacent tooling that still expects a -queue directory, but the active queue records live in the host database. This -creates the shared host database that later forge/native lifecycle work can -extend in separate PRDs. +No per-slug queue directory is mounted into the sidecar. This creates the shared +host database that later forge/native lifecycle work can extend in separate +PRDs. ### Tables @@ -113,9 +109,8 @@ CREATE TABLE supervise_audit_entries ( ### Compatibility -The existing helper functions keep accepting `Path` arguments for queue -directories. Internally, they map the queue directory to a queue key and perform -equivalent operations against `~/.bot-bottle/bot-bottle.db`: +The queue helpers take a bottle slug / queue key and perform equivalent +operations against `~/.bot-bottle/bot-bottle.db`: - `list_pending_proposals` returns non-archived proposals without a non-archived response, sorted by arrival time. @@ -123,9 +118,9 @@ equivalent operations against `~/.bot-bottle/bot-bottle.db`: moving files into `processed/`. - `wait_for_response` keeps the current polling behavior but polls SQLite. -The old path helpers (`queue_dir_for_slug`, `audit_dir`, `audit_log_path`) stay -available for compatibility. `audit_log_path` no longer describes the active -storage location; callers should use `read_audit_entries`. +The old audit path helpers (`audit_dir`, `audit_log_path`) stay available for +compatibility. `audit_log_path` no longer describes the active storage location; +callers should use `read_audit_entries`. ## Implementation chunks diff --git a/tests/unit/test_compose.py b/tests/unit/test_compose.py index e874423..fa83cb3 100644 --- a/tests/unit/test_compose.py +++ b/tests/unit/test_compose.py @@ -107,7 +107,6 @@ def _egress_plan( def _supervise_plan() -> SupervisePlan: return SupervisePlan( slug=SLUG, - queue_dir=STATE / "supervise" / "queue", db_path=STATE / "bot-bottle.db", internal_network=f"bot-bottle-net-{SLUG}", ) @@ -394,7 +393,6 @@ class TestSidecarBundleShape(unittest.TestCase): env_strings = sc["environment"] self.assertIn(f"SUPERVISE_BOTTLE_SLUG={SLUG}", env_strings) self.assertIn("SUPERVISE_DB_PATH=/run/supervise/bot-bottle.db", env_strings) - self.assertTrue(any(e.startswith("SUPERVISE_QUEUE_DIR=") for e in env_strings)) self.assertTrue(any(e.startswith("SUPERVISE_PORT=") for e in env_strings)) def test_volumes_always_includes_egress_ca(self): @@ -411,8 +409,6 @@ class TestSidecarBundleShape(unittest.TestCase): self.assertIn("/git-gate-entrypoint.sh", targets) self.assertIn("/git-gate/creds/upstream-known_hosts", targets) self.assertIn("/run/supervise/bot-bottle.db", targets) - self.assertTrue(any("supervise/queue" in t or t.startswith("/run/supervise") - for t in targets)) def test_extra_hosts_omitted_for_git_upstreams(self): sc = self._render(with_git=True)["services"]["sidecars"] diff --git a/tests/unit/test_contrib_claude_provider.py b/tests/unit/test_contrib_claude_provider.py index 86d5231..0e8589b 100644 --- a/tests/unit/test_contrib_claude_provider.py +++ b/tests/unit/test_contrib_claude_provider.py @@ -74,7 +74,6 @@ def _plan( if supervise: supervise_plan = SupervisePlan( slug="demo-abc12", - queue_dir=Path("/tmp/queue"), db_path=Path("/tmp/bot-bottle.db"), ) return DockerBottlePlan( diff --git a/tests/unit/test_contrib_codex_provider.py b/tests/unit/test_contrib_codex_provider.py index 97c45c5..3ccfbfb 100644 --- a/tests/unit/test_contrib_codex_provider.py +++ b/tests/unit/test_contrib_codex_provider.py @@ -77,7 +77,6 @@ def _plan( if supervise: supervise_plan = SupervisePlan( slug="demo-abc12", - queue_dir=Path("/tmp/queue"), db_path=Path("/tmp/bot-bottle.db"), ) return DockerBottlePlan( diff --git a/tests/unit/test_egress_addon_log_redaction.py b/tests/unit/test_egress_addon_log_redaction.py index 87290c3..dbff136 100644 --- a/tests/unit/test_egress_addon_log_redaction.py +++ b/tests/unit/test_egress_addon_log_redaction.py @@ -47,7 +47,6 @@ def _addon() -> EgressAddon: a: EgressAddon = EgressAddon.__new__(EgressAddon) a.config = Config(routes=(), log=LOG_FULL) a.safe_tokens = set() - a._supervise_queue_dir = "" a._supervise_slug = "" a._token_allow_timeout = 300.0 return a diff --git a/tests/unit/test_egress_addon_request_flow.py b/tests/unit/test_egress_addon_request_flow.py index f374dd5..ed8d8b2 100644 --- a/tests/unit/test_egress_addon_request_flow.py +++ b/tests/unit/test_egress_addon_request_flow.py @@ -212,7 +212,6 @@ def _addon(config: Config) -> EgressAddon: a: EgressAddon = EgressAddon.__new__(EgressAddon) a.config = config a.safe_tokens = set() - a._supervise_queue_dir = "" a._supervise_slug = "" a._token_allow_timeout = 300.0 a.routes_path = "/nonexistent/routes.yaml" @@ -386,10 +385,10 @@ def _fake_sv(response_status: str | None) -> types.SimpleNamespace: def _sha256_hex(_payload: Any) -> str: return "hash" - def _noop(_a: Any, _b: Any) -> None: + def _noop(*_args: Any) -> None: return None - def _read_response(_qd: Any, _pid: Any) -> Any: + def _read_response(_slug: Any, _pid: Any) -> Any: if response_status is None: raise OSError("not written yet") # forces poll -> timeout return types.SimpleNamespace(status=response_status) @@ -409,7 +408,6 @@ def _fake_sv(response_status: str | None) -> types.SimpleNamespace: class TestSuperviseBranch(unittest.TestCase): def _supervised_addon(self) -> EgressAddon: addon = _addon(Config(routes=(Route(host="api.example.com"),))) - addon._supervise_queue_dir = "/tmp/egress-queue" addon._supervise_slug = "test-bottle" addon._token_allow_timeout = 0.05 return addon @@ -632,14 +630,13 @@ class TestRedactSurfaces(unittest.TestCase): class TestSuperviseWriteFailure(unittest.TestCase): def test_write_proposal_oserror_blocks(self) -> None: addon = _addon(Config(routes=(Route(host="api.example.com"),))) - addon._supervise_queue_dir = "/tmp/egress-queue" addon._supervise_slug = "test-bottle" addon._token_allow_timeout = 0.05 flow = _Flow(_Request(host="api.example.com", method="POST", body=f"k={_OPENAI_KEY}")) fake = _fake_sv("approved") - def _raise(_qd: Any, _p: Any) -> None: + def _raise(_p: Any) -> None: raise OSError("disk full") fake.write_proposal = _raise diff --git a/tests/unit/test_git_gate.py b/tests/unit/test_git_gate.py index 2f4e2f0..6cc0a99 100644 --- a/tests/unit/test_git_gate.py +++ b/tests/unit/test_git_gate.py @@ -213,7 +213,6 @@ class TestHookRender(unittest.TestCase): self.assertIn("tool=_sv.TOOL_GITLEAKS_ALLOW", hook) self.assertIn("_sv.write_proposal", hook) self.assertIn("_sv.read_response", hook) - self.assertIn("SUPERVISE_QUEUE_DIR", hook) self.assertIn("SUPERVISE_BOTTLE_SLUG", hook) self.assertIn("supervisor approved # gitleaks:allow", hook) self.assertIn("supervisor rejected # gitleaks:allow", hook) diff --git a/tests/unit/test_macos_container_launch.py b/tests/unit/test_macos_container_launch.py index d96a055..4d1ecb0 100644 --- a/tests/unit/test_macos_container_launch.py +++ b/tests/unit/test_macos_container_launch.py @@ -72,7 +72,6 @@ def _plan( git_gate_plan = SimpleNamespace(upstreams=()) supervise_plan = ( SimpleNamespace( - queue_dir=Path("/state/supervise/queue"), db_path=Path("/state/bot-bottle.db"), ) if supervise else None @@ -143,10 +142,6 @@ class TestMacosContainerLaunchArgv(unittest.TestCase): "type=bind,source=/state/bot-bottle.db,target=/run/supervise/bot-bottle.db", argv, ) - self.assertIn( - "type=bind,source=/state/supervise/queue,target=/run/supervise/queue", - argv, - ) def test_sidecar_argv_registers_canary_env_as_sensitive(self): plan = _plan(stage_dir=self.stage_dir, canary=True) diff --git a/tests/unit/test_smolmachines_provision.py b/tests/unit/test_smolmachines_provision.py index df92867..2f93c13 100644 --- a/tests/unit/test_smolmachines_provision.py +++ b/tests/unit/test_smolmachines_provision.py @@ -130,7 +130,6 @@ def _plan( if supervise: supervise_plan = SupervisePlan( slug="demo-abc12", - queue_dir=Path("/tmp/queue"), db_path=Path("/tmp/bot-bottle.db"), ) return SmolmachinesBottlePlan( diff --git a/tests/unit/test_supervise.py b/tests/unit/test_supervise.py index 3d7419e..88a594d 100644 --- a/tests/unit/test_supervise.py +++ b/tests/unit/test_supervise.py @@ -112,33 +112,44 @@ class TestResponseRoundtrip(unittest.TestCase): class TestQueueIO(unittest.TestCase): def setUp(self): self._tmp = tempfile.TemporaryDirectory(prefix="bot-bottle-supervise-test.") - self.queue_dir = Path(self._tmp.name) + self._home_patch = self._patch_home(Path(self._tmp.name)) + self.slug = "dev" def tearDown(self): + self._home_patch() self._tmp.cleanup() + def _patch_home(self, fake_home: Path): + original = supervise.bot_bottle_root + + def fake_root() -> Path: + return fake_home / ".bot-bottle" + + supervise.bot_bottle_root = fake_root # type: ignore[assignment] + return lambda: setattr(supervise, "bot_bottle_root", original) + def test_write_and_read_proposal(self): p = _proposal() - path = write_proposal(self.queue_dir, p) + path = write_proposal(p) self.assertTrue(path.exists()) - self.assertEqual(queue_db_path(self.queue_dir), path) + self.assertEqual(queue_db_path(), path) self.assertEqual(0o600, path.stat().st_mode & 0o777) - loaded = read_proposal(self.queue_dir, p.id) + loaded = read_proposal(self.slug, p.id) self.assertEqual(p, loaded) def test_list_pending_excludes_responded(self): a = _proposal(justification="first") b = _proposal(justification="second") - write_proposal(self.queue_dir, a) - write_proposal(self.queue_dir, b) - write_response(self.queue_dir, Response( + write_proposal(a) + write_proposal(b) + write_response(self.slug, Response( proposal_id=a.id, status=STATUS_APPROVED, notes="", )) - pending = list_pending_proposals(self.queue_dir) + pending = list_pending_proposals(self.slug) self.assertEqual([b.id], [p.id for p in pending]) - def test_list_pending_returns_empty_for_missing_dir(self): - self.assertEqual([], list_pending_proposals(self.queue_dir / "nope")) + def test_list_pending_returns_empty_for_missing_slug(self): + self.assertEqual([], list_pending_proposals("nope")) def test_list_pending_sorted_by_arrival(self): # Fabricate two with explicit timestamps. @@ -155,30 +166,30 @@ class TestQueueIO(unittest.TestCase): now=datetime(2026, 5, 25, 14, 0, 0, tzinfo=timezone.utc), ) # Write in reverse order. - write_proposal(self.queue_dir, b) - write_proposal(self.queue_dir, a) - ordered = list_pending_proposals(self.queue_dir) + write_proposal(b) + write_proposal(a) + ordered = list_pending_proposals(self.slug) self.assertEqual([a.id, b.id], [p.id for p in ordered]) def test_write_and_read_response(self): r = Response(proposal_id="xyz", status=STATUS_REJECTED, notes="no") - write_response(self.queue_dir, r) - self.assertEqual(r, read_response(self.queue_dir, "xyz")) + write_response(self.slug, r) + self.assertEqual(r, read_response(self.slug, "xyz")) def test_wait_for_response_returns_when_file_appears(self): p = _proposal() - write_proposal(self.queue_dir, p) + write_proposal(p) def write_after_delay(): time.sleep(0.05) - write_response(self.queue_dir, Response( + write_response(self.slug, Response( proposal_id=p.id, status=STATUS_APPROVED, notes="ok", )) t = threading.Thread(target=write_after_delay) t.start() try: - r = wait_for_response(self.queue_dir, p.id, poll_interval=0.01) + r = wait_for_response(self.slug, p.id, poll_interval=0.01) finally: t.join() self.assertEqual(STATUS_APPROVED, r.status) @@ -188,24 +199,24 @@ class TestQueueIO(unittest.TestCase): deadline = time.monotonic() + 0.05 with self.assertRaises(TimeoutError): wait_for_response( - self.queue_dir, "never", + self.slug, "never", poll_interval=0.01, deadline=deadline, ) - def test_archive_proposal_moves_both_files(self): + def test_archive_proposal_hides_rows(self): p = _proposal() - write_proposal(self.queue_dir, p) - write_response(self.queue_dir, Response( + write_proposal(p) + write_response(self.slug, Response( proposal_id=p.id, status=STATUS_APPROVED, notes="", )) - archive_proposal(self.queue_dir, p.id) - self.assertEqual([], list_pending_proposals(self.queue_dir)) + archive_proposal(self.slug, p.id) + self.assertEqual([], list_pending_proposals(self.slug)) with self.assertRaises(FileNotFoundError): - read_response(self.queue_dir, p.id) + read_response(self.slug, p.id) def test_archive_is_idempotent_on_missing_files(self): # Should not raise. - archive_proposal(self.queue_dir, "nope") + archive_proposal(self.slug, "nope") class TestAuditLog(unittest.TestCase): @@ -381,7 +392,6 @@ class TestSupervisePrepare(unittest.TestCase): def test_prepare_creates_queue(self): plan = _StubSupervise().prepare("dev", self.stage_dir) - self.assertTrue(plan.queue_dir.is_dir()) self.assertTrue(plan.db_path.is_file()) self.assertEqual("dev", plan.slug) self.assertEqual("", plan.internal_network) diff --git a/tests/unit/test_supervise_cli.py b/tests/unit/test_supervise_cli.py index 47de267..836a221 100644 --- a/tests/unit/test_supervise_cli.py +++ b/tests/unit/test_supervise_cli.py @@ -77,9 +77,7 @@ class TestDiscoverPending(_FakeHomeMixin, unittest.TestCase): def test_walks_all_slug_subdirs(self): for slug in ("dev", "api"): - qdir = supervise.queue_dir_for_slug(slug) - qdir.mkdir(parents=True) - supervise.write_proposal(qdir, _proposal(slug=slug)) + supervise.write_proposal(_proposal(slug=slug)) pending = supervise_cli.discover_pending() self.assertEqual({"dev", "api"}, {qp.proposal.bottle_slug for qp in pending}) @@ -97,18 +95,14 @@ class TestDiscoverPending(_FakeHomeMixin, unittest.TestCase): now=datetime(2026, 5, 25, 14, 0, 0, tzinfo=timezone.utc), ) for p in (late, early): - qdir = supervise.queue_dir_for_slug(p.bottle_slug) - qdir.mkdir(parents=True, exist_ok=True) - supervise.write_proposal(qdir, p) + supervise.write_proposal(p) pending = supervise_cli.discover_pending() self.assertEqual([early.id, late.id], [qp.proposal.id for qp in pending]) def test_excludes_already_responded(self): p = _proposal() - qdir = supervise.queue_dir_for_slug("dev") - qdir.mkdir(parents=True) - supervise.write_proposal(qdir, p) - supervise.write_response(qdir, supervise.Response( + supervise.write_proposal(p) + supervise.write_response("dev", supervise.Response( proposal_id=p.id, status=STATUS_APPROVED, notes="", )) self.assertEqual([], supervise_cli.discover_pending()) @@ -123,10 +117,8 @@ class TestApproveReject(_FakeHomeMixin, unittest.TestCase): def _enqueue(self, tool: str = TOOL_EGRESS_ALLOW): p = _proposal(tool=tool) - qdir = supervise.queue_dir_for_slug("dev") - qdir.mkdir(parents=True, exist_ok=True) - supervise.write_proposal(qdir, p) - return supervise_cli.QueuedProposal(proposal=p, queue_dir=qdir) + supervise.write_proposal(p) + return supervise_cli.QueuedProposal(proposal=p) def test_approve_writes_response(self): qp = self._enqueue() @@ -135,7 +127,7 @@ class TestApproveReject(_FakeHomeMixin, unittest.TestCase): return_value=("routes: []\n", "routes:\n - host: example.com\n"), ): supervise_cli.approve(qp) - resp = read_response(qp.queue_dir, qp.proposal.id) + resp = read_response(qp.proposal.bottle_slug, qp.proposal.id) self.assertEqual(STATUS_APPROVED, resp.status) self.assertIsNone(resp.final_file) @@ -150,7 +142,7 @@ class TestApproveReject(_FakeHomeMixin, unittest.TestCase): final_file="routes:\n - host: edited.example.com\n", notes="tweaked", ) - resp = read_response(qp.queue_dir, qp.proposal.id) + resp = read_response(qp.proposal.bottle_slug, qp.proposal.id) self.assertEqual(STATUS_MODIFIED, resp.status) self.assertEqual("routes:\n - host: edited.example.com\n", resp.final_file) self.assertEqual("tweaked", resp.notes) @@ -158,7 +150,7 @@ class TestApproveReject(_FakeHomeMixin, unittest.TestCase): def test_reject_writes_rejection(self): qp = self._enqueue() supervise_cli.reject(qp, reason="nope") - resp = read_response(qp.queue_dir, qp.proposal.id) + resp = read_response(qp.proposal.bottle_slug, qp.proposal.id) self.assertEqual(STATUS_REJECTED, resp.status) self.assertEqual("nope", resp.notes) @@ -181,36 +173,33 @@ class TestApproveReject(_FakeHomeMixin, unittest.TestCase): def test_approve_gitleaks_allow_leaves_response_for_gate(self): qp = self._enqueue(tool=TOOL_GITLEAKS_ALLOW) supervise_cli.approve(qp, notes="dummy fixture") - # Gate polls the queue dir for the response; TUI must not archive it. - resp = read_response(qp.queue_dir, qp.proposal.id) + # Gate polls the DB for the response; TUI must not archive it. + resp = read_response(qp.proposal.bottle_slug, qp.proposal.id) self.assertEqual(STATUS_APPROVED, resp.status) self.assertEqual("dummy fixture", resp.notes) - self.assertFalse((qp.queue_dir / "processed").exists()) def test_tui_gitleaks_allow_requires_reason(self): qp = self._enqueue(tool=TOOL_GITLEAKS_ALLOW) with patch.object(supervise_cli, "_prompt", return_value=""): status = supervise_cli._approve_from_tui(None, qp) # type: ignore[arg-type] self.assertEqual("approve aborted (empty reason)", status) - self.assertFalse((qp.queue_dir / "processed").exists()) def test_tui_gitleaks_allow_writes_reason(self): qp = self._enqueue(tool=TOOL_GITLEAKS_ALLOW) with patch.object(supervise_cli, "_prompt", return_value="test fixture"): status = supervise_cli._approve_from_tui(None, qp) # type: ignore[arg-type] self.assertIn("approved gitleaks-allow", status) - resp = read_response(qp.queue_dir, qp.proposal.id) + resp = read_response(qp.proposal.bottle_slug, qp.proposal.id) self.assertEqual("test fixture", resp.notes) def test_approve_token_allow_leaves_response_for_egress(self): qp = self._enqueue(tool=TOOL_EGRESS_TOKEN_ALLOW) supervise_cli.approve(qp, notes="false positive") - # The egress addon polls the queue dir for the response; the TUI must + # The egress addon polls the DB for the response; the TUI must # not archive it (the addon archives after reading). - resp = read_response(qp.queue_dir, qp.proposal.id) + resp = read_response(qp.proposal.bottle_slug, qp.proposal.id) self.assertEqual(STATUS_APPROVED, resp.status) self.assertEqual("false positive", resp.notes) - self.assertFalse((qp.queue_dir / "processed").exists()) def test_token_allow_writes_no_audit_log(self): qp = self._enqueue(tool=TOOL_EGRESS_TOKEN_ALLOW) @@ -222,14 +211,13 @@ class TestApproveReject(_FakeHomeMixin, unittest.TestCase): with patch.object(supervise_cli, "_prompt", return_value=""): status = supervise_cli._approve_from_tui(None, qp) # type: ignore[arg-type] self.assertEqual("approve aborted (empty reason)", status) - self.assertFalse((qp.queue_dir / "processed").exists()) def test_tui_token_allow_writes_reason(self): qp = self._enqueue(tool=TOOL_EGRESS_TOKEN_ALLOW) with patch.object(supervise_cli, "_prompt", return_value="legit"): status = supervise_cli._approve_from_tui(None, qp) # type: ignore[arg-type] self.assertIn("approved egress-token-allow", status) - resp = read_response(qp.queue_dir, qp.proposal.id) + resp = read_response(qp.proposal.bottle_slug, qp.proposal.id) self.assertEqual("legit", resp.notes) def test_suffix_for_token_allow_is_txt(self): diff --git a/tests/unit/test_supervise_edge.py b/tests/unit/test_supervise_edge.py index d709a3f..87fb289 100644 --- a/tests/unit/test_supervise_edge.py +++ b/tests/unit/test_supervise_edge.py @@ -7,7 +7,6 @@ from __future__ import annotations import tempfile import time import unittest -from pathlib import Path from unittest.mock import patch from bot_bottle import supervise @@ -39,61 +38,56 @@ class TestPathHelpers(unittest.TestCase): def test_bot_bottle_root(self) -> None: self.assertTrue(str(supervise.bot_bottle_root()).endswith(".bot-bottle")) - def test_queue_dir_for_slug(self) -> None: - self.assertIn("slug", str(supervise.queue_dir_for_slug("slug"))) - - def test_queue_db_path_for_slug_dir(self) -> None: - self.assertEqual( - supervise.host_db_path(), - supervise.queue_db_path(Path("/tmp/queue")), - ) + def test_queue_db_path_is_host_db_path(self) -> None: + self.assertEqual(supervise.host_db_path(), supervise.queue_db_path()) class TestReadMalformed(unittest.TestCase): def test_read_proposal_missing_row(self) -> None: with tempfile.TemporaryDirectory() as d: - with self.assertRaises(FileNotFoundError): - read_proposal(Path(d), "p") + with patch.dict("os.environ", {"HOME": d}), \ + self.assertRaises(FileNotFoundError): + read_proposal("slug", "p") def test_read_response_missing_row(self) -> None: with tempfile.TemporaryDirectory() as d: - with self.assertRaises(FileNotFoundError): - read_response(Path(d), "p") + with patch.dict("os.environ", {"HOME": d}), \ + self.assertRaises(FileNotFoundError): + read_response("slug", "p") - def test_list_pending_ignores_legacy_json_files(self) -> None: + def test_list_pending_reads_db_only(self) -> None: with tempfile.TemporaryDirectory() as d: - qd = Path(d) - (qd / "bad.proposal.json").write_text("{ not json") - (qd / "arr.proposal.json").write_text("[]") - supervise.write_proposal(qd, _proposal()) # one valid - pending = list_pending_proposals(qd) + with patch.dict("os.environ", {"HOME": d}): + supervise.write_proposal(_proposal()) + pending = list_pending_proposals("slug") self.assertEqual(1, len(pending)) self.assertEqual("slug", pending[0].bottle_slug) def test_list_pending_skips_when_response_present(self) -> None: with tempfile.TemporaryDirectory() as d: - qd = Path(d) - p = _proposal() - supervise.write_proposal(qd, p) - supervise.write_response(qd, supervise.Response( - proposal_id=p.id, - status=STATUS_APPROVED, - notes="", - )) - self.assertEqual([], list_pending_proposals(qd)) + with patch.dict("os.environ", {"HOME": d}): + p = _proposal() + supervise.write_proposal(p) + supervise.write_response("slug", supervise.Response( + proposal_id=p.id, + status=STATUS_APPROVED, + notes="", + )) + self.assertEqual([], list_pending_proposals("slug")) class TestWaitForResponse(unittest.TestCase): def test_missing_response_times_out(self) -> None: with tempfile.TemporaryDirectory() as d: - with self.assertRaises(TimeoutError): - wait_for_response(Path(d), "p", deadline=time.monotonic()) + with patch.dict("os.environ", {"HOME": d}), \ + self.assertRaises(TimeoutError): + wait_for_response("slug", "p", deadline=time.monotonic()) - def test_legacy_response_file_does_not_count(self) -> None: + def test_empty_db_response_does_not_count(self) -> None: with tempfile.TemporaryDirectory() as d: - (Path(d) / "p.response.json").write_text("{}") # dict but from_dict raises - with self.assertRaises(TimeoutError): - wait_for_response(Path(d), "p", deadline=time.monotonic()) + with patch.dict("os.environ", {"HOME": d}), \ + self.assertRaises(TimeoutError): + wait_for_response("slug", "p", deadline=time.monotonic()) class TestReadAuditEntries(unittest.TestCase): diff --git a/tests/unit/test_supervise_server.py b/tests/unit/test_supervise_server.py index 8a95aa6..e41894f 100644 --- a/tests/unit/test_supervise_server.py +++ b/tests/unit/test_supervise_server.py @@ -112,7 +112,7 @@ class TestRpcErrorTaxonomy(unittest.TestCase): validate_proposed_file(_sv.TOOL_EGRESS_ALLOW, "routes: nope\n") def test_unknown_tool_in_tools_call_is_client_error(self): - config = ServerConfig(bottle_slug="dev", queue_dir=Path("/unused")) + config = ServerConfig(bottle_slug="dev") with self.assertRaises(_RpcClientError) as cm: handle_tools_call({"name": "no-such-tool", "arguments": {}}, config) self.assertEqual(ERR_INVALID_PARAMS, cm.exception.code) @@ -122,7 +122,6 @@ class TestRpcInternalErrorOnIoFailure(unittest.TestCase): def test_write_proposal_os_error_raises_internal(self): config = ServerConfig( bottle_slug="dev", - queue_dir=Path("/unused"), ) with patch.object(_sv, "write_proposal", side_effect=OSError("disk full")), \ self.assertRaises(_RpcInternalError) as cm: @@ -266,21 +265,31 @@ class TestHandleToolsList(unittest.TestCase): class TestHandleToolsCall(unittest.TestCase): def setUp(self): self._tmp = tempfile.TemporaryDirectory(prefix="supervise-server-test.") - self.queue_dir = Path(self._tmp.name) - self.config = ServerConfig(bottle_slug="dev", queue_dir=self.queue_dir) + self._home_patch = self._patch_home(Path(self._tmp.name)) + self.config = ServerConfig(bottle_slug="dev") def tearDown(self): + self._home_patch() self._tmp.cleanup() + def _patch_home(self, fake_home: Path): + original = _sv.bot_bottle_root + + def fake_root() -> Path: + return fake_home / ".bot-bottle" + + _sv.bot_bottle_root = fake_root # type: ignore[assignment] + return lambda: setattr(_sv, "bot_bottle_root", original) + def _respond_when_proposal_appears(self, status: str, notes: str = "") -> threading.Thread: """Background thread: poll the queue for a fresh proposal, write a matching response. Returns the thread so the test can join it.""" def runner(): for _ in range(200): - pending = _sv.list_pending_proposals(self.queue_dir) + pending = _sv.list_pending_proposals("dev") if pending: p = pending[0] - _sv.write_response(self.queue_dir, _sv.Response( + _sv.write_response("dev", _sv.Response( proposal_id=p.id, status=status, notes=notes, )) return @@ -413,13 +422,11 @@ class TestHandleToolsCall(unittest.TestCase): finally: responder.join() # No pending proposals left after archive. - self.assertEqual([], _sv.list_pending_proposals(self.queue_dir)) - self.assertFalse((self.queue_dir / "processed").exists()) + self.assertEqual([], _sv.list_pending_proposals("dev")) def test_pending_response_times_out_without_archive(self): config = ServerConfig( bottle_slug="dev", - queue_dir=self.queue_dir, response_timeout_seconds=0.05, ) result = handle_tools_call( @@ -437,8 +444,7 @@ class TestHandleToolsCall(unittest.TestCase): text = result["content"][0]["text"] # type: ignore[index] self.assertIn("status: pending", text) self.assertIn("proposal remains queued", text) - self.assertEqual(1, len(_sv.list_pending_proposals(self.queue_dir))) - self.assertFalse((self.queue_dir / "processed").exists()) + self.assertEqual(1, len(_sv.list_pending_proposals("dev"))) class TestHandleListEgressRoutes(unittest.TestCase): @@ -460,7 +466,7 @@ class TestHandleListEgressRoutes(unittest.TestCase): with patch.object(supervise_server.urllib.request, "build_opener", return_value=_Opener()): result = handle_list_egress_routes( {}, - ServerConfig(bottle_slug="dev", queue_dir=Path("/unused")), + ServerConfig(bottle_slug="dev"), ) self.assertFalse(result["isError"]) # type: ignore[index] @@ -475,7 +481,7 @@ class TestHandleListEgressRoutes(unittest.TestCase): with patch.object(supervise_server.urllib.request, "build_opener", return_value=_Opener()): result = handle_list_egress_routes( {}, - ServerConfig(bottle_slug="dev", queue_dir=Path("/unused")), + ServerConfig(bottle_slug="dev"), ) self.assertTrue(result["isError"]) # type: ignore[index] @@ -543,7 +549,6 @@ class TestHttpEndToEnd(unittest.TestCase): def setUp(self): self._tmp = tempfile.TemporaryDirectory(prefix="supervise-http-test.") - self.queue_dir = Path(self._tmp.name) # Pick a random port by binding to :0 first. import socket s = socket.socket() @@ -551,7 +556,7 @@ class TestHttpEndToEnd(unittest.TestCase): self.port = s.getsockname()[1] s.close() self.server = MCPServer(("127.0.0.1", self.port), MCPHandler) - self.server.config = ServerConfig(bottle_slug="dev", queue_dir=self.queue_dir) + self.server.config = ServerConfig(bottle_slug="dev") self.thread = threading.Thread( target=self.server.serve_forever, daemon=True, ) -- 2.52.0 From 244ad6a91428a8b2d863b99633159ddf226befd2 Mon Sep 17 00:00:00 2001 From: claude Date: Wed, 1 Jul 2026 21:45:08 +0000 Subject: [PATCH 07/10] refactor: extract QueueStore and AuditStore to their own modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moves _QueueStore → bot_bottle/queue_store.py (public QueueStore) and _AuditStore → bot_bottle/audit_store.py (public AuditStore). Removes the public queue_db_path() function; QueueStore resolves the DB path via host_db_path() on the host, or via the SUPERVISE_DB_PATH env var in the sidecar container (internal mechanism, not public API). Adds queue_store.py and audit_store.py to Dockerfile.sidecars so the sidecar bundle picks them up. Updates __all__ in supervise.py. Co-Authored-By: Claude Sonnet 4.6 --- Dockerfile.sidecars | 2 + bot_bottle/audit_store.py | 120 +++++++++++ bot_bottle/queue_store.py | 248 ++++++++++++++++++++++ bot_bottle/supervise.py | 330 +++--------------------------- tests/unit/test_supervise.py | 3 +- tests/unit/test_supervise_edge.py | 3 - 6 files changed, 394 insertions(+), 312 deletions(-) create mode 100644 bot_bottle/audit_store.py create mode 100644 bot_bottle/queue_store.py diff --git a/Dockerfile.sidecars b/Dockerfile.sidecars index ba13551..2b43ee1 100644 --- a/Dockerfile.sidecars +++ b/Dockerfile.sidecars @@ -66,6 +66,8 @@ COPY bot_bottle/egress_dlp_config.py /app/egress_dlp_config.py COPY bot_bottle/egress_addon.py /app/egress_addon.py COPY bot_bottle/dlp_detectors.py /app/dlp_detectors.py COPY bot_bottle/yaml_subset.py /app/yaml_subset.py +COPY bot_bottle/queue_store.py /app/queue_store.py +COPY bot_bottle/audit_store.py /app/audit_store.py COPY bot_bottle/supervise.py /app/supervise.py COPY bot_bottle/supervise_server.py /app/supervise_server.py COPY bot_bottle/sidecar_init.py /app/sidecar_init.py diff --git a/bot_bottle/audit_store.py b/bot_bottle/audit_store.py new file mode 100644 index 0000000..2682217 --- /dev/null +++ b/bot_bottle/audit_store.py @@ -0,0 +1,120 @@ +"""SQLite-backed audit store for supervise (PRD 0013).""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .supervise import AuditEntry + + +def _sv() -> object: + """Lazy import of supervise to avoid a circular-import at module init time. + Mirrors our own module identity so patches on supervise.bot_bottle_root + propagate correctly in both flat (sidecar / sys.path-injection tests) and + package contexts.""" + import sys + sv_name = "supervise" if __name__ == "audit_store" else "bot_bottle.supervise" + if sv_name in sys.modules: + return sys.modules[sv_name] + try: + import bot_bottle.supervise as _m + except ImportError: + import supervise as _m # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module + return _m + + +def _audit_entry_from_row(row: sqlite3.Row) -> AuditEntry: + m = _sv() + return m.AuditEntry( # type: ignore[attr-defined] + timestamp=row["timestamp"], + bottle_slug=row["bottle_slug"], + component=row["component"], + operator_action=row["operator_action"], + operator_notes=row["operator_notes"], + justification=row["justification"], + diff=row["diff"], + ) + + +def _host_db_path() -> Path: + return _sv().host_db_path() # type: ignore[attr-defined,no-any-return] + + +class AuditStore: + """SQLite-backed persistent store for supervise audit entries.""" + + def __init__(self, db_path: Path | None = None) -> None: + self.db_path = db_path or _host_db_path() + self.db_path.parent.mkdir(parents=True, exist_ok=True) + self._init() + + def write_audit_entry(self, entry: AuditEntry) -> Path: + with self._connect() as conn: + conn.execute( + """ + INSERT INTO supervise_audit_entries ( + timestamp, bottle_slug, component, operator_action, + operator_notes, justification, diff + ) VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ( + entry.timestamp, + entry.bottle_slug, + entry.component, + entry.operator_action, + entry.operator_notes, + entry.justification, + entry.diff, + ), + ) + self._chmod() + return self.db_path + + def read_audit_entries(self, component: str, slug: str) -> list[AuditEntry]: + if not self.db_path.is_file(): + return [] + with self._connect() as conn: + rows = conn.execute( + """ + SELECT * FROM supervise_audit_entries + WHERE component = ? AND bottle_slug = ? + ORDER BY id + """, + (component, slug), + ).fetchall() + return [_audit_entry_from_row(row) for row in rows] + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + return conn + + def _init(self) -> None: + with self._connect() as conn: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS supervise_audit_entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + bottle_slug TEXT NOT NULL, + component TEXT NOT NULL, + operator_action TEXT NOT NULL, + operator_notes TEXT NOT NULL, + justification TEXT NOT NULL, + diff TEXT NOT NULL + ) + """ + ) + self._chmod() + + def _chmod(self) -> None: + try: + self.db_path.chmod(0o600) + except OSError: + pass + + +__all__ = ["AuditStore"] diff --git a/bot_bottle/queue_store.py b/bot_bottle/queue_store.py new file mode 100644 index 0000000..1ede262 --- /dev/null +++ b/bot_bottle/queue_store.py @@ -0,0 +1,248 @@ +"""SQLite-backed queue store for supervise proposals and responses (PRD 0013).""" + +from __future__ import annotations + +import os +import sqlite3 +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .supervise import Proposal, Response + + +def _sv() -> object: + """Lazy import of supervise to avoid a circular-import at module init time. + By the time any QueueStore method is called, both modules are fully loaded. + + Mirrors our own module identity: when we are 'queue_store' (sidecar flat + context or tests that inject bot_bottle/ into sys.path) we use the flat + 'supervise' module so that patches on supervise.bot_bottle_root propagate + correctly. When we are 'bot_bottle.queue_store' we use 'bot_bottle.supervise'.""" + import sys + sv_name = "supervise" if __name__ == "queue_store" else "bot_bottle.supervise" + if sv_name in sys.modules: + return sys.modules[sv_name] + try: + import bot_bottle.supervise as _m + except ImportError: + import supervise as _m # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module + return _m + + +def _proposal_from_row(row: sqlite3.Row) -> Proposal: + m = _sv() + return m.Proposal( # type: ignore[attr-defined] + id=row["id"], + bottle_slug=row["bottle_slug"], + tool=row["tool"], + proposed_file=row["proposed_file"], + justification=row["justification"], + arrival_timestamp=row["arrival_timestamp"], + current_file_hash=row["current_file_hash"], + ) + + +def _response_from_row(row: sqlite3.Row) -> Response: + m = _sv() + return m.Response( # type: ignore[attr-defined] + proposal_id=row["proposal_id"], + status=row["status"], + notes=row["notes"], + final_file=row["final_file"], + ) + + +def _host_db_path() -> Path: + return _sv().host_db_path() # type: ignore[attr-defined,no-any-return] + + +class QueueStore: + """SQLite-backed persistent store for supervise proposals and responses.""" + + def __init__(self, queue_key: str, db_path: Path | None = None) -> None: + self.queue_key = queue_key + if db_path is not None: + self.db_path = db_path + else: + # In the sidecar container SUPERVISE_DB_PATH points at the + # bind-mounted host DB. On the host this env var is never set, + # so we always fall through to host_db_path(). + env_path = os.environ.get("SUPERVISE_DB_PATH", "").strip() + self.db_path = Path(env_path) if env_path else _host_db_path() + self.db_path.parent.mkdir(parents=True, exist_ok=True) + self._init() + + def write_proposal(self, proposal: Proposal) -> Path: + with self._connect() as conn: + conn.execute( + """ + INSERT OR REPLACE INTO supervise_proposals ( + queue_key, id, bottle_slug, tool, proposed_file, justification, + arrival_timestamp, current_file_hash, archived + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0) + """, + ( + self.queue_key, + proposal.id, + proposal.bottle_slug, + proposal.tool, + proposal.proposed_file, + proposal.justification, + proposal.arrival_timestamp, + proposal.current_file_hash, + ), + ) + self._chmod() + return self.db_path + + def read_proposal(self, proposal_id: str) -> Proposal: + with self._connect() as conn: + row = conn.execute( + """ + SELECT * FROM supervise_proposals + WHERE queue_key = ? AND id = ? AND archived = 0 + """, + (self.queue_key, proposal_id), + ).fetchone() + if row is None: + raise FileNotFoundError(proposal_id) + return _proposal_from_row(row) + + def list_pending_proposals(self) -> list[Proposal]: + if not self.db_path.is_file(): + return [] + with self._connect() as conn: + rows = conn.execute( + """ + SELECT p.* FROM supervise_proposals p + WHERE p.archived = 0 + AND p.queue_key = ? + AND NOT EXISTS ( + SELECT 1 FROM supervise_responses r + WHERE r.queue_key = p.queue_key + AND r.proposal_id = p.id + AND r.archived = 0 + ) + ORDER BY p.arrival_timestamp, p.id + """, + (self.queue_key,), + ).fetchall() + return [_proposal_from_row(row) for row in rows] + + def list_all_pending_proposals(self) -> list[Proposal]: + if not self.db_path.is_file(): + return [] + with self._connect() as conn: + rows = conn.execute( + """ + SELECT p.* FROM supervise_proposals p + WHERE p.archived = 0 + AND NOT EXISTS ( + SELECT 1 FROM supervise_responses r + WHERE r.queue_key = p.queue_key + AND r.proposal_id = p.id + AND r.archived = 0 + ) + ORDER BY p.arrival_timestamp, p.id + """ + ).fetchall() + return [_proposal_from_row(row) for row in rows] + + def write_response(self, response: Response) -> Path: + with self._connect() as conn: + conn.execute( + """ + INSERT OR REPLACE INTO supervise_responses ( + queue_key, proposal_id, status, notes, final_file, archived + ) VALUES (?, ?, ?, ?, ?, 0) + """, + ( + self.queue_key, + response.proposal_id, + response.status, + response.notes, + response.final_file, + ), + ) + self._chmod() + return self.db_path + + def read_response(self, proposal_id: str) -> Response: + with self._connect() as conn: + row = conn.execute( + """ + SELECT * FROM supervise_responses + WHERE queue_key = ? AND proposal_id = ? AND archived = 0 + """, + (self.queue_key, proposal_id), + ).fetchone() + if row is None: + raise FileNotFoundError(proposal_id) + return _response_from_row(row) + + def archive_proposal(self, proposal_id: str) -> None: + if not self.db_path.is_file(): + return + with self._connect() as conn: + conn.execute( + """ + UPDATE supervise_proposals SET archived = 1 + WHERE queue_key = ? AND id = ? + """, + (self.queue_key, proposal_id), + ) + conn.execute( + """ + UPDATE supervise_responses SET archived = 1 + WHERE queue_key = ? AND proposal_id = ? + """, + (self.queue_key, proposal_id), + ) + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + return conn + + def _init(self) -> None: + with self._connect() as conn: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS supervise_proposals ( + queue_key TEXT NOT NULL, + id TEXT NOT NULL, + bottle_slug TEXT NOT NULL, + tool TEXT NOT NULL, + proposed_file TEXT NOT NULL, + justification TEXT NOT NULL, + arrival_timestamp TEXT NOT NULL, + current_file_hash TEXT NOT NULL, + archived INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (queue_key, id) + ) + """ + ) + conn.execute( + """ + CREATE TABLE IF NOT EXISTS supervise_responses ( + queue_key TEXT NOT NULL, + proposal_id TEXT NOT NULL, + status TEXT NOT NULL, + notes TEXT NOT NULL, + final_file TEXT, + archived INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (queue_key, proposal_id) + ) + """ + ) + self._chmod() + + def _chmod(self) -> None: + try: + self.db_path.chmod(0o600) + except OSError: + pass + + +__all__ = ["QueueStore"] diff --git a/bot_bottle/supervise.py b/bot_bottle/supervise.py index 6605bb3..ba00b50 100644 --- a/bot_bottle/supervise.py +++ b/bot_bottle/supervise.py @@ -33,8 +33,6 @@ from __future__ import annotations import dataclasses import difflib import hashlib -import os -import sqlite3 import time import uuid from abc import ABC @@ -109,11 +107,6 @@ def host_db_path() -> Path: return bot_bottle_root() / HOST_DB_FILENAME -def queue_db_path() -> Path: - env_path = os.environ.get("SUPERVISE_DB_PATH", "").strip() - return Path(env_path) if env_path else host_db_path() - - # --- Dataclasses ----------------------------------------------------------- @@ -226,37 +219,46 @@ class AuditEntry: return dataclasses.asdict(self) +try: + from .queue_store import QueueStore + from .audit_store import AuditStore +except ImportError: + # Sidecar bundle: files are flat-copied under /app, not a package. + from queue_store import QueueStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module + from audit_store import AuditStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module + + # --- Queue I/O ------------------------------------------------------------- def write_proposal(proposal: Proposal) -> Path: """Persist `proposal` in the queue database, mode 0o600. Directory is created if missing.""" - return _QueueStore(proposal.bottle_slug).write_proposal(proposal) + return QueueStore(proposal.bottle_slug).write_proposal(proposal) def read_proposal(bottle_slug: str, proposal_id: str) -> Proposal: - return _QueueStore(bottle_slug).read_proposal(proposal_id) + return QueueStore(bottle_slug).read_proposal(proposal_id) def list_pending_proposals(bottle_slug: str) -> list[Proposal]: """All proposals for `bottle_slug` that do not yet have a matching response. Sorted by `arrival_timestamp` so the operator sees the queue FIFO.""" - return _QueueStore(bottle_slug).list_pending_proposals() + return QueueStore(bottle_slug).list_pending_proposals() def list_all_pending_proposals() -> list[Proposal]: """All pending proposals across bottles, sorted FIFO.""" - return _QueueStore("").list_all_pending_proposals() + return QueueStore("").list_all_pending_proposals() def write_response(bottle_slug: str, response: Response) -> Path: - return _QueueStore(bottle_slug).write_response(response) + return QueueStore(bottle_slug).write_response(response) def read_response(bottle_slug: str, proposal_id: str) -> Response: - return _QueueStore(bottle_slug).read_response(proposal_id) + return QueueStore(bottle_slug).read_response(proposal_id) def wait_for_response( @@ -272,7 +274,7 @@ def wait_for_response( natural shape, since the operator's response time is unbounded. Polls SQLite so the implementation stays portable and stdlib-only.""" - store = _QueueStore(bottle_slug) + store = QueueStore(bottle_slug) while True: try: return store.read_response(proposal_id) @@ -286,7 +288,7 @@ def wait_for_response( def archive_proposal(bottle_slug: str, proposal_id: str) -> None: """Mark both proposal and response rows processed. Idempotent — missing rows are silently skipped.""" - _QueueStore(bottle_slug).archive_proposal(proposal_id) + QueueStore(bottle_slug).archive_proposal(proposal_id) # --- Audit log ------------------------------------------------------------- @@ -294,12 +296,12 @@ def archive_proposal(bottle_slug: str, proposal_id: str) -> None: def write_audit_entry(entry: AuditEntry) -> Path: """Append `entry` to the host supervise audit table.""" - return _AuditStore().write_audit_entry(entry) + return AuditStore().write_audit_entry(entry) def read_audit_entries(component: str, slug: str) -> list[AuditEntry]: """Load all audit entries for the given component+slug.""" - return _AuditStore().read_audit_entries(component, slug) + return AuditStore().read_audit_entries(component, slug) # --- Diff rendering -------------------------------------------------------- @@ -325,293 +327,6 @@ def sha256_hex(content: str) -> str: return hashlib.sha256(content.encode("utf-8")).hexdigest() -# --- SQLite storage -------------------------------------------------------- - - -class _QueueStore: - def __init__(self, queue_key: str) -> None: - self.queue_key = queue_key - self.db_path = queue_db_path() - self.db_path.parent.mkdir(parents=True, exist_ok=True) - self._init() - - def write_proposal(self, proposal: Proposal) -> Path: - with self._connect() as conn: - conn.execute( - """ - INSERT OR REPLACE INTO supervise_proposals ( - queue_key, id, bottle_slug, tool, proposed_file, justification, - arrival_timestamp, current_file_hash, archived - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0) - """, - ( - self.queue_key, - proposal.id, - proposal.bottle_slug, - proposal.tool, - proposal.proposed_file, - proposal.justification, - proposal.arrival_timestamp, - proposal.current_file_hash, - ), - ) - self._chmod() - return self.db_path - - def read_proposal(self, proposal_id: str) -> Proposal: - with self._connect() as conn: - row = conn.execute( - """ - SELECT * FROM supervise_proposals - WHERE queue_key = ? AND id = ? AND archived = 0 - """, - (self.queue_key, proposal_id), - ).fetchone() - if row is None: - raise FileNotFoundError(proposal_id) - return _proposal_from_row(row) - - def list_pending_proposals(self) -> list[Proposal]: - if not self.db_path.is_file(): - return [] - with self._connect() as conn: - rows = conn.execute( - """ - SELECT p.* FROM supervise_proposals p - WHERE p.archived = 0 - AND p.queue_key = ? - AND NOT EXISTS ( - SELECT 1 FROM supervise_responses r - WHERE r.queue_key = p.queue_key - AND r.proposal_id = p.id - AND r.archived = 0 - ) - ORDER BY p.arrival_timestamp, p.id - """, - (self.queue_key,), - ).fetchall() - return [_proposal_from_row(row) for row in rows] - - def list_all_pending_proposals(self) -> list[Proposal]: - if not self.db_path.is_file(): - return [] - with self._connect() as conn: - rows = conn.execute( - """ - SELECT p.* FROM supervise_proposals p - WHERE p.archived = 0 - AND NOT EXISTS ( - SELECT 1 FROM supervise_responses r - WHERE r.queue_key = p.queue_key - AND r.proposal_id = p.id - AND r.archived = 0 - ) - ORDER BY p.arrival_timestamp, p.id - """ - ).fetchall() - return [_proposal_from_row(row) for row in rows] - - def write_response(self, response: Response) -> Path: - with self._connect() as conn: - conn.execute( - """ - INSERT OR REPLACE INTO supervise_responses ( - queue_key, proposal_id, status, notes, final_file, archived - ) VALUES (?, ?, ?, ?, ?, 0) - """, - ( - self.queue_key, - response.proposal_id, - response.status, - response.notes, - response.final_file, - ), - ) - self._chmod() - return self.db_path - - def read_response(self, proposal_id: str) -> Response: - with self._connect() as conn: - row = conn.execute( - """ - SELECT * FROM supervise_responses - WHERE queue_key = ? AND proposal_id = ? AND archived = 0 - """, - (self.queue_key, proposal_id), - ).fetchone() - if row is None: - raise FileNotFoundError(proposal_id) - return _response_from_row(row) - - def archive_proposal(self, proposal_id: str) -> None: - if not self.db_path.is_file(): - return - with self._connect() as conn: - conn.execute( - """ - UPDATE supervise_proposals SET archived = 1 - WHERE queue_key = ? AND id = ? - """, - (self.queue_key, proposal_id), - ) - conn.execute( - """ - UPDATE supervise_responses SET archived = 1 - WHERE queue_key = ? AND proposal_id = ? - """, - (self.queue_key, proposal_id), - ) - - def _connect(self) -> sqlite3.Connection: - conn = sqlite3.connect(self.db_path) - conn.row_factory = sqlite3.Row - return conn - - def _init(self) -> None: - with self._connect() as conn: - conn.execute( - """ - CREATE TABLE IF NOT EXISTS supervise_proposals ( - queue_key TEXT NOT NULL, - id TEXT NOT NULL, - bottle_slug TEXT NOT NULL, - tool TEXT NOT NULL, - proposed_file TEXT NOT NULL, - justification TEXT NOT NULL, - arrival_timestamp TEXT NOT NULL, - current_file_hash TEXT NOT NULL, - archived INTEGER NOT NULL DEFAULT 0, - PRIMARY KEY (queue_key, id) - ) - """ - ) - conn.execute( - """ - CREATE TABLE IF NOT EXISTS supervise_responses ( - queue_key TEXT NOT NULL, - proposal_id TEXT NOT NULL, - status TEXT NOT NULL, - notes TEXT NOT NULL, - final_file TEXT, - archived INTEGER NOT NULL DEFAULT 0, - PRIMARY KEY (queue_key, proposal_id) - ) - """ - ) - self._chmod() - - def _chmod(self) -> None: - try: - self.db_path.chmod(0o600) - except OSError: - pass - - -class _AuditStore: - def __init__(self, db_path: Path | None = None) -> None: - self.db_path = db_path or host_db_path() - self.db_path.parent.mkdir(parents=True, exist_ok=True) - self._init() - - def write_audit_entry(self, entry: AuditEntry) -> Path: - with self._connect() as conn: - conn.execute( - """ - INSERT INTO supervise_audit_entries ( - timestamp, bottle_slug, component, operator_action, - operator_notes, justification, diff - ) VALUES (?, ?, ?, ?, ?, ?, ?) - """, - ( - entry.timestamp, - entry.bottle_slug, - entry.component, - entry.operator_action, - entry.operator_notes, - entry.justification, - entry.diff, - ), - ) - self._chmod() - return self.db_path - - def read_audit_entries(self, component: str, slug: str) -> list[AuditEntry]: - if not self.db_path.is_file(): - return [] - with self._connect() as conn: - rows = conn.execute( - """ - SELECT * FROM supervise_audit_entries - WHERE component = ? AND bottle_slug = ? - ORDER BY id - """, - (component, slug), - ).fetchall() - return [_audit_entry_from_row(row) for row in rows] - - def _connect(self) -> sqlite3.Connection: - conn = sqlite3.connect(self.db_path) - conn.row_factory = sqlite3.Row - return conn - - def _init(self) -> None: - with self._connect() as conn: - conn.execute( - """ - CREATE TABLE IF NOT EXISTS supervise_audit_entries ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - timestamp TEXT NOT NULL, - bottle_slug TEXT NOT NULL, - component TEXT NOT NULL, - operator_action TEXT NOT NULL, - operator_notes TEXT NOT NULL, - justification TEXT NOT NULL, - diff TEXT NOT NULL - ) - """ - ) - self._chmod() - - def _chmod(self) -> None: - try: - self.db_path.chmod(0o600) - except OSError: - pass - - -def _proposal_from_row(row: sqlite3.Row) -> Proposal: - return Proposal( - id=row["id"], - bottle_slug=row["bottle_slug"], - tool=row["tool"], - proposed_file=row["proposed_file"], - justification=row["justification"], - arrival_timestamp=row["arrival_timestamp"], - current_file_hash=row["current_file_hash"], - ) - - -def _response_from_row(row: sqlite3.Row) -> Response: - return Response( - proposal_id=row["proposal_id"], - status=row["status"], - notes=row["notes"], - final_file=row["final_file"], - ) - - -def _audit_entry_from_row(row: sqlite3.Row) -> AuditEntry: - return AuditEntry( - timestamp=row["timestamp"], - bottle_slug=row["bottle_slug"], - component=row["component"], - operator_action=row["operator_action"], - operator_notes=row["operator_notes"], - justification=row["justification"], - diff=row["diff"], - ) - - # --- Sidecar plan + abstract lifecycle ------------------------------------- @@ -642,8 +357,8 @@ class Supervise(ABC): must be set by the launch step before .start runs.""" del stage_dir db_path = host_db_path() - _QueueStore(slug) - _AuditStore(db_path) + QueueStore(slug) + AuditStore(db_path) return SupervisePlan( slug=slug, db_path=db_path, @@ -662,10 +377,12 @@ def _require_str(raw: dict[str, object], key: str) -> str: __all__ = [ "ACTION_OPERATOR_EDIT", "AuditEntry", + "AuditStore", "COMPONENT_FOR_TOOL", "DEFAULT_POLL_INTERVAL_SEC", "DB_PATH_IN_CONTAINER", "Proposal", + "QueueStore", "Response", "STATUSES", "STATUS_APPROVED", @@ -690,7 +407,6 @@ __all__ = [ "host_db_path", "list_pending_proposals", "list_all_pending_proposals", - "queue_db_path", "read_audit_entries", "read_proposal", "read_response", diff --git a/tests/unit/test_supervise.py b/tests/unit/test_supervise.py index 88a594d..f41ceba 100644 --- a/tests/unit/test_supervise.py +++ b/tests/unit/test_supervise.py @@ -20,7 +20,6 @@ from bot_bottle.supervise import ( archive_proposal, host_db_path, list_pending_proposals, - queue_db_path, read_audit_entries, read_proposal, read_response, @@ -132,7 +131,7 @@ class TestQueueIO(unittest.TestCase): p = _proposal() path = write_proposal(p) self.assertTrue(path.exists()) - self.assertEqual(queue_db_path(), path) + self.assertEqual(host_db_path(), path) self.assertEqual(0o600, path.stat().st_mode & 0o777) loaded = read_proposal(self.slug, p.id) self.assertEqual(p, loaded) diff --git a/tests/unit/test_supervise_edge.py b/tests/unit/test_supervise_edge.py index 87fb289..421c5ba 100644 --- a/tests/unit/test_supervise_edge.py +++ b/tests/unit/test_supervise_edge.py @@ -38,9 +38,6 @@ class TestPathHelpers(unittest.TestCase): def test_bot_bottle_root(self) -> None: self.assertTrue(str(supervise.bot_bottle_root()).endswith(".bot-bottle")) - def test_queue_db_path_is_host_db_path(self) -> None: - self.assertEqual(supervise.host_db_path(), supervise.queue_db_path()) - class TestReadMalformed(unittest.TestCase): def test_read_proposal_missing_row(self) -> None: -- 2.52.0 From 5f0fc0d540d81a1987aa5913264d4b446bd57ad3 Mon Sep 17 00:00:00 2001 From: claude Date: Thu, 2 Jul 2026 02:59:26 +0000 Subject: [PATCH 08/10] test: cover QueueStore/AuditStore guard branches and supervise bundle spec Add 8 tests covering the branches that were keeping diff-coverage below 90%: explicit db_path constructor arg, early-return guards when the DB file is absent, _chmod OSError swallowing in both store classes, and the supervise volume/env/daemon path in _bundle_launch_spec. Diff-coverage rises from 89.2% to 94.6% (176/186 changed lines). --- tests/unit/test_smolmachines_provision.py | 8 ++++ tests/unit/test_supervise_edge.py | 54 +++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/tests/unit/test_smolmachines_provision.py b/tests/unit/test_smolmachines_provision.py index 2f93c13..3c7c25e 100644 --- a/tests/unit/test_smolmachines_provision.py +++ b/tests/unit/test_smolmachines_provision.py @@ -422,6 +422,14 @@ class TestBundleLaunchSpec(unittest.TestCase): spec.environment, ) + def test_supervise_adds_daemon_volume_and_env(self): + from bot_bottle.supervise import DB_PATH_IN_CONTAINER + plan = _plan(supervise=True) + spec = _bundle_launch_spec(plan, "net", "127.0.0.16") + self.assertIn("supervise", spec.daemons_csv) + self.assertIn(f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}", spec.environment) + self.assertIn(("/tmp/bot-bottle.db", DB_PATH_IN_CONTAINER, False), spec.volumes) + def test_canary_env_visible_to_smolvm_guest(self): plan = _plan(canary=True) with patch.object( diff --git a/tests/unit/test_supervise_edge.py b/tests/unit/test_supervise_edge.py index 421c5ba..2f69920 100644 --- a/tests/unit/test_supervise_edge.py +++ b/tests/unit/test_supervise_edge.py @@ -7,9 +7,12 @@ from __future__ import annotations import tempfile import time import unittest +from pathlib import Path from unittest.mock import patch from bot_bottle import supervise +from bot_bottle.audit_store import AuditStore +from bot_bottle.queue_store import QueueStore from bot_bottle.supervise import ( AuditEntry, Proposal, @@ -132,5 +135,56 @@ class TestReadAuditEntries(unittest.TestCase): self.assertEqual([], entries) +class TestStoreGuardBranches(unittest.TestCase): + """Direct QueueStore / AuditStore construction and early-return guard branches.""" + + def test_queue_store_explicit_db_path(self): + with tempfile.TemporaryDirectory() as d: + db = Path(d) / "q.db" + store = QueueStore("key", db_path=db) + self.assertTrue(db.is_file()) + self.assertEqual(db, store.db_path) + + def test_queue_store_missing_db_list_pending_returns_empty(self): + with tempfile.TemporaryDirectory() as d: + db = Path(d) / "q.db" + store = QueueStore("key", db_path=db) + db.unlink() + self.assertEqual([], store.list_pending_proposals()) + + def test_queue_store_missing_db_list_all_returns_empty(self): + with tempfile.TemporaryDirectory() as d: + db = Path(d) / "q.db" + store = QueueStore("key", db_path=db) + db.unlink() + self.assertEqual([], store.list_all_pending_proposals()) + + def test_queue_store_missing_db_archive_is_noop(self): + with tempfile.TemporaryDirectory() as d: + db = Path(d) / "q.db" + store = QueueStore("key", db_path=db) + db.unlink() + store.archive_proposal("anything") # must not raise + + def test_queue_store_chmod_oserror_is_swallowed(self): + with tempfile.TemporaryDirectory() as d: + db = Path(d) / "q.db" + with patch("pathlib.Path.chmod", side_effect=OSError("ro")): + QueueStore("key", db_path=db) # must not raise + + def test_audit_store_missing_db_read_returns_empty(self): + with tempfile.TemporaryDirectory() as d: + db = Path(d) / "a.db" + store = AuditStore(db_path=db) + db.unlink() + self.assertEqual([], store.read_audit_entries("egress", "slug")) + + def test_audit_store_chmod_oserror_is_swallowed(self): + with tempfile.TemporaryDirectory() as d: + db = Path(d) / "a.db" + with patch("pathlib.Path.chmod", side_effect=OSError("ro")): + AuditStore(db_path=db) # must not raise + + if __name__ == "__main__": unittest.main() -- 2.52.0 From e8e4f6f7c70b62051188b2f39421fd114fb2955c Mon Sep 17 00:00:00 2001 From: claude Date: Thu, 2 Jul 2026 03:27:02 +0000 Subject: [PATCH 09/10] =?UTF-8?q?refactor:=20address=20PR=20review=20?= =?UTF-8?q?=E2=80=94=20rename,=20move=20helpers,=20add=20migration=20runne?= =?UTF-8?q?r?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review #320 comments: - Rename _sv() → get_supervise_mod() in both store files (review 206/211) - Move _audit_entry_from_row onto AuditStore as _row_to_entry static method (review 208); move _proposal/_response_from_row onto QueueStore (review 211) - Remove _host_db_path() free function; inline into __init__ (review 209/211) - Add stdlib migration runner using a shared schema_versions table; each store tracks its own version under a module key so they can coexist in the same DB without clobbering a shared PRAGMA user_version (reviews 210/212/213) - PRD: add goal 6 (migration runner), narrow non-goal to third-party ORM only --- bot_bottle/audit_store.py | 75 ++++++++----- bot_bottle/queue_store.py | 126 +++++++++++++--------- docs/prds/prd-new-sqlite-local-storage.md | 5 +- 3 files changed, 126 insertions(+), 80 deletions(-) diff --git a/bot_bottle/audit_store.py b/bot_bottle/audit_store.py index 2682217..9d5518d 100644 --- a/bot_bottle/audit_store.py +++ b/bot_bottle/audit_store.py @@ -10,7 +10,7 @@ if TYPE_CHECKING: from .supervise import AuditEntry -def _sv() -> object: +def get_supervise_mod() -> object: """Lazy import of supervise to avoid a circular-import at module init time. Mirrors our own module identity so patches on supervise.bot_bottle_root propagate correctly in both flat (sidecar / sys.path-injection tests) and @@ -26,28 +26,31 @@ def _sv() -> object: return _m -def _audit_entry_from_row(row: sqlite3.Row) -> AuditEntry: - m = _sv() - return m.AuditEntry( # type: ignore[attr-defined] - timestamp=row["timestamp"], - bottle_slug=row["bottle_slug"], - component=row["component"], - operator_action=row["operator_action"], - operator_notes=row["operator_notes"], - justification=row["justification"], - diff=row["diff"], +# One entry per schema version: _MIGRATIONS[0] brings a fresh DB (user_version=0) +# to version 1, _MIGRATIONS[1] to version 2, and so on. Add new migrations at +# the end; never edit existing ones. +_MIGRATIONS: list[str] = [ + # v1 — initial schema + """ + CREATE TABLE IF NOT EXISTS supervise_audit_entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + bottle_slug TEXT NOT NULL, + component TEXT NOT NULL, + operator_action TEXT NOT NULL, + operator_notes TEXT NOT NULL, + justification TEXT NOT NULL, + diff TEXT NOT NULL ) - - -def _host_db_path() -> Path: - return _sv().host_db_path() # type: ignore[attr-defined,no-any-return] + """, +] class AuditStore: """SQLite-backed persistent store for supervise audit entries.""" def __init__(self, db_path: Path | None = None) -> None: - self.db_path = db_path or _host_db_path() + self.db_path = db_path or get_supervise_mod().host_db_path() # type: ignore[attr-defined] self.db_path.parent.mkdir(parents=True, exist_ok=True) self._init() @@ -85,29 +88,49 @@ class AuditStore: """, (component, slug), ).fetchall() - return [_audit_entry_from_row(row) for row in rows] + return [self._row_to_entry(row) for row in rows] + + @staticmethod + def _row_to_entry(row: sqlite3.Row) -> AuditEntry: + m = get_supervise_mod() + return m.AuditEntry( # type: ignore[attr-defined] + timestamp=row["timestamp"], + bottle_slug=row["bottle_slug"], + component=row["component"], + operator_action=row["operator_action"], + operator_notes=row["operator_notes"], + justification=row["justification"], + diff=row["diff"], + ) def _connect(self) -> sqlite3.Connection: conn = sqlite3.connect(self.db_path) conn.row_factory = sqlite3.Row return conn + _SCHEMA_KEY = "audit_store" + def _init(self) -> None: with self._connect() as conn: conn.execute( """ - CREATE TABLE IF NOT EXISTS supervise_audit_entries ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - timestamp TEXT NOT NULL, - bottle_slug TEXT NOT NULL, - component TEXT NOT NULL, - operator_action TEXT NOT NULL, - operator_notes TEXT NOT NULL, - justification TEXT NOT NULL, - diff TEXT NOT NULL + CREATE TABLE IF NOT EXISTS schema_versions ( + module TEXT PRIMARY KEY, + version INTEGER NOT NULL DEFAULT 0 ) """ ) + row = conn.execute( + "SELECT version FROM schema_versions WHERE module = ?", + (self._SCHEMA_KEY,), + ).fetchone() + version = row[0] if row else 0 + for i, sql in enumerate(_MIGRATIONS[version:], start=version + 1): + conn.execute(sql) + conn.execute( + "INSERT OR REPLACE INTO schema_versions (module, version) VALUES (?, ?)", + (self._SCHEMA_KEY, i), + ) self._chmod() def _chmod(self) -> None: diff --git a/bot_bottle/queue_store.py b/bot_bottle/queue_store.py index 1ede262..6ef5dcb 100644 --- a/bot_bottle/queue_store.py +++ b/bot_bottle/queue_store.py @@ -11,7 +11,7 @@ if TYPE_CHECKING: from .supervise import Proposal, Response -def _sv() -> object: +def get_supervise_mod() -> object: """Lazy import of supervise to avoid a circular-import at module init time. By the time any QueueStore method is called, both modules are fully loaded. @@ -30,31 +30,38 @@ def _sv() -> object: return _m -def _proposal_from_row(row: sqlite3.Row) -> Proposal: - m = _sv() - return m.Proposal( # type: ignore[attr-defined] - id=row["id"], - bottle_slug=row["bottle_slug"], - tool=row["tool"], - proposed_file=row["proposed_file"], - justification=row["justification"], - arrival_timestamp=row["arrival_timestamp"], - current_file_hash=row["current_file_hash"], +# One entry per schema version: _MIGRATIONS[0] brings a fresh DB (user_version=0) +# to version 1, _MIGRATIONS[1] to version 2, and so on. Add new migrations at +# the end; never edit existing ones. +_MIGRATIONS: list[str] = [ + # v1 — proposals table + """ + CREATE TABLE IF NOT EXISTS supervise_proposals ( + queue_key TEXT NOT NULL, + id TEXT NOT NULL, + bottle_slug TEXT NOT NULL, + tool TEXT NOT NULL, + proposed_file TEXT NOT NULL, + justification TEXT NOT NULL, + arrival_timestamp TEXT NOT NULL, + current_file_hash TEXT NOT NULL, + archived INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (queue_key, id) ) - - -def _response_from_row(row: sqlite3.Row) -> Response: - m = _sv() - return m.Response( # type: ignore[attr-defined] - proposal_id=row["proposal_id"], - status=row["status"], - notes=row["notes"], - final_file=row["final_file"], + """, + # v2 — responses table + """ + CREATE TABLE IF NOT EXISTS supervise_responses ( + queue_key TEXT NOT NULL, + proposal_id TEXT NOT NULL, + status TEXT NOT NULL, + notes TEXT NOT NULL, + final_file TEXT, + archived INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (queue_key, proposal_id) ) - - -def _host_db_path() -> Path: - return _sv().host_db_path() # type: ignore[attr-defined,no-any-return] + """, +] class QueueStore: @@ -69,7 +76,7 @@ class QueueStore: # bind-mounted host DB. On the host this env var is never set, # so we always fall through to host_db_path(). env_path = os.environ.get("SUPERVISE_DB_PATH", "").strip() - self.db_path = Path(env_path) if env_path else _host_db_path() + self.db_path = Path(env_path) if env_path else get_supervise_mod().host_db_path() # type: ignore[attr-defined] self.db_path.parent.mkdir(parents=True, exist_ok=True) self._init() @@ -107,7 +114,7 @@ class QueueStore: ).fetchone() if row is None: raise FileNotFoundError(proposal_id) - return _proposal_from_row(row) + return self._row_to_proposal(row) def list_pending_proposals(self) -> list[Proposal]: if not self.db_path.is_file(): @@ -128,7 +135,7 @@ class QueueStore: """, (self.queue_key,), ).fetchall() - return [_proposal_from_row(row) for row in rows] + return [self._row_to_proposal(row) for row in rows] def list_all_pending_proposals(self) -> list[Proposal]: if not self.db_path.is_file(): @@ -147,7 +154,7 @@ class QueueStore: ORDER BY p.arrival_timestamp, p.id """ ).fetchall() - return [_proposal_from_row(row) for row in rows] + return [self._row_to_proposal(row) for row in rows] def write_response(self, response: Response) -> Path: with self._connect() as conn: @@ -179,7 +186,7 @@ class QueueStore: ).fetchone() if row is None: raise FileNotFoundError(proposal_id) - return _response_from_row(row) + return self._row_to_response(row) def archive_proposal(self, proposal_id: str) -> None: if not self.db_path.is_file(): @@ -200,42 +207,57 @@ class QueueStore: (self.queue_key, proposal_id), ) + @staticmethod + def _row_to_proposal(row: sqlite3.Row) -> Proposal: + m = get_supervise_mod() + return m.Proposal( # type: ignore[attr-defined] + id=row["id"], + bottle_slug=row["bottle_slug"], + tool=row["tool"], + proposed_file=row["proposed_file"], + justification=row["justification"], + arrival_timestamp=row["arrival_timestamp"], + current_file_hash=row["current_file_hash"], + ) + + @staticmethod + def _row_to_response(row: sqlite3.Row) -> Response: + m = get_supervise_mod() + return m.Response( # type: ignore[attr-defined] + proposal_id=row["proposal_id"], + status=row["status"], + notes=row["notes"], + final_file=row["final_file"], + ) + def _connect(self) -> sqlite3.Connection: conn = sqlite3.connect(self.db_path) conn.row_factory = sqlite3.Row return conn + _SCHEMA_KEY = "queue_store" + def _init(self) -> None: with self._connect() as conn: conn.execute( """ - CREATE TABLE IF NOT EXISTS supervise_proposals ( - queue_key TEXT NOT NULL, - id TEXT NOT NULL, - bottle_slug TEXT NOT NULL, - tool TEXT NOT NULL, - proposed_file TEXT NOT NULL, - justification TEXT NOT NULL, - arrival_timestamp TEXT NOT NULL, - current_file_hash TEXT NOT NULL, - archived INTEGER NOT NULL DEFAULT 0, - PRIMARY KEY (queue_key, id) + CREATE TABLE IF NOT EXISTS schema_versions ( + module TEXT PRIMARY KEY, + version INTEGER NOT NULL DEFAULT 0 ) """ ) - conn.execute( - """ - CREATE TABLE IF NOT EXISTS supervise_responses ( - queue_key TEXT NOT NULL, - proposal_id TEXT NOT NULL, - status TEXT NOT NULL, - notes TEXT NOT NULL, - final_file TEXT, - archived INTEGER NOT NULL DEFAULT 0, - PRIMARY KEY (queue_key, proposal_id) + row = conn.execute( + "SELECT version FROM schema_versions WHERE module = ?", + (self._SCHEMA_KEY,), + ).fetchone() + version = row[0] if row else 0 + for i, sql in enumerate(_MIGRATIONS[version:], start=version + 1): + conn.execute(sql) + conn.execute( + "INSERT OR REPLACE INTO schema_versions (module, version) VALUES (?, ?)", + (self._SCHEMA_KEY, i), ) - """ - ) self._chmod() def _chmod(self) -> None: diff --git a/docs/prds/prd-new-sqlite-local-storage.md b/docs/prds/prd-new-sqlite-local-storage.md index df5981e..1055a8a 100644 --- a/docs/prds/prd-new-sqlite-local-storage.md +++ b/docs/prds/prd-new-sqlite-local-storage.md @@ -32,7 +32,8 @@ one-off persistence. 4. The sidecar receives the host database mount across docker, smolmachines, and macOS-container backends. 5. The implementation stays stdlib-only. -6. Unit tests cover queue round-trips, pending discovery, response waits, +6. Schema migrations use a `PRAGMA user_version` runner — no third-party deps. +7. Unit tests cover queue round-trips, pending discovery, response waits, archive semantics, audit round-trips, and path creation. ## Non-goals @@ -41,7 +42,7 @@ one-off persistence. - Adding forge orchestration state tables. - Adding egress metering or budget tables. - Changing the supervise TUI workflow or remediation behavior. -- Introducing a third-party ORM or migration framework. +- Introducing a third-party ORM or migration library. ## Design -- 2.52.0 From a4d82e5ff2ff41da7d2dc56867ad986fe9c6e249 Mon Sep 17 00:00:00 2001 From: claude Date: Thu, 2 Jul 2026 21:04:34 +0000 Subject: [PATCH 10/10] refactor: extract TableMigrations and DbStore base class Adds bot_bottle/migrations.py (TableMigrations) and bot_bottle/db_store.py (DbStore) per PR review. Both stores now inherit from DbStore and hold a TableMigrations instance instead of duplicating schema-version logic inline. --- Dockerfile.sidecars | 2 ++ bot_bottle/audit_store.py | 60 ++++++++++--------------------------- bot_bottle/db_store.py | 40 +++++++++++++++++++++++++ bot_bottle/migrations.py | 37 +++++++++++++++++++++++ bot_bottle/queue_store.py | 62 ++++++++++----------------------------- 5 files changed, 110 insertions(+), 91 deletions(-) create mode 100644 bot_bottle/db_store.py create mode 100644 bot_bottle/migrations.py diff --git a/Dockerfile.sidecars b/Dockerfile.sidecars index 2b43ee1..2e28b1a 100644 --- a/Dockerfile.sidecars +++ b/Dockerfile.sidecars @@ -66,6 +66,8 @@ COPY bot_bottle/egress_dlp_config.py /app/egress_dlp_config.py COPY bot_bottle/egress_addon.py /app/egress_addon.py COPY bot_bottle/dlp_detectors.py /app/dlp_detectors.py COPY bot_bottle/yaml_subset.py /app/yaml_subset.py +COPY bot_bottle/migrations.py /app/migrations.py +COPY bot_bottle/db_store.py /app/db_store.py COPY bot_bottle/queue_store.py /app/queue_store.py COPY bot_bottle/audit_store.py /app/audit_store.py COPY bot_bottle/supervise.py /app/supervise.py diff --git a/bot_bottle/audit_store.py b/bot_bottle/audit_store.py index 9d5518d..5f848d0 100644 --- a/bot_bottle/audit_store.py +++ b/bot_bottle/audit_store.py @@ -9,6 +9,13 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: from .supervise import AuditEntry +try: + from .db_store import DbStore + from .migrations import TableMigrations +except ImportError: + from db_store import DbStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module + from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module + def get_supervise_mod() -> object: """Lazy import of supervise to avoid a circular-import at module init time. @@ -26,10 +33,10 @@ def get_supervise_mod() -> object: return _m -# One entry per schema version: _MIGRATIONS[0] brings a fresh DB (user_version=0) -# to version 1, _MIGRATIONS[1] to version 2, and so on. Add new migrations at -# the end; never edit existing ones. -_MIGRATIONS: list[str] = [ +# One entry per schema version: _MIGRATIONS.migrations[0] brings a fresh DB +# to version 1, [1] to version 2, and so on. Add new migrations at the end; +# never edit existing ones. +_MIGRATIONS = TableMigrations("audit_store", [ # v1 — initial schema """ CREATE TABLE IF NOT EXISTS supervise_audit_entries ( @@ -43,16 +50,15 @@ _MIGRATIONS: list[str] = [ diff TEXT NOT NULL ) """, -] +]) -class AuditStore: +class AuditStore(DbStore): """SQLite-backed persistent store for supervise audit entries.""" def __init__(self, db_path: Path | None = None) -> None: - self.db_path = db_path or get_supervise_mod().host_db_path() # type: ignore[attr-defined] - self.db_path.parent.mkdir(parents=True, exist_ok=True) - self._init() + resolved = db_path or get_supervise_mod().host_db_path() # type: ignore[attr-defined] + super().__init__(resolved, _MIGRATIONS) def write_audit_entry(self, entry: AuditEntry) -> Path: with self._connect() as conn: @@ -103,41 +109,5 @@ class AuditStore: diff=row["diff"], ) - def _connect(self) -> sqlite3.Connection: - conn = sqlite3.connect(self.db_path) - conn.row_factory = sqlite3.Row - return conn - - _SCHEMA_KEY = "audit_store" - - def _init(self) -> None: - with self._connect() as conn: - conn.execute( - """ - CREATE TABLE IF NOT EXISTS schema_versions ( - module TEXT PRIMARY KEY, - version INTEGER NOT NULL DEFAULT 0 - ) - """ - ) - row = conn.execute( - "SELECT version FROM schema_versions WHERE module = ?", - (self._SCHEMA_KEY,), - ).fetchone() - version = row[0] if row else 0 - for i, sql in enumerate(_MIGRATIONS[version:], start=version + 1): - conn.execute(sql) - conn.execute( - "INSERT OR REPLACE INTO schema_versions (module, version) VALUES (?, ?)", - (self._SCHEMA_KEY, i), - ) - self._chmod() - - def _chmod(self) -> None: - try: - self.db_path.chmod(0o600) - except OSError: - pass - __all__ = ["AuditStore"] diff --git a/bot_bottle/db_store.py b/bot_bottle/db_store.py new file mode 100644 index 0000000..7cc1849 --- /dev/null +++ b/bot_bottle/db_store.py @@ -0,0 +1,40 @@ +"""Shared SQLite-backed store base class for bot-bottle (PRD 0013).""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +try: + from .migrations import TableMigrations +except ImportError: + from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module + + +class DbStore: + """Base for SQLite-backed stores. Subclasses resolve db_path then call super().__init__.""" + + def __init__(self, db_path: Path, migrations: TableMigrations) -> None: + self.db_path = db_path + self._migrations = migrations + self.db_path.parent.mkdir(parents=True, exist_ok=True) + self._init() + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + return conn + + def _init(self) -> None: + with self._connect() as conn: + self._migrations.apply(conn) + self._chmod() + + def _chmod(self) -> None: + try: + self.db_path.chmod(0o600) + except OSError: + pass + + +__all__ = ["DbStore"] diff --git a/bot_bottle/migrations.py b/bot_bottle/migrations.py new file mode 100644 index 0000000..83fccf6 --- /dev/null +++ b/bot_bottle/migrations.py @@ -0,0 +1,37 @@ +"""SQLite migration runner for bot-bottle stores.""" + +from __future__ import annotations + +import sqlite3 + + +class TableMigrations: + """Runs a sequential list of DDL migrations tracked by schema_key in schema_versions.""" + + def __init__(self, schema_key: str, migrations: list[str]) -> None: + self.schema_key = schema_key + self.migrations = migrations + + def apply(self, conn: sqlite3.Connection) -> None: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_versions ( + module TEXT PRIMARY KEY, + version INTEGER NOT NULL DEFAULT 0 + ) + """ + ) + row = conn.execute( + "SELECT version FROM schema_versions WHERE module = ?", + (self.schema_key,), + ).fetchone() + version = row[0] if row else 0 + for i, sql in enumerate(self.migrations[version:], start=version + 1): + conn.execute(sql) + conn.execute( + "INSERT OR REPLACE INTO schema_versions (module, version) VALUES (?, ?)", + (self.schema_key, i), + ) + + +__all__ = ["TableMigrations"] diff --git a/bot_bottle/queue_store.py b/bot_bottle/queue_store.py index 6ef5dcb..76a02da 100644 --- a/bot_bottle/queue_store.py +++ b/bot_bottle/queue_store.py @@ -10,6 +10,13 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: from .supervise import Proposal, Response +try: + from .db_store import DbStore + from .migrations import TableMigrations +except ImportError: + from db_store import DbStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module + from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module + def get_supervise_mod() -> object: """Lazy import of supervise to avoid a circular-import at module init time. @@ -30,10 +37,10 @@ def get_supervise_mod() -> object: return _m -# One entry per schema version: _MIGRATIONS[0] brings a fresh DB (user_version=0) -# to version 1, _MIGRATIONS[1] to version 2, and so on. Add new migrations at -# the end; never edit existing ones. -_MIGRATIONS: list[str] = [ +# One entry per schema version: _MIGRATIONS.migrations[0] brings a fresh DB +# to version 1, [1] to version 2, and so on. Add new migrations at the end; +# never edit existing ones. +_MIGRATIONS = TableMigrations("queue_store", [ # v1 — proposals table """ CREATE TABLE IF NOT EXISTS supervise_proposals ( @@ -61,24 +68,23 @@ _MIGRATIONS: list[str] = [ PRIMARY KEY (queue_key, proposal_id) ) """, -] +]) -class QueueStore: +class QueueStore(DbStore): """SQLite-backed persistent store for supervise proposals and responses.""" def __init__(self, queue_key: str, db_path: Path | None = None) -> None: self.queue_key = queue_key if db_path is not None: - self.db_path = db_path + resolved = db_path else: # In the sidecar container SUPERVISE_DB_PATH points at the # bind-mounted host DB. On the host this env var is never set, # so we always fall through to host_db_path(). env_path = os.environ.get("SUPERVISE_DB_PATH", "").strip() - self.db_path = Path(env_path) if env_path else get_supervise_mod().host_db_path() # type: ignore[attr-defined] - self.db_path.parent.mkdir(parents=True, exist_ok=True) - self._init() + resolved = Path(env_path) if env_path else get_supervise_mod().host_db_path() # type: ignore[attr-defined] + super().__init__(resolved, _MIGRATIONS) def write_proposal(self, proposal: Proposal) -> Path: with self._connect() as conn: @@ -230,41 +236,5 @@ class QueueStore: final_file=row["final_file"], ) - def _connect(self) -> sqlite3.Connection: - conn = sqlite3.connect(self.db_path) - conn.row_factory = sqlite3.Row - return conn - - _SCHEMA_KEY = "queue_store" - - def _init(self) -> None: - with self._connect() as conn: - conn.execute( - """ - CREATE TABLE IF NOT EXISTS schema_versions ( - module TEXT PRIMARY KEY, - version INTEGER NOT NULL DEFAULT 0 - ) - """ - ) - row = conn.execute( - "SELECT version FROM schema_versions WHERE module = ?", - (self._SCHEMA_KEY,), - ).fetchone() - version = row[0] if row else 0 - for i, sql in enumerate(_MIGRATIONS[version:], start=version + 1): - conn.execute(sql) - conn.execute( - "INSERT OR REPLACE INTO schema_versions (module, version) VALUES (?, ?)", - (self._SCHEMA_KEY, i), - ) - self._chmod() - - def _chmod(self) -> None: - try: - self.db_path.chmod(0o600) - except OSError: - pass - __all__ = ["QueueStore"] -- 2.52.0