From 124b1f473c21350caa6486d645d976ad81908c17 Mon Sep 17 00:00:00 2001 From: didericis Date: Mon, 13 Jul 2026 13:26:03 -0400 Subject: [PATCH] feat(orchestrator): registry co-tenants the shared bot-bottle.db (#352) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review: use one shared bot-bottle.db for all runtime state including the registry (DbStore namespaces by schema_key), so it's one queryable file for backup/console. default_db_path() -> host_db_path(). Drop the unilateral WAL flip — WAL on the shared DB affects supervise/audit and is finicky over guest shares, so it's a deliberate future change; keep a busy_timeout for lock contention. PRD State section updated: integrity now by SOLE ownership (only the orchestrator opens bot-bottle.db; data plane + console reach state via the control-plane RPC, never a file handle) rather than ro/rw mount-splitting, which one shared file can't do. Notes the transitional caveat that the supervise sidecar currently rw-mounts bot-bottle.db. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck --- bot_bottle/orchestrator/registry.py | 25 +++++++----- docs/prds/0070-per-host-orchestrator.md | 54 ++++++++++++++----------- 2 files changed, 46 insertions(+), 33 deletions(-) diff --git a/bot_bottle/orchestrator/registry.py b/bot_bottle/orchestrator/registry.py index df35a12..b8655ae 100644 --- a/bot_bottle/orchestrator/registry.py +++ b/bot_bottle/orchestrator/registry.py @@ -1,10 +1,15 @@ """Orchestrator bottle registry — the per-host runtime-state store (PRD 0070). -SQLite-backed (WAL) registry mapping each live bottle to its source IP and +SQLite-backed registry mapping each live bottle to its source IP and per-bottle identity token, plus the **fail-closed attribution** the data plane relies on: a request is attributed to a bottle only when its source IP *and* its identity token both match a single active record. +The registry co-tenants the shared host `bot-bottle.db` (the `DbStore` +framework namespaces each store by `schema_key`), so all bot-bottle +runtime state lives in one queryable file — one place to back up, inspect, +and integrate a console against. + This is the "runtime state" tier of PRD 0070 (leases / approvals / registry) — deliberately NOT config (which stays declarative under `~/.bot-bottle/`) and NOT the build-time constants (a flat file). It is the @@ -32,6 +37,7 @@ from pathlib import Path from ..db_store import DbStore from ..migrations import TableMigrations +from ..supervise_types import host_db_path # 256 bits of urandom, URL-safe — unguessable per-bottle identity token. IDENTITY_TOKEN_BYTES = 32 @@ -43,9 +49,10 @@ def new_identity_token() -> str: def default_db_path() -> Path: - """Host location for the orchestrator's runtime-state DB. Runtime state - (not config), so it lives under the cache dir like the pool state.""" - return Path.home() / ".cache" / "bot-bottle" / "orchestrator" / "registry.db" + """The shared host state DB (`bot-bottle.db`) — all bot-bottle runtime + state in one file, co-tenanted via schema_key. Host-resident so it + survives orchestrator restarts (re-adoption sweeps it).""" + return host_db_path() @dataclass(frozen=True) @@ -113,12 +120,12 @@ class RegistryStore(DbStore): super().__init__(db_path or default_db_path(), _MIGRATIONS) def _connect(self) -> sqlite3.Connection: - """Open a WAL connection (concurrent data-plane reads + writer).""" + """Open a connection with a busy timeout for the shared DB.""" conn = super()._connect() - # WAL lets the data-plane attribution reads run concurrently with the - # control-plane writer without blocking; busy_timeout avoids spurious - # "database is locked" under that concurrency (PRD 0070 state tier). - conn.execute("PRAGMA journal_mode=WAL") + # The registry co-tenants the shared bot-bottle.db, so a busy_timeout + # rides out brief lock contention with the other stores. WAL for the + # shared DB is a deliberate future change (it affects supervise/audit + # and is finicky over guest shares) — not flipped here. conn.execute("PRAGMA busy_timeout=5000") return conn diff --git a/docs/prds/0070-per-host-orchestrator.md b/docs/prds/0070-per-host-orchestrator.md index c79bc18..08b8bd5 100644 --- a/docs/prds/0070-per-host-orchestrator.md +++ b/docs/prds/0070-per-host-orchestrator.md @@ -261,7 +261,7 @@ piling methods on — the same discipline, recursed. The orchestrator is the natural owner of per-host **runtime state**: - pool **slot leases** (which bottle holds slot *i*) — replaces today's - `fcntl`-locked files with WAL-mode transactions; + `fcntl`-locked files with SQLite transactions; - the **supervise approval queue** + remembered approvals; - the **live bottle registry** (source IP → bottle → policy/secrets refs), the lookup table the attribution invariant reads. @@ -273,38 +273,44 @@ Config splits into three tiers with different homes: |---|---|---| | Build-time constants | pool size, IP base, nft table | flat `.env` (PR #350) — must be readable by Nix eval + root bash, zero runtime | | User-authored config | bottle manifests, egress routes, secret refs | declarative files under `~/.bot-bottle/` — trust boundary at `$HOME`, git-trackable, "unknown keys die at load" | -| Runtime state | slot leases, approvals, registry | **SQLite**, owned by the orchestrator | +| Runtime state | slot leases, approvals, registry | one shared **`bot-bottle.db`**, solely owned by the orchestrator | SQLite is right for the runtime tier (mutable, concurrent, queried) and wrong for the other two (Nix can't read it at eval time; it fights the declarative manifest trust model). Keep the tiers separate. -**The DB file is host-resident, not owned inside the orchestrator unit** -(decided in review). Two reasons: +**One shared `bot-bottle.db` for all runtime state** (decided in review). +The registry co-tenants the existing host `bot-bottle.db` — the `DbStore` +framework already namespaces each store by `schema_key`, so slot +leases / approvals / registry share one file. One place to query, back up, +and integrate a console against. -- **Durability across restarts.** Re-adoption sweeps the registry after an - orchestrator restart, so the state *must* outlive the orchestrator - instance's lifecycle. A host-resident file (the slice-1 default under - `~/.cache/bot-bottle/orchestrator/`) is the simplest durable store; the - orchestrator VM/container mounts it rather than carrying it. -- **Integrity via access-scoping, not location.** Agents can't touch the - DB directly regardless of where it lives — they're network-isolated in - their bottles and only ever speak to the control/data plane. The real - risk is a *compromised agent-facing data-plane service* (egress/git-gate, - which parse hostile bytes) writing the registry and forging attribution. - The control is a **read/write split**: writes (register/deregister) come - only from the **control plane**; the **data plane gets the DB read-only** - for attribution lookups. A host-resident file makes that split - enforceable (mount `ro` into the data plane, `rw` into the control - plane) — which owning it "entirely inside" a monolithic orchestrator - would not. +- **Host-resident, for durability.** Re-adoption sweeps the registry after + an orchestrator restart, so state *must* outlive the orchestrator + instance. The file lives on the host (`bot_bottle_root()/db/bot-bottle.db`); + the orchestrator unit reaches it, it doesn't carry it. +- **Integrity by sole ownership, not mount permissions.** Agents can't + touch the DB directly wherever it lives (network-isolated in their + bottles). The risk is a *compromised agent-facing data-plane service* + (egress/git-gate, which parse hostile bytes) writing the registry and + forging attribution. Because it's now one shared file, coarse `ro`/`rw` + mount-splitting no longer isolates the registry — so the rule is stronger + and simpler: **only the orchestrator (control plane) opens `bot-bottle.db`; + the data plane and the console reach state through the control-plane RPC, + never a direct file handle.** No agent-facing component gets the file, so + none can forge attribution. (This supersedes the earlier `ro`-mount idea.) + - *Transitional caveat:* today the per-bottle **supervise sidecar + rw-bind-mounts `bot-bottle.db`** to write proposals — exactly the + pattern the orchestrator removes (supervise consolidates into the + orchestrator; sidecar writes become RPC calls). Until that lands, don't + put the attribution registry behind a data-plane-writable mount. Implementation note for the VM slices: SQLite **WAL** over a guest share (virtiofs/9p) is finicky (the `-shm`/`-wal` files need real mmap/locking), -so the durable-DB-on-host may want a small **host-side owner** the -orchestrator reaches over the control-plane RPC, rather than a shared -mount. Decide this at the Firecracker slice; `sqlite3` itself is stdlib, so -"the host needs SQLite" is a non-cost. +which is a second reason the DB wants a **host-side owner** the orchestrator +reaches over the RPC rather than a shared mount into the VM. WAL on the +shared DB is therefore a deliberate, tested future change — not enabled ad +hoc. `sqlite3` itself is stdlib, so "the host needs SQLite" is a non-cost. ## Sequencing