didericis/bot-bottle
1
0
Fork 0
Code Issues 32 Pull Requests 8 Actions Packages Releases Wiki Activity

refactor(orchestrator): uniform control-plane auth provisioning per trust domain

Browse Source 
Hoist control-plane auth provisioning out of the per-backend launchers into
one shared contract, parameterized per trust domain (#476). Every blocking
finding in PR #471 was the same integration-bug class: each launcher
re-derived, by hand, how to generate the signing key, scope it to the
orchestrator, mint the gateway JWT, and keep the host key canonical.

Introduces `trust_domain.py`:

  * `TrustDomain` — one credential boundary (host-canonical key file + role
    set + env vars). `mint`/`verify` are scoped to the domain's roles, so a
    future host-controller domain (#468) uses its own key/verifier/roles
    rather than a `host` role on the control plane's frozenset (which the
    orchestrator key could then forge).
  * `ControlPlaneProvisioning` — the single seam answering the four
    invariants: host-canonical key, split key-vs-token credential, CLI token
    valid across co-running backends, and fail-closed (no open mode) for any
    co-located topology.
  * `Topology` — the backend declares what it is; the default is co-located +
    fail-closed, so a backend need not redeclare it.

The `Orchestrator` ABC gets `control_plane_key()` (fail-closed) and routes
`mint_gateway_token()` through the contract; docker/macOS/firecracker
orchestrators, the server (verify), and the host CLI client (mint cli) all go
through the domain instead of reading the host key directly. `orchestrator_auth`
gains an optional `roles=` arg (default unchanged) so a domain scopes its own
role set; `paths.host_signing_key(filename)` generalizes host_orchestrator_token.

Adds unit coverage for the domain boundary + provisioning invariants and a PRD
capturing the durable rationale. No change to the auth primitive's HMAC, the
plane split, or the server's documented open-mode fallback.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Closes #476
This commit is contained in:
didericis (claude)
2026-07-26 00:41:44 +00:00
parent 0e70d26af4
commit 45f3cefbc5
16 changed files with 553 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bot_bottle/backend/docker/orchestrator.py
+3 -2
View File
@@ -19,7 +19,6 @@ from .util import run_docker
from ...paths import (
ORCHESTRATOR_TOKEN_ENV,
bot_bottle_root,
host_orchestrator_token,
)
from ...gateway import GatewayError
from ...orchestrator.lifecycle import (
@@ -171,7 +170,9 @@ class DockerOrchestrator(Orchestrator):
fixed-name container first)."""
self._ensure_control_network()
run_docker(["docker", "rm", "--force", self.name])
_signing_key = host_orchestrator_token()
# The signing key comes through the shared provisioning contract (#476),
# which fail-closes rather than yield an empty key that would run OPEN.
_signing_key = self.control_plane_key()
proc = run_docker([
"docker", "run", "--detach",
"--name", self.name,
bot_bottle/backend/firecracker/orchestrator.py
+6 -5
View File
@@ -21,7 +21,6 @@ import time
from pathlib import Path
from ...log import die, info
from ...paths import host_orchestrator_token
from ...orchestrator.lifecycle import (
DEFAULT_STARTUP_TIMEOUT_SECONDS,
Orchestrator,
@@ -108,11 +107,13 @@ class FirecrackerOrchestrator(Orchestrator):
data_drive=self._ensure_registry_volume(),
)
# Push the host-canonical signing key (the init waits for it before
# starting the control plane). The host token file stays the single
# source of truth, so a co-running docker/macOS control plane keeps
# working; the guest verifies tokens with the same key the CLI signs from.
# starting the control plane). It comes through the shared provisioning
# contract (#476) — the same host token file every backend uses, so a
# co-running docker/macOS control plane keeps working and the guest
# verifies tokens with the same key the CLI signs from; fail-closed, so
# the guest is never handed an empty key that would run it OPEN.
infra_vm.push_secret(
vm, host_orchestrator_token(), infra_vm._GUEST_SIGNING_KEY_PATH,
vm, self.control_plane_key(), infra_vm._GUEST_SIGNING_KEY_PATH,
"the control-plane signing key to the orchestrator VM "
"(its control plane will not start)",
)
bot_bottle/backend/macos_container/orchestrator.py
+4 -5
View File
@@ -18,10 +18,7 @@ import urllib.request
from pathlib import Path
from ... import log
from ...paths import (
ORCHESTRATOR_TOKEN_ENV,
host_orchestrator_token,
)
from ...paths import ORCHESTRATOR_TOKEN_ENV
from ...orchestrator.lifecycle import (
DEFAULT_HEALTH_TIMEOUT_SECONDS,
DEFAULT_PORT,
@@ -134,7 +131,9 @@ class MacosOrchestrator(Orchestrator):
def _run_container(self, current_hash: str) -> None:
container_mod.force_remove_container(self.name)
_signing_key = host_orchestrator_token()
# The signing key comes through the shared provisioning contract (#476),
# which fail-closes rather than yield an empty key that would run OPEN.
_signing_key = self.control_plane_key()
argv = [
"container", "run", "--detach",
"--name", self.name,
bot_bottle/orchestrator/client.py
+3 -3
View File
@@ -18,8 +18,8 @@ import urllib.request
from collections.abc import Iterable
from dataclasses import dataclass
from ..orchestrator_auth import ROLE_CLI, mint
from ..paths import host_orchestrator_token
from ..orchestrator_auth import ROLE_CLI
from ..trust_domain import CONTROL_PLANE
from .server import ORCHESTRATOR_AUTH_HEADER
DEFAULT_TIMEOUT_SECONDS = 5.0
@@ -32,7 +32,7 @@ def _host_auth_token() -> str:
"" means 'send no auth header' — correct against an open (unconfigured)
control plane, and harmlessly rejected by a secured one."""
try:
return mint(ROLE_CLI, host_orchestrator_token())
return CONTROL_PLANE.mint(ROLE_CLI)
except (OSError, ValueError):
return ""
bot_bottle/orchestrator/lifecycle.py
+20 -5
View File
@@ -21,8 +21,7 @@ import urllib.error
import urllib.request
from pathlib import Path
from ..orchestrator_auth import ROLE_GATEWAY, mint
from ..paths import host_orchestrator_token
from ..trust_domain import ControlPlaneProvisioning
DEFAULT_PORT = 8099
DEFAULT_STARTUP_TIMEOUT_SECONDS = 45.0
@@ -58,6 +57,13 @@ class Orchestrator(abc.ABC):
so it — not the gateway — mints the gateway's role-scoped token.
Backend-neutral."""
# The shared control-plane auth provisioning contract (#476). Every backend
# gets its signing key + gateway token through this one seam rather than
# re-deriving the wiring; the default topology is co-located + fail-closed,
# so a backend need not redeclare it (a truly isolated control plane would
# override this with a different `Topology`).
provisioning: ControlPlaneProvisioning = ControlPlaneProvisioning()
def ensure_built(self) -> None:
"""Ensure the orchestrator's image / rootfs exists, building it if
needed. Default: nothing to build (e.g. a pre-pulled image). Call before
@@ -105,9 +111,17 @@ class Orchestrator(abc.ABC):
def mint_gateway_token(self) -> str:
"""Mint a role-scoped `gateway` JWT from the host signing key for the
gateway to present. The orchestrator holds the key; the gateway never
does (#469). Backend-neutral — the same host token file is the single
source of truth across backends."""
return mint(ROLE_GATEWAY, host_orchestrator_token())
does (#469). Routed through the shared provisioning contract (#476), so
the same host token file is the single source of truth across backends."""
return self.provisioning.gateway_token()
def control_plane_key(self) -> str:
"""The raw signing key the control-plane *process* must receive — the ONE
place a backend obtains it (docker/macOS inject it as `key_env`;
firecracker pushes it to the guest). Fail-closed via the provisioning
contract: it raises rather than yield an empty key that would run the
server OPEN (#476)."""
return self.provisioning.orchestrator_key()
__all__ = [
@@ -117,4 +131,5 @@ __all__ = [
"OrchestratorStartError",
"source_hash",
"Orchestrator",
"ControlPlaneProvisioning",
]
bot_bottle/orchestrator/server.py
+7 -5
View File
@@ -63,8 +63,8 @@ import sys
import typing
from urllib.parse import urlsplit
from ..orchestrator_auth import ROLE_CLI, ROLES, verify
from ..paths import ORCHESTRATOR_TOKEN_ENV
from ..orchestrator_auth import ROLE_CLI, ROLES
from ..trust_domain import CONTROL_PLANE
from ..supervisor.types import TOOLS
from .service import OrchestratorCore
@@ -413,11 +413,13 @@ class OrchestratorServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
def __init__(self, address: tuple[str, int], orchestrator: OrchestratorCore) -> None:
self.orchestrator = orchestrator
self._signing_key = os.environ.get(ORCHESTRATOR_TOKEN_ENV, "").strip()
# The control-plane trust domain's signing key, as injected into THIS
# (the owning) process by the launcher (#476). Unset → open mode below.
self._signing_key = CONTROL_PLANE.key_from_env()
if not self._signing_key:
sys.stderr.write(
"orchestrator: WARNING — no control-plane signing key "
f"(${ORCHESTRATOR_TOKEN_ENV}); running WITHOUT caller "
f"(${CONTROL_PLANE.key_env}); running WITHOUT caller "
"authentication. Any client that can reach this port can drive "
"it. Backends that put the control plane on an agent-reachable "
"network MUST set this.\n"
@@ -433,7 +435,7 @@ class OrchestratorServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
role (→ per-route 401/403 in `dispatch`)."""
if not self._signing_key:
return ROLE_CLI
return verify(presented, self._signing_key)
return CONTROL_PLANE.verify(presented, self._signing_key)
def make_server(
bot_bottle/orchestrator_auth.py
+15 -7
View File
@@ -59,12 +59,19 @@ _HEADER_SEGMENT = _b64url_encode(
)
def mint(role: str, secret: str) -> str:
def mint(role: str, secret: str, *, roles: frozenset[str] = ROLES) -> str:
"""A compact HS256 token asserting `role`, signed with `secret`.
Raises ValueError for an unknown role (mint only what the control plane will
`roles` is the role set the caller's *trust domain* recognises (default: the
orchestrator control plane's `{gateway, cli}`). A domain names its own set so
each credential boundary mints only its own roles — a separate boundary
(e.g. a host controller) instantiates a distinct domain with a distinct key
and role set rather than adding a role here, so its key cannot forge the
other domain's tokens (see `trust_domain.py`, issues #476/#468).
Raises ValueError for a role outside `roles` (mint only what that domain will
accept) or an empty signing key (an unsigned credential is never valid)."""
if role not in ROLES:
if role not in roles:
raise ValueError(f"unknown control-plane role {role!r}")
if not secret:
raise ValueError("cannot mint a control-plane token without a signing key")
@@ -73,10 +80,11 @@ def mint(role: str, secret: str) -> str:
return f"{signing_input}.{_sign(secret, signing_input)}"
def verify(token: str, secret: str) -> str | None:
def verify(token: str, secret: str, *, roles: frozenset[str] = ROLES) -> str | None:
"""The role a valid `token` carries, or None if it is malformed, wrongly
signed, or names an unknown role. Constant-time signature check; rejects any
header whose alg isn't HS256 (no alg-confusion / `none`)."""
signed, or names a role outside `roles` (the verifying trust domain's set —
default `{gateway, cli}`). Constant-time signature check; rejects any header
whose alg isn't HS256 (no alg-confusion / `none`)."""
if not token or not secret:
return None
parts = token.split(".")
@@ -94,7 +102,7 @@ def verify(token: str, secret: str) -> str | None:
if not isinstance(header, dict) or header.get("alg") != _ALG:
return None
role = payload.get("role") if isinstance(payload, dict) else None
return role if isinstance(role, str) and role in ROLES else None
return role if isinstance(role, str) and role in roles else None
__all__ = ["ROLE_GATEWAY", "ROLE_CLI", "ROLES", "mint", "verify"]
bot_bottle/paths.py
+21 -9
View File
@@ -97,16 +97,19 @@ def host_gateway_ca_dir() -> Path:
return ca_dir
def host_orchestrator_token() -> str:
"""The per-host control-plane secret, minted (256-bit, url-safe) and
persisted 0600 on first use, then reused.
def host_signing_key(filename: str) -> str:
"""A per-host signing key at `<root>/<filename>`, minted (256-bit, url-safe)
and persisted 0600 on first use, then reused.
This is the shared secret the launchers inject into the control-plane and
gateway containers and that the host CLI presents on every call. It is a
*host* artifact — the file lives under the root the agent never mounts, and
the env var is set only on the trusted containers — so reading it here is
safe on the host launch path but the value never reaches a bottle."""
path = bot_bottle_root() / ORCHESTRATOR_TOKEN_FILENAME
The generic form of `host_orchestrator_token()`: a *trust domain*
(`trust_domain.py`) names its own key file so each credential boundary gets a
distinct host-canonical key — the orchestrator control plane names one file,
a separate boundary (e.g. a host controller) names another, and neither can
read the other's key (issues #476/#468). It is a *host* artifact: the file
lives under the root the agent never mounts, and its value is injected only
into the trusted control-plane process, so reading it here is safe on the
host launch path but the value never reaches a bottle."""
path = bot_bottle_root() / filename
try:
existing = path.read_text().strip()
if existing:
@@ -128,6 +131,14 @@ def host_orchestrator_token() -> str:
return token
def host_orchestrator_token() -> str:
"""The per-host control-plane signing key — the host-canonical key the
launchers inject into the control-plane process and the host CLI mints its
own `cli` token from. The `control-plane` trust domain's specialization of
`host_signing_key()`."""
return host_signing_key(ORCHESTRATOR_TOKEN_FILENAME)
__all__ = [
"HOST_DB_FILENAME",
"ORCHESTRATOR_TOKEN_FILENAME",
@@ -138,5 +149,6 @@ __all__ = [
"host_db_path",
"host_db_dir",
"host_gateway_ca_dir",
"host_signing_key",
"host_orchestrator_token",
]
bot_bottle/trust_domain.py
+189
View File
@@ -0,0 +1,189 @@
"""Trust domains: the unit of control-plane auth provisioning (issue #476).
A *trust domain* is one **credential boundary** — a single host-canonical
signing key plus the role set that key is allowed to sign, plus the env vars the
key and a pre-minted token are carried in. Provisioning is parameterized *per
domain*, not per key: the orchestrator control plane is one domain
(`CONTROL_PLANE` — key `orchestrator-token`, roles `{gateway, cli}`); a future
boundary (e.g. the host controller of #468) instantiates its **own** domain with
its **own** key, verifier, and role set rather than adding a role to this one.
That distinction is the security invariant behind the split: adding a `host`
role to the control plane's `ROLES` frozenset would let anything holding the
control-plane signing key (the orchestrator itself) mint host-controller tokens,
collapsing the boundary #468 needs — the host controller owns the orchestrator's
lifecycle, so it must not be forgeable *by* the orchestrator. Two keys, two
verifiers, two role sets.
`ControlPlaneProvisioning` is the single shared contract every backend launcher
satisfies instead of re-deriving, by hand, how to generate the signing key,
scope it to the orchestrator process, mint the gateway JWT, and keep the host
key canonical (the bug class that took PR #471 three review rounds — see
`docs/prds/prd-new-control-plane-auth-provisioning.md`).
Stdlib-only; the crypto lives in `orchestrator_auth` (untouched HMAC), the key
file lives in `paths` (no bot-bottle imports, safe to copy flat), and this module
composes the two into the provisioning seam.
"""
from __future__ import annotations
import os
from collections.abc import Mapping
from dataclasses import dataclass, field
from . import orchestrator_auth
from .orchestrator_auth import ROLE_GATEWAY
from .paths import (
ORCHESTRATOR_AUTH_JWT_ENV,
ORCHESTRATOR_TOKEN_ENV,
ORCHESTRATOR_TOKEN_FILENAME,
host_signing_key,
)
class ProvisioningError(RuntimeError):
"""A control-plane auth invariant would be violated (e.g. starting a
co-located control plane without its signing key — which would run OPEN)."""
@dataclass(frozen=True)
class TrustDomain:
"""One credential boundary: a host-canonical signing key + the role set it
signs + the env vars its key and a pre-minted token ride in.
`signing_key()` reads-or-mints the host-canonical key (never a guest's); the
orchestrator process that *owns* the domain receives that raw key (via
`key_env`), while a delegate (the data plane) receives only a pre-minted,
role-scoped token (via `token_env`) it cannot rewrite. `mint`/`verify` are
scoped to this domain's `roles`, so a token minted here neither carries nor
verifies a role from another domain."""
name: str
key_filename: str
roles: frozenset[str]
key_env: str
token_env: str
def signing_key(self) -> str:
"""The host-canonical signing key for this domain (minted 0600 on first
use). Host-side only — the value is injected into the owning process, not
read there."""
return host_signing_key(self.key_filename)
def key_from_env(self, environ: Mapping[str, str] | None = None) -> str:
"""The signing key as seen by the *owning process* — read from `key_env`
in the environment (default `os.environ`). "" when unset: the caller
decides whether that is fatal (see `OrchestratorServer`'s open-mode
fallback) or fail-closed (see `ControlPlaneProvisioning`)."""
env = os.environ if environ is None else environ
return env.get(self.key_env, "").strip()
def mint(self, role: str) -> str:
"""A role-scoped token for a delegate, signed with this domain's key.
Raises ValueError for a role outside this domain (mint only what this
boundary accepts)."""
if role not in self.roles:
raise ValueError(f"role {role!r} is not in trust domain {self.name!r}")
return orchestrator_auth.mint(role, self.signing_key(), roles=self.roles)
def verify(self, token: str, key: str) -> str | None:
"""The role `token` carries under `key`, or None. `key` is passed
explicitly (not read from the host file) because the verifier — the
control-plane process — holds it in `key_env`, not on disk in its guest."""
return orchestrator_auth.verify(token, key, roles=self.roles)
# The orchestrator control-plane domain: the signing key held by the
# orchestrator + host CLI, the `gateway` token handed to the data plane, and the
# `cli` token the CLI mints for itself.
CONTROL_PLANE = TrustDomain(
name="control-plane",
key_filename=ORCHESTRATOR_TOKEN_FILENAME,
roles=orchestrator_auth.ROLES,
key_env=ORCHESTRATOR_TOKEN_ENV,
token_env=ORCHESTRATOR_AUTH_JWT_ENV,
)
@dataclass(frozen=True)
class Topology:
"""What a backend *is*, for control-plane auth provisioning (#476) — the
backend declares this instead of encoding the provisioning decision by hand
in its launcher.
`data_plane_shares_control_host` — the data plane runs on the same host/VM as
the control plane, so a reachable-but-OPEN control plane would hand that
co-located data plane full `cli`. This is the default and the case that makes
the signing key **mandatory** (open mode unreachable). Every current backend
is co-located (docker/macOS: two containers on one host; firecracker: two VMs
on one host, agents L3-isolated). A backend with a genuinely isolated control
plane on a separate trusted host may declare False.
`combined_guest` — control plane and data plane share a single guest (the
retired combined infra VM). Informational today; kept so a future combined
backend *declares* it rather than rediscovering the provisioning."""
data_plane_shares_control_host: bool = True
combined_guest: bool = False
# The default posture: data plane co-located with the control plane. Fail-closed
# — the signing key is mandatory. A backend opts out only by declaring otherwise.
COLOCATED = Topology(data_plane_shares_control_host=True)
@dataclass(frozen=True)
class ControlPlaneProvisioning:
"""The single shared control-plane auth provisioning contract (#476).
A backend launcher satisfies THIS instead of re-deriving the four invariants
that each took a PR #471 review round to get right:
1. **Host-canonical key.** The signing key is `domain.signing_key()` — a
guest is *handed* it, never generates or overwrites it (round 3's bug:
the Firecracker guest clobbered the host key).
2. **Split credential.** Only the orchestrator process gets the raw key
(`orchestrator_key`); the data plane gets a pre-minted, role-scoped
token (`gateway_token`) it cannot rewrite into `cli` (round 1's bug).
3. **CLI validity across backends.** The host CLI mints its `cli` token
from this same canonical key, so it stays valid no matter which backend
(or how many) are co-running.
4. **No open mode.** `orchestrator_key` fail-closes for any topology whose
data plane shares the control plane's host/VM, so a launcher cannot
start the control plane OPEN (round 2's bug)."""
domain: TrustDomain = CONTROL_PLANE
topology: Topology = field(default=COLOCATED)
def orchestrator_key(self) -> str:
"""The raw signing key the control-plane *process* must receive (carry it
in `domain.key_env`). Fail-closed: raises `ProvisioningError` rather than
returning "" for a co-located topology, because an empty key makes the
server run OPEN and hand the co-located data plane full `cli` (#476
invariant 4)."""
key = self.domain.signing_key()
if not key and self.topology.data_plane_shares_control_host:
raise ProvisioningError(
f"refusing to provision the {self.domain.name} control plane "
"without a signing key: its data plane shares this host/VM, so "
"an OPEN control plane would grant that data plane full `cli` "
"(#476)"
)
return key
def gateway_token(self) -> str:
"""The pre-minted `gateway`-role token the data plane receives (carry it
in `domain.token_env`) — minted from the canonical key, never the key
itself, so a compromised data plane cannot forge a `cli` token."""
return self.domain.mint(ROLE_GATEWAY)
__all__ = [
"ProvisioningError",
"TrustDomain",
"CONTROL_PLANE",
"Topology",
"COLOCATED",
"ControlPlaneProvisioning",
]