refactor(orchestrator): uniform control-plane auth provisioning per trust domain #482
@@ -19,7 +19,6 @@ from .util import run_docker
|
||||
from ...paths import (
|
||||
ORCHESTRATOR_TOKEN_ENV,
|
||||
bot_bottle_root,
|
||||
host_orchestrator_token,
|
||||
)
|
||||
from ...gateway import GatewayError
|
||||
from ...orchestrator.lifecycle import (
|
||||
@@ -171,7 +170,9 @@ class DockerOrchestrator(Orchestrator):
|
||||
fixed-name container first)."""
|
||||
self._ensure_control_network()
|
||||
run_docker(["docker", "rm", "--force", self.name])
|
||||
_signing_key = host_orchestrator_token()
|
||||
# The signing key comes through the shared provisioning contract (#476),
|
||||
# which fail-closes rather than yield an empty key that would run OPEN.
|
||||
_signing_key = self.control_plane_key()
|
||||
proc = run_docker([
|
||||
"docker", "run", "--detach",
|
||||
"--name", self.name,
|
||||
|
||||
@@ -21,7 +21,6 @@ import time
|
||||
from pathlib import Path
|
||||
|
||||
from ...log import die, info
|
||||
from ...paths import host_orchestrator_token
|
||||
from ...orchestrator.lifecycle import (
|
||||
DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||
Orchestrator,
|
||||
@@ -108,11 +107,13 @@ class FirecrackerOrchestrator(Orchestrator):
|
||||
data_drive=self._ensure_registry_volume(),
|
||||
)
|
||||
# Push the host-canonical signing key (the init waits for it before
|
||||
# starting the control plane). The host token file stays the single
|
||||
# source of truth, so a co-running docker/macOS control plane keeps
|
||||
# working; the guest verifies tokens with the same key the CLI signs from.
|
||||
# starting the control plane). It comes through the shared provisioning
|
||||
# contract (#476) — the same host token file every backend uses, so a
|
||||
# co-running docker/macOS control plane keeps working and the guest
|
||||
# verifies tokens with the same key the CLI signs from; fail-closed, so
|
||||
# the guest is never handed an empty key that would run it OPEN.
|
||||
infra_vm.push_secret(
|
||||
vm, host_orchestrator_token(), infra_vm._GUEST_SIGNING_KEY_PATH,
|
||||
vm, self.control_plane_key(), infra_vm._GUEST_SIGNING_KEY_PATH,
|
||||
"the control-plane signing key to the orchestrator VM "
|
||||
"(its control plane will not start)",
|
||||
)
|
||||
|
||||
@@ -18,10 +18,7 @@ import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from ... import log
|
||||
from ...paths import (
|
||||
ORCHESTRATOR_TOKEN_ENV,
|
||||
host_orchestrator_token,
|
||||
)
|
||||
from ...paths import ORCHESTRATOR_TOKEN_ENV
|
||||
from ...orchestrator.lifecycle import (
|
||||
DEFAULT_HEALTH_TIMEOUT_SECONDS,
|
||||
DEFAULT_PORT,
|
||||
@@ -134,7 +131,9 @@ class MacosOrchestrator(Orchestrator):
|
||||
|
||||
def _run_container(self, current_hash: str) -> None:
|
||||
container_mod.force_remove_container(self.name)
|
||||
_signing_key = host_orchestrator_token()
|
||||
# The signing key comes through the shared provisioning contract (#476),
|
||||
# which fail-closes rather than yield an empty key that would run OPEN.
|
||||
_signing_key = self.control_plane_key()
|
||||
argv = [
|
||||
"container", "run", "--detach",
|
||||
"--name", self.name,
|
||||
|
||||
@@ -18,8 +18,8 @@ import urllib.request
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ..orchestrator_auth import ROLE_CLI, mint
|
||||
from ..paths import host_orchestrator_token
|
||||
from ..orchestrator_auth import ROLE_CLI
|
||||
from ..trust_domain import CONTROL_PLANE
|
||||
from .server import ORCHESTRATOR_AUTH_HEADER
|
||||
|
||||
DEFAULT_TIMEOUT_SECONDS = 5.0
|
||||
@@ -32,7 +32,7 @@ def _host_auth_token() -> str:
|
||||
"" means 'send no auth header' — correct against an open (unconfigured)
|
||||
control plane, and harmlessly rejected by a secured one."""
|
||||
try:
|
||||
return mint(ROLE_CLI, host_orchestrator_token())
|
||||
return CONTROL_PLANE.mint(ROLE_CLI)
|
||||
except (OSError, ValueError):
|
||||
return ""
|
||||
|
||||
|
||||
@@ -21,8 +21,7 @@ import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from ..orchestrator_auth import ROLE_GATEWAY, mint
|
||||
from ..paths import host_orchestrator_token
|
||||
from ..trust_domain import ControlPlaneProvisioning
|
||||
|
||||
DEFAULT_PORT = 8099
|
||||
DEFAULT_STARTUP_TIMEOUT_SECONDS = 45.0
|
||||
@@ -58,6 +57,13 @@ class Orchestrator(abc.ABC):
|
||||
so it — not the gateway — mints the gateway's role-scoped token.
|
||||
Backend-neutral."""
|
||||
|
||||
# The shared control-plane auth provisioning contract (#476). Every backend
|
||||
# gets its signing key + gateway token through this one seam rather than
|
||||
# re-deriving the wiring; the default topology is co-located + fail-closed,
|
||||
# so a backend need not redeclare it (a truly isolated control plane would
|
||||
# override this with a different `Topology`).
|
||||
provisioning: ControlPlaneProvisioning = ControlPlaneProvisioning()
|
||||
|
||||
def ensure_built(self) -> None:
|
||||
"""Ensure the orchestrator's image / rootfs exists, building it if
|
||||
needed. Default: nothing to build (e.g. a pre-pulled image). Call before
|
||||
@@ -105,9 +111,17 @@ class Orchestrator(abc.ABC):
|
||||
def mint_gateway_token(self) -> str:
|
||||
"""Mint a role-scoped `gateway` JWT from the host signing key for the
|
||||
gateway to present. The orchestrator holds the key; the gateway never
|
||||
does (#469). Backend-neutral — the same host token file is the single
|
||||
source of truth across backends."""
|
||||
return mint(ROLE_GATEWAY, host_orchestrator_token())
|
||||
does (#469). Routed through the shared provisioning contract (#476), so
|
||||
the same host token file is the single source of truth across backends."""
|
||||
return self.provisioning.gateway_token()
|
||||
|
||||
def control_plane_key(self) -> str:
|
||||
"""The raw signing key the control-plane *process* must receive — the ONE
|
||||
place a backend obtains it (docker/macOS inject it as `key_env`;
|
||||
firecracker pushes it to the guest). Fail-closed via the provisioning
|
||||
contract: it raises rather than yield an empty key that would run the
|
||||
server OPEN (#476)."""
|
||||
return self.provisioning.orchestrator_key()
|
||||
|
||||
|
||||
__all__ = [
|
||||
@@ -117,4 +131,5 @@ __all__ = [
|
||||
"OrchestratorStartError",
|
||||
"source_hash",
|
||||
"Orchestrator",
|
||||
"ControlPlaneProvisioning",
|
||||
]
|
||||
|
||||
@@ -63,8 +63,8 @@ import sys
|
||||
import typing
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from ..orchestrator_auth import ROLE_CLI, ROLES, verify
|
||||
from ..paths import ORCHESTRATOR_TOKEN_ENV
|
||||
from ..orchestrator_auth import ROLE_CLI, ROLES
|
||||
from ..trust_domain import CONTROL_PLANE
|
||||
from ..supervisor.types import TOOLS
|
||||
from .service import OrchestratorCore
|
||||
|
||||
@@ -413,11 +413,13 @@ class OrchestratorServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
|
||||
|
||||
def __init__(self, address: tuple[str, int], orchestrator: OrchestratorCore) -> None:
|
||||
self.orchestrator = orchestrator
|
||||
self._signing_key = os.environ.get(ORCHESTRATOR_TOKEN_ENV, "").strip()
|
||||
# The control-plane trust domain's signing key, as injected into THIS
|
||||
# (the owning) process by the launcher (#476). Unset → open mode below.
|
||||
self._signing_key = CONTROL_PLANE.key_from_env()
|
||||
if not self._signing_key:
|
||||
sys.stderr.write(
|
||||
"orchestrator: WARNING — no control-plane signing key "
|
||||
f"(${ORCHESTRATOR_TOKEN_ENV}); running WITHOUT caller "
|
||||
f"(${CONTROL_PLANE.key_env}); running WITHOUT caller "
|
||||
"authentication. Any client that can reach this port can drive "
|
||||
"it. Backends that put the control plane on an agent-reachable "
|
||||
"network MUST set this.\n"
|
||||
@@ -433,7 +435,7 @@ class OrchestratorServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
|
||||
role (→ per-route 401/403 in `dispatch`)."""
|
||||
if not self._signing_key:
|
||||
return ROLE_CLI
|
||||
return verify(presented, self._signing_key)
|
||||
return CONTROL_PLANE.verify(presented, self._signing_key)
|
||||
|
||||
|
||||
def make_server(
|
||||
|
||||
@@ -59,12 +59,19 @@ _HEADER_SEGMENT = _b64url_encode(
|
||||
)
|
||||
|
||||
|
||||
def mint(role: str, secret: str) -> str:
|
||||
def mint(role: str, secret: str, *, roles: frozenset[str] = ROLES) -> str:
|
||||
"""A compact HS256 token asserting `role`, signed with `secret`.
|
||||
|
||||
Raises ValueError for an unknown role (mint only what the control plane will
|
||||
`roles` is the role set the caller's *trust domain* recognises (default: the
|
||||
orchestrator control plane's `{gateway, cli}`). A domain names its own set so
|
||||
each credential boundary mints only its own roles — a separate boundary
|
||||
(e.g. a host controller) instantiates a distinct domain with a distinct key
|
||||
and role set rather than adding a role here, so its key cannot forge the
|
||||
other domain's tokens (see `trust_domain.py`, issues #476/#468).
|
||||
|
||||
Raises ValueError for a role outside `roles` (mint only what that domain will
|
||||
accept) or an empty signing key (an unsigned credential is never valid)."""
|
||||
if role not in ROLES:
|
||||
if role not in roles:
|
||||
raise ValueError(f"unknown control-plane role {role!r}")
|
||||
if not secret:
|
||||
raise ValueError("cannot mint a control-plane token without a signing key")
|
||||
@@ -73,10 +80,11 @@ def mint(role: str, secret: str) -> str:
|
||||
return f"{signing_input}.{_sign(secret, signing_input)}"
|
||||
|
||||
|
||||
def verify(token: str, secret: str) -> str | None:
|
||||
def verify(token: str, secret: str, *, roles: frozenset[str] = ROLES) -> str | None:
|
||||
"""The role a valid `token` carries, or None if it is malformed, wrongly
|
||||
signed, or names an unknown role. Constant-time signature check; rejects any
|
||||
header whose alg isn't HS256 (no alg-confusion / `none`)."""
|
||||
signed, or names a role outside `roles` (the verifying trust domain's set —
|
||||
default `{gateway, cli}`). Constant-time signature check; rejects any header
|
||||
whose alg isn't HS256 (no alg-confusion / `none`)."""
|
||||
if not token or not secret:
|
||||
return None
|
||||
parts = token.split(".")
|
||||
@@ -94,7 +102,7 @@ def verify(token: str, secret: str) -> str | None:
|
||||
if not isinstance(header, dict) or header.get("alg") != _ALG:
|
||||
return None
|
||||
role = payload.get("role") if isinstance(payload, dict) else None
|
||||
return role if isinstance(role, str) and role in ROLES else None
|
||||
return role if isinstance(role, str) and role in roles else None
|
||||
|
||||
|
||||
__all__ = ["ROLE_GATEWAY", "ROLE_CLI", "ROLES", "mint", "verify"]
|
||||
|
||||
+21
-9
@@ -97,16 +97,19 @@ def host_gateway_ca_dir() -> Path:
|
||||
return ca_dir
|
||||
|
||||
|
||||
def host_orchestrator_token() -> str:
|
||||
"""The per-host control-plane secret, minted (256-bit, url-safe) and
|
||||
persisted 0600 on first use, then reused.
|
||||
def host_signing_key(filename: str) -> str:
|
||||
"""A per-host signing key at `<root>/<filename>`, minted (256-bit, url-safe)
|
||||
and persisted 0600 on first use, then reused.
|
||||
|
||||
This is the shared secret the launchers inject into the control-plane and
|
||||
gateway containers and that the host CLI presents on every call. It is a
|
||||
*host* artifact — the file lives under the root the agent never mounts, and
|
||||
the env var is set only on the trusted containers — so reading it here is
|
||||
safe on the host launch path but the value never reaches a bottle."""
|
||||
path = bot_bottle_root() / ORCHESTRATOR_TOKEN_FILENAME
|
||||
The generic form of `host_orchestrator_token()`: a *trust domain*
|
||||
(`trust_domain.py`) names its own key file so each credential boundary gets a
|
||||
distinct host-canonical key — the orchestrator control plane names one file,
|
||||
a separate boundary (e.g. a host controller) names another, and neither can
|
||||
read the other's key (issues #476/#468). It is a *host* artifact: the file
|
||||
lives under the root the agent never mounts, and its value is injected only
|
||||
into the trusted control-plane process, so reading it here is safe on the
|
||||
host launch path but the value never reaches a bottle."""
|
||||
path = bot_bottle_root() / filename
|
||||
try:
|
||||
existing = path.read_text().strip()
|
||||
if existing:
|
||||
@@ -128,6 +131,14 @@ def host_orchestrator_token() -> str:
|
||||
return token
|
||||
|
||||
|
||||
def host_orchestrator_token() -> str:
|
||||
"""The per-host control-plane signing key — the host-canonical key the
|
||||
launchers inject into the control-plane process and the host CLI mints its
|
||||
own `cli` token from. The `control-plane` trust domain's specialization of
|
||||
`host_signing_key()`."""
|
||||
return host_signing_key(ORCHESTRATOR_TOKEN_FILENAME)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"HOST_DB_FILENAME",
|
||||
"ORCHESTRATOR_TOKEN_FILENAME",
|
||||
@@ -138,5 +149,6 @@ __all__ = [
|
||||
"host_db_path",
|
||||
"host_db_dir",
|
||||
"host_gateway_ca_dir",
|
||||
"host_signing_key",
|
||||
"host_orchestrator_token",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
"""Trust domains: the unit of control-plane auth provisioning (issue #476).
|
||||
|
||||
A *trust domain* is one **credential boundary** — a single host-canonical
|
||||
signing key plus the role set that key is allowed to sign, plus the env vars the
|
||||
key and a pre-minted token are carried in. Provisioning is parameterized *per
|
||||
domain*, not per key: the orchestrator control plane is one domain
|
||||
(`CONTROL_PLANE` — key `orchestrator-token`, roles `{gateway, cli}`); a future
|
||||
boundary (e.g. the host controller of #468) instantiates its **own** domain with
|
||||
its **own** key, verifier, and role set rather than adding a role to this one.
|
||||
|
||||
That distinction is the security invariant behind the split: adding a `host`
|
||||
role to the control plane's `ROLES` frozenset would let anything holding the
|
||||
control-plane signing key (the orchestrator itself) mint host-controller tokens,
|
||||
collapsing the boundary #468 needs — the host controller owns the orchestrator's
|
||||
lifecycle, so it must not be forgeable *by* the orchestrator. Two keys, two
|
||||
verifiers, two role sets.
|
||||
|
||||
`ControlPlaneProvisioning` is the single shared contract every backend launcher
|
||||
satisfies instead of re-deriving, by hand, how to generate the signing key,
|
||||
scope it to the orchestrator process, mint the gateway JWT, and keep the host
|
||||
key canonical (the bug class that took PR #471 three review rounds — see
|
||||
`docs/prds/prd-new-control-plane-auth-provisioning.md`).
|
||||
|
||||
Stdlib-only; the crypto lives in `orchestrator_auth` (untouched HMAC), the key
|
||||
file lives in `paths` (no bot-bottle imports, safe to copy flat), and this module
|
||||
composes the two into the provisioning seam.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from . import orchestrator_auth
|
||||
from .orchestrator_auth import ROLE_GATEWAY
|
||||
from .paths import (
|
||||
ORCHESTRATOR_AUTH_JWT_ENV,
|
||||
ORCHESTRATOR_TOKEN_ENV,
|
||||
ORCHESTRATOR_TOKEN_FILENAME,
|
||||
host_signing_key,
|
||||
)
|
||||
|
||||
|
||||
class ProvisioningError(RuntimeError):
|
||||
"""A control-plane auth invariant would be violated (e.g. starting a
|
||||
co-located control plane without its signing key — which would run OPEN)."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TrustDomain:
|
||||
"""One credential boundary: a host-canonical signing key + the role set it
|
||||
signs + the env vars its key and a pre-minted token ride in.
|
||||
|
||||
`signing_key()` reads-or-mints the host-canonical key (never a guest's); the
|
||||
orchestrator process that *owns* the domain receives that raw key (via
|
||||
`key_env`), while a delegate (the data plane) receives only a pre-minted,
|
||||
role-scoped token (via `token_env`) it cannot rewrite. `mint`/`verify` are
|
||||
scoped to this domain's `roles`, so a token minted here neither carries nor
|
||||
verifies a role from another domain."""
|
||||
|
||||
name: str
|
||||
key_filename: str
|
||||
roles: frozenset[str]
|
||||
key_env: str
|
||||
token_env: str
|
||||
|
||||
def signing_key(self) -> str:
|
||||
"""The host-canonical signing key for this domain (minted 0600 on first
|
||||
use). Host-side only — the value is injected into the owning process, not
|
||||
read there."""
|
||||
return host_signing_key(self.key_filename)
|
||||
|
||||
def key_from_env(self, environ: Mapping[str, str] | None = None) -> str:
|
||||
"""The signing key as seen by the *owning process* — read from `key_env`
|
||||
in the environment (default `os.environ`). "" when unset: the caller
|
||||
decides whether that is fatal (see `OrchestratorServer`'s open-mode
|
||||
fallback) or fail-closed (see `ControlPlaneProvisioning`)."""
|
||||
env = os.environ if environ is None else environ
|
||||
return env.get(self.key_env, "").strip()
|
||||
|
||||
def mint(self, role: str) -> str:
|
||||
"""A role-scoped token for a delegate, signed with this domain's key.
|
||||
Raises ValueError for a role outside this domain (mint only what this
|
||||
boundary accepts)."""
|
||||
if role not in self.roles:
|
||||
raise ValueError(f"role {role!r} is not in trust domain {self.name!r}")
|
||||
return orchestrator_auth.mint(role, self.signing_key(), roles=self.roles)
|
||||
|
||||
def verify(self, token: str, key: str) -> str | None:
|
||||
"""The role `token` carries under `key`, or None. `key` is passed
|
||||
explicitly (not read from the host file) because the verifier — the
|
||||
control-plane process — holds it in `key_env`, not on disk in its guest."""
|
||||
return orchestrator_auth.verify(token, key, roles=self.roles)
|
||||
|
||||
|
||||
# The orchestrator control-plane domain: the signing key held by the
|
||||
# orchestrator + host CLI, the `gateway` token handed to the data plane, and the
|
||||
# `cli` token the CLI mints for itself.
|
||||
CONTROL_PLANE = TrustDomain(
|
||||
name="control-plane",
|
||||
key_filename=ORCHESTRATOR_TOKEN_FILENAME,
|
||||
roles=orchestrator_auth.ROLES,
|
||||
key_env=ORCHESTRATOR_TOKEN_ENV,
|
||||
token_env=ORCHESTRATOR_AUTH_JWT_ENV,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Topology:
|
||||
"""What a backend *is*, for control-plane auth provisioning (#476) — the
|
||||
backend declares this instead of encoding the provisioning decision by hand
|
||||
in its launcher.
|
||||
|
||||
`data_plane_shares_control_host` — the data plane runs on the same host/VM as
|
||||
the control plane, so a reachable-but-OPEN control plane would hand that
|
||||
co-located data plane full `cli`. This is the default and the case that makes
|
||||
the signing key **mandatory** (open mode unreachable). Every current backend
|
||||
is co-located (docker/macOS: two containers on one host; firecracker: two VMs
|
||||
on one host, agents L3-isolated). A backend with a genuinely isolated control
|
||||
plane on a separate trusted host may declare False.
|
||||
|
||||
`combined_guest` — control plane and data plane share a single guest (the
|
||||
retired combined infra VM). Informational today; kept so a future combined
|
||||
backend *declares* it rather than rediscovering the provisioning."""
|
||||
|
||||
data_plane_shares_control_host: bool = True
|
||||
combined_guest: bool = False
|
||||
|
||||
|
||||
# The default posture: data plane co-located with the control plane. Fail-closed
|
||||
# — the signing key is mandatory. A backend opts out only by declaring otherwise.
|
||||
COLOCATED = Topology(data_plane_shares_control_host=True)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ControlPlaneProvisioning:
|
||||
"""The single shared control-plane auth provisioning contract (#476).
|
||||
|
||||
A backend launcher satisfies THIS instead of re-deriving the four invariants
|
||||
that each took a PR #471 review round to get right:
|
||||
|
||||
1. **Host-canonical key.** The signing key is `domain.signing_key()` — a
|
||||
guest is *handed* it, never generates or overwrites it (round 3's bug:
|
||||
the Firecracker guest clobbered the host key).
|
||||
2. **Split credential.** Only the orchestrator process gets the raw key
|
||||
(`orchestrator_key`); the data plane gets a pre-minted, role-scoped
|
||||
token (`gateway_token`) it cannot rewrite into `cli` (round 1's bug).
|
||||
3. **CLI validity across backends.** The host CLI mints its `cli` token
|
||||
from this same canonical key, so it stays valid no matter which backend
|
||||
(or how many) are co-running.
|
||||
4. **No open mode.** `orchestrator_key` fail-closes for any topology whose
|
||||
data plane shares the control plane's host/VM, so a launcher cannot
|
||||
start the control plane OPEN (round 2's bug)."""
|
||||
|
||||
domain: TrustDomain = CONTROL_PLANE
|
||||
topology: Topology = field(default=COLOCATED)
|
||||
|
||||
def orchestrator_key(self) -> str:
|
||||
"""The raw signing key the control-plane *process* must receive (carry it
|
||||
in `domain.key_env`). Fail-closed: raises `ProvisioningError` rather than
|
||||
returning "" for a co-located topology, because an empty key makes the
|
||||
server run OPEN and hand the co-located data plane full `cli` (#476
|
||||
invariant 4)."""
|
||||
key = self.domain.signing_key()
|
||||
if not key and self.topology.data_plane_shares_control_host:
|
||||
raise ProvisioningError(
|
||||
f"refusing to provision the {self.domain.name} control plane "
|
||||
"without a signing key: its data plane shares this host/VM, so "
|
||||
"an OPEN control plane would grant that data plane full `cli` "
|
||||
"(#476)"
|
||||
)
|
||||
return key
|
||||
|
||||
def gateway_token(self) -> str:
|
||||
"""The pre-minted `gateway`-role token the data plane receives (carry it
|
||||
in `domain.token_env`) — minted from the canonical key, never the key
|
||||
itself, so a compromised data plane cannot forge a `cli` token."""
|
||||
return self.domain.mint(ROLE_GATEWAY)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ProvisioningError",
|
||||
"TrustDomain",
|
||||
"CONTROL_PLANE",
|
||||
"Topology",
|
||||
"COLOCATED",
|
||||
"ControlPlaneProvisioning",
|
||||
]
|
||||
@@ -0,0 +1,141 @@
|
||||
# PRD prd-new: Uniform control-plane auth provisioning
|
||||
|
||||
- **Status:** Draft
|
||||
- **Author:** claude
|
||||
- **Created:** 2026-07-26
|
||||
- **Issue:** #476
|
||||
|
||||
## Summary
|
||||
|
||||
Hoist control-plane auth provisioning out of the per-backend launchers into a
|
||||
single shared contract, parameterized per **trust domain**. A backend obtains
|
||||
its signing key and the data plane's token through one seam
|
||||
(`trust_domain.ControlPlaneProvisioning`) instead of re-deriving, by hand, how
|
||||
to generate the signing key, scope it to the orchestrator, mint the `gateway`
|
||||
JWT, and keep the host key canonical. This removes the integration-bug class
|
||||
that took PR #471 three review rounds to land, and gives issue #468's host
|
||||
controller a clean seam to instantiate its **own** domain (own key, own roles)
|
||||
without weakening the control plane's.
|
||||
|
||||
## Problem
|
||||
|
||||
Each launcher (`docker` gateway, `docker` infra, `macos` infra, `firecracker`
|
||||
infra) implemented the control-plane auth invariants independently. Every
|
||||
blocking finding in PR #471 was the same class of *integration* bug — not a flaw
|
||||
in the auth primitive (`orchestrator_auth.mint`/`verify`), but in how each
|
||||
backend wired it:
|
||||
|
||||
- **Round 1 (High):** the data plane got the full-power control-plane token, so
|
||||
a compromised egress/git-gate could approve its own supervise proposals. Fixed
|
||||
with role-scoped JWTs (`gateway` vs `cli`).
|
||||
- **Round 2 (High):** the Firecracker infra VM never provisioned the signing key
|
||||
or a `gateway` JWT, so the control plane fell into **open mode** and handed
|
||||
every unauthenticated caller the `cli` role.
|
||||
- **Round 3 (High):** the Firecracker fix then clobbered the host-canonical
|
||||
`orchestrator-token` with its guest key, 401'ing every already-running
|
||||
Docker/macOS orchestrator.
|
||||
|
||||
Miss one step per bespoke launcher and it's either a security hole or a
|
||||
cross-backend coexistence regression — and the tests didn't catch it because each
|
||||
backend's provisioning was hand-rolled.
|
||||
|
||||
## Goals / Success Criteria
|
||||
|
||||
- One shared seam every backend satisfies for control-plane auth provisioning;
|
||||
no launcher re-derives the four invariants.
|
||||
- The signing key is **host-canonical** — a guest is handed it, never generates
|
||||
or overwrites it.
|
||||
- Only the orchestrator process receives the raw key; the data plane receives a
|
||||
pre-minted, role-scoped `gateway` token it cannot rewrite into `cli`.
|
||||
- The host CLI's `cli` token is minted from the same canonical key, so it stays
|
||||
valid across simultaneously-running backends.
|
||||
- Open mode is unreachable for any backend whose data plane shares a host/VM
|
||||
with the control plane (fail-closed by default).
|
||||
- Provisioning is parameterized **per trust domain**, so #468 adds a separate
|
||||
host-controller domain (own key, own verifier, own role set) rather than a
|
||||
`host` role on the control plane's frozenset.
|
||||
- Adding a backend or a data-plane daemon means *implementing the contract*, not
|
||||
rediscovering the invariants.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Not a rewrite of the auth primitive. `orchestrator_auth`'s HMAC mint/verify is
|
||||
unchanged except for an optional `roles=` argument (default preserved) so a
|
||||
domain can scope its own role set.
|
||||
- Not the #468 host-controller domain itself — this only provides the seam it
|
||||
will instantiate.
|
||||
- Not a change to network topology, the plane split (#469), or the server's
|
||||
documented open-mode fallback for tests/isolated control planes.
|
||||
- The complementary #469 hardening (distinct non-root UIDs for co-located
|
||||
daemons) stays separate.
|
||||
|
||||
## Design
|
||||
|
||||
### Trust domain — the unit of provisioning
|
||||
|
||||
A **trust domain** (`trust_domain.TrustDomain`) is one credential boundary: a
|
||||
host-canonical signing key file, the role set that key may sign, and the env
|
||||
vars the raw key and a pre-minted token ride in. `mint`/`verify` are scoped to
|
||||
the domain's roles, so a token minted in one domain neither carries nor verifies
|
||||
a role from another.
|
||||
|
||||
The orchestrator control plane is one domain, `CONTROL_PLANE` (key
|
||||
`orchestrator-token`, roles `{gateway, cli}`). The security reason provisioning
|
||||
is per-domain and not per-key: adding a `host` role to `CONTROL_PLANE.roles`
|
||||
would let anything holding the control-plane key (the orchestrator itself) mint
|
||||
host-controller tokens, collapsing the boundary #468 needs — the host controller
|
||||
owns the orchestrator's lifecycle, so it must not be forgeable *by* the
|
||||
orchestrator. Two keys, two verifiers, two role sets.
|
||||
|
||||
`paths.host_signing_key(filename)` generalizes the old
|
||||
`host_orchestrator_token()` (now a thin specialization) so each domain names its
|
||||
own host-canonical key file.
|
||||
|
||||
### The provisioning contract
|
||||
|
||||
`ControlPlaneProvisioning` composes a domain with a declared `Topology` and
|
||||
answers the four invariants once:
|
||||
|
||||
- `orchestrator_key()` → the raw key the control-plane **process** receives.
|
||||
Fail-closed: raises `ProvisioningError` for a co-located topology when the key
|
||||
is empty (which would run the server OPEN).
|
||||
- `gateway_token()` → the pre-minted `gateway` token the data plane receives,
|
||||
minted from the canonical key, never the key itself.
|
||||
|
||||
The `Orchestrator` ABC (`orchestrator/lifecycle.py`) holds one
|
||||
`ControlPlaneProvisioning` and exposes `control_plane_key()` and
|
||||
`mint_gateway_token()` over it. Each backend's orchestrator obtains its key
|
||||
through `control_plane_key()` and applies it via its own transport (docker/macOS:
|
||||
env var `key_env`; firecracker: SSH push to the guest) — the transport differs,
|
||||
the derivation no longer does.
|
||||
|
||||
### Topology — the backend declares what it is
|
||||
|
||||
`Topology` captures the provisioning-relevant dimensions the issue names
|
||||
(combined-guest vs standalone, data plane co-located vs isolated). The default,
|
||||
`COLOCATED`, makes the signing key mandatory (fail-closed). Every current backend
|
||||
is co-located (docker/macOS: two containers on one host; firecracker: two VMs on
|
||||
one host, agents L3-isolated), so none needs to redeclare it — the safe posture
|
||||
is the default, and a genuinely isolated control plane opts out explicitly.
|
||||
|
||||
### Data flow
|
||||
|
||||
```
|
||||
host key file (per-domain, 0600, host-canonical)
|
||||
│ paths.host_signing_key(domain.key_filename)
|
||||
▼
|
||||
TrustDomain ── mint(role) ─────────────► gateway token ─► data-plane process (token_env / SSH)
|
||||
│ signing_key() (gateway role, unrewritable)
|
||||
▼
|
||||
ControlPlaneProvisioning.orchestrator_key() ─► control-plane process (key_env / SSH)
|
||||
│ (fail-closed for co-located topology)
|
||||
▼
|
||||
OrchestratorClient ── CONTROL_PLANE.mint(cli) ─► host CLI's own operator token
|
||||
```
|
||||
|
||||
## Open questions
|
||||
|
||||
None blocking. #468 will add its host-controller domain as a second
|
||||
`TrustDomain` + `ControlPlaneProvisioning`-shaped consumer; whether the
|
||||
provisioning class is renamed to a domain-neutral `DomainProvisioning` at that
|
||||
point is a cosmetic call to make when #468 lands.
|
||||
@@ -19,7 +19,9 @@ _ORCH = "bot_bottle.backend.docker.orchestrator"
|
||||
_RUN = f"{_ORCH}.run_docker"
|
||||
_SLEEP = f"{_ORCH}.time.sleep"
|
||||
_MONOTONIC = f"{_ORCH}.time.monotonic"
|
||||
_TOKEN = f"{_ORCH}.host_orchestrator_token"
|
||||
# The signing key is read through the shared provisioning contract (#476); patch
|
||||
# its host-canonical key file read to keep it off the real host file.
|
||||
_TOKEN = "bot_bottle.trust_domain.host_signing_key"
|
||||
# The ABC's is_healthy probes /health via urllib in the lifecycle module.
|
||||
_URLOPEN = "bot_bottle.orchestrator.lifecycle.urllib.request.urlopen"
|
||||
|
||||
|
||||
@@ -44,7 +44,7 @@ class TestEnsureRunning(unittest.TestCase):
|
||||
vm = infra_vm.InfraVm(guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock())
|
||||
with patch.object(infra_vm, "boot_vm", return_value=vm) as boot, \
|
||||
patch.object(infra_vm, "push_secret") as push, \
|
||||
patch(f"{_ORCH}.host_orchestrator_token", return_value="host-key"), \
|
||||
patch("bot_bottle.trust_domain.host_signing_key", return_value="host-key"), \
|
||||
patch.object(orch, "is_running", return_value=False), \
|
||||
patch.object(orch, "_ensure_registry_volume", return_value=Path("/reg")), \
|
||||
patch.object(orch, "_wait_for_health"):
|
||||
|
||||
@@ -33,7 +33,7 @@ class TestMacosOrchestratorRun(unittest.TestCase):
|
||||
def _run(self) -> list[str]:
|
||||
run = Mock(return_value=_ok())
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch(f"{_ORCH}.host_orchestrator_token", return_value="k"):
|
||||
patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
|
||||
mod.dns_server.return_value = "1.1.1.1"
|
||||
mod.bind_mount_spec.side_effect = _spec
|
||||
mod.run_container_argv = run
|
||||
@@ -65,7 +65,7 @@ class TestMacosOrchestratorRun(unittest.TestCase):
|
||||
|
||||
def test_start_failure_raises(self) -> None:
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch(f"{_ORCH}.host_orchestrator_token", return_value="k"):
|
||||
patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
|
||||
mod.dns_server.return_value = "1.1.1.1"
|
||||
mod.run_container_argv = Mock(return_value=_fail())
|
||||
with self.assertRaises(OrchestratorStartError):
|
||||
|
||||
@@ -82,5 +82,26 @@ class TestMintVerify(unittest.TestCase):
|
||||
mint(ROLE_GATEWAY, "")
|
||||
|
||||
|
||||
class TestCustomRoleSet(unittest.TestCase):
|
||||
"""The `roles=` seam a trust domain other than the control plane uses (#476):
|
||||
mint/verify are scoped to the passed role set, not the module default."""
|
||||
|
||||
_ROLES = frozenset({"host"})
|
||||
|
||||
def test_round_trips_a_role_in_the_custom_set(self) -> None:
|
||||
tok = mint("host", _KEY, roles=self._ROLES)
|
||||
self.assertEqual("host", verify(tok, _KEY, roles=self._ROLES))
|
||||
|
||||
def test_mint_rejects_a_role_outside_the_custom_set(self) -> None:
|
||||
with self.assertRaises(ValueError):
|
||||
mint(ROLE_CLI, _KEY, roles=self._ROLES)
|
||||
|
||||
def test_verify_rejects_a_role_outside_the_verifiers_set(self) -> None:
|
||||
# A validly signed token whose role isn't in the verifier's set fails —
|
||||
# this is what keeps one domain's key from asserting another's role.
|
||||
tok = mint(ROLE_CLI, _KEY) # a control-plane `cli` token
|
||||
self.assertIsNone(verify(tok, _KEY, roles=self._ROLES))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -20,13 +20,15 @@ _URLOPEN = "bot_bottle.orchestrator.client.urllib.request.urlopen"
|
||||
|
||||
class TestHostAuthToken(unittest.TestCase):
|
||||
def test_mints_a_cli_token_from_the_host_key(self) -> None:
|
||||
with patch("bot_bottle.orchestrator.client.host_orchestrator_token",
|
||||
# The CLI mints its `cli` token from the control-plane trust domain's
|
||||
# host-canonical key (#476) — patch the key file read underneath it.
|
||||
with patch("bot_bottle.trust_domain.host_signing_key",
|
||||
return_value="signing-key"):
|
||||
tok = _host_auth_token()
|
||||
self.assertEqual(ROLE_CLI, verify(tok, "signing-key"))
|
||||
|
||||
def test_returns_empty_when_key_unreadable(self) -> None:
|
||||
with patch("bot_bottle.orchestrator.client.host_orchestrator_token",
|
||||
with patch("bot_bottle.trust_domain.host_signing_key",
|
||||
side_effect=OSError("no host root")):
|
||||
self.assertEqual("", _host_auth_token())
|
||||
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Unit: trust domains + the shared control-plane provisioning contract (#476)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from bot_bottle import orchestrator_auth
|
||||
from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY
|
||||
from bot_bottle.trust_domain import (
|
||||
CONTROL_PLANE,
|
||||
ControlPlaneProvisioning,
|
||||
ProvisioningError,
|
||||
Topology,
|
||||
TrustDomain,
|
||||
)
|
||||
|
||||
# A second, unrelated domain — the shape #468's host controller would take: its
|
||||
# own key file, its own role, its own env vars. Distinct from CONTROL_PLANE.
|
||||
_HOST_CTRL = TrustDomain(
|
||||
name="host-controller",
|
||||
key_filename="host-controller-token",
|
||||
roles=frozenset({"host"}),
|
||||
key_env="BOT_BOTTLE_HOST_CONTROLLER_TOKEN",
|
||||
token_env="BOT_BOTTLE_HOST_CONTROLLER_JWT",
|
||||
)
|
||||
|
||||
|
||||
class TestTrustDomainMintVerify(unittest.TestCase):
|
||||
def test_mint_verify_round_trips_within_a_domain(self) -> None:
|
||||
with patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
|
||||
tok = CONTROL_PLANE.mint(ROLE_GATEWAY)
|
||||
self.assertEqual(ROLE_GATEWAY, CONTROL_PLANE.verify(tok, "k"))
|
||||
|
||||
def test_mint_rejects_a_role_outside_the_domain(self) -> None:
|
||||
# `host` is a valid role in _HOST_CTRL but not in the control plane —
|
||||
# the control-plane key must refuse to mint it.
|
||||
with patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
|
||||
with self.assertRaises(ValueError):
|
||||
CONTROL_PLANE.mint("host")
|
||||
|
||||
def test_a_custom_role_set_verifies_its_own_role(self) -> None:
|
||||
with patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
|
||||
tok = _HOST_CTRL.mint("host")
|
||||
self.assertEqual("host", _HOST_CTRL.verify(tok, "k"))
|
||||
|
||||
def test_key_from_env_reads_the_domains_env_var(self) -> None:
|
||||
self.assertEqual(
|
||||
"secret", CONTROL_PLANE.key_from_env({CONTROL_PLANE.key_env: " secret "}))
|
||||
self.assertEqual("", CONTROL_PLANE.key_from_env({}))
|
||||
|
||||
|
||||
class TestDomainBoundary(unittest.TestCase):
|
||||
"""The #476/#468 invariant: two domains, two keys, two role sets — one
|
||||
domain's key can neither mint nor verify the other's tokens."""
|
||||
|
||||
def test_a_control_plane_token_does_not_verify_under_another_domains_key(
|
||||
self,
|
||||
) -> None:
|
||||
# Even with an IDENTICAL underlying key, a `cli` token minted under the
|
||||
# control-plane domain must not verify as a role in the host-controller
|
||||
# domain — the role isn't in that domain's set.
|
||||
cli_tok = orchestrator_auth.mint(ROLE_CLI, "shared-bytes")
|
||||
self.assertIsNone(_HOST_CTRL.verify(cli_tok, "shared-bytes"))
|
||||
|
||||
def test_distinct_keys_do_not_cross_verify(self) -> None:
|
||||
# The realistic case: distinct host-canonical keys per domain. A token
|
||||
# signed by one key never verifies under the other.
|
||||
host_tok = orchestrator_auth.mint(
|
||||
"host", "host-ctrl-key", roles=_HOST_CTRL.roles)
|
||||
self.assertIsNone(_HOST_CTRL.verify(host_tok, "control-plane-key"))
|
||||
self.assertEqual("host", _HOST_CTRL.verify(host_tok, "host-ctrl-key"))
|
||||
|
||||
|
||||
class TestControlPlaneProvisioning(unittest.TestCase):
|
||||
def test_orchestrator_key_returns_the_canonical_key(self) -> None:
|
||||
prov = ControlPlaneProvisioning()
|
||||
with patch("bot_bottle.trust_domain.host_signing_key", return_value="key"):
|
||||
self.assertEqual("key", prov.orchestrator_key())
|
||||
|
||||
def test_orchestrator_key_fail_closes_when_colocated_and_empty(self) -> None:
|
||||
# Invariant 4: a co-located backend must never start the control plane
|
||||
# without a key (it would run OPEN and hand the co-located data plane
|
||||
# full `cli`).
|
||||
prov = ControlPlaneProvisioning() # default topology: co-located
|
||||
with patch("bot_bottle.trust_domain.host_signing_key", return_value=""):
|
||||
with self.assertRaises(ProvisioningError):
|
||||
prov.orchestrator_key()
|
||||
|
||||
def test_orchestrator_key_allows_empty_when_isolated(self) -> None:
|
||||
# A genuinely isolated control plane (no co-located data plane) may run
|
||||
# without a key — the only topology where open mode is permissible.
|
||||
prov = ControlPlaneProvisioning(
|
||||
topology=Topology(data_plane_shares_control_host=False))
|
||||
with patch("bot_bottle.trust_domain.host_signing_key", return_value=""):
|
||||
self.assertEqual("", prov.orchestrator_key())
|
||||
|
||||
def test_gateway_token_is_a_verifiable_gateway_role_token(self) -> None:
|
||||
prov = ControlPlaneProvisioning()
|
||||
with patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
|
||||
tok = prov.gateway_token()
|
||||
self.assertEqual(ROLE_GATEWAY, CONTROL_PLANE.verify(tok, "k"))
|
||||
|
||||
def test_gateway_token_cannot_be_reused_as_cli(self) -> None:
|
||||
# The data plane's token is `gateway`-scoped: it never carries `cli`.
|
||||
prov = ControlPlaneProvisioning()
|
||||
with patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
|
||||
tok = prov.gateway_token()
|
||||
self.assertNotEqual(ROLE_CLI, CONTROL_PLANE.verify(tok, "k"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user