feat(orchestrator): durable launch-broker secret via TrustDomain (#468)

Chunk 2 of the host-control-server stack: close the PRD's **durable
secret** gap and replace chunk 1's BOT_BOTTLE_BROKER_SECRET stopgap.

- trust_domain.py: two new domains. LAUNCH_BROKER holds the durable
  HS256 key both the orchestrator (signer) and the host control server
  (verifier) share for the broker's launch JWT — a host-canonical key
  file minted 0600 on first use, so a restarted orchestrator re-verifies
  against the same key (re-adoption). HOST_CONTROLLER is the separate
  domain for the controller's own lifecycle endpoints, keyed by a key the
  orchestrator never holds (its lifecycle role is `host`, deliberately
  outside the control-plane ROLES). LaunchBrokerProvisioning is the
  fail-closed seam, mirroring ControlPlaneProvisioning.
- orchestrator_auth.py: ROLE_HOST, outside ROLES so the orchestrator's
  control-plane key can neither mint nor accept it.
- paths.py: key-file + env-var constants for both domains.
- host_server.py / __main__.py: broker_secret() now resolves the durable
  LAUNCH_BROKER key — env-injected for a containerized launcher, else the
  host key file for a host-side dev-harness process — so `--broker http`
  and the host controller "just work" on one host without exporting a
  secret, and stay fail-closed when the root is unwritable.

Tested: domain boundary (host role unmintable by the control plane,
cross-domain tokens don't verify), distinct keys/env vars per domain,
provisioning fail-closed, and broker_secret env/file resolution +
durability. Full unit suite green; pyright clean; pylint 9.90.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

bot_bottle/orchestrator/__main__.py

@@ -17,9 +17,10 @@ from pathlib import Path
 
 from .. import log
 from .store.store_manager import StoreManager
+from ..paths import LAUNCH_BROKER_KEY_ENV
 from .broker import StubBroker, SubmitBroker
 from .broker_client import BrokerClient
-from .host_server import BROKER_SECRET_ENV, DEFAULT_PORT, broker_secret_from_env
+from .host_server import DEFAULT_PORT, broker_secret
 from .server import make_server
 from .docker_broker import DockerBroker
 from .store.registry_store import RegistryStore, default_db_path
@@ -57,17 +58,17 @@ def main(argv: list[str] | None = None) -> int:
     # A signing secret ties the orchestrator (signer) to its broker (verifier).
     # 'stub' records launches instead of starting anything; 'docker' runs real
     # containers in-process; 'http' relays signed requests to a separate host
-    # control server, which verifies and launches. For 'stub'/'docker' the
-    # secret is ephemeral (signer and verifier share this process); for 'http'
-    # it must be the SAME secret the host controller holds, so it is read from
-    # the shared env var (the chunk-1 stand-in for out-of-band provisioning).
+    # control server, which verifies and launches. For 'stub'/'docker' the secret
+    # is ephemeral (signer and verifier share this process); for 'http' it must be
+    # the SAME key the host controller holds, so both resolve the durable
+    # launch-broker TrustDomain key (env-injected, or the host key file).
     broker: SubmitBroker
     if args.broker == "http":
-        secret = broker_secret_from_env()
+        secret = broker_secret()
         if secret is None:
             parser.error(
-                f"--broker http requires a shared signing secret in "
-                f"${BROKER_SECRET_ENV} (hex), matching the host control server"
+                f"--broker http requires the launch-broker key (${LAUNCH_BROKER_KEY_ENV}, "
+                "or a writable host root to mint it), matching the host control server"
             )
         broker = BrokerClient(args.host_controller_url)
     else:

bot_bottle/orchestrator/host_server.py

@@ -23,12 +23,15 @@ controller down.
 
 The signed launch token *is* the endpoint's authentication (its provenance is the
 whole point of the JWS), so `/broker` needs no separate caller credential; the
-host controller's own lifecycle endpoints, which do, arrive with the durable
-`TrustDomain` key in a later chunk.
+host controller's own lifecycle endpoints, which do, arrive with the `host`-role
+tokens of the separate `HOST_CONTROLLER` trust domain in a later chunk.
 
-The shared signing secret is read from `$BOT_BOTTLE_BROKER_SECRET` (hex). That is
-a **chunk-1 stopgap**: it must be provisioned to signer and verifier out of band,
-which is exactly what the durable `TrustDomain` key in chunk 2 (#476) replaces.
+The shared signing secret is the durable **launch-broker `TrustDomain` key**
+(#468/#476): a host-canonical key file minted 0600 on first use, provisioned to
+the orchestrator (signer) and this server (verifier). A backend launcher injects
+it via `$BOT_BOTTLE_LAUNCH_BROKER_KEY`; a host-side dev-harness process reads the
+key file directly. Durability is the point — a restarted orchestrator re-verifies
+against the same key, so re-adoption works.
 """
 
 from __future__ import annotations
@@ -43,16 +46,14 @@ import typing
 from urllib.parse import urlsplit
 
 from .. import log
+from ..paths import LAUNCH_BROKER_KEY_ENV
+from ..trust_domain import LAUNCH_BROKER
 from .broker import BrokerAuthError, LaunchBroker
 from .docker_broker import DockerBroker
 
 # JSON body payload type (parsed request / rendered response).
 Json = dict[str, object]
 
-# The hex-encoded HS256 secret shared with the request signer (the orchestrator).
-# Chunk-1 stopgap for the durable, out-of-band `TrustDomain` key of chunk 2.
-BROKER_SECRET_ENV = "BOT_BOTTLE_BROKER_SECRET"
-
 # Default host-controller port. Distinct from the orchestrator control plane
 # (8099) — a separate privileged component listening on its own socket.
 DEFAULT_PORT = 8091
@@ -68,19 +69,22 @@ def _parse_json_object(body: bytes) -> Json:
     return obj
 
 
-def broker_secret_from_env(environ: typing.Mapping[str, str] | None = None) -> bytes | None:
-    """The shared HS256 secret from `$BOT_BOTTLE_BROKER_SECRET` (hex), or None
-    when unset or not valid hex. The signer (orchestrator, `--broker http`) and
-    the verifier (this server) read the same env var so both hold the same key —
-    the chunk-1 stand-in for out-of-band provisioning."""
-    env = os.environ if environ is None else environ
-    raw = env.get(BROKER_SECRET_ENV, "").strip()
-    if not raw:
-        return None
-    try:
-        return bytes.fromhex(raw)
-    except ValueError:
-        return None
+def broker_secret(environ: typing.Mapping[str, str] | None = None) -> bytes | None:
+    """The shared launch-broker HS256 secret, as this process should use it.
+
+    Prefers the key injected into this process's env (`$BOT_BOTTLE_LAUNCH_BROKER_KEY`)
+    — the path a containerized backend launcher uses — and falls back to the
+    durable host key file (`bot_bottle_root()/launch-broker-key`, minted 0600 on
+    first use) for a host-side process like the dev-harness. The signer
+    (orchestrator, `--broker http`) resolves it the same way, so both hold the
+    same key. None only when neither is available (an unwritable host root)."""
+    key = LAUNCH_BROKER.key_from_env(environ)
+    if not key:
+        try:
+            key = LAUNCH_BROKER.signing_key()  # host-canonical, minted on first use
+        except OSError:
+            return None
+    return key.encode("utf-8") if key else None
 
 
 def dispatch(  # pylint: disable=too-many-return-statements
@@ -192,20 +196,21 @@ def main(argv: list[str] | None = None) -> int:
 
         python -m bot_bottle.orchestrator.host_server [--host H] [--port P]
 
-    Fail-closed: without a shared `$BOT_BOTTLE_BROKER_SECRET` the server can
-    verify no request's provenance, so it refuses to start rather than run a
-    launcher that accepts unsigned input."""
+    Fail-closed: without the launch-broker key the server can verify no request's
+    provenance, so it refuses to start rather than run a launcher that accepts
+    unsigned input."""
     parser = argparse.ArgumentParser(prog="bot_bottle.orchestrator.host_server")
     parser.add_argument("--host", default="127.0.0.1", help="bind address")
     parser.add_argument("--port", type=int, default=DEFAULT_PORT, help="bind port (0 = ephemeral)")
     args = parser.parse_args(argv)
 
-    secret = broker_secret_from_env()
+    secret = broker_secret()
     if secret is None:
         sys.stderr.write(
-            f"host controller: refusing to start without a shared signing secret "
-            f"(${BROKER_SECRET_ENV}, hex) — it could verify no request's "
-            "provenance and would relay unsigned launches to the backend\n"
+            f"host controller: refusing to start without the launch-broker key "
+            f"(${LAUNCH_BROKER_KEY_ENV}, or a writable host root to mint it) — it "
+            "could verify no request's provenance and would relay unsigned "
+            "launches to the backend\n"
         )
         sys.stderr.flush()
         return 2
@@ -231,10 +236,9 @@ __all__ = [
     "Handler",
     "HostControlServer",
     "make_host_server",
-    "broker_secret_from_env",
+    "broker_secret",
     "main",
     "Json",
-    "BROKER_SECRET_ENV",
     "DEFAULT_PORT",
 ]
 

bot_bottle/orchestrator_auth.py

@@ -36,6 +36,13 @@ ROLE_GATEWAY = "gateway"
 ROLE_CLI = "cli"
 ROLES: frozenset[str] = frozenset({ROLE_GATEWAY, ROLE_CLI})
 
+# The host controller's own lifecycle role (#468). Deliberately OUTSIDE `ROLES`:
+# it belongs to a separate trust domain (`HOST_CONTROLLER`) signed by a key the
+# orchestrator never holds, so the orchestrator's control-plane key can neither
+# mint nor accept it — the orchestrator must not be able to forge the credential
+# used to start and stop it.
+ROLE_HOST = "host"
+
 _ALG = "HS256"
 
 
@@ -103,4 +110,4 @@ def verify(token: str, secret: str, *, roles: frozenset[str] = ROLES) -> str | N
     return role if isinstance(role, str) and role in roles else None
 
 
-__all__ = ["ROLE_GATEWAY", "ROLE_CLI", "ROLES", "mint", "verify"]
+__all__ = ["ROLE_GATEWAY", "ROLE_CLI", "ROLE_HOST", "ROLES", "mint", "verify"]

bot_bottle/paths.py

@@ -47,6 +47,22 @@ ORCHESTRATOR_TOKEN_ENV = "BOT_BOTTLE_ORCHESTRATOR_TOKEN"
 # cannot forge a higher-privilege `cli` token (issue #469 review).
 ORCHESTRATOR_AUTH_JWT_ENV = "BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT"
 
+# The durable launch-broker signing key: the HS256 secret the orchestrator
+# (signer) and the host control server (verifier) share to sign/verify launch
+# requests (#468). A host-canonical key file (minted 0600 on first use) so it
+# survives orchestrator restarts — re-adoption re-verifies against the same key —
+# instead of the ephemeral per-process secret of the in-process broker.
+LAUNCH_BROKER_KEY_FILENAME = "launch-broker-key"
+LAUNCH_BROKER_KEY_ENV = "BOT_BOTTLE_LAUNCH_BROKER_KEY"
+# The host controller's OWN key, for its lifecycle endpoints (the direct
+# cli -> host controller path that starts/stops the orchestrator). Separate from
+# the launch-broker key and never held by the orchestrator: the controller starts
+# and stops the orchestrator, so the orchestrator must not be able to mint the
+# credentials used to drive it (#468/#476).
+HOST_CONTROLLER_KEY_FILENAME = "host-controller-key"
+HOST_CONTROLLER_KEY_ENV = "BOT_BOTTLE_HOST_CONTROLLER_KEY"
+HOST_CONTROLLER_AUTH_JWT_ENV = "BOT_BOTTLE_HOST_CONTROLLER_AUTH_JWT"
+
 # The host directory holding the gateway's persistent mitmproxy CA. Bind-mounted
 # into the infra/gateway container at mitmproxy's confdir so the self-generated
 # CA survives container recreation — every agent installs this one CA to trust
@@ -142,6 +158,11 @@ __all__ = [
     "ORCHESTRATOR_TOKEN_FILENAME",
     "ORCHESTRATOR_TOKEN_ENV",
     "ORCHESTRATOR_AUTH_JWT_ENV",
+    "LAUNCH_BROKER_KEY_FILENAME",
+    "LAUNCH_BROKER_KEY_ENV",
+    "HOST_CONTROLLER_KEY_FILENAME",
+    "HOST_CONTROLLER_KEY_ENV",
+    "HOST_CONTROLLER_AUTH_JWT_ENV",
     "GATEWAY_CA_DIRNAME",
     "bot_bottle_root",
     "host_db_path",

bot_bottle/trust_domain.py

@@ -29,8 +29,13 @@ from collections.abc import Mapping
 from dataclasses import dataclass
 
 from . import orchestrator_auth
-from .orchestrator_auth import ROLE_GATEWAY
+from .orchestrator_auth import ROLE_GATEWAY, ROLE_HOST
 from .paths import (
+    HOST_CONTROLLER_AUTH_JWT_ENV,
+    HOST_CONTROLLER_KEY_ENV,
+    HOST_CONTROLLER_KEY_FILENAME,
+    LAUNCH_BROKER_KEY_ENV,
+    LAUNCH_BROKER_KEY_FILENAME,
     ORCHESTRATOR_AUTH_JWT_ENV,
     ORCHESTRATOR_TOKEN_ENV,
     ORCHESTRATOR_TOKEN_FILENAME,
@@ -99,6 +104,40 @@ CONTROL_PLANE = TrustDomain(
 )
 
 
+# The launch-broker domain (#468): durable key material for the broker's own
+# signed launch requests (`broker.py`'s HS256 launch JWT), shared by the
+# orchestrator (signer) and the host control server (verifier). Unlike
+# `CONTROL_PLANE` it mints no role tokens — the broker's provenance is the launch
+# JWT, not a role token — so its `roles` set is empty and it is used only as a
+# provider of durable, host-canonical key material (`signing_key` / `key_from_env`).
+# The durability is the point: the key survives orchestrator restarts, so a
+# restarted orchestrator re-verifies against the same key instead of the
+# ephemeral per-process secret the in-process broker used.
+LAUNCH_BROKER = TrustDomain(
+    name="launch-broker",
+    key_filename=LAUNCH_BROKER_KEY_FILENAME,
+    roles=frozenset(),
+    key_env=LAUNCH_BROKER_KEY_ENV,
+    token_env="",
+)
+
+# The host controller's own domain (#468) — the SECOND domain #476 reserves. Its
+# key, which the orchestrator never holds, signs the `host`-role tokens the CLI
+# presents on the host controller's lifecycle endpoints (start / restart / status
+# of the orchestrator itself). Keeping it separate from `CONTROL_PLANE` is the
+# whole point: the host controller starts and stops the orchestrator, so the
+# orchestrator must not be able to mint the credentials used to drive it. (The
+# lifecycle endpoints themselves arrive in a later chunk; the domain is
+# established here alongside the durable launch-broker key.)
+HOST_CONTROLLER = TrustDomain(
+    name="host-controller",
+    key_filename=HOST_CONTROLLER_KEY_FILENAME,
+    roles=frozenset({ROLE_HOST}),
+    key_env=HOST_CONTROLLER_KEY_ENV,
+    token_env=HOST_CONTROLLER_AUTH_JWT_ENV,
+)
+
+
 @dataclass(frozen=True)
 class ControlPlaneProvisioning:
     """The one seam every backend launcher uses to provision control-plane auth,
@@ -132,9 +171,56 @@ class ControlPlaneProvisioning:
         return self.domain.mint(ROLE_GATEWAY)
 
 
+@dataclass(frozen=True)
+class LaunchBrokerProvisioning:
+    """The seam that provisions the host-side launch broker's durable keys (#468),
+    the counterpart to `ControlPlaneProvisioning`. Both the orchestrator (signer)
+    and the host control server (verifier) receive the SAME launch-broker key
+    (carry it in `broker_domain.key_env`); the host controller ALSO receives its
+    own lifecycle key (`controller_domain.key_env`) the orchestrator never holds.
+
+    Fail-closed like the control-plane seam: minting returns "" only if the host
+    root is unwritable, and an empty launch-broker key would leave the verifier
+    unable to authenticate any launch — so we raise rather than hand back a key
+    that would make the host controller reject (or, if a caller defaulted it,
+    accept) unsigned input."""
+
+    broker_domain: TrustDomain = LAUNCH_BROKER
+    controller_domain: TrustDomain = HOST_CONTROLLER
+
+    def broker_key(self) -> str:
+        """The durable launch-broker key both the orchestrator and the host
+        control server must receive (in `broker_domain.key_env`). Raises rather
+        than return ""."""
+        key = self.broker_domain.signing_key()
+        if not key:
+            raise ProvisioningError(
+                f"refusing to provision the {self.broker_domain.name} broker "
+                "without a signing key: the host controller could then verify no "
+                "launch request's provenance"
+            )
+        return key
+
+    def controller_key(self) -> str:
+        """The host controller's own lifecycle key — provisioned ONLY to the host
+        controller (in `controller_domain.key_env`), never to the orchestrator, so
+        the orchestrator cannot mint the `host`-role tokens that start and stop
+        it. Raises rather than return ""."""
+        key = self.controller_domain.signing_key()
+        if not key:
+            raise ProvisioningError(
+                f"refusing to provision the {self.controller_domain.name} without "
+                "a signing key: its lifecycle endpoints would authenticate no one"
+            )
+        return key
+
+
 __all__ = [
     "ProvisioningError",
     "TrustDomain",
     "CONTROL_PLANE",
+    "LAUNCH_BROKER",
+    "HOST_CONTROLLER",
     "ControlPlaneProvisioning",
+    "LaunchBrokerProvisioning",
 ]

tests/unit/test_orchestrator_host_server.py

@@ -8,9 +8,12 @@ the full sign -> POST -> verify -> act seam over HTTP.
 from __future__ import annotations
 
 import json
+import os
 import secrets
+import tempfile
 import threading
 import unittest
+from unittest.mock import patch
 
 from bot_bottle.orchestrator.broker import (
     BrokerAuthError,
@@ -21,10 +24,11 @@ from bot_bottle.orchestrator.broker import (
 )
 from bot_bottle.orchestrator.broker_client import BrokerClient
 from bot_bottle.orchestrator.host_server import (
-    broker_secret_from_env,
+    broker_secret,
     dispatch,
     make_host_server,
 )
+from bot_bottle.paths import LAUNCH_BROKER_KEY_ENV
 
 
 def _body(obj: object) -> bytes:
@@ -106,16 +110,24 @@ class TestDispatch(unittest.TestCase):
         self.assertEqual(200, status)
 
 
-class TestBrokerSecretFromEnv(unittest.TestCase):
-    def test_reads_hex_secret(self) -> None:
-        s = secrets.token_bytes(16)
-        self.assertEqual(s, broker_secret_from_env({"BOT_BOTTLE_BROKER_SECRET": s.hex()}))
+class TestBrokerSecret(unittest.TestCase):
+    """The durable launch-broker key (#468/#476): prefer the env-injected key,
+    else the durable host key file, so signer and verifier resolve the same one."""
 
-    def test_unset_is_none(self) -> None:
-        self.assertIsNone(broker_secret_from_env({}))
+    def test_reads_injected_key_from_env(self) -> None:
+        self.assertEqual(
+            b"injected-key", broker_secret({LAUNCH_BROKER_KEY_ENV: "injected-key"}))
 
-    def test_invalid_hex_is_none(self) -> None:
-        self.assertIsNone(broker_secret_from_env({"BOT_BOTTLE_BROKER_SECRET": "not-hex"}))
+    def test_falls_back_to_the_durable_host_key_file(self) -> None:
+        with tempfile.TemporaryDirectory() as root:
+            with patch.dict("os.environ", {"BOT_BOTTLE_ROOT": root}, clear=False):
+                os.environ.pop(LAUNCH_BROKER_KEY_ENV, None)
+                first = broker_secret()
+                second = broker_secret()
+        self.assertTrue(first)
+        # Durable: minted once, the same key on every subsequent call — a
+        # restarted orchestrator re-verifies against it.
+        self.assertEqual(first, second)
 
 
 class TestSeamRoundTrip(unittest.TestCase):

tests/unit/test_trust_domain.py

@@ -6,10 +6,13 @@ import unittest
 from unittest.mock import patch
 
 from bot_bottle import orchestrator_auth
-from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY
+from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY, ROLE_HOST
 from bot_bottle.trust_domain import (
     CONTROL_PLANE,
+    HOST_CONTROLLER,
+    LAUNCH_BROKER,
     ControlPlaneProvisioning,
+    LaunchBrokerProvisioning,
     ProvisioningError,
     TrustDomain,
 )
@@ -101,5 +104,73 @@ class TestControlPlaneProvisioning(unittest.TestCase):
         self.assertNotEqual(ROLE_CLI, CONTROL_PLANE.verify(tok, "k"))
 
 
+class TestLaunchBrokerAndHostControllerDomains(unittest.TestCase):
+    """The real #468 domains: the launch-broker key (shared by orchestrator +
+    host controller) and the host controller's own lifecycle key."""
+
+    def test_launch_broker_mints_no_role_tokens(self) -> None:
+        # Empty role set — it provides durable key material for the broker's own
+        # launch JWT, not orchestrator_auth role tokens.
+        self.assertEqual(frozenset(), LAUNCH_BROKER.roles)
+        with patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
+            with self.assertRaises(ValueError):
+                LAUNCH_BROKER.mint(ROLE_CLI)
+
+    def test_host_controller_signs_host_role_only(self) -> None:
+        with patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
+            tok = HOST_CONTROLLER.mint(ROLE_HOST)
+        self.assertEqual(ROLE_HOST, HOST_CONTROLLER.verify(tok, "k"))
+        # A control-plane `cli` token (the orchestrator's key) never verifies as a
+        # host-controller role — the orchestrator can't forge lifecycle creds.
+        cli_tok = orchestrator_auth.mint(ROLE_CLI, "k")
+        self.assertIsNone(HOST_CONTROLLER.verify(cli_tok, "k"))
+
+    def test_control_plane_cannot_mint_the_host_role(self) -> None:
+        # `host` is outside the control-plane role set on purpose.
+        with patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
+            with self.assertRaises(ValueError):
+                CONTROL_PLANE.mint(ROLE_HOST)
+
+    def test_the_three_domains_use_distinct_keys_and_env_vars(self) -> None:
+        self.assertEqual(3, len({
+            CONTROL_PLANE.key_filename,
+            LAUNCH_BROKER.key_filename,
+            HOST_CONTROLLER.key_filename,
+        }))
+        self.assertEqual(3, len({
+            CONTROL_PLANE.key_env, LAUNCH_BROKER.key_env, HOST_CONTROLLER.key_env,
+        }))
+
+
+class TestLaunchBrokerProvisioning(unittest.TestCase):
+    def test_broker_key_returns_the_durable_key(self) -> None:
+        prov = LaunchBrokerProvisioning()
+        with patch("bot_bottle.trust_domain.host_signing_key", return_value="bk"):
+            self.assertEqual("bk", prov.broker_key())
+
+    def test_broker_key_fail_closes_when_empty(self) -> None:
+        # An empty key would leave the host controller unable to verify any
+        # launch — fail-closed rather than hand back a useless/dangerous key.
+        prov = LaunchBrokerProvisioning()
+        with patch("bot_bottle.trust_domain.host_signing_key", return_value=""):
+            with self.assertRaises(ProvisioningError):
+                prov.broker_key()
+
+    def test_controller_key_is_distinct_from_the_broker_key(self) -> None:
+        # The orchestrator holds the broker key but NEVER the controller key.
+        prov = LaunchBrokerProvisioning()
+        keys = {"launch-broker-key": "bk", "host-controller-key": "ck"}
+        with patch("bot_bottle.trust_domain.host_signing_key",
+                   side_effect=lambda fn: keys[fn]):
+            self.assertEqual("bk", prov.broker_key())
+            self.assertEqual("ck", prov.controller_key())
+
+    def test_controller_key_fail_closes_when_empty(self) -> None:
+        prov = LaunchBrokerProvisioning()
+        with patch("bot_bottle.trust_domain.host_signing_key", return_value=""):
+            with self.assertRaises(ProvisioningError):
+                prov.controller_key()
+
+
 if __name__ == "__main__":
     unittest.main()