feat(orchestrator): durable launch-broker secret via TrustDomain (#468)

Chunk 2 of the host-control-server stack: close the PRD's **durable
secret** gap and replace chunk 1's BOT_BOTTLE_BROKER_SECRET stopgap.

- trust_domain.py: two new domains. LAUNCH_BROKER holds the durable
  HS256 key both the orchestrator (signer) and the host control server
  (verifier) share for the broker's launch JWT — a host-canonical key
  file minted 0600 on first use, so a restarted orchestrator re-verifies
  against the same key. HOST_CONTROLLER is the separate domain for the
  controller's own lifecycle endpoints, keyed by a key the orchestrator
  never holds (its role is `host`, deliberately outside control-plane
  ROLES). LaunchBrokerProvisioning is the fail-closed seam.
- orchestrator_auth.py: ROLE_HOST, outside ROLES.
- paths.py: key-file + env-var constants for both domains.

Key resolution is split by owner (addresses codex review on #497):
broker_secret(allow_host_file=...) — the host controller / dev-harness
(True) may mint/read the durable host key file it owns; the GUEST
orchestrator (--broker http, default False) must be *injected* the key
and fails closed if it isn't. A guest that fell back to the host file
would mint a process-local key unrelated to the host controller's, so
startup would succeed but every launch would 401 — this prevents that
silent divergence.

Tested: domain boundary + separation, provisioning fail-closed, and
broker_secret env-only (guest) vs host-file (host) resolution. pyright
clean; pylint 9.86.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

bot_bottle/orchestrator/__main__.py

@@ -17,9 +17,10 @@ from pathlib import Path
 
 from .. import log
 from .store.store_manager import StoreManager
+from ..paths import LAUNCH_BROKER_KEY_ENV
 from .broker import StubBroker, SubmitBroker
 from .broker_client import BrokerClient
-from .host_server import BROKER_SECRET_ENV, DEFAULT_PORT, broker_secret_from_env
+from .host_server import DEFAULT_PORT, broker_secret
 from .server import make_server
 from .docker_broker import DockerBroker
 from .store.registry_store import RegistryStore, default_db_path
@@ -57,17 +58,19 @@ def main(argv: list[str] | None = None) -> int:
     # A signing secret ties the orchestrator (signer) to its broker (verifier).
     # 'stub' records launches instead of starting anything; 'docker' runs real
     # containers in-process; 'http' relays signed requests to a separate host
-    # control server, which verifies and launches. For 'stub'/'docker' the
-    # secret is ephemeral (signer and verifier share this process); for 'http'
-    # it must be the SAME secret the host controller holds, so it is read from
-    # the shared env var (the chunk-1 stand-in for out-of-band provisioning).
+    # control server, which verifies and launches. For 'stub'/'docker' the secret
+    # is ephemeral (signer and verifier share this process). For 'http' it must be
+    # the SAME key the host controller holds — and this process is the *guest*
+    # (signer), so it must be given that key by injection, NOT mint its own
+    # process-local one (which would diverge from the host's and 401 every launch).
     broker: SubmitBroker
     if args.broker == "http":
-        secret = broker_secret_from_env()
+        secret = broker_secret()  # env-injected only; no host-file fallback here
         if secret is None:
             parser.error(
-                f"--broker http requires a shared signing secret in "
-                f"${BROKER_SECRET_ENV} (hex), matching the host control server"
+                f"--broker http requires the launch-broker key injected as "
+                f"${LAUNCH_BROKER_KEY_ENV} (the host controller owns/mints it); the "
+                "orchestrator must not mint its own or it would diverge from the host's"
             )
         broker = BrokerClient(args.host_controller_url)
     else:

bot_bottle/orchestrator/host_server.py

@@ -23,12 +23,15 @@ controller down.
 
 The signed launch token *is* the endpoint's authentication (its provenance is the
 whole point of the JWS), so `/broker` needs no separate caller credential; the
-host controller's own lifecycle endpoints, which do, arrive with the durable
-`TrustDomain` key in a later chunk.
+host controller's own lifecycle endpoints, which do, arrive with the `host`-role
+tokens of the separate `HOST_CONTROLLER` trust domain in a later chunk.
 
-The shared signing secret is read from `$BOT_BOTTLE_BROKER_SECRET` (hex). That is
-a **chunk-1 stopgap**: it must be provisioned to signer and verifier out of band,
-which is exactly what the durable `TrustDomain` key in chunk 2 (#476) replaces.
+The shared signing secret is the durable **launch-broker `TrustDomain` key**
+(#468/#476): a host-canonical key file minted 0600 on first use, provisioned to
+the orchestrator (signer) and this server (verifier). A backend launcher injects
+it via `$BOT_BOTTLE_LAUNCH_BROKER_KEY`; a host-side dev-harness process reads the
+key file directly. Durability is the point — a restarted orchestrator re-verifies
+against the same key, so re-adoption works.
 """
 
 from __future__ import annotations
@@ -43,16 +46,14 @@ import typing
 from urllib.parse import urlsplit
 
 from .. import log
+from ..paths import LAUNCH_BROKER_KEY_ENV
+from ..trust_domain import LAUNCH_BROKER
 from .broker import BrokerAuthError, LaunchBroker
 from .docker_broker import DockerBroker
 
 # JSON body payload type (parsed request / rendered response).
 Json = dict[str, object]
 
-# The hex-encoded HS256 secret shared with the request signer (the orchestrator).
-# Chunk-1 stopgap for the durable, out-of-band `TrustDomain` key of chunk 2.
-BROKER_SECRET_ENV = "BOT_BOTTLE_BROKER_SECRET"
-
 # Default host-controller port. Distinct from the orchestrator control plane
 # (8099) — a separate privileged component listening on its own socket.
 DEFAULT_PORT = 8091
@@ -78,19 +79,34 @@ def _parse_json_object(body: bytes) -> Json:
     return obj
 
 
-def broker_secret_from_env(environ: typing.Mapping[str, str] | None = None) -> bytes | None:
-    """The shared HS256 secret from `$BOT_BOTTLE_BROKER_SECRET` (hex), or None
-    when unset or not valid hex. The signer (orchestrator, `--broker http`) and
-    the verifier (this server) read the same env var so both hold the same key —
-    the chunk-1 stand-in for out-of-band provisioning."""
-    env = os.environ if environ is None else environ
-    raw = env.get(BROKER_SECRET_ENV, "").strip()
-    if not raw:
-        return None
-    try:
-        return bytes.fromhex(raw)
-    except ValueError:
-        return None
+def broker_secret(
+    environ: typing.Mapping[str, str] | None = None, *, allow_host_file: bool = False,
+) -> bytes | None:
+    """The shared launch-broker HS256 secret, as this process should use it.
+
+    Always prefers the key injected into this process's env
+    (`$BOT_BOTTLE_LAUNCH_BROKER_KEY`). `allow_host_file` decides the fallback when
+    it is absent, and the distinction is a security boundary:
+
+    - **Host-side** processes — the host controller and the host dev-harness — pass
+      ``allow_host_file=True`` to read (minting on first use) the durable host key
+      file (``bot_bottle_root()/launch-broker-key``) they legitimately own.
+    - The **guest orchestrator** (``--broker http``) keeps the default ``False``.
+      It runs inside a container/VM whose ``bot_bottle_root()`` is process-local,
+      so minting a file there would silently create a key UNRELATED to the host
+      controller's — startup would succeed but every launch would be rejected 401.
+      It must instead be *given* the key by its launcher, and fail closed (None)
+      if it wasn't, rather than diverge.
+
+    None when no key is available (a guest with no injection, or an unwritable
+    host root)."""
+    key = LAUNCH_BROKER.key_from_env(environ)
+    if not key and allow_host_file:
+        try:
+            key = LAUNCH_BROKER.signing_key()  # host-canonical, minted on first use
+        except OSError:
+            return None
+    return key.encode("utf-8") if key else None
 
 
 def dispatch(  # pylint: disable=too-many-return-statements
@@ -219,20 +235,22 @@ def main(argv: list[str] | None = None) -> int:
 
         python -m bot_bottle.orchestrator.host_server [--host H] [--port P]
 
-    Fail-closed: without a shared `$BOT_BOTTLE_BROKER_SECRET` the server can
-    verify no request's provenance, so it refuses to start rather than run a
-    launcher that accepts unsigned input."""
+    Fail-closed: without the launch-broker key the server can verify no request's
+    provenance, so it refuses to start rather than run a launcher that accepts
+    unsigned input. As the host-side owner of the key, it may mint/read the host
+    key file (`allow_host_file=True`)."""
     parser = argparse.ArgumentParser(prog="bot_bottle.orchestrator.host_server")
     parser.add_argument("--host", default="127.0.0.1", help="bind address")
     parser.add_argument("--port", type=int, default=DEFAULT_PORT, help="bind port (0 = ephemeral)")
     args = parser.parse_args(argv)
 
-    secret = broker_secret_from_env()
+    secret = broker_secret(allow_host_file=True)
     if secret is None:
         sys.stderr.write(
-            f"host controller: refusing to start without a shared signing secret "
-            f"(${BROKER_SECRET_ENV}, hex) — it could verify no request's "
-            "provenance and would relay unsigned launches to the backend\n"
+            f"host controller: refusing to start without the launch-broker key "
+            f"(${LAUNCH_BROKER_KEY_ENV}, or a writable host root to mint it) — it "
+            "could verify no request's provenance and would relay unsigned "
+            "launches to the backend\n"
         )
         sys.stderr.flush()
         return 2
@@ -258,10 +276,9 @@ __all__ = [
     "Handler",
     "HostControlServer",
     "make_host_server",
-    "broker_secret_from_env",
+    "broker_secret",
     "main",
     "Json",
-    "BROKER_SECRET_ENV",
     "DEFAULT_PORT",
 ]