fix(firecracker): keep the host key canonical instead of clobbering the host token file

Reimplements the dropped firecracker fix on the refactored code, addressing
didericis-codex's blocking review on #471.

The previous firecracker path let the VM generate its own signing key on the
guest volume and had `_with_signing_key()` unconditionally overwrite the single
host-wide orchestrator-token with it. That breaks a co-running Docker/macOS
control plane: their orchestrators still verify with the old key while new host
clients start signing `cli` tokens with the guest key, so
supervise/teardown/policy calls against those backends begin returning 401
(cross-backend operation is supported).

Keep the host token file the single source of truth. The launcher now pushes the
host-canonical key into the freshly booted infra VM over SSH (atomic write,
mirroring persist_env_var_secret) via `_push_signing_key()`; the VM's init waits
for it and refuses to start the control plane — rather than run OPEN — if it
never arrives. Nothing writes back to the host file, so other backends are
untouched, and the best-effort silent path is gone (both the push and the guest
fail loudly).

Ported onto the reorg'd names (host_orchestrator_token, BOT_BOTTLE_ORCHESTRATOR_
TOKEN, orchestrator-token, bot_bottle.orchestrator_auth, bot_bottle.gateway.
bootstrap). pyright: 0 errors. Full unit suite green (2243).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

bot_bottle/backend/firecracker/infra_vm.py

@@ -33,15 +33,17 @@ from pathlib import Path
 from typing import Generator
 
 from ...log import die, info
-from ...paths import ORCHESTRATOR_TOKEN_FILENAME, bot_bottle_root
+from ...paths import host_orchestrator_token
 from .. import util as backend_util
 from ..docker import util as docker_mod
 from ..docker.gateway_provision import GatewayProvisionError
 from . import firecracker_vm, infra_artifact, netpool, util
 
-# Where the infra VM keeps its control-plane signing key (generated on the
+# The infra VM's control-plane signing-key path on its persistent /dev/vdb
-# persistent /dev/vdb volume mounted at BOT_BOTTLE_ROOT). The host mirrors it
+# volume (mounted at BOT_BOTTLE_ROOT=/var/lib/bot-bottle). The launcher seeds
-# back so the CLI signs `cli` tokens the VM verifies (issue #469 review).
+# this file with the host-canonical key BEFORE boot, so the VM verifies tokens
+# with the same key the host CLI signs with — the host token file stays the
+# single source of truth, never clobbered per-backend (issue #469 review).
 _GUEST_SIGNING_KEY_PATH = "/var/lib/bot-bottle/orchestrator-token"
 
 # The single infra-VM image: gateway data plane + baked control-plane source
@@ -70,6 +72,10 @@ _INFRA_RESOLVER = "1.1.1.1"
 _HEALTH_TIMEOUT_SECONDS = 45.0
 _HEALTH_POLL_SECONDS = 0.5
 _CA_TIMEOUT_SECONDS = 30.0
+# How long the launcher retries pushing the signing key while the guest's SSH
+# comes up. Below the init's own wait window, so a failed push dies here first.
+_SIGNING_KEY_PUSH_TIMEOUT_SECONDS = 30.0
+_SIGNING_KEY_PUSH_POLL_SECONDS = 0.5
 
 
 @dataclass
@@ -173,45 +179,21 @@ def ensure_running() -> InfraVm:
     want = _expected_version()
     if _adoptable(key, url, want):
         info(f"adopting running infra VM at {url}")
-        return _with_signing_key(InfraVm(guest_ip=slot.guest_ip, private_key=key))
+        return InfraVm(guest_ip=slot.guest_ip, private_key=key)
 
     with _singleton_lock():
         # Re-check under the lock: another launcher may have booted it while
         # we waited for the lock (double-checked, so we adopt not re-boot).
         if _adoptable(key, url, want):
             info(f"adopting running infra VM at {url}")
-            return _with_signing_key(InfraVm(guest_ip=slot.guest_ip, private_key=key))
+            return InfraVm(guest_ip=slot.guest_ip, private_key=key)
         # Clear a stale/hung/OUTDATED VM holding the link before booting fresh.
         stop()
         ensure_built()
         infra = boot()
         wait_for_health(infra)
         _record_booted_version(want)
-        return _with_signing_key(infra)
-
-
-def _with_signing_key(infra: InfraVm) -> InfraVm:
-    """Mirror the infra VM's control-plane signing key (generated on its
-    persistent volume) into the host's orchestrator-token file, so the host CLI
-    signs `cli` tokens the VM verifies (issue #469 review). Best-effort: an
-    unreadable key is logged, not fatal — the VM still enforces auth, but the CLI
-    may then be rejected until the key is readable. Returns `infra` for chaining."""
-    proc = subprocess.run(
-        util.ssh_base_argv(infra.private_key, infra.guest_ip)
-        + [f"cat {_GUEST_SIGNING_KEY_PATH}"],
-        capture_output=True, text=True, check=False,
-    )
-    signing_key = proc.stdout.strip()
-    if proc.returncode != 0 or not signing_key:
-        info("infra signing key not yet readable; control-plane auth may fail")
         return infra
-    path = bot_bottle_root() / ORCHESTRATOR_TOKEN_FILENAME
-    path.parent.mkdir(parents=True, exist_ok=True)
-    fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
-    with os.fdopen(fd, "w") as f:
-        f.write(signing_key)
-    os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)
-    return infra
 
 
 @contextmanager
@@ -267,7 +249,13 @@ def boot() -> InfraVm:
         data_drive=_ensure_registry_volume(),
     )
     _pid_file().write_text(str(vm.process.pid))
-    return InfraVm(guest_ip=slot.guest_ip, private_key=private_key, vm=vm)
+    infra = InfraVm(guest_ip=slot.guest_ip, private_key=private_key, vm=vm)
+    # Push the host-canonical control-plane signing key into the guest over SSH
+    # (the init waits for it before starting the control plane). The host token
+    # file stays the single source of truth, so a co-running Docker/macOS control
+    # plane keeps working (issue #469 review).
+    _push_signing_key(infra)
+    return infra
 
 
 def _infra_dir() -> Path:
@@ -339,6 +327,33 @@ def _ensure_registry_volume() -> Path:
     return vol
 
 
+def _push_signing_key(infra: InfraVm) -> None:
+    """Push the host-canonical control-plane signing key into the freshly booted
+    infra VM over SSH (atomic write), so its orchestrator verifies tokens with
+    the same key the host CLI signs from. Mirrors the existing
+    `persist_env_var_secret` transport. Retries until the guest's SSH is up (it
+    comes up before the init waits for this file); dies if it never lands, since
+    the VM then refuses to start its control plane rather than run OPEN."""
+    key = host_orchestrator_token()
+    push = (
+        f"umask 077; cat > {_GUEST_SIGNING_KEY_PATH}.tmp "
+        f"&& mv {_GUEST_SIGNING_KEY_PATH}.tmp {_GUEST_SIGNING_KEY_PATH}"
+    )
+    deadline = time.monotonic() + _SIGNING_KEY_PUSH_TIMEOUT_SECONDS
+    last = ""
+    while time.monotonic() < deadline:
+        proc = subprocess.run(
+            util.ssh_base_argv(infra.private_key, infra.guest_ip) + [push],
+            input=key, capture_output=True, text=True, check=False,
+        )
+        if proc.returncode == 0:
+            return
+        last = proc.stderr.strip()
+        time.sleep(_SIGNING_KEY_PUSH_POLL_SECONDS)
+    die("could not push the control-plane signing key to the infra VM "
+        f"(its control plane will not start): {last or '<no stderr>'}")
+
+
 def _stable_keypair() -> tuple[Path, str]:
     """The infra VM's SSH keypair — generated once and reused, so any later
     launcher can SSH in (fetch CA / provision) even though a different process
@@ -517,32 +532,43 @@ mount -t ext4 /dev/vdb /var/lib/bot-bottle 2>/dev/null || true
 
 # Control plane. Source is baked at /app; the package is stdlib-only.
 cd /app
-# Control-plane signing key + a pre-minted `gateway` JWT, scoped per-process
+# Wait for the launcher to push the host-canonical control-plane signing key
-# (issue #469 review). The key is generated once on the persistent volume and
+# over SSH, then hand it ONLY to the orchestrator (to verify tokens); the
-# handed ONLY to the orchestrator (to verify tokens); the data-plane daemons get
+# data-plane daemons get a pre-minted `gateway` JWT they present, never the key.
-# the `gateway` JWT they present, never the key. Without this the control plane
+# If it never arrives, REFUSE to start the control plane rather than run OPEN —
-# would run OPEN and a compromised egress / supervise / git-http daemon in this
+# open mode would grant every unauthenticated caller the `cli` role, and a
-# same VM — reaching the orchestrator over 127.0.0.1, past the nft boundary that
+# compromised egress / supervise / git-http daemon in this same VM (reaching the
-# only fences off the separate agent VM — could drive the operator routes
+# orchestrator over 127.0.0.1, past the nft boundary that only fences off the
-# (approve its own supervise proposals, rewrite policy, read injected tokens).
+# separate agent VM) could then drive the operator routes (issue #469).
-CP_KEY=$(BOT_BOTTLE_ROOT=/var/lib/bot-bottle python3 -c 'from bot_bottle.paths import host_orchestrator_token as t; print(t())')
+CP_KEY=""
-GW_JWT=$(BB_SIGNING_KEY="$CP_KEY" python3 -c 'import os; from bot_bottle.orchestrator_auth import mint, ROLE_GATEWAY; print(mint(ROLE_GATEWAY, os.environ["BB_SIGNING_KEY"]))')
+i=0
+while [ "$i" -lt 600 ]; do
+    CP_KEY=$(cat /var/lib/bot-bottle/orchestrator-token 2>/dev/null)
+    [ -n "$CP_KEY" ] && break
+    i=$((i + 1))
+    sleep 0.1
+done
+if [ -z "$CP_KEY" ]; then
+    echo "infra: control-plane signing key never arrived; refusing to start the control plane (would run OPEN)" >&2
+else
+    chmod 600 /var/lib/bot-bottle/orchestrator-token 2>/dev/null || true
+    GW_JWT=$(BB_SIGNING_KEY="$CP_KEY" python3 -c 'import os; from bot_bottle.orchestrator_auth import mint, ROLE_GATEWAY; print(mint(ROLE_GATEWAY, os.environ["BB_SIGNING_KEY"]))')
+    BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_ORCHESTRATOR_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator \\
+        --host 0.0.0.0 --port {ORCHESTRATOR_PORT} --broker stub &
 
-BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_ORCHESTRATOR_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator \\
+    # Gateway data plane, multi-tenant: each request resolves source-IP ->
-    --host 0.0.0.0 --port {ORCHESTRATOR_PORT} --broker stub &
+    # policy against the local control plane. The VM backend reaches git over
-
+    # git-http (9420), so the git:// daemon (git-gate, needs a per-bottle
-# Gateway data plane, multi-tenant: each request resolves source-IP ->
+    # entrypoint the consolidated model doesn't use) is left out. No
-# policy against the local control plane. The VM backend reaches git over
+    # SUPERVISE_DB_PATH: the data plane reaches the supervise queue over the
-# git-http (9420), so the git:// daemon (git-gate, needs a per-bottle
+    # control-plane RPC and never opens bot-bottle.db (PRD 0070 / #469). It
-# entrypoint the consolidated model doesn't use) is left out. No
+    # presents the pre-minted `gateway` JWT; gateway_init keeps the signing key
-# SUPERVISE_DB_PATH: the data plane reaches the supervise queue over the
+    # out of the data-plane daemons' env.
-# control-plane RPC and never opens bot-bottle.db (PRD 0070 / #469). It presents
+    BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise \\
-# the pre-minted `gateway` JWT; gateway_init keeps the signing key out of the
+    BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{ORCHESTRATOR_PORT} \\
-# data-plane daemons' env.
+    BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT="$GW_JWT" \\
-BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise \\
+        python3 -m bot_bottle.gateway.bootstrap &
-BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{ORCHESTRATOR_PORT} \\
+fi
-BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT="$GW_JWT" \\
-    python3 -m bot_bottle.gateway.bootstrap &
 
 # Reap as PID 1; children are backgrounded, so `wait` blocks.
 while : ; do wait ; done

tests/unit/test_firecracker_infra_vm.py

@@ -8,7 +8,6 @@ decisions that must hold without a VM.
 from __future__ import annotations
 
 import os
-import tempfile
 import unittest
 from pathlib import Path
 from unittest.mock import MagicMock, patch
@@ -16,35 +15,36 @@ from unittest.mock import MagicMock, patch
 from bot_bottle.backend.firecracker import infra_vm
 
 
-class TestSigningKeySync(unittest.TestCase):
+class TestPushSigningKey(unittest.TestCase):
-    """`_with_signing_key` mirrors the VM's control-plane signing key into the
+    """`_push_signing_key` pushes the host-canonical key into the guest over SSH
-    host token file so the CLI signs `cli` tokens the VM verifies (issue #469)."""
+    (the init waits for it). The host token file stays the single source of
+    truth, never clobbered per-backend (issue #469)."""
 
     def _infra(self) -> infra_vm.InfraVm:
         return infra_vm.InfraVm(guest_ip="10.0.0.1", private_key=Path("/k"))
 
-    def test_mirrors_guest_key_to_host_file(self):
+    def test_writes_host_key_over_ssh_atomically(self):
-        with tempfile.TemporaryDirectory() as d:
+        proc = MagicMock(returncode=0, stderr="")
-            root = Path(d)
+        with patch.object(infra_vm, "host_orchestrator_token", return_value="host-key"), \
-            proc = MagicMock(returncode=0, stdout="the-signing-key\n")
+                patch.object(infra_vm.subprocess, "run", return_value=proc) as run:
-            with patch.object(infra_vm.subprocess, "run", return_value=proc) as run, \
+            infra_vm._push_signing_key(self._infra())
-                    patch.object(infra_vm, "bot_bottle_root", return_value=root):
+        # Piped the key to an atomic write of the guest token path.
-                out = infra_vm._with_signing_key(self._infra())
+        self.assertEqual("host-key", run.call_args.kwargs["input"])
-            self.assertIsInstance(out, infra_vm.InfraVm)  # returned for chaining
+        remote_cmd = run.call_args.args[0][-1]
-            token = root / infra_vm.ORCHESTRATOR_TOKEN_FILENAME
+        self.assertIn(f"cat > {infra_vm._GUEST_SIGNING_KEY_PATH}.tmp", remote_cmd)
-            self.assertEqual("the-signing-key", token.read_text())
+        self.assertIn(f"mv {infra_vm._GUEST_SIGNING_KEY_PATH}.tmp "
-            self.assertEqual(0o600, token.stat().st_mode & 0o777)
+                      f"{infra_vm._GUEST_SIGNING_KEY_PATH}", remote_cmd)
-            # It cat'd the guest volume path over SSH.
-            self.assertIn(infra_vm._GUEST_SIGNING_KEY_PATH, run.call_args.args[0][-1])
 
-    def test_unreadable_key_is_not_fatal(self):
+    def test_dies_when_push_never_succeeds(self):
-        with tempfile.TemporaryDirectory() as d:
+        proc = MagicMock(returncode=255, stderr="ssh: connect refused")
-            root = Path(d)
+        with patch.object(infra_vm, "host_orchestrator_token", return_value="host-key"), \
-            proc = MagicMock(returncode=255, stdout="")
+                patch.object(infra_vm, "_SIGNING_KEY_PUSH_TIMEOUT_SECONDS", 0.05), \
-            with patch.object(infra_vm.subprocess, "run", return_value=proc), \
+                patch.object(infra_vm, "_SIGNING_KEY_PUSH_POLL_SECONDS", 0.0), \
-                    patch.object(infra_vm, "bot_bottle_root", return_value=root):
+                patch.object(infra_vm.subprocess, "run", return_value=proc), \
-                infra_vm._with_signing_key(self._infra())  # no raise
+                patch.object(infra_vm, "die", side_effect=SystemExit) as die:
-            self.assertFalse((root / infra_vm.ORCHESTRATOR_TOKEN_FILENAME).exists())
+            with self.assertRaises(SystemExit):
+                infra_vm._push_signing_key(self._infra())
+        die.assert_called_once()
 
 
 class TestOrchestratorUrl(unittest.TestCase):
@@ -79,10 +79,11 @@ class TestBuildInfraRootfs(unittest.TestCase):
         # VM backend uses git-http (9420); the git:// daemon is left out.
         self.assertIn("BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise", init)
         # Role-scoped control-plane auth (issue #469 review): the orchestrator
-        # gets the signing key, the gateway daemons get a pre-minted `gateway`
+        # gets the host-seeded signing key, the gateway daemons get a pre-minted
-        # JWT — never open mode in the infra VM.
+        # `gateway` JWT, and the VM refuses to run OPEN if the key is missing.
-        self.assertIn("host_orchestrator_token", init)  # key generated on the volume
+        self.assertIn("cat /var/lib/bot-bottle/orchestrator-token", init)  # host-seeded key
-        self.assertIn("mint, ROLE_GATEWAY", init)         # gateway JWT minted from it
+        self.assertIn("refusing to start the control plane", init)          # no open mode
+        self.assertIn("mint, ROLE_GATEWAY", init)                           # gateway JWT
         self.assertIn('BOT_BOTTLE_ORCHESTRATOR_TOKEN="$CP_KEY" python3 -m bot_bottle.orchestrator',
                       init)  # key -> orchestrator only
         self.assertIn('BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT="$GW_JWT"', init)  # JWT -> gateway daemons