fix(secrets): recover encrypted tokens on all backends

bot_bottle/backend/firecracker/bottle_plan.py

@@ -22,6 +22,9 @@ class FirecrackerBottlePlan(BottlePlan):
     # (egress proxy credentials, git-gate/supervise headers); set by launch
     # from the orchestrator registration. Empty pre-registration.
     identity_token: str = ""
+    # Applied to every agent SSH exec and mirrored into /run inside the VM so
+    # the host can recover it after the infra VM restarts.
+    env_var_secret: str = ""
 
     @property
     def container_name(self) -> str:

bot_bottle/backend/firecracker/cleanup.py

@@ -88,6 +88,12 @@ def _scan_processes(run_root: Path) -> tuple[set[str], list[int]]:
     return live, orphan_pids
 
 
+def live_run_dirs() -> tuple[Path, ...]:
+    """Run directories backed by currently running agent microVMs."""
+    live, _ = _scan_processes(_run_root())
+    return tuple(Path(path) for path in sorted(live))
+
+
 def _orphan_run_dirs(run_root: Path, live: set[str]) -> list[str]:
     """Run dirs with no live VM behind them — the leaked ones to remove."""
     if not run_root.is_dir():

bot_bottle/backend/firecracker/consolidated_launch.py

@@ -25,16 +25,27 @@ The TAP slot allocation, rootfs build, and VM boot are the caller's job.
 
 from __future__ import annotations
 
+import json
+import subprocess
 from dataclasses import dataclass
+from pathlib import Path
 
 from ...egress import EgressPlan
 from ...git_gate import GitGatePlan
-from ...orchestrator.client import OrchestratorClient
+from ...log import info
+from ...orchestrator.client import OrchestratorClient, OrchestratorClientError
 from ...orchestrator.lifecycle import (
     OrchestratorStartError,  # re-exported so callers can catch it
 )
-from ..consolidated_util import provision_bottle, teardown_consolidated as _teardown_util
-from . import infra_vm
+from ...orchestrator.reprovision import reprovision_bottles
+from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME
+from ..consolidated_util import (
+    provision_bottle,
+    teardown_consolidated as _teardown_util,
+)
+from . import cleanup, infra_vm, util
+
+_ENV_VAR_SECRET_PATH = "/run/bot-bottle/env-var-secret"
 
 
 class ConsolidatedLaunchError(RuntimeError):
@@ -53,6 +64,54 @@ class LaunchContext:
     env_var_secret: str = ""  # encryption key injected into the agent's env
 
 
+def _guest_ip_from_config(config_path: Path) -> str:
+    """Read the kernel's configured guest IP from a Firecracker config."""
+    try:
+        config = json.loads(config_path.read_text())
+        args = config["boot-source"]["boot_args"]
+        ip_arg = next(part for part in args.split() if part.startswith("ip="))
+        return ip_arg.removeprefix("ip=").split(":", 1)[0]
+    except (OSError, ValueError, KeyError, TypeError, StopIteration):
+        return ""
+
+
+def persist_env_var_secret(private_key: Path, guest_ip: str, secret: str) -> None:
+    """Mirror the exec-time key into guest tmpfs for restart recovery."""
+    proc = subprocess.run(
+        util.ssh_base_argv(private_key, guest_ip)
+        + [f"umask 077; mkdir -p /run/bot-bottle; cat > {_ENV_VAR_SECRET_PATH}"],
+        input=secret, capture_output=True, text=True, check=False,
+    )
+    if proc.returncode != 0:
+        raise ConsolidatedLaunchError(
+            f"failed to persist {ENV_VAR_SECRET_NAME} in agent VM: "
+            f"{proc.stderr.strip() or '<no stderr>'}"
+        )
+
+
+def _reprovision_running_bottles(client: OrchestratorClient) -> None:
+    """Read keys from live agent VMs and restore the restarted gateway."""
+    try:
+        secrets_by_ip: dict[str, str] = {}
+        for run_dir in cleanup.live_run_dirs():
+            guest_ip = _guest_ip_from_config(run_dir / "config.json")
+            private_key = run_dir / "bottle_id_ed25519"
+            if not guest_ip or not private_key.is_file():
+                continue
+            proc = subprocess.run(
+                util.ssh_base_argv(private_key, guest_ip)
+                + [f"cat {_ENV_VAR_SECRET_PATH}"],
+                capture_output=True, text=True, check=False,
+            )
+            if proc.returncode == 0 and proc.stdout.strip():
+                secrets_by_ip[guest_ip] = proc.stdout.strip()
+        count = reprovision_bottles(client, secrets_by_ip)
+        if count:
+            info(f"reprovisioned egress tokens for {count} Firecracker bottle(s)")
+    except (OSError, OrchestratorClientError) as exc:
+        info(f"egress token reprovision skipped: {exc}")
+
+
 def launch_consolidated(
     egress_plan: EgressPlan,
     git_gate_plan: GitGatePlan,
@@ -67,6 +126,7 @@ def launch_consolidated(
     infra = infra_vm.ensure_running()
     url = infra.control_plane_url
     client = OrchestratorClient(url)
+    _reprovision_running_bottles(client)
 
     transport = infra_vm.gateway_transport()
     reg = provision_bottle(

bot_bottle/backend/firecracker/launch.py

@@ -55,8 +55,10 @@ from . import firecracker_vm, image_builder, isolation_probe, netpool, util
 from .bottle import FirecrackerBottle
 from .bottle_plan import FirecrackerBottlePlan
 from ...orchestrator.config_store import resolve_teardown_timeout
+from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME
 from .consolidated_launch import (
     launch_consolidated,
+    persist_env_var_secret,
     teardown_consolidated,
 )
 
@@ -153,6 +155,7 @@ def launch(
             git_gate_plan=git_gate_plan,
             egress_plan=egress_plan,
             identity_token=ctx.identity_token,
+            env_var_secret=ctx.env_var_secret,
             # Deliver the identity token as egress proxy credentials — clients
             # honor `HTTPS_PROXY=http://id:token@gw` without app changes; the
             # gateway reads Proxy-Authorization, validates the (source_ip,
@@ -187,6 +190,7 @@ def launch(
         )
         stack.callback(vm.terminate)
         firecracker_vm.wait_for_ssh(vm, private_key)
+        persist_env_var_secret(private_key, slot.guest_ip, ctx.env_var_secret)
 
         # Authoritative fail-closed egress-boundary check, before the agent
         # runs: prove the VM cannot reach the host directly.
@@ -281,6 +285,8 @@ def _agent_guest_env(plan: FirecrackerBottlePlan, host_ip: str) -> dict[str, str
         env["GIT_GATE_URL"] = plan.agent_git_gate_url
     if plan.agent_supervise_url:
         env["MCP_SUPERVISE_URL"] = plan.agent_supervise_url
+    if plan.env_var_secret:
+        env[ENV_VAR_SECRET_NAME] = plan.env_var_secret
     for entry in egress_agent_env_entries(plan.egress_plan):
         key, _, value = entry.partition("=")
         env[key] = value

bot_bottle/backend/firecracker/util.py

@@ -399,6 +399,9 @@ fi
 chown -R 0:0 /root 2>/dev/null || true
 
 mkdir -p /etc/dropbear /run
+# Keep restart-recovery key material memory-backed, separate from both the
+# agent rootfs and the infra VM's persistent registry volume.
+mount -t tmpfs -o mode=0755 tmpfs /run 2>/dev/null || true
 # -R: generate host keys on demand. -E: log auth failures to stderr,
 # captured in the host-side console.log for debugging.
 /bb-dropbear -R -E -p 22 &