feat(secrets): encrypt egress tokens at rest with per-bottle ENV_VAR_SECRET

Implements the interim secret-provider design (PRD prd-new-secret-provider):
each agent receives a random ENV_VAR_SECRET injected into its container env
at launch. The host uses this key to encrypt each egress auth token value
(HMAC-SHA256 CTR mode, stdlib-only) and store it in a new
bottled_agent_secrets table (one row per env var, key column plaintext for
auditing). The key never touches the DB.

On infra container restart the in-memory token map is lost. launch_consolidated
now calls _reprovision_running_bottles after ensure_running: for each
registered bottle still alive on the gateway network it execs
`printenv ENV_VAR_SECRET` into the agent container and posts the result to the
new POST /bottles/<id>/reprovision_gateway control-plane endpoint, which
decrypts the stored rows and restores _tokens — no manual intervention needed.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

.gitea/workflows/test.yml

@@ -1,5 +1,4 @@
-# Run the project's test suite when package or runtime inputs change on a PR
+# Run the project's test suite on every PR push and on push to main.
-# or on push to main.
 #
 # The suite uses stdlib `unittest` discovery — no external Python
 # dependencies are required to execute it. Tests are split by directory:
@@ -24,34 +23,22 @@ on:
     branches:
       - main
     paths:
-      - 'bot_bottle/**'
+      - '**.py'
-      - 'tests/**/*.py'
+      - '.gitea/workflows/**.yml'
-      - 'cli.py'
+      - 'scripts/**'
-      - 'scripts/coverage.sh'
+      - 'README.md'
-      - 'scripts/critical-modules.txt'
+      # Dockerfiles and pyproject.toml are baked into the infra rootfs; a
-      - 'scripts/diff_coverage.py'
+      # change here alters what the integration/coverage jobs build locally.
-      - 'scripts/tracker_policy.py'
-      - 'scripts/firecracker-netpool.sh'
       - 'Dockerfile*'
       - 'pyproject.toml'
-      - 'requirements-dev.txt'
-      - '.coveragerc'
-      - '.dockerignore'
   pull_request:
     paths:
-      - 'bot_bottle/**'
+      - '**.py'
-      - 'tests/**/*.py'
+      - '.gitea/workflows/**.yml'
-      - 'cli.py'
+      - 'scripts/**'
-      - 'scripts/coverage.sh'
+      - 'README.md'
-      - 'scripts/critical-modules.txt'
-      - 'scripts/diff_coverage.py'
-      - 'scripts/tracker_policy.py'
-      - 'scripts/firecracker-netpool.sh'
       - 'Dockerfile*'
       - 'pyproject.toml'
-      - 'requirements-dev.txt'
-      - '.coveragerc'
-      - '.dockerignore'
   workflow_dispatch:
 
 jobs:

.gitea/workflows/update-badges.yml

@@ -21,7 +21,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-          token: ${{ secrets.BADGE_PUSH_TOKEN }}
+          token: ${{ secrets.GITHUB_TOKEN }}
 
       # No actions/setup-python: the runner image ships Python 3.12 and older
       # act_runner engines mishandle setup-python's PATH. Install into the

README.md

@@ -5,7 +5,7 @@
 # bot-bottle
 
 [![test](https://gitea.dideric.is/didericis/bot-bottle/actions/workflows/test.yml/badge.svg?branch=main)](https://gitea.dideric.is/didericis/bot-bottle/actions?workflow=test.yml)
-[![coverage](https://img.shields.io/badge/coverage-83%25-brightgreen)](https://coverage.readthedocs.io/)
+[![coverage](https://img.shields.io/badge/coverage-81%25-brightgreen)](https://coverage.readthedocs.io/)
 [![core coverage](https://img.shields.io/badge/core%20coverage-94%25-brightgreen)](https://gitea.dideric.is/didericis/bot-bottle/src/branch/main/docs/decisions/0004-coverage-policy.md)
 
 **Problem:** Developer wants to run a coding agent without supervision, but they don't want a prompt injected or misbehaving agent wrecking their environment or exfiltrating sensitive data.

bot_bottle/backend/consolidated_util.py

@@ -7,10 +7,13 @@ imports it rather than re-implementing it.
 
 from __future__ import annotations
 
+import dataclasses
+
 from ..egress import EgressPlan
 from ..git_gate import GitGatePlan
-from ..orchestrator.client import OrchestratorClient
+from ..orchestrator.client import OrchestratorClient, RegisteredBottle
 from ..orchestrator.registration import registration_inputs
+from ..orchestrator.secret_store import new_env_var_secret
 from .docker.gateway_provision import GatewayTransport, deprovision_git_gate, provision_git_gate
 
 
@@ -23,21 +26,26 @@ def provision_bottle(
     *,
     image_ref: str = "",
     tokens: dict[str, str] | None = None,
-):
+) -> RegisteredBottle:
     """Register the bottle and provision its git-gate state. Rolls back the
-    registration if provisioning fails so no orphan is left. Returns the
+    registration if provisioning fails so no orphan is left.
-    `RegisteredBottle` from the orchestrator."""
+
+    Generates a fresh ENV_VAR_SECRET, passes it to the orchestrator so it can
+    encrypt the token values at rest, and stamps the secret onto the returned
+    ``RegisteredBottle`` so callers can inject it into the agent container's
+    environment."""
     inputs = registration_inputs(egress_plan)
+    env_var_secret = new_env_var_secret()
     reg = client.register_bottle(
         source_ip, image_ref=image_ref, policy=inputs.policy,
-        metadata=inputs.metadata, tokens=tokens,
+        metadata=inputs.metadata, tokens=tokens, env_var_secret=env_var_secret,
     )
     try:
         provision_git_gate(transport, reg.bottle_id, git_gate_plan)
     except Exception:
         client.teardown_bottle(reg.bottle_id)
         raise
-    return reg
+    return dataclasses.replace(reg, env_var_secret=env_var_secret)
 
 
 def teardown_consolidated(

bot_bottle/backend/docker/bottle_plan.py

@@ -39,6 +39,10 @@ class DockerBottlePlan(BottlePlan):
     # (egress proxy credentials, git-gate/supervise headers); set by launch
     # from the orchestrator registration. Empty pre-registration.
     identity_token: str = ""
+    # Encryption key for the agent's stored egress secrets; injected into the
+    # agent container as ENV_VAR_SECRET via the compose subprocess env (bare
+    # name — value never written to the compose file). Empty pre-registration.
+    env_var_secret: str = ""
 
     @property
     def container_name(self) -> str:

bot_bottle/backend/docker/consolidated_compose.py

@@ -17,6 +17,7 @@ from __future__ import annotations
 from typing import Any
 
 from ...egress import egress_agent_env_entries
+from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME
 from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
 from .bottle_plan import DockerBottlePlan
 from .egress import EGRESS_PORT
@@ -58,6 +59,10 @@ def consolidated_agent_compose(
     # the secret value never lands on argv or in the compose file.
     for name in sorted(plan.forwarded_env.keys()):
         env.append(name)
+    # ENV_VAR_SECRET: bare name so the value comes from the compose subprocess
+    # env (set in launch.py) and is never written to the compose file on disk.
+    if getattr(plan, "env_var_secret", ""):
+        env.append(ENV_VAR_SECRET_NAME)
     env.extend(egress_agent_env_entries(plan.egress_plan))
 
     service: dict[str, Any] = {

bot_bottle/backend/docker/consolidated_launch.py

@@ -15,12 +15,14 @@ from __future__ import annotations
 
 from dataclasses import dataclass
 
+from ... import log
 from ...docker_cmd import run_docker
 from ...egress import EgressPlan
 from ...git_gate import GitGatePlan
 from ...orchestrator.client import OrchestratorClient
 from ...orchestrator.gateway import GATEWAY_NETWORK
 from ...orchestrator.lifecycle import INFRA_NAME, OrchestratorService
+from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME
 from ..consolidated_util import provision_bottle
 from ..consolidated_util import teardown_consolidated as _teardown_util
 from .gateway_provision import DockerGatewayTransport
@@ -41,6 +43,7 @@ class LaunchContext:
     network: str            # the shared gateway network to attach to
     gateway_ip: str         # the gateway's address — the agent's proxy target
     orchestrator_url: str
+    env_var_secret: str = ""  # encryption key injected into the agent's env
 
 
 def _network_cidr(network: str) -> str:
@@ -85,6 +88,66 @@ def _network_container_ips(network: str) -> list[str]:
     return ips
 
 
+def _reprovision_running_bottles(
+    orchestrator_url: str,
+    network: str = GATEWAY_NETWORK,
+    infra_name: str = INFRA_NAME,
+) -> None:
+    """Re-inject egress tokens for any registered bottles that lost their
+    in-memory tokens (e.g., after an infra container restart).
+
+    For each registered bottle whose source IP maps to a live container on the
+    gateway network, reads ENV_VAR_SECRET via ``docker exec … printenv`` and
+    calls ``POST /bottles/<id>/reprovision_gateway``. Idempotent — a no-op
+    when the orchestrator already has all tokens loaded. Best-effort: a single
+    container exec failure never blocks a new bottle launch."""
+    client = OrchestratorClient(orchestrator_url)
+    bottles = client.list_bottles()
+    if not bottles:
+        return
+
+    # Build {source_ip: container_name} from live containers on the gateway
+    # network, excluding the infra container itself.
+    proc = run_docker([
+        "docker", "network", "inspect",
+        "--format", "{{range .Containers}}{{.Name}} {{.IPv4Address}}\n{{end}}",
+        network,
+    ])
+    ip_to_container: dict[str, str] = {}
+    for line in proc.stdout.splitlines():
+        parts = line.strip().split()
+        if len(parts) >= 2 and parts[0] != infra_name:
+            ip = parts[1].split("/", 1)[0]
+            if ip:
+                ip_to_container[ip] = parts[0]
+
+    reprovisioned = 0
+    for bottle in bottles:
+        bottle_id = bottle.get("bottle_id")
+        source_ip = bottle.get("source_ip")
+        if not isinstance(bottle_id, str) or not isinstance(source_ip, str):
+            continue
+        container_name = ip_to_container.get(source_ip)
+        if not container_name:
+            continue
+        proc = run_docker(
+            ["docker", "exec", container_name, "printenv", ENV_VAR_SECRET_NAME]
+        )
+        if proc.returncode != 0 or not proc.stdout.strip():
+            continue
+        try:
+            if client.reprovision_gateway(bottle_id, proc.stdout.strip()):
+                reprovisioned += 1
+        except Exception:  # noqa: BLE001 — best-effort, never block a launch
+            pass
+
+    if reprovisioned:
+        log.info(
+            "reprovisioned egress tokens",
+            context={"count": reprovisioned},
+        )
+
+
 def launch_consolidated(
     egress_plan: EgressPlan,
     git_gate_plan: GitGatePlan,
@@ -96,9 +159,14 @@ def launch_consolidated(
     network: str = GATEWAY_NETWORK,
 ) -> LaunchContext:
     """Ensure the infra container is up, allocate + register the bottle, and
-    provision its git-gate state. Returns the agent's attach context."""
+    provision its git-gate state. Returns the agent's attach context.
+
+    Also reprovisiones egress tokens for any already-running bottles that lost
+    their in-memory credentials (e.g. after an infra container restart), so
+    they regain egress access before the new bottle is registered."""
     service = service or OrchestratorService()
     url = service.ensure_running()
+    _reprovision_running_bottles(url, network=network, infra_name=infra_name)
     client = OrchestratorClient(url)
 
     cidr = _network_cidr(network)
@@ -117,6 +185,7 @@ def launch_consolidated(
         network=network,
         gateway_ip=gateway_ip,
         orchestrator_url=url,
+        env_var_secret=reg.env_var_secret,
     )
 
 

bot_bottle/backend/docker/launch.py

@@ -186,6 +186,7 @@ def launch(
             agent_git_gate_url=git_gate_url,
             agent_supervise_url=supervise_url,
             identity_token=ctx.identity_token,
+            env_var_secret=ctx.env_var_secret,
         )
 
         # Step 5: render + up the agent-only compose, pinned on the shared
@@ -198,7 +199,12 @@ def launch(
         project = compose_project_name(plan.slug)
         # Forwarded vars (OAuth token, host interpolations) flow through the
         # subprocess env as bare names so values never land in the file.
+        # ENV_VAR_SECRET follows the same pattern: bare name in the compose
+        # spec, value only in the subprocess env so it is never written to disk.
         compose_env: dict[str, str] = {**os.environ, **plan.forwarded_env}
+        if plan.env_var_secret:
+            from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME
+            compose_env[ENV_VAR_SECRET_NAME] = plan.env_var_secret
         info(
             f"docker compose up -d  (project {project}, agent on shared "
             f"gateway {ctx.gateway_ip}, ip {ctx.source_ip})"

bot_bottle/backend/firecracker/cleanup.py

@@ -1,23 +1,8 @@
 """Cleanup for the Firecracker backend.
 
-Reaps *orphans* only — resources with no live VM behind them:
+Orphans are: firecracker VMM processes whose config lives under our run
-
+dir, and the per-bottle run dirs. TAP slots free themselves (the flock
-  * orphan run dirs: a per-bottle run dir (holding the ~1G rootfs.ext4)
+drops when the launcher exits), so there is nothing to reclaim there.
-    whose firecracker process has exited. These leak when a launch is
-    hard-killed before its teardown runs (host OOM/crash, a cancelled CI
-    job, `kill -9`); the clean-exit path already removes its own dir in
-    launch.py.
-  * orphan VM pids: a firecracker process whose run dir is already gone
-    — a VMM left lingering after its dir was removed.
-
-A run dir with a *live* firecracker process is a running bottle and is
-left strictly alone: it is neither killed nor removed. (The backend's
-`enumerate_active` registry is still a stub — #354 — so a live process
-is the only reliable "this bottle is in use" signal we have. Once the
-registry lands, registry-orphaned-but-running VMs can be reaped too.)
-
-TAP slots free themselves (the flock drops when the launcher exits), so
-there is nothing to reclaim there.
 """
 
 from __future__ import annotations
@@ -37,73 +22,38 @@ def _run_root() -> Path:
     return util.cache_dir() / "run"
 
 
-def _run_dir_of(cmd: str, run_root: Path) -> Path | None:
+def _orphan_vm_pids() -> list[int]:
-    """The bottle run dir a firecracker cmdline belongs to, or None.
+    """firecracker processes whose --config-file is under our run dir."""
-
+    run_root = str(_run_root())
-    A bottle VM is launched with `--config-file <run_root>/<slug>/config.json`,
-    so the run dir is the config file's parent when it sits directly under
-    the run root. Anything else (a builder VM, the infra VM elsewhere) is
-    not ours to reap here.
-    """
-    toks = cmd.split()
-    for i, tok in enumerate(toks):
-        if tok == "--config-file" and i + 1 < len(toks):
-            parent = Path(toks[i + 1]).parent
-            if parent.parent == run_root:
-                return parent
-    return None
-
-
-def _scan_processes(run_root: Path) -> tuple[set[str], list[int]]:
-    """Inspect running firecracker VMs under ``run_root``.
-
-    Returns ``(live_run_dirs, orphan_pids)``:
-      * ``live_run_dirs`` — run dirs backed by a running VM (never reaped);
-      * ``orphan_pids`` — firecracker pids whose run dir no longer exists
-        (a lingering VMM to kill).
-    """
     result = subprocess.run(
         ["pgrep", "-a", "firecracker"],
         capture_output=True, text=True, check=False,
     )
     if result.returncode != 0:
-        return set(), []
+        return []
-    live: set[str] = set()
+    pids: list[int] = []
-    orphan_pids: list[int] = []
     for line in result.stdout.splitlines():
         parts = line.split(None, 1)
-        if len(parts) != 2:
+        if len(parts) != 2 or run_root not in parts[1]:
             continue
         try:
-            pid = int(parts[0])
+            pids.append(int(parts[0]))
         except ValueError:
             continue
-        run_dir = _run_dir_of(parts[1], run_root)
+    return pids
-        if run_dir is None:
-            continue
-        if run_dir.is_dir():
-            live.add(str(run_dir))
-        else:
-            orphan_pids.append(pid)
-    return live, orphan_pids
 
 
-def _orphan_run_dirs(run_root: Path, live: set[str]) -> list[str]:
+def _run_dirs() -> list[str]:
-    """Run dirs with no live VM behind them — the leaked ones to remove."""
+    run_root = _run_root()
     if not run_root.is_dir():
         return []
-    return sorted(
+    return sorted(str(p) for p in run_root.iterdir() if p.is_dir())
-        str(p) for p in run_root.iterdir()
-        if p.is_dir() and str(p) not in live
-    )
 
 
 def prepare_cleanup() -> FirecrackerBottleCleanupPlan:
-    run_root = _run_root()
-    live, orphan_pids = _scan_processes(run_root)
     return FirecrackerBottleCleanupPlan(
-        vm_pids=tuple(orphan_pids),
+        vm_pids=tuple(_orphan_vm_pids()),
-        run_dirs=tuple(_orphan_run_dirs(run_root, live)),
+        run_dirs=tuple(_run_dirs()),
     )
 
 

bot_bottle/backend/firecracker/consolidated_launch.py

@@ -50,6 +50,7 @@ class LaunchContext:
     source_ip: str          # the VM's guest IP — the attribution key
     gateway_ca_pem: str     # the shared gateway CA the provisioner installs
     orchestrator_url: str
+    env_var_secret: str = ""  # encryption key injected into the agent's env
 
 
 def launch_consolidated(
@@ -80,6 +81,7 @@ def launch_consolidated(
         source_ip=guest_ip,
         gateway_ca_pem=infra.gateway_ca_pem(),
         orchestrator_url=url,
+        env_var_secret=reg.env_var_secret,
     )
 
 

bot_bottle/backend/firecracker/launch.py

@@ -26,7 +26,6 @@ from __future__ import annotations
 
 import dataclasses
 import os
-import shutil
 from contextlib import ExitStack, contextmanager
 from pathlib import Path
 from typing import Callable, Generator
@@ -168,10 +167,6 @@ def launch(
         # Step 6: build the per-bottle rootfs + SSH key, then boot.
         run_dir = util.cache_dir() / "run" / plan.slug
         run_dir.mkdir(parents=True, exist_ok=True)
-        # Remove the run dir on teardown so the per-bottle rootfs.ext4 (~1G)
-        # doesn't leak. Registered before vm.terminate below so it runs *after*
-        # it (ExitStack is LIFO): the VM is gone before we rm its rootfs.
-        stack.callback(lambda: shutil.rmtree(run_dir, ignore_errors=True))
         rootfs = run_dir / "rootfs.ext4"
         util.build_rootfs_ext4(agent_base, rootfs)
         private_key, pubkey = util.generate_keypair(run_dir)

bot_bottle/backend/macos_container/consolidated_launch.py

@@ -72,6 +72,7 @@ class LaunchContext:
     gateway_ip: str
     network: str
     orchestrator_url: str
+    env_var_secret: str = ""  # encryption key injected into the agent's env
 
 
 def ensure_gateway(
@@ -152,6 +153,7 @@ def register_agent(
         gateway_ip=endpoint.gateway_ip,
         network=endpoint.network,
         orchestrator_url=endpoint.orchestrator_url,
+        env_var_secret=reg.env_var_secret,
     )
 
 

bot_bottle/orchestrator/client.py

@@ -41,10 +41,13 @@ class OrchestratorClientError(RuntimeError):
 @dataclass(frozen=True)
 class RegisteredBottle:
     """What `POST /bottles` returns: the minted bottle id and the per-bottle
-    identity token the agent presents for app-layer attribution."""
+    identity token the agent presents for app-layer attribution. `env_var_secret`
+    is set by the caller (not from the server response) and carries the
+    encryption key so it can be injected into the agent container's env."""
 
     bottle_id: str
     identity_token: str
+    env_var_secret: str = ""
 
 
 class OrchestratorClient:
@@ -120,17 +123,21 @@ class OrchestratorClient:
         metadata: str = "",
         policy: str = "",
         tokens: dict[str, str] | None = None,
+        env_var_secret: str = "",
     ) -> RegisteredBottle:
         """Register a bottle and broker its launch (`POST /bottles`). `tokens`
         are the per-bottle egress auth values (env_name -> value) the
-        orchestrator holds in memory for the gateway to inject. Returns the
+        orchestrator holds in memory for the gateway to inject. When
-        minted id + identity token."""
+        *env_var_secret* is provided, the orchestrator also encrypts the token
+        values and stores them in ``bottled_agent_secrets`` for restart
+        recovery. Returns the minted id + identity token."""
         payload = self._ok("POST", "/bottles", {
             "source_ip": source_ip,
             "image_ref": image_ref,
             "metadata": metadata,
             "policy": policy,
             "tokens": tokens or {},
+            "env_var_secret": env_var_secret,
         })
         bottle_id = payload.get("bottle_id")
         token = payload.get("identity_token")
@@ -138,6 +145,24 @@ class OrchestratorClient:
             raise OrchestratorClientError("register: response missing bottle_id/identity_token")
         return RegisteredBottle(bottle_id=bottle_id, identity_token=token)
 
+    def reprovision_gateway(self, bottle_id: str, env_var_secret: str) -> bool:
+        """Re-inject a bottle's egress tokens from its ENV_VAR_SECRET
+        (`POST /bottles/<id>/reprovision_gateway`). Returns True when the
+        orchestrator successfully decrypted and restored the tokens, False
+        when it had no stored secrets for this bottle (404)."""
+        status, _ = self._request(
+            "POST",
+            f"/bottles/{bottle_id}/reprovision_gateway",
+            {"env_var_secret": env_var_secret},
+        )
+        if status == 404:
+            return False
+        if not 200 <= status < 300:
+            raise OrchestratorClientError(
+                f"reprovision_gateway {bottle_id}: HTTP {status}"
+            )
+        return True
+
     def teardown_bottle(self, bottle_id: str) -> bool:
         """Tear a bottle down (`DELETE /bottles/<id>`). False if the
         orchestrator didn't know it (404) — idempotent for cleanup paths."""

bot_bottle/orchestrator/control_plane.py

@@ -9,9 +9,13 @@ vsock / unix-socket portability caveats):
     GET    /bottles                   -> 200 {"bottles": [ <redacted record>, ...]}
     POST   /bottles                   -> 201 {"bottle_id","identity_token"}  (launch)
                                          body: {"source_ip", ["image_ref"],
-                                                ["metadata"], ["policy"]}
+                                                ["metadata"], ["policy"],
+                                                ["tokens"], ["env_var_secret"]}
     PUT    /bottles/<bottle_id>/policy -> 200 {"updated": true} | 404  (live reload)
                                          body: {"policy"}
+    POST   /bottles/<bottle_id>/reprovision_gateway
+                                      -> 200 {"reprovisioned": true} | 404
+                                         body: {"env_var_secret"}
     DELETE /bottles/<bottle_id>        -> 200 {"torn_down": true} | 404  (teardown)
     POST   /reconcile                 -> 200 {"reaped": [bottle_id, ...]}
                                          body: {"live_source_ips": [...],
@@ -116,12 +120,14 @@ def dispatch(  # pylint: disable=too-many-return-statements,too-many-branches
         tokens = {
             k: v for k, v in raw_tokens.items() if isinstance(k, str) and isinstance(v, str)
         } if isinstance(raw_tokens, dict) else {}
+        env_var_secret = data.get("env_var_secret", "")
         rec = orch.launch_bottle(
             source_ip,
             image_ref=image_ref if isinstance(image_ref, str) else "",
             metadata=metadata if isinstance(metadata, str) else "",
             policy=policy if isinstance(policy, str) else "",
             tokens=tokens,
+            env_var_secret=env_var_secret if isinstance(env_var_secret, str) else "",
         )
         return 201, {"bottle_id": rec.bottle_id, "identity_token": rec.identity_token}
 
@@ -138,6 +144,23 @@ def dispatch(  # pylint: disable=too-many-return-statements,too-many-branches
             return 200, {"updated": True}
         return 404, {"error": "no such bottle"}
 
+    if (
+        method == "POST"
+        and route.startswith("/bottles/")
+        and route.endswith("/reprovision_gateway")
+    ):
+        bottle_id = route[len("/bottles/") : -len("/reprovision_gateway")]
+        try:
+            data = _parse_json_object(body)
+        except ValueError as e:
+            return 400, {"error": f"invalid JSON: {e}"}
+        env_var_secret = data.get("env_var_secret")
+        if not isinstance(env_var_secret, str) or not env_var_secret:
+            return 400, {"error": "env_var_secret (string) is required"}
+        if orch.reprovision_from_secret(bottle_id, env_var_secret):
+            return 200, {"reprovisioned": True}
+        return 404, {"error": "no stored secrets for this bottle"}
+
     if method == "DELETE" and route.startswith("/bottles/"):
         bottle_id = route[len("/bottles/"):]
         if orch.teardown_bottle(bottle_id):

bot_bottle/orchestrator/registry.py

@@ -113,6 +113,22 @@ _MIGRATIONS = TableMigrations(
         # egress allowlist / routes / git config selected by source IP. The
         # multi-tenant gateway resolves it per request via `attribute`.
         "ALTER TABLE orchestrator_bottles ADD COLUMN policy TEXT NOT NULL DEFAULT ''",
+        # v4 — per-bottle encrypted egress secrets (PRD prd-new-secret-provider).
+        # One row per env-var: key (env-var name) is plaintext for auditing;
+        # value is the encrypted token string. The encryption key (ENV_VAR_SECRET)
+        # lives only in the agent's environment — a row alone cannot recover the
+        # credential.
+        """
+        CREATE TABLE IF NOT EXISTS bottled_agent_secrets (
+            bottled_agent_id TEXT NOT NULL,
+            key              TEXT NOT NULL,
+            value            TEXT NOT NULL,
+            type             TEXT NOT NULL DEFAULT 'injected_env_var'
+        )
+        """,
+        # v5 — index for fast per-bottle lookups and bulk DELETE on teardown.
+        "CREATE INDEX IF NOT EXISTS idx_bottled_agent_secrets_id "
+        "ON bottled_agent_secrets (bottled_agent_id, type)",
     ],
 )
 
@@ -326,6 +342,57 @@ class RegistryStore(DbStore):
             return None
         return rec
 
+    # --- encrypted egress secret store ------------------------------------
+
+    def store_agent_secrets(
+        self,
+        bottle_id: str,
+        encrypted_values: dict[str, str],
+        secret_type: str = "injected_env_var",
+    ) -> None:
+        """Replace all stored secrets for *bottle_id* with *encrypted_values*
+        (env-var name → encrypted ciphertext). Deletes then re-inserts so a
+        re-registration is always consistent with the current token set."""
+        with self._connection() as conn:
+            conn.execute(
+                "DELETE FROM bottled_agent_secrets "
+                "WHERE bottled_agent_id = ? AND type = ?",
+                (bottle_id, secret_type),
+            )
+            conn.executemany(
+                "INSERT INTO bottled_agent_secrets "
+                "(bottled_agent_id, key, value, type) VALUES (?, ?, ?, ?)",
+                [(bottle_id, k, v, secret_type) for k, v in encrypted_values.items()],
+            )
+        self._chmod()
+
+    def get_agent_secrets(
+        self,
+        bottle_id: str,
+        secret_type: str = "injected_env_var",
+    ) -> dict[str, str]:
+        """Return {env_var_name: encrypted_value} for *bottle_id*, or {} if none."""
+        with self._connection() as conn:
+            rows = conn.execute(
+                "SELECT key, value FROM bottled_agent_secrets "
+                "WHERE bottled_agent_id = ? AND type = ?",
+                (bottle_id, secret_type),
+            ).fetchall()
+        return {row[0]: row[1] for row in rows}
+
+    def delete_agent_secrets(
+        self,
+        bottle_id: str,
+        secret_type: str = "injected_env_var",
+    ) -> None:
+        """Remove all stored secrets for *bottle_id* (e.g. on teardown)."""
+        with self._connection() as conn:
+            conn.execute(
+                "DELETE FROM bottled_agent_secrets "
+                "WHERE bottled_agent_id = ? AND type = ?",
+                (bottle_id, secret_type),
+            )
+
 
 __all__ = [
     "BottleRecord",

bot_bottle/orchestrator/secret_store.py

@@ -0,0 +1,94 @@
+"""Symmetric encryption for per-bottle egress secrets (PRD prd-new-secret-provider).
+
+Each agent receives a random ENV_VAR_SECRET at startup — passed as an env var,
+never logged or persisted. The host uses this key to encrypt each egress auth
+token value before writing it to the bottled_agent_secrets table; the DB rows
+(ciphertext, plaintext env-var name) without the key are insufficient to
+recover the credentials.
+
+On orchestrator restart the in-memory token map is lost. The host-side
+reattachment path reads ENV_VAR_SECRET from the running agent container via
+``docker exec … printenv ENV_VAR_SECRET`` and posts it to
+``POST /bottles/<id>/reprovision_gateway``; the orchestrator decrypts the
+stored rows and re-populates ``_tokens``.
+
+Encryption scheme: HMAC-SHA256 used as a PRF in CTR mode (stdlib-only,
+no external deps). Each value is encrypted independently. The output blob is
+``nonce (16 bytes) || ciphertext`` encoded as URL-safe base64 (no padding).
+
+    keystream_block_i = HMAC-SHA256(key, nonce || i.to_bytes(4, "big"))
+    ciphertext_i      = plaintext_i XOR keystream_block_i[:len(plaintext_i)]
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import hmac
+import secrets
+
+_KEY_BYTES = 32    # 256-bit key from ENV_VAR_SECRET
+_NONCE_BYTES = 16  # 128-bit random nonce per encrypt call
+_BLOCK = 32        # HMAC-SHA256 output width == one keystream block
+
+# Env-var name the agent container receives at startup.
+ENV_VAR_SECRET_NAME = "ENV_VAR_SECRET"
+
+
+def new_env_var_secret() -> str:
+    """Generate a fresh ENV_VAR_SECRET: 32 random bytes as URL-safe base64."""
+    return base64.urlsafe_b64encode(secrets.token_bytes(_KEY_BYTES)).rstrip(b"=").decode()
+
+
+def _b64dec(s: str) -> bytes:
+    return base64.urlsafe_b64decode(s + "=" * (-len(s) % 4))
+
+
+def _keystream(key: bytes, nonce: bytes, block_index: int) -> bytes:
+    return hmac.new(
+        key, nonce + block_index.to_bytes(4, "big"), hashlib.sha256
+    ).digest()
+
+
+def encrypt_value(secret_b64: str, plaintext: str) -> str:
+    """Encrypt a single string value with *secret_b64* (the ENV_VAR_SECRET).
+
+    Returns a URL-safe base64 blob ``nonce || ciphertext`` suitable for
+    the ``bottled_agent_secrets.value`` column."""
+    key = _b64dec(secret_b64)
+    pt = plaintext.encode()
+    nonce = secrets.token_bytes(_NONCE_BYTES)
+    ct = bytearray()
+    for i in range(0, len(pt), _BLOCK):
+        chunk = pt[i : i + _BLOCK]
+        ks = _keystream(key, nonce, i)[: len(chunk)]
+        ct.extend(p ^ k for p, k in zip(chunk, ks))
+    return base64.urlsafe_b64encode(nonce + bytes(ct)).rstrip(b"=").decode()
+
+
+def decrypt_value(secret_b64: str, blob_b64: str) -> str:
+    """Decrypt a blob produced by :func:`encrypt_value`.
+
+    Returns the original plaintext string. Raises ``ValueError`` for malformed
+    input or a key mismatch (wrong key produces garbage, not an error, unless
+    the plaintext is non-UTF-8 — treat all such failures as wrong key)."""
+    key = _b64dec(secret_b64)
+    try:
+        blob = _b64dec(blob_b64)
+    except Exception as exc:
+        raise ValueError(f"invalid ciphertext blob: {exc}") from exc
+    if len(blob) < _NONCE_BYTES:
+        raise ValueError("ciphertext blob too short")
+    nonce, ciphertext = blob[:_NONCE_BYTES], blob[_NONCE_BYTES:]
+    pt = bytearray()
+    for i in range(0, len(ciphertext), _BLOCK):
+        chunk = ciphertext[i : i + _BLOCK]
+        ks = _keystream(key, nonce, i)[: len(chunk)]
+        pt.extend(c ^ k for c, k in zip(chunk, ks))
+    try:
+        return bytes(pt).decode()
+    except UnicodeDecodeError as exc:
+        raise ValueError(f"decryption produced non-UTF-8 output (wrong key?): {exc}") from exc
+
+
+__all__ = ["ENV_VAR_SECRET_NAME", "new_env_var_secret", "encrypt_value", "decrypt_value"]

bot_bottle/orchestrator/service.py

@@ -87,13 +87,22 @@ class Orchestrator:
         metadata: str = "",
         policy: str = "",
         tokens: dict[str, str] | None = None,
+        env_var_secret: str = "",
     ) -> BottleRecord:
         """Register a bottle (with its gateway policy + in-memory egress auth
         tokens) and broker its launch. Rolls the registry entry back if the
-        launch doesn't take, so a failure leaves no orphan."""
+        launch doesn't take, so a failure leaves no orphan.
+
+        When *env_var_secret* is provided alongside *tokens*, the token values
+        are also encrypted and written to ``bottled_agent_secrets`` so they can
+        survive an orchestrator restart (see ``reprovision_from_secret``)."""
         rec = self.registry.register(source_ip, metadata=metadata, policy=policy)
         if tokens:
             self._tokens[rec.bottle_id] = dict(tokens)
+            if env_var_secret:
+                from .secret_store import encrypt_value
+                encrypted = {k: encrypt_value(env_var_secret, v) for k, v in tokens.items()}
+                self.registry.store_agent_secrets(rec.bottle_id, encrypted)
         req = LaunchRequest(
             op="launch",
             bottle_id=rec.bottle_id,
@@ -284,6 +293,26 @@ class Orchestrator:
             ))
         return True, ""
 
+    # --- secret reprovision -----------------------------------------------
+
+    def reprovision_from_secret(self, bottle_id: str, env_var_secret: str) -> bool:
+        """Re-inject a bottle's egress tokens from its ENV_VAR_SECRET.
+
+        Reads the encrypted rows from ``bottled_agent_secrets``, decrypts each
+        value with *env_var_secret*, and restores ``_tokens[bottle_id]``.
+        Returns True on success, False when no stored secrets exist for this
+        bottle or decryption fails (wrong key / corrupt data)."""
+        from .secret_store import decrypt_value
+        encrypted = self.registry.get_agent_secrets(bottle_id)
+        if not encrypted:
+            return False
+        try:
+            self._tokens[bottle_id] = {k: decrypt_value(env_var_secret, v)
+                                       for k, v in encrypted.items()}
+        except ValueError:
+            return False
+        return True
+
     # --- consolidated gateway ----------------------------------------------
 
     def ensure_gateway(self) -> None:

docs/research/firecracker-image-remote-store.md

@@ -1,178 +0,0 @@
-# Firecracker Image Remote Store
-
-**Date:** 2026-07-22  
-**Context:** PR #459 (run-dir leak fix) surfaced that committed Firecracker snapshots
-currently live only on the host machine. Once a host is wiped or a run-dir is
-evicted, a user's preserved bottle is gone. This note investigates a secure remote
-store so committed images survive host turnover and can be restored without the
-user having to pre-flag which sessions to keep.
-
-## Verdict
-
-Backblaze B2 + Cloudflare CDN is the cost-optimal choice for most deployments.
-Cloudflare R2 is the simpler zero-config option at slightly higher storage cost.
-Self-hosted MinIO is the right call for air-gapped or on-premises installs.
-
-On the image-size front, zstd-compressing the committed tar before upload
-produces roughly a 60–70% reduction with negligible impact on restore latency.
-OverlayFS (for in-flight working rootfs, not for the committed artifact) cuts
-per-instance disk use to ~10–50 MB per extra bottle sharing the same base.
-
----
-
-## What Gets Stored
-
-The Firecracker backend produces two artifact types:
-
-| Artifact | Created by | Size | Lifetime |
-|----------|-----------|------|---------|
-| `rootfs/agent-<digest>/` (dir) | `image_builder.py` → `mke2fs -d` | ~1 GB as ext4 | Cached per Dockerfile hash; evictable |
-| `committed/<slug>/rootfs.tar` | `FirecrackerFreezer._freeze` via SSH tar | 500 MB–1 GB | User-preserved; must survive host wipe |
-
-The committed artifact is a tar of the guest's live filesystem streamed out over
-SSH (`freezer.py:58–90`). At resume time `launch.py` calls `mke2fs -d` to
-rebuild a fresh ext4 from this tar. The tar — not the ext4 — is what needs to be
-pushed to remote storage and pulled back at restore time.
-
-The base image cache (`rootfs/agent-<digest>/`) is derivable from the Dockerfile
-and can be rebuilt on demand; it is lower priority for remote storage.
-
----
-
-## Storage Candidates
-
-### Object storage
-
-| Provider | Storage | Egress | Notes |
-|----------|---------|--------|-------|
-| **Backblaze B2** | $0.006/GB | Free (via Cloudflare Bandwidth Alliance) | Cheapest storage; pairs with Cloudflare CDN to eliminate egress |
-| **Cloudflare R2** | $0.015/GB | $0 always | Zero-config egress; no lifecycle transitions (limitation) |
-| **Wasabi** | $0.0069/GB | Free (1:1 ratio) | 90-day minimum retention; good for archival; lifecycle evaluated daily |
-| **AWS S3** | $0.023/GB | $0.09/GB | Richest lifecycle support; expensive at scale; avoid unless already in AWS |
-| **MinIO** (self-hosted) | Host cost only | None | S3-compatible; best for private/on-prem deployments |
-
-**B2 + Cloudflare CDN** is effectively $0.006/GB with zero egress — about 18×
-cheaper than S3 for restore-heavy workloads. **R2** is the zero-config choice
-($0 egress by default, no Bandwidth Alliance pairing needed) at a slightly
-higher storage rate.
-
-**Cloudflare R2's missing lifecycle support** is the main caveat: auto-eviction
-rules (evict images older than N days) cannot currently be expressed natively in
-R2. Wasabi and S3 both support declarative lifecycle policies.
-
-### Retention policy recommendation
-
-The comment proposes:
-- Retain images for ~1 week by default
-- Warn when approaching a capacity threshold
-- Auto-evict oldest images once threshold is exceeded
-
-This maps cleanly to an application-level policy (not a provider lifecycle rule),
-which avoids the R2 limitation and works consistently across providers:
-
-1. On `commit`: upload tar, record `(slug, size_bytes, uploaded_at)` in a local
-   or remote manifest file.
-2. On startup / on `list`: scan the manifest, warn if total stored size exceeds
-   e.g. 80% of the configured threshold.
-3. On eviction run (CLI or cron): delete objects older than `retention_days`
-   (default 7) that push total over `max_capacity`; oldest-first.
-
-This keeps the policy logic in bot-bottle and the storage provider as a dumb
-object store — no vendor-specific lifecycle API required.
-
----
-
-## Image Size Reduction
-
-### Current artifact sizes
-
-A typical committed tar for a Claude Code agent image is 500 MB–1 GB uncompressed.
-The per-run ext4 (copy of the base, written at `start`) adds another ~1 GB of
-local disk. The leak fix in this PR addresses the ext4 copies; the remote store
-addresses the committed tars.
-
-### Compression
-
-zstd compression of the committed tar before upload is the highest-leverage
-single change:
-
-| Codec | Typical size (1 GB rootfs) | Compress speed | Decompress speed |
-|-------|---------------------------|---------------|-----------------|
-| gzip | 350–430 MB | ~100 MB/s | ~500 MB/s |
-| **zstd (default)** | **330–360 MB** | **~400 MB/s** | **~2 GB/s** |
-| xz | 290–320 MB | ~20 MB/s | ~200 MB/s |
-
-**zstd is the best trade-off**: 65–67% size reduction, near-instantaneous
-decompression. The `tar` call in `freezer.py` could pipe through `zstd` before
-writing to disk and to the remote; `resume` decompresses on the way back. A
-`.tar.zst` suffix marks compressed artifacts so old tars remain restorable
-without the codec.
-
-### SquashFS for the base image cache
-
-The `rootfs/agent-<digest>/` directory (the buildah-exported tree) is rebuilt by
-`image_builder.py` and turned into per-run ext4 by `mke2fs -d`. Storing the
-base as a SquashFS image instead of a flat directory tree would reduce it from
-~1 GB to ~330–360 MB and make the cache remote-friendly. Firecracker does not
-directly boot SquashFS, but the existing `mke2fs -d` path reads a directory tree
-— a SquashFS mount could serve as the source. This is a larger change and lower
-priority than tar compression.
-
-### OverlayFS for per-run rootfs
-
-Multiple simultaneous bottles sharing the same agent image today each get a full
-`mke2fs -d` copy (~1 GB). OverlayFS (read-only base + writable sparse overlay)
-would reduce this to ~10–50 MB per instance beyond the first:
-
-- Mount the base image directory as read-only lower layer
-- Attach a sparse ext4 or tmpfs writable layer per bottle
-- Pass the merged overlay to Firecracker as the block device
-
-E2B's public write-up on Firecracker + OverlayFS confirms this approach works
-at scale. The `launch.py` changes would be non-trivial (device mapper or
-`fuse-overlayfs` plumbing), so this is a follow-up rather than a prerequisite
-for the remote store.
-
----
-
-## Recommended Approach
-
-**Phase 1 — remote store with zstd (tight scope, actionable now)**
-
-1. Add `--zstd` to the `tar` call in `FirecrackerFreezer._freeze`; name the
-   artifact `rootfs.tar.zst`. Keep uncompressed restore path for legacy tars.
-2. Add a `bb firecracker upload <slug>` / `bb firecracker pull <slug>` pair that
-   pushes/fetches the compressed tar to the configured object store (S3-compatible
-   API, so B2, R2, MinIO, and Wasabi all work with the same client).
-3. Store a `manifest.json` in the bucket (or a local mirror) tracking slug →
-   `{size, uploaded_at}`. Use it for threshold warnings and eviction.
-4. Default retention: 7 days, configurable via `firecracker.image_retention_days`
-   in `~/.config/bot-bottle/config.toml` (or equivalent).
-5. Warn at 80% of `max_capacity` (default e.g. 50 GB); evict oldest on commit
-   once at 100%.
-
-**Storage recommendation:** Cloudflare R2 for hosted deployments (zero egress,
-zero config), MinIO for private/on-premises.
-
-**Phase 2 — base image cache compression**
-
-Compress the `agent-<digest>` cache dir as a `.tar.zst` to save ~65% on repeated
-image uploads. Low urgency since the base image is rebuildable.
-
-**Phase 3 — OverlayFS per-run disk**
-
-Replace the full per-run ext4 copy with an OverlayFS sparse layer. Largest disk
-impact (~90–95% savings per concurrent bottle) but highest implementation
-complexity. Track as a separate PRD.
-
----
-
-## Open Questions
-
-- Does the host have a configured object-store credential path, or should the
-  remote store be an opt-in with an explicit `bb config set image-store.url ...`?
-- Should `commit` automatically upload, or should upload be an explicit step to
-  avoid surprise egress?
-- What is the acceptable cold-start latency for a restore from remote? A 330 MB
-  zstd tar at 100 Mbit/s takes ~26 s; at 1 Gbit/s, ~2.6 s. This bounds the
-  retention strategy (evict from local after successful upload vs keep local copy).

tests/unit/test_firecracker_cleanup.py

@@ -10,9 +10,7 @@ classmethods forward to their module.
 from __future__ import annotations
 
 import subprocess
-import tempfile
 import unittest
-from pathlib import Path
 from unittest.mock import patch
 
 from bot_bottle.backend.firecracker import cleanup as fc_cleanup
@@ -25,69 +23,32 @@ def _proc(stdout: str = "", returncode: int = 0) -> "subprocess.CompletedProcess
     return subprocess.CompletedProcess([], returncode, stdout=stdout, stderr="")
 
 
-class TestProcessScan(unittest.TestCase):
+class TestOrphanEnumeration(unittest.TestCase):
-    def test_run_dir_of_matches_only_direct_children(self):
+    def test_orphan_vm_pids_filters_by_run_dir(self):
-        run_root = Path("/cache/run")
+        run_root = str(fc_cleanup._run_root())
-        self.assertEqual(
-            Path("/cache/run/dev-a"),
-            fc_cleanup._run_dir_of(
-                f"firecracker --config-file {run_root}/dev-a/config.json", run_root
-            ),
-        )
-        # infra/builder VMs elsewhere, or nested paths, are not ours.
-        self.assertIsNone(
-            fc_cleanup._run_dir_of("firecracker --config-file /elsewhere/config.json", run_root)
-        )
-        self.assertIsNone(
-            fc_cleanup._run_dir_of("firecracker --no-config", run_root)
-        )
-
-    def test_scan_splits_live_dirs_from_orphan_pids(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            run_root = Path(tmp)
-            (run_root / "live-a").mkdir()  # dir present -> live VM, protected
-            # "gone-b" dir intentionally absent -> lingering VMM, orphan pid
         out = (
-                f"111 firecracker --config-file {run_root}/live-a/config.json\n"
+            f"111 firecracker --config-file {run_root}/dev-a/config.json\n"
-                f"222 firecracker --config-file {run_root}/gone-b/config.json\n"
+            "222 firecracker --config-file /somewhere/else/config.json\n"
-                "333 firecracker --config-file /elsewhere/config.json\n"
+            "notanint firecracker --config-file " + run_root + "/x\n"
-                "notanint firecracker --config-file x\n"
         )
         with patch.object(fc_cleanup.subprocess, "run", return_value=_proc(out)):
-                live, orphan_pids = fc_cleanup._scan_processes(run_root)
+            self.assertEqual([111], fc_cleanup._orphan_vm_pids())
-        self.assertEqual({str(run_root / "live-a")}, live)
-        self.assertEqual([222], orphan_pids)
 
-    def test_scan_empty_when_pgrep_fails(self):
+    def test_orphan_vm_pids_empty_when_pgrep_fails(self):
         with patch.object(fc_cleanup.subprocess, "run", return_value=_proc(returncode=1)):
-            self.assertEqual((set(), []), fc_cleanup._scan_processes(Path("/x")))
+            self.assertEqual([], fc_cleanup._orphan_vm_pids())
 
-    def test_orphan_run_dirs_excludes_live_and_missing_root(self):
+    def test_run_dirs_empty_when_absent(self):
-        with tempfile.TemporaryDirectory() as tmp:
+        with patch.object(fc_cleanup.util, "cache_dir") as cache:
-            run_root = Path(tmp)
+            cache.return_value.__truediv__.return_value.is_dir.return_value = False
-            (run_root / "live-a").mkdir()
+            self.assertEqual([], fc_cleanup._run_dirs())
-            (run_root / "dead-b").mkdir()
-            live = {str(run_root / "live-a")}
-            self.assertEqual(
-                [str(run_root / "dead-b")],
-                fc_cleanup._orphan_run_dirs(run_root, live),
-            )
-        # absent run root -> nothing to reap
-        self.assertEqual([], fc_cleanup._orphan_run_dirs(Path("/nope/run"), set()))
 
-    def test_prepare_cleanup_reaps_orphans_only(self):
+    def test_prepare_cleanup_assembles_plan(self):
-        """The live VM's dir is never in the plan; the dead one is."""
+        with patch.object(fc_cleanup, "_orphan_vm_pids", return_value=[7]), \
-        with tempfile.TemporaryDirectory() as tmp:
+             patch.object(fc_cleanup, "_run_dirs", return_value=["/run/x"]):
-            run_root = Path(tmp)
-            (run_root / "live-a").mkdir()
-            (run_root / "dead-b").mkdir()
-            out = f"111 firecracker --config-file {run_root}/live-a/config.json\n"
-            with patch.object(fc_cleanup, "_run_root", return_value=run_root), \
-                 patch.object(fc_cleanup.subprocess, "run", return_value=_proc(out)):
             plan = fc_cleanup.prepare_cleanup()
-        self.assertEqual((), plan.vm_pids)
+        self.assertEqual((7,), plan.vm_pids)
-        self.assertEqual((str(run_root / "dead-b"),), plan.run_dirs)
+        self.assertEqual(("/run/x",), plan.run_dirs)
-        self.assertNotIn(str(run_root / "live-a"), plan.run_dirs)
 
 
 class TestCleanupRemoval(unittest.TestCase):