feat(prd-0081): reprovision CA, git-gate, and egress tokens on gateway bring-up

On a Firecracker gateway cold boot, reconcile every live agent VM against
the fresh gateway: push the new mitmproxy CA into each agent's trust store,
re-provision git-gate repos/creds from the persisted upstreams snapshot,
and restore egress tokens. Per-bottle failures are logged and skipped so
one unreachable VM does not block the rest.

New modules / changes:
- backend/firecracker/reconcile.py: attach_bottled_agents_to_gateway,
  _push_ca, _reprovision_git_gate, _guest_ip_from_config
- git_gate/provision.py: write upstreams.json after key provisioning so
  the bring-up reconcile can reconstruct the upstream table without the
  manifest
- backend/firecracker/infra.py: call attach_bottled_agents_to_gateway in
  the cold-boot branch of ensure_running()
- backend/base.py: no-op default on BottleBackend
- backend/firecracker/consolidated_launch.py: remove superseded
  _reprovision_running_bottles / _guest_ip_from_config
- orchestrator: OrchestratorCore.update_agent_secret + POST
  /bottles/<id>/secret + client.update_agent_secret (single-secret
  in-place update, the reusable primitive from the 2026-07-26 hotfix)
- tests: 15 new tests in test_firecracker_reconcile.py; updated
  test_firecracker_infra.py cold-boot cases; stale FC reprovision tests
  removed from test_backend_secret_reprovision.py

Closes #516.

bot_bottle/backend/base.py

@@ -496,6 +496,20 @@ class BottleBackend(ABC, Generic[PlanT, CleanupT]):
         (macos-container) die with a pointer — the default here."""
         die(f"backend {self.name!r} has no orchestrator control plane")
 
+    def attach_bottled_agents_to_gateway(self) -> None:
+        """Reconcile all running bottles against the current gateway.
+
+        Called when the gateway is (re)brought up (cold-boot path) to restore
+        the three gateway-dependent services for every live bottle: CA trust,
+        git-gate repos/creds, and egress tokens. Per-bottle failures must be
+        logged and skipped — one unreachable agent must not block the rest.
+
+        Default: no-op. Backends that run a gateway (Firecracker, docker,
+        macOS) override this with a backend-native implementation that reaches
+        running agents via their transport (SSH for Firecracker, exec/cp for
+        docker and macOS). PRD 0081."""
+        return
+
     @abstractmethod
     def prepare_cleanup(self) -> CleanupT:
         """Enumerate orphaned resources from previous bottles. No side

bot_bottle/backend/firecracker/consolidated_launch.py

@@ -26,22 +26,19 @@ The TAP slot allocation, rootfs build, and VM boot are the caller's job.
 
 from __future__ import annotations
 
-import json
 import subprocess
 from dataclasses import dataclass
 from pathlib import Path
 
 from ...egress import EgressPlan
 from ...git_gate import GitGatePlan
-from ...log import info
+from ...orchestrator.client import OrchestratorClient
-from ...orchestrator.client import OrchestratorClient, OrchestratorClientError
 from ...orchestrator.lifecycle import (
     OrchestratorStartError,  # re-exported so callers can catch it
 )
-from ...orchestrator.reprovision import reprovision_bottles
 from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
 from ..provision_bottle import deprovision_bottle, provision_bottle
-from . import cleanup, util
+from . import util
 from .gateway import FirecrackerGateway
 from .infra import FirecrackerInfraService
 
@@ -64,17 +61,6 @@ class LaunchContext:
     env_var_secret: str = ""  # encryption key injected into the agent's env
 
 
-def _guest_ip_from_config(config_path: Path) -> str:
-    """Read the kernel's configured guest IP from a Firecracker config."""
-    try:
-        config = json.loads(config_path.read_text())
-        args = config["boot-source"]["boot_args"]
-        ip_arg = next(part for part in args.split() if part.startswith("ip="))
-        return ip_arg.removeprefix("ip=").split(":", 1)[0]
-    except (OSError, ValueError, KeyError, TypeError, StopIteration):
-        return ""
-
-
 def persist_env_var_secret(private_key: Path, guest_ip: str, secret: str) -> None:
     """Mirror the exec-time key into guest tmpfs for restart recovery."""
     proc = subprocess.run(
@@ -89,29 +75,6 @@ def persist_env_var_secret(private_key: Path, guest_ip: str, secret: str) -> Non
         )
 
 
-def _reprovision_running_bottles(client: OrchestratorClient) -> None:
-    """Read keys from live agent VMs and restore the restarted gateway."""
-    try:
-        secrets_by_ip: dict[str, str] = {}
-        for run_dir in cleanup.live_run_dirs():
-            guest_ip = _guest_ip_from_config(run_dir / "config.json")
-            private_key = run_dir / "bottle_id_ed25519"
-            if not guest_ip or not private_key.is_file():
-                continue
-            proc = subprocess.run(
-                util.ssh_base_argv(private_key, guest_ip)
-                + [f"cat {_ENV_VAR_SECRET_PATH}"],
-                capture_output=True, text=True, check=False,
-            )
-            if proc.returncode == 0 and proc.stdout.strip():
-                secrets_by_ip[guest_ip] = proc.stdout.strip()
-        count = reprovision_bottles(client, secrets_by_ip)
-        if count:
-            info(f"reprovisioned egress tokens for {count} Firecracker bottle(s)")
-    except (OSError, OrchestratorClientError) as exc:
-        info(f"egress token reprovision skipped: {exc}")
-
-
 def launch_consolidated(
     egress_plan: EgressPlan,
     git_gate_plan: GitGatePlan,
@@ -126,7 +89,6 @@ def launch_consolidated(
     service = FirecrackerInfraService()
     url = service.ensure_running()
     client = OrchestratorClient(url)
-    _reprovision_running_bottles(client)
 
     # Read the gateway's provisioning transport + CA off the Gateway service.
     gateway = service.gateway()

bot_bottle/backend/firecracker/infra.py

@@ -17,6 +17,7 @@ from ...orchestrator.lifecycle import DEFAULT_STARTUP_TIMEOUT_SECONDS
 from . import infra_vm
 from .gateway import FirecrackerGateway
 from .orchestrator import FirecrackerOrchestrator
+from .reconcile import attach_bottled_agents_to_gateway
 
 
 class FirecrackerInfraService(InfraService):
@@ -67,9 +68,11 @@ class FirecrackerInfraService(InfraService):
             # holds the signing key) mints the role-scoped `gateway` JWT for the
             # gateway, which never sees the key (#469).
             orchestrator.ensure_running(startup_timeout=startup_timeout)
-            self.gateway().connect_to_orchestrator(
+            gateway = self.gateway()
+            gateway.connect_to_orchestrator(
                 orchestrator.gateway_url(), orchestrator.mint_gateway_token())
             infra_vm.record_booted_version(want)
+            attach_bottled_agents_to_gateway(url, gateway)
             return url
 
     def stop(self) -> None:

bot_bottle/backend/firecracker/reconcile.py

@@ -0,0 +1,199 @@
+"""Bring-up reconcile: re-attach all running agent VMs to a freshly-booted gateway.
+
+Called from the cold-boot branch of `FirecrackerInfraService.ensure_running()`
+after `gateway.connect_to_orchestrator()` completes. Restores the three
+gateway-dependent services for every live bottle:
+
+  - **CA** — push the current (freshly-minted) gateway CA to each agent's
+    trust store and run `update-ca-certificates`, so the agent trusts the
+    new CA on its next egress call.
+  - **git-gate** — re-provision each bottle's bare repos and per-repo creds
+    in the gateway, rebuilt from the persisted host-side state dir.
+  - **egress tokens** — read the agent's `ENV_VAR_SECRET` and feed
+    `reprovision_bottles` to restore the orchestrator's in-memory tokens.
+
+Per-bottle failures are caught, logged, and skipped so one unreachable VM
+does not block the others or the gateway coming up (PRD 0081).
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+from pathlib import Path
+
+from ...bottle_state import git_gate_state_dir
+from ...gateway import GatewayError
+from ...gateway.git_gate.render import GitGateUpstream
+from ...git_gate.plan import GitGatePlan
+from ...log import info
+from ...orchestrator.client import OrchestratorClient, OrchestratorClientError
+from ...orchestrator.reprovision import reprovision_bottles
+from ..provision_gateway import provision_git_gate
+from ..util import AGENT_CA_PATH
+from . import cleanup, util
+from .gateway import FirecrackerGateway
+
+# Where persist_env_var_secret writes the key on the agent VM (matches
+# consolidated_launch._ENV_VAR_SECRET_PATH — duplicated to avoid a
+# circular import through infra.py).
+_ENV_VAR_SECRET_PATH = "/run/bot-bottle/env-var-secret"
+
+
+def _guest_ip_from_config(config_path: Path) -> str:
+    """Read the agent VM's guest IP from its Firecracker config file."""
+    try:
+        config = json.loads(config_path.read_text())
+        args = config["boot-source"]["boot_args"]
+        ip_arg = next(p for p in args.split() if p.startswith("ip="))
+        return ip_arg.removeprefix("ip=").split(":", 1)[0]
+    except (OSError, ValueError, KeyError, TypeError, StopIteration):
+        return ""
+
+
+def attach_bottled_agents_to_gateway(
+    orchestrator_url: str, gateway: FirecrackerGateway,
+) -> None:
+    """Reconcile all live agent VMs against the freshly-booted gateway.
+
+    Fetches the new CA, lists registered bottles, then for each live run dir
+    pushes the CA, re-provisions git-gate, and restores egress tokens. Each
+    per-bottle step is wrapped so a failure is logged and skipped."""
+    try:
+        ca_pem = gateway.ca_cert_pem()
+    except GatewayError as exc:
+        info(f"bring-up reconcile: could not fetch gateway CA, skipping: {exc}")
+        return
+
+    client = OrchestratorClient(orchestrator_url)
+    try:
+        bottles = client.list_bottles()
+    except OrchestratorClientError as exc:
+        info(f"bring-up reconcile: could not list bottles, skipping: {exc}")
+        return
+
+    source_ip_to_bottle_id: dict[str, str] = {}
+    for b in bottles:
+        source_ip = b.get("source_ip")
+        bottle_id = b.get("bottle_id")
+        if isinstance(source_ip, str) and isinstance(bottle_id, str):
+            source_ip_to_bottle_id[source_ip] = bottle_id
+
+    transport = gateway.provisioning_transport()
+    secrets_by_ip: dict[str, str] = {}
+
+    for run_dir in cleanup.live_run_dirs():
+        slug = run_dir.name
+        guest_ip = _guest_ip_from_config(run_dir / "config.json")
+        private_key = run_dir / "bottle_id_ed25519"
+        if not guest_ip or not private_key.is_file():
+            continue
+
+        # CA: push the gateway's current certificate to the agent's trust store.
+        try:
+            _push_ca(private_key, guest_ip, ca_pem)
+        except Exception as exc:
+            info(f"bring-up reconcile: CA push to {slug!r} failed: {exc}")
+
+        # git-gate: re-provision repos + creds from the persisted state dir.
+        bottle_id = source_ip_to_bottle_id.get(guest_ip)
+        if bottle_id:
+            try:
+                _reprovision_git_gate(transport, bottle_id, slug)
+            except Exception as exc:
+                info(f"bring-up reconcile: git-gate for {slug!r} failed: {exc}")
+
+        # egress tokens: read ENV_VAR_SECRET from the agent's tmpfs.
+        proc = subprocess.run(
+            util.ssh_base_argv(private_key, guest_ip) + [f"cat {_ENV_VAR_SECRET_PATH}"],
+            capture_output=True, text=True, check=False,
+        )
+        if proc.returncode == 0 and proc.stdout.strip():
+            secrets_by_ip[guest_ip] = proc.stdout.strip()
+
+    # Restore in-memory egress tokens for all bottles that exposed a key.
+    if secrets_by_ip:
+        try:
+            count = reprovision_bottles(client, secrets_by_ip)
+            if count:
+                info(
+                    f"bring-up reconcile: restored egress tokens for {count} bottle(s)"
+                )
+        except OrchestratorClientError as exc:
+            info(f"bring-up reconcile: egress token restore failed: {exc}")
+
+
+def _push_ca(private_key: Path, guest_ip: str, ca_pem: str) -> None:
+    """SSH the gateway CA PEM into the agent VM and update its trust store.
+
+    Runs as the SSH root user (the agent VM's dropbear accepts root). Each
+    step uses check=True so a failure raises and the caller's except clause
+    logs and continues."""
+    ssh = util.ssh_base_argv(private_key, guest_ip)
+    # mkdir is idempotent; rootfs builds may not preserve the target dir.
+    subprocess.run(
+        ssh + ["mkdir -p /usr/local/share/ca-certificates"],
+        capture_output=True, check=True,
+    )
+    subprocess.run(
+        ssh + [f"cat > {AGENT_CA_PATH}"],
+        input=ca_pem, text=True, capture_output=True, check=True,
+    )
+    subprocess.run(
+        ssh + [f"chmod 644 {AGENT_CA_PATH} && update-ca-certificates"],
+        capture_output=True, check=True,
+    )
+
+
+def _reprovision_git_gate(
+    transport: object, bottle_id: str, slug: str,
+) -> None:
+    """Re-provision one bottle's git-gate repos and creds from its state dir.
+
+    Reads `upstreams.json` (written by `provision_git_gate_dynamic_keys` at
+    launch) to reconstruct the GitGateUpstream table, then calls
+    `provision_git_gate` which places the credential files and inits the bare
+    repos in the gateway. No-op when there is no `upstreams.json` (the bottle
+    has no git upstreams)."""
+    state_dir = git_gate_state_dir(slug)
+    upstreams_file = state_dir / "upstreams.json"
+    if not upstreams_file.exists():
+        return
+
+    raw = json.loads(upstreams_file.read_text())
+    upstreams: list[GitGateUpstream] = []
+    for u in raw:
+        name = u["name"]
+        # The key in the state dir is preferred: gitea deploy keys are written
+        # there by provision_git_gate_dynamic_keys; static keys keep their
+        # original manifest path.
+        key_in_state = state_dir / f"{name}-key"
+        identity_file = (
+            str(key_in_state) if key_in_state.is_file()
+            else u.get("identity_file", "")
+        )
+        known_hosts = state_dir / f"{name}-known_hosts"
+        upstreams.append(GitGateUpstream(
+            name=name,
+            upstream_url=u["upstream_url"],
+            upstream_host=u.get("upstream_host", ""),
+            upstream_port=u.get("upstream_port", ""),
+            identity_file=identity_file,
+            known_host_key=u.get("known_host_key", ""),
+            known_hosts_file=known_hosts if known_hosts.is_file() else Path(),
+        ))
+
+    if not upstreams:
+        return
+
+    plan = GitGatePlan(
+        slug=slug,
+        entrypoint_script=state_dir / "git_gate_entrypoint.sh",
+        hook_script=state_dir / "git_gate_pre_receive.sh",
+        access_hook_script=state_dir / "git_gate_access_hook.sh",
+        upstreams=tuple(upstreams),
+    )
+    provision_git_gate(transport, bottle_id, plan)  # type: ignore[arg-type]
+
+
+__all__ = ["attach_bottled_agents_to_gateway"]

bot_bottle/git_gate/provision.py

@@ -8,6 +8,7 @@ imported (`deploy_key_provisioner`) to keep its cost off the host path.
 
 from __future__ import annotations
 
+import json
 import os
 import dataclasses
 from pathlib import Path
@@ -138,7 +139,32 @@ def provision_git_gate_dynamic_keys(
             if upstream.name not in updated_names:
                 updated.append(upstream)
 
-    return dataclasses.replace(plan, upstreams=tuple(updated))
+    final_plan = dataclasses.replace(plan, upstreams=tuple(updated))
+    _write_upstreams_snapshot(stage_dir, final_plan.upstreams)
+    return final_plan
+
+
+def _write_upstreams_snapshot(
+    stage_dir: Path, upstreams: tuple[GitGateUpstream, ...]
+) -> None:
+    """Persist the fully-resolved upstream table to `upstreams.json` in
+    `stage_dir` so the bring-up reconcile can re-provision git-gate without
+    the manifest. Written after dynamic keys are provisioned so every
+    identity_file is set."""
+    data = [
+        {
+            "name": u.name,
+            "upstream_url": u.upstream_url,
+            "upstream_host": u.upstream_host,
+            "upstream_port": u.upstream_port,
+            "identity_file": u.identity_file,
+            "known_host_key": u.known_host_key,
+        }
+        for u in upstreams
+    ]
+    snapshot = stage_dir / "upstreams.json"
+    snapshot.write_text(json.dumps(data, indent=2))
+    snapshot.chmod(0o600)
 
 
 __all__ = [

bot_bottle/orchestrator/api.py

@@ -50,6 +50,12 @@ class ReprovisionBody(_StrictModel):
     env_var_secret: StrictStr
 
 
+class SecretBody(_StrictModel):
+    name: StrictStr
+    value: StrictStr
+    env_var_secret: StrictStr
+
+
 class ReconcileBody(_StrictModel):
     live_source_ips: list[StrictStr]
     grace_seconds: float | None = None
@@ -259,6 +265,21 @@ def create_app(orch: OrchestratorCore, *, signing_key: str) -> FastAPI:
             return {"reprovisioned": True}
         raise HTTPException(404, "no stored secrets for this bottle")
 
+    @app.post("/bottles/{bottle_id}/secret")
+    def update_secret(bottle_id: str, body: SecretBody) -> dict[str, object]:
+        # Update ONE egress token for a running bottle in place — the
+        # single-secret form of reprovision, for refreshing a short-lived host
+        # credential (e.g. the Codex access token) without a relaunch. cli-only
+        # (not in _GATEWAY_ROUTES): a bottle must never set its own tokens.
+        if orch.update_agent_secret(
+            bottle_id,
+            _required(body.name, "name"),
+            _required(body.value, "value"),
+            _required(body.env_var_secret, "env_var_secret"),
+        ):
+            return {"updated": True}
+        raise HTTPException(404, "no such bottle")
+
     @app.delete("/bottles/{bottle_id}")
     def teardown(bottle_id: str) -> dict[str, object]:
         if orch.teardown_bottle(bottle_id):

bot_bottle/orchestrator/client.py

@@ -184,6 +184,27 @@ class OrchestratorClient:
             )
         return True
 
+    def update_agent_secret(
+        self, bottle_id: str, name: str, value: str, env_var_secret: str,
+    ) -> bool:
+        """Update ONE egress token for a running bottle in place
+        (`POST /bottles/<id>/secret`) — the single-secret form of
+        `reprovision_gateway`, for pushing a freshly-refreshed host credential
+        into a bottle without a relaunch. Returns True on success, False when the
+        orchestrator doesn't know the bottle (404)."""
+        status, _ = self._request(
+            "POST",
+            f"/bottles/{bottle_id}/secret",
+            {"name": name, "value": value, "env_var_secret": env_var_secret},
+        )
+        if status == 404:
+            return False
+        if not 200 <= status < 300:
+            raise OrchestratorClientError(
+                f"update_agent_secret {bottle_id}: HTTP {status}"
+            )
+        return True
+
     def teardown_bottle(self, bottle_id: str) -> bool:
         """Tear a bottle down (`DELETE /bottles/<id>`). False if the
         orchestrator didn't know it (404) — idempotent for cleanup paths."""

bot_bottle/orchestrator/service.py

@@ -379,6 +379,30 @@ class OrchestratorCore:
         self._tokens[bottle_id] = decrypted
         return True
 
+    def update_agent_secret(
+        self, bottle_id: str, name: str, value: str, env_var_secret: str,
+    ) -> bool:
+        """Update ONE egress token for a known bottle in place — the
+        single-secret form of ``reprovision_from_secret``.
+
+        Sets the in-memory token AND upserts the single re-encrypted row under
+        *env_var_secret* (the same key the rest of the rows are encrypted with, so
+        the whole set stays decryptable by a later ``reprovision_from_secret``),
+        leaving every other token untouched. Returns False if the bottle is
+        unknown.
+
+        Unlike ``reprovision_from_secret`` (which restores the values captured at
+        launch), this pushes a *caller-supplied* value — used to refresh a
+        short-lived host credential (e.g. the Codex access token) into a
+        still-running bottle without a relaunch."""
+        from .store.secret_store import encrypt_value
+        if self.registry.get(bottle_id) is None:
+            return False
+        self._tokens.setdefault(bottle_id, {})[name] = value
+        self.registry.store_agent_secret(
+            bottle_id, name, encrypt_value(env_var_secret, value))
+        return True
+
     # --- consolidated gateway ----------------------------------------------
 
     def gateway_status(self) -> dict[str, object]:

bot_bottle/orchestrator/store/registry_store.py

@@ -370,6 +370,30 @@ class RegistryStore(DbStore):
             )
         self._chmod()
 
+    def store_agent_secret(
+        self,
+        bottle_id: str,
+        key: str,
+        encrypted_value: str,
+        secret_type: str = "injected_env_var",
+    ) -> None:
+        """Upsert ONE encrypted secret row (env-var name → ciphertext) for
+        *bottle_id*, leaving the bottle's other secrets untouched — the per-key
+        counterpart of ``store_agent_secrets``' replace-all. Delete-then-insert
+        because the table carries no unique constraint to `ON CONFLICT` against."""
+        with self._connection() as conn:
+            conn.execute(
+                "DELETE FROM bottled_agent_secrets "
+                "WHERE bottled_agent_id = ? AND key = ? AND type = ?",
+                (bottle_id, key, secret_type),
+            )
+            conn.execute(
+                "INSERT INTO bottled_agent_secrets "
+                "(bottled_agent_id, key, value, type) VALUES (?, ?, ?, ?)",
+                (bottle_id, key, encrypted_value, secret_type),
+            )
+        self._chmod()
+
     def get_agent_secrets(
         self,
         bottle_id: str,

docs/prds/0081-reprovision-gateway-state-on-bringup.md

@@ -0,0 +1,117 @@
+# PRD 0081: Reprovision gateway-dependent state on gateway bring-up
+
+- **Status:** Active
+- **Author:** claude
+- **Created:** 2026-07-26
+- **Issue:** #516
+
+## Summary
+
+When the gateway is (re)built, reconcile every already-running bottle against the
+fresh gateway instead of persisting the gateway's state. On a gateway cold boot,
+the gateway reconciles all live bottles in one flow: **replace** each agent's
+trusted CA with the freshly-minted gateway CA, **re-provision** each bottle's
+git-gate repos + creds onto the gateway, and **restore** each bottle's egress
+tokens. One mechanism across all three services; the CA rotates for free on every
+bring-up.
+
+## Problem
+
+A gateway rebuild/restart silently breaks every already-running bottle:
+
+- **CA (#510).** The gateway's mitmproxy mints a new CA on a fresh rootfs; agents
+  still trust the old one, so egress fails TLS verification (`SSL certificate
+  verification failed`).
+- **git-gate (#512).** Per-bottle bare repos (`/git/<id>`) + deploy creds
+  (`/git-gate/creds/<id>`) live in the gateway's ephemeral rootfs; a rebuild wipes
+  them and the agent 404s on fetch/push.
+
+Both are the same root cause: per-bottle gateway-dependent state is provisioned
+**once, at bottle launch**, and nothing restores it for already-running bottles
+when the gateway comes back. The existing launch-time reprovision
+(`reprovision_bottles`) restores **only** egress tokens, and only as a side effect
+of the *next* launch.
+
+Persisting the state (a host bind-mount / a per-VM data volume per service) was
+prototyped and rejected: it differs per service (a volume for the CA, another for
+git-gate, reprovision for tokens), it pins the CA static forever (no rotation),
+and it adds volume surface that `docker volume prune` / a wiped cache can silently
+destroy (the original #450 failure mode).
+
+## Goals / Success Criteria
+
+- A gateway (re)boot restores **all** running bottles' gateway-dependent state
+  with no manual step and no relaunch — the agent's next egress / fetch / push
+  just works.
+- **One** reconcile flow covering CA, git-gate, and egress tokens, rather than a
+  different mechanism per service.
+- The CA **rotates** on every gateway bring-up (no long-lived CA), distributed to
+  running agents by the same reconcile.
+- Per-bottle failures are tolerated: one unreachable or malformed bottle does not
+  block the others or the gateway coming up.
+- **All backends** (Firecracker, docker, macOS) reconcile through the *same*
+  contract — an abstract method on the backend base class, so a new backend
+  cannot forget to implement it and none drifts onto a bespoke mechanism.
+
+## Non-goals
+
+- Deliberate mid-session CA rotation *without* a gateway restart — `rotate_ca`
+  stays for that operator action.
+- Changing source-IP attribution, `/resolve`, or the plane split (#469).
+
+## Design
+
+**The contract — `attach_bottled_agents_to_gateway()` on the backend ABC.**
+Reconciling running bottles against the current gateway is a backend
+responsibility (only the backend can enumerate its agents and reach them —
+firecracker over SSH, docker/macOS over `exec`/`cp`), so it is an
+`@abc.abstractmethod` on `BottleBackend` (`backend/base.py`). Every backend
+implements it; the host calls it whenever the gateway is (re)brought up. This is
+what makes the fix cross-backend by construction rather than a per-backend
+follow-up. It reprovisions **all** registered bottles' gateway-dependent state:
+CA, git-gate, and egress tokens.
+
+**Trigger — the gateway bring-up path.** The host calls
+`attach_bottled_agents_to_gateway()` only when the gateway was actually
+(re)brought up — the cold-boot branch of the infra bring-up (e.g.
+`FirecrackerInfraService.ensure_running` after it boots a fresh pair), never on an
+adopt of a healthy, current gateway (state intact). So it fires exactly when the
+gateway was (re)booted — including orchestrator restarts, since the pair boots
+together — and there is no bare-restart path that bypasses bring-up.
+
+**Per-backend implementation.** Each backend's `attach_bottled_agents_to_gateway`
+enumerates its live bottles and, for each, reconciles the three services against
+the current gateway. The firecracker implementation, once the gateway VM is up
+and its CA is available:
+
+1. Map each live bottle's guest IP → `bottle_id` from the orchestrator registry
+   (`list_bottles`).
+2. Install the **current** shared git-gate hooks (`git_gate_render_hook` /
+   `git_gate_render_access_hook`) into the fresh gateway once — rendered from
+   code, never from a bottle's possibly-stale state dir.
+3. For each live agent VM (enumerated from its run dir):
+   - **CA:** SSH the current gateway CA into the agent's trust store and run
+     `update-ca-certificates` (unconditional replace — there is one gateway, so no
+     fingerprint match is needed).
+   - **git-gate:** rebuild the bottle's upstreams from its persisted git-gate
+     state dir (deploy key, known_hosts, upstream URL) and re-init its bare repos
+     + per-repo creds under `/git/<bottle_id>`.
+   - **egress token:** read the agent's `ENV_VAR_SECRET` and feed
+     `reprovision_bottles`, restoring the orchestrator's in-memory tokens.
+4. Every per-bottle step is wrapped so one failure is logged and skipped.
+
+**Retire the persistence prototype.** No CA data volume, no git-gate data volume
+(the abandoned PRs #511 / #513). The launch-time `_reprovision_running_bottles`
+call folds into this bring-up reconcile, so egress tokens are restored on the same
+cold-boot trigger (an adopt needs no restore — the orchestrator never restarted).
+
+**CA rotation.** Because the gateway rootfs is ephemeral, every cold boot mints a
+fresh CA; reconcile is what distributes it, so a routine gateway rebuild doubles
+as a CA rotation with zero extra machinery.
+
+## Open questions
+
+- Docker's existing host-bind-mounted CA (`host_gateway_ca_dir`): once docker's
+  `attach_bottled_agents_to_gateway` pushes the CA to running agents on bring-up,
+  the bind-mount is redundant — drop it (so docker rotates like firecracker) or
+  keep it as belt-and-suspenders? Leaning drop, for one behaviour across backends.

tests/unit/test_backend_secret_reprovision.py

@@ -2,9 +2,7 @@
 
 from __future__ import annotations
 
-import json
 import subprocess
-import tempfile
 import unittest
 from pathlib import Path
 from types import SimpleNamespace
@@ -107,21 +105,6 @@ class TestDockerReprovision(unittest.TestCase):
 
 
 class TestFirecrackerReprovision(unittest.TestCase):
-    def _run_dir(self, root: Path, ip: str = "10.243.0.3") -> Path:
-        run_dir = root / "demo"
-        run_dir.mkdir()
-        (run_dir / "bottle_id_ed25519").write_text("key")
-        (run_dir / "config.json").write_text(json.dumps({
-            "boot-source": {"boot_args": f"root=/dev/vda ip={ip}::gw:mask::eth0:off"}
-        }))
-        return run_dir
-
-    def test_extracts_guest_ip_from_config(self) -> None:
-        with tempfile.TemporaryDirectory() as tmp:
-            run_dir = self._run_dir(Path(tmp))
-            self.assertEqual("10.243.0.3", fc._guest_ip_from_config(run_dir / "config.json"))
-            self.assertEqual("", fc._guest_ip_from_config(run_dir / "missing.json"))
-
     def test_persists_key_over_stdin_not_argv(self) -> None:
         with patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
              patch.object(fc.subprocess, "run", return_value=_proc()) as run:
@@ -135,29 +118,6 @@ class TestFirecrackerReprovision(unittest.TestCase):
             with self.assertRaisesRegex(fc.ConsolidatedLaunchError, "denied"):
                 fc.persist_env_var_secret(Path("/key"), "10.0.0.1", "secret")
 
-    def test_reads_live_vm_key_and_reprovisions(self) -> None:
-        with tempfile.TemporaryDirectory() as tmp:
-            run_dir = self._run_dir(Path(tmp))
-            client = Mock()
-            with patch.object(fc.cleanup, "live_run_dirs", return_value=(run_dir,)), \
-                 patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
-                 patch.object(fc.subprocess, "run", return_value=_proc(stdout="secret\n")), \
-                 patch.object(fc, "reprovision_bottles", return_value=1) as restore, \
-                 patch.object(fc, "info"):
-                fc._reprovision_running_bottles(client)
-        restore.assert_called_once_with(client, {"10.243.0.3": "secret"})
-
-    def test_unreadable_vm_is_skipped(self) -> None:
-        with tempfile.TemporaryDirectory() as tmp:
-            run_dir = self._run_dir(Path(tmp))
-            client = Mock()
-            with patch.object(fc.cleanup, "live_run_dirs", return_value=(run_dir,)), \
-                 patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
-                 patch.object(fc.subprocess, "run", return_value=_proc(1)), \
-                 patch.object(fc, "reprovision_bottles", return_value=0) as restore:
-                fc._reprovision_running_bottles(client)
-        restore.assert_called_once_with(client, {})
-
 
 if __name__ == "__main__":
     unittest.main()

tests/unit/test_firecracker_infra.py

@@ -21,6 +21,7 @@ from bot_bottle.backend.firecracker.orchestrator import FirecrackerOrchestrator
 # there (not at their home modules).
 _ORCH_CLS = "bot_bottle.backend.firecracker.infra.FirecrackerOrchestrator"
 _GW_CLS = "bot_bottle.backend.firecracker.infra.FirecrackerGateway"
+_RECONCILE = "bot_bottle.backend.firecracker.infra.attach_bottled_agents_to_gateway"
 
 
 class TestAccessors(unittest.TestCase):
@@ -53,10 +54,13 @@ class TestEnsureRunning(unittest.TestCase):
                     patch.object(infra_vm, "_health_ok", return_value=True), \
                     patch.object(infra_vm, "_pidfile_alive", return_value=True), \
                     patch.object(infra_vm, "stop") as stop, \
-                    patch.object(infra_vm, "ensure_built") as built:
+                    patch.object(infra_vm, "ensure_built") as built, \
+                    patch(_RECONCILE) as reconcile:
                 url = FirecrackerInfraService().ensure_running()
             stop.assert_not_called()
             built.assert_not_called()
+            # Adopt path: no reconcile — gateway state is intact.
+            reconcile.assert_not_called()
             ip = infra_vm.netpool.orch_slot().guest_ip
             self.assertEqual(f"http://{ip}:{infra_vm.ORCHESTRATOR_PORT}", url)
 
@@ -73,7 +77,8 @@ class TestEnsureRunning(unittest.TestCase):
                     patch.object(infra_vm, "_pidfile_alive", return_value=True), \
                     patch.object(infra_vm, "stop") as stop, \
                     patch.object(infra_vm, "ensure_built"), \
-                    patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls:
+                    patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls, \
+                    patch(_RECONCILE) as reconcile:
                 orch_cls.return_value.gateway_url.return_value = "http://10.243.255.1:8099"
                 orch_cls.return_value.mint_gateway_token.return_value = "gw.jwt"
                 FirecrackerInfraService().ensure_running()
@@ -85,6 +90,8 @@ class TestEnsureRunning(unittest.TestCase):
                 "http://10.243.255.1:8099", "gw.jwt")
             # The fresh boot records the current version for the next launcher.
             self.assertEqual("v-current\n", (d / "booted-version").read_text())
+            # Cold boot: reconcile fires after gateway is up.
+            reconcile.assert_called_once()
 
     def test_reboots_when_gateway_dead(self):
         # Orchestrator healthy + marker current, but the gateway VM is gone ->
@@ -99,11 +106,13 @@ class TestEnsureRunning(unittest.TestCase):
                     patch.object(infra_vm, "_pidfile_alive", return_value=False), \
                     patch.object(infra_vm, "stop") as stop, \
                     patch.object(infra_vm, "ensure_built"), \
-                    patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls:
+                    patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls, \
+                    patch(_RECONCILE) as reconcile:
                 FirecrackerInfraService().ensure_running()
             stop.assert_called_once()
             orch_cls.return_value.ensure_running.assert_called_once()
             gw_cls.return_value.connect_to_orchestrator.assert_called_once()
+            reconcile.assert_called_once()
 
     def test_boots_both_when_no_running_pair(self):
         with tempfile.TemporaryDirectory() as td:
@@ -112,7 +121,8 @@ class TestEnsureRunning(unittest.TestCase):
                     patch.object(infra_vm, "_health_ok", return_value=False), \
                     patch.object(infra_vm, "stop") as stop, \
                     patch.object(infra_vm, "ensure_built") as built, \
-                    patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls:
+                    patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls, \
+                    patch(_RECONCILE) as reconcile:
                 FirecrackerInfraService().ensure_running()
         stop.assert_called_once()      # clear stale VMs first
         built.assert_called_once()
@@ -120,6 +130,27 @@ class TestEnsureRunning(unittest.TestCase):
         # reach the control plane at startup).
         orch_cls.return_value.ensure_running.assert_called_once()
         gw_cls.return_value.connect_to_orchestrator.assert_called_once()
+        # Cold boot: reconcile fires to restore CA, git-gate, and egress tokens.
+        reconcile.assert_called_once()
+
+    def test_reconcile_receives_url_and_fresh_gateway(self):
+        # reconcile gets the orchestrator URL and the gateway service (not the
+        # class). Verify the args to ensure it can reach the right endpoints.
+        with tempfile.TemporaryDirectory() as td:
+            with patch.object(infra_vm, "_infra_dir", return_value=Path(td)), \
+                    patch.object(infra_vm, "expected_version", return_value="v-current"), \
+                    patch.object(infra_vm, "_health_ok", return_value=False), \
+                    patch.object(infra_vm, "stop"), \
+                    patch.object(infra_vm, "ensure_built"), \
+                    patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls, \
+                    patch(_RECONCILE) as reconcile:
+                orch_cls.return_value.url.return_value = "http://10.243.255.1:8099"
+                FirecrackerInfraService().ensure_running()
+        call_args = reconcile.call_args
+        # First arg: the orchestrator's host URL.
+        self.assertEqual("http://10.243.255.1:8099", call_args.args[0])
+        # Second arg: the gateway service instance (not the class).
+        self.assertIs(gw_cls.return_value, call_args.args[1])
 
 
 if __name__ == "__main__":

tests/unit/test_firecracker_reconcile.py

@@ -0,0 +1,280 @@
+"""Unit: bring-up reconcile — CA push, git-gate reprovision, egress tokens.
+
+Covers attach_bottled_agents_to_gateway (reconcile.py): each per-bottle step
+fires with correct args, failures on one bottle don't block others, and the
+egress-token restore path works end-to-end. Does NOT spin up VMs or real SSH.
+"""
+
+from __future__ import annotations
+
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from subprocess import CalledProcessError, CompletedProcess
+from unittest.mock import MagicMock, patch
+
+_RECONCILE = "bot_bottle.backend.firecracker.reconcile"
+
+
+class TestGuestIpFromConfig(unittest.TestCase):
+    def test_reads_ip_from_boot_args(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import _guest_ip_from_config
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
+            json.dump({
+                "boot-source": {"boot_args": "console=ttyS0 ip=10.243.0.5:10.243.0.6:255.255.255.254"},
+            }, f)
+            name = f.name
+        self.assertEqual("10.243.0.5", _guest_ip_from_config(Path(name)))
+
+    def test_returns_empty_on_missing_file(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import _guest_ip_from_config
+        self.assertEqual("", _guest_ip_from_config(Path("/nonexistent/config.json")))
+
+    def test_returns_empty_on_bad_json(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import _guest_ip_from_config
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
+            f.write("not-json")
+            name = f.name
+        self.assertEqual("", _guest_ip_from_config(Path(name)))
+
+
+class TestPushCa(unittest.TestCase):
+    def test_runs_three_ssh_commands_in_order(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import _push_ca
+        with tempfile.TemporaryDirectory() as td:
+            key = Path(td) / "key"
+            key.write_text("k")
+            with patch(f"{_RECONCILE}.subprocess.run") as run:
+                run.return_value = CompletedProcess([], 0)
+                _push_ca(key, "10.0.0.1", "-----BEGIN CERTIFICATE-----\n")
+        calls = run.call_args_list
+        self.assertEqual(3, len(calls))
+        # Each call's argv is a list; the remote command is the last element.
+        self.assertIn("mkdir", calls[0].args[0][-1])
+        self.assertIn("cat >", calls[1].args[0][-1])
+        self.assertIn("update-ca-certificates", calls[2].args[0][-1])
+
+    def test_passes_ca_pem_via_stdin(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import _push_ca
+        with tempfile.TemporaryDirectory() as td:
+            key = Path(td) / "key"
+            key.write_text("k")
+            with patch(f"{_RECONCILE}.subprocess.run") as run:
+                run.return_value = CompletedProcess([], 0)
+                _push_ca(key, "10.0.0.1", "MY-CA-PEM")
+        cat_call = run.call_args_list[1]
+        self.assertEqual("MY-CA-PEM", cat_call.kwargs.get("input"))
+
+    def test_raises_on_ssh_failure(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import _push_ca
+        with tempfile.TemporaryDirectory() as td:
+            key = Path(td) / "key"
+            key.write_text("k")
+            with patch(f"{_RECONCILE}.subprocess.run") as run:
+                run.side_effect = CalledProcessError(1, "ssh")
+                with self.assertRaises(CalledProcessError):
+                    _push_ca(key, "10.0.0.1", "pem")
+
+
+class TestReprovisionGitGate(unittest.TestCase):
+    def _make_state_dir(self, upstreams: list[dict[str, str]]) -> Path:
+        d = Path(tempfile.mkdtemp())
+        (d / "upstreams.json").write_text(json.dumps(upstreams))
+        (d / "git_gate_pre_receive.sh").write_text("#!/bin/sh")
+        (d / "git_gate_access_hook.sh").write_text("#!/bin/sh")
+        (d / "git_gate_entrypoint.sh").write_text("#!/bin/sh")
+        return d
+
+    def test_no_op_when_no_upstreams_json(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import _reprovision_git_gate
+        transport = MagicMock()
+        with tempfile.TemporaryDirectory() as td:
+            # The state dir exists but has no upstreams.json
+            with patch(f"{_RECONCILE}.git_gate_state_dir", return_value=Path(td)):
+                _reprovision_git_gate(transport, "bottle-123", "my-slug")
+        transport.exec.assert_not_called()
+
+    def test_calls_provision_git_gate_with_upstreams(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import _reprovision_git_gate
+        transport = MagicMock()
+        upstreams = [{
+            "name": "bot-bottle",
+            "upstream_url": "ssh://git@gitea.example/org/bot-bottle.git",
+            "upstream_host": "gitea.example",
+            "upstream_port": "22",
+            "identity_file": "/home/node/.ssh/id_ed25519",
+            "known_host_key": "ssh-ed25519 AAAA...",
+        }]
+        state_dir = self._make_state_dir(upstreams)
+        # Write the key file so identity_file falls back to state dir path.
+        (state_dir / "bot-bottle-key").write_bytes(b"PRIVATE")
+        try:
+            with patch(f"{_RECONCILE}.git_gate_state_dir", return_value=state_dir), \
+                    patch(f"{_RECONCILE}.provision_git_gate") as prov:
+                _reprovision_git_gate(transport, "bottle-abc", "my-slug")
+            prov.assert_called_once()
+            _, bottle_id, plan = prov.call_args.args
+            self.assertEqual("bottle-abc", bottle_id)
+            self.assertEqual(1, len(plan.upstreams))
+            self.assertEqual("bot-bottle", plan.upstreams[0].name)
+            # Key in state dir takes priority over identity_file in JSON.
+            self.assertEqual(str(state_dir / "bot-bottle-key"), plan.upstreams[0].identity_file)
+        finally:
+            import shutil; shutil.rmtree(state_dir, ignore_errors=True)
+
+    def test_falls_back_to_manifest_identity_file_when_no_key_in_state(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import _reprovision_git_gate
+        transport = MagicMock()
+        upstreams = [{
+            "name": "repo",
+            "upstream_url": "ssh://git@github.com/org/repo.git",
+            "upstream_host": "github.com",
+            "upstream_port": "22",
+            "identity_file": "/home/node/.ssh/id_ed25519",
+            "known_host_key": "",
+        }]
+        state_dir = self._make_state_dir(upstreams)
+        # No `repo-key` in state dir — static manifest path should be used.
+        try:
+            with patch(f"{_RECONCILE}.git_gate_state_dir", return_value=state_dir), \
+                    patch(f"{_RECONCILE}.provision_git_gate") as prov:
+                _reprovision_git_gate(transport, "bottle-xyz", "my-slug")
+            prov.assert_called_once()
+            _, _, plan = prov.call_args.args
+            self.assertEqual("/home/node/.ssh/id_ed25519", plan.upstreams[0].identity_file)
+        finally:
+            import shutil; shutil.rmtree(state_dir, ignore_errors=True)
+
+
+class TestAttachBottledAgentsToGateway(unittest.TestCase):
+    """Integration-level: the full reconcile loop against mocked SSH + gateway."""
+
+    def _make_run_dir(self, tmp: Path, slug: str, guest_ip: str) -> Path:
+        run_dir = tmp / slug
+        run_dir.mkdir()
+        key = run_dir / "bottle_id_ed25519"
+        key.write_text("PRIVATE")
+        cfg = {
+            "boot-source": {
+                "boot_args": f"console=ttyS0 ip={guest_ip}:{guest_ip}:255.255.255.254",
+            }
+        }
+        (run_dir / "config.json").write_text(json.dumps(cfg))
+        return run_dir
+
+    def _make_gateway(self, ca_pem: str = "-----BEGIN CERTIFICATE-----\n") -> MagicMock:
+        gw = MagicMock()
+        gw.ca_cert_pem.return_value = ca_pem
+        gw.provisioning_transport.return_value = MagicMock()
+        return gw
+
+    def test_skips_all_when_ca_fetch_fails(self) -> None:
+        from bot_bottle.backend.firecracker.gateway import FirecrackerGateway
+        from bot_bottle.backend.firecracker.reconcile import attach_bottled_agents_to_gateway
+        from bot_bottle.gateway import GatewayError
+        gw = MagicMock(spec=FirecrackerGateway)
+        gw.ca_cert_pem.side_effect = GatewayError("timeout")
+        with patch(f"{_RECONCILE}.cleanup.live_run_dirs", return_value=()):
+            attach_bottled_agents_to_gateway("http://orch:8099", gw)
+        # Nothing else is called when CA fetch fails.
+        gw.provisioning_transport.assert_not_called()
+
+    def test_ca_push_called_per_live_bottle(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import attach_bottled_agents_to_gateway
+        gw = self._make_gateway()
+        with tempfile.TemporaryDirectory() as td:
+            tmp = Path(td)
+            rd1 = self._make_run_dir(tmp, "agent-abc12", "10.243.0.5")
+            rd2 = self._make_run_dir(tmp, "agent-xyz99", "10.243.0.7")
+            with patch(f"{_RECONCILE}.cleanup.live_run_dirs", return_value=(rd1, rd2)), \
+                    patch(f"{_RECONCILE}.OrchestratorClient") as client_cls, \
+                    patch(f"{_RECONCILE}._push_ca") as push_ca, \
+                    patch(f"{_RECONCILE}._reprovision_git_gate"), \
+                    patch(f"{_RECONCILE}.subprocess.run",
+                          return_value=CompletedProcess([], 1)):
+                client_cls.return_value.list_bottles.return_value = []
+                attach_bottled_agents_to_gateway("http://orch:8099", gw)
+        # CA pushed to both bottles.
+        self.assertEqual(2, push_ca.call_count)
+
+    def test_per_bottle_ca_failure_does_not_block_others(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import attach_bottled_agents_to_gateway
+        gw = self._make_gateway()
+        with tempfile.TemporaryDirectory() as td:
+            tmp = Path(td)
+            rd1 = self._make_run_dir(tmp, "agent-fail1", "10.243.0.5")
+            rd2 = self._make_run_dir(tmp, "agent-ok99", "10.243.0.7")
+            with patch(f"{_RECONCILE}.cleanup.live_run_dirs", return_value=(rd1, rd2)), \
+                    patch(f"{_RECONCILE}.OrchestratorClient") as client_cls, \
+                    patch(f"{_RECONCILE}._push_ca",
+                          side_effect=[CalledProcessError(1, "ssh"), None]) as push_ca, \
+                    patch(f"{_RECONCILE}._reprovision_git_gate"), \
+                    patch(f"{_RECONCILE}.subprocess.run",
+                          return_value=CompletedProcess([], 1)):
+                client_cls.return_value.list_bottles.return_value = []
+                # Must not raise even though the first bottle's CA push fails.
+                attach_bottled_agents_to_gateway("http://orch:8099", gw)
+        # Both bottles were attempted.
+        self.assertEqual(2, push_ca.call_count)
+
+    def test_egress_token_restored_for_bottles_with_secret(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import attach_bottled_agents_to_gateway
+        gw = self._make_gateway()
+        with tempfile.TemporaryDirectory() as td:
+            tmp = Path(td)
+            rd = self._make_run_dir(tmp, "agent-tok11", "10.243.0.5")
+            bottles = [{"bottle_id": "bid-001", "source_ip": "10.243.0.5"}]
+            with patch(f"{_RECONCILE}.cleanup.live_run_dirs", return_value=(rd,)), \
+                    patch(f"{_RECONCILE}.OrchestratorClient") as client_cls, \
+                    patch(f"{_RECONCILE}._push_ca"), \
+                    patch(f"{_RECONCILE}._reprovision_git_gate"), \
+                    patch(f"{_RECONCILE}.reprovision_bottles") as reprov, \
+                    patch(f"{_RECONCILE}.subprocess.run",
+                          return_value=CompletedProcess([], 0, stdout="secret-key\n")):
+                client_cls.return_value.list_bottles.return_value = bottles
+                reprov.return_value = 1
+                attach_bottled_agents_to_gateway("http://orch:8099", gw)
+        reprov.assert_called_once()
+        _, secrets = reprov.call_args.args
+        self.assertIn("10.243.0.5", secrets)
+        self.assertEqual("secret-key", secrets["10.243.0.5"])
+
+    def test_git_gate_reprovision_called_with_bottle_id(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import attach_bottled_agents_to_gateway
+        gw = self._make_gateway()
+        with tempfile.TemporaryDirectory() as td:
+            tmp = Path(td)
+            rd = self._make_run_dir(tmp, "agent-git11", "10.243.0.5")
+            bottles = [{"bottle_id": "bid-git", "source_ip": "10.243.0.5"}]
+            with patch(f"{_RECONCILE}.cleanup.live_run_dirs", return_value=(rd,)), \
+                    patch(f"{_RECONCILE}.OrchestratorClient") as client_cls, \
+                    patch(f"{_RECONCILE}._push_ca"), \
+                    patch(f"{_RECONCILE}._reprovision_git_gate") as reprov_gw, \
+                    patch(f"{_RECONCILE}.subprocess.run",
+                          return_value=CompletedProcess([], 1)):
+                client_cls.return_value.list_bottles.return_value = bottles
+                attach_bottled_agents_to_gateway("http://orch:8099", gw)
+        reprov_gw.assert_called_once()
+        _, bottle_id_arg, slug_arg = reprov_gw.call_args.args
+        self.assertEqual("bid-git", bottle_id_arg)
+        self.assertEqual("agent-git11", slug_arg)
+
+    def test_run_dir_without_key_file_is_skipped(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import attach_bottled_agents_to_gateway
+        gw = self._make_gateway()
+        with tempfile.TemporaryDirectory() as td:
+            tmp = Path(td)
+            rd = self._make_run_dir(tmp, "agent-nokey", "10.243.0.5")
+            (rd / "bottle_id_ed25519").unlink()  # remove the key
+            with patch(f"{_RECONCILE}.cleanup.live_run_dirs", return_value=(rd,)), \
+                    patch(f"{_RECONCILE}.OrchestratorClient") as client_cls, \
+                    patch(f"{_RECONCILE}._push_ca") as push_ca, \
+                    patch(f"{_RECONCILE}._reprovision_git_gate"):
+                client_cls.return_value.list_bottles.return_value = []
+                attach_bottled_agents_to_gateway("http://orch:8099", gw)
+        push_ca.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/unit/test_orchestrator_client.py

@@ -151,6 +151,31 @@ class TestReprovisionGateway(unittest.TestCase):
                 self.c.reprovision_gateway("b1", "key")
 
 
+class TestUpdateAgentSecret(unittest.TestCase):
+    def setUp(self) -> None:
+        self.c = OrchestratorClient("http://orch:8080")
+
+    def test_success_posts_single_secret(self) -> None:
+        with patch(_URLOPEN, return_value=_resp(200, {"updated": True})) as opened:
+            self.assertTrue(self.c.update_agent_secret("b1", "A", "fresh", "key"))
+        request = opened.call_args.args[0]
+        self.assertEqual("POST", request.get_method())
+        self.assertTrue(request.full_url.endswith("/bottles/b1/secret"))
+        self.assertEqual(
+            {"name": "A", "value": "fresh", "env_var_secret": "key"},
+            json.loads(request.data),
+        )
+
+    def test_unknown_bottle_is_false(self) -> None:
+        with patch(_URLOPEN, side_effect=_http_error(404)):
+            self.assertFalse(self.c.update_agent_secret("b1", "A", "v", "key"))
+
+    def test_other_status_raises(self) -> None:
+        with patch(_URLOPEN, side_effect=_http_error(400)):
+            with self.assertRaises(OrchestratorClientError):
+                self.c.update_agent_secret("b1", "A", "v", "key")
+
+
 class TestHealthAndPolicy(unittest.TestCase):
     def setUp(self) -> None:
         self.c = OrchestratorClient("http://orch:8080")

tests/unit/test_orchestrator_registry_store.py

@@ -199,6 +199,20 @@ class TestAgentSecrets(unittest.TestCase):
         got = self.store.get_agent_secrets("bottle-1")
         self.assertEqual({"K": "new", "K2": "v2"}, got)
 
+    def test_store_agent_secret_upserts_one_key_leaving_others(self) -> None:
+        self.store.store_agent_secrets("bottle-1", {"K": "v1", "K2": "v2"})
+        self.store.store_agent_secret("bottle-1", "K", "v1-new")  # update existing
+        self.store.store_agent_secret("bottle-1", "K3", "v3")     # insert new
+        self.assertEqual(
+            {"K": "v1-new", "K2": "v2", "K3": "v3"},
+            self.store.get_agent_secrets("bottle-1"),
+        )
+
+    def test_store_agent_secret_does_not_duplicate_rows(self) -> None:
+        self.store.store_agent_secret("bottle-1", "K", "v1")
+        self.store.store_agent_secret("bottle-1", "K", "v2")
+        self.assertEqual({"K": "v2"}, self.store.get_agent_secrets("bottle-1"))
+
     def test_delete_removes_secrets(self) -> None:
         self.store.store_agent_secrets("bottle-1", {"K": "v"})
         self.store.delete_agent_secrets("bottle-1")

tests/unit/test_orchestrator_server.py

@@ -125,6 +125,47 @@ class TestDispatch(unittest.TestCase):
             {"EGRESS_TOKEN_0": "upstream-secret"}, self.orch.tokens_for(bottle_id),
         )
 
+    def test_update_agent_secret_in_place(self) -> None:
+        key = base64.urlsafe_b64encode(b"unit-test-key").rstrip(b"=").decode()
+        status, payload = dispatch(
+            self.orch, "POST", "/bottles", _body({
+                "source_ip": "10.243.0.21",
+                "tokens": {"A": "old", "B": "keep"},
+                "env_var_secret": key,
+            }),
+        )
+        self.assertEqual(201, status)
+        bottle_id = payload["bottle_id"]
+        assert isinstance(bottle_id, str)
+        status, response = dispatch(
+            self.orch, "POST", f"/bottles/{bottle_id}/secret",
+            _body({"name": "A", "value": "fresh", "env_var_secret": key}),
+        )
+        self.assertEqual((200, {"updated": True}), (status, response))
+        self.assertEqual({"A": "fresh", "B": "keep"}, self.orch.tokens_for(bottle_id))
+
+    def test_update_agent_secret_validates_and_unknown_bottle(self) -> None:
+        status, _ = dispatch(self.orch, "POST", "/bottles/b1/secret", b"not-json")
+        self.assertEqual(400, status)
+        status, _ = dispatch(
+            self.orch, "POST", "/bottles/b1/secret", _body({"name": "A"}),
+        )
+        self.assertEqual(400, status)  # value + env_var_secret missing
+        status, _ = dispatch(
+            self.orch, "POST", "/bottles/ghost/secret",
+            _body({"name": "A", "value": "v", "env_var_secret": "k"}),
+        )
+        self.assertEqual(404, status)
+
+    def test_update_agent_secret_is_cli_only(self) -> None:
+        # A data-plane (`gateway`) caller must not set a bottle's tokens.
+        status, _ = dispatch(
+            self.orch, "POST", "/bottles/b1/secret",
+            _body({"name": "A", "value": "v", "env_var_secret": "k"}),
+            role=ROLE_GATEWAY,
+        )
+        self.assertEqual(403, status)
+
     def test_reprovision_validates_request_and_missing_rows(self) -> None:
         status, _ = dispatch(
             self.orch, "POST", "/bottles/b1/reprovision_gateway", b"not-json",

tests/unit/test_orchestrator_service.py

@@ -103,6 +103,25 @@ class TestOrchestrator(unittest.TestCase):
         self.assertTrue(self.orch.reprovision_from_secret(rec.bottle_id, key))
         self.assertEqual({"EGRESS_TOKEN_0": "secret"}, self.orch.tokens_for(rec.bottle_id))
 
+    def test_update_agent_secret_refreshes_one_token_in_place(self) -> None:
+        key = new_env_var_secret()
+        rec = self.orch.launch_bottle(
+            "10.243.0.20", tokens={"A": "old-a", "B": "keep-b"}, env_var_secret=key,
+        )
+        self.assertTrue(self.orch.update_agent_secret(rec.bottle_id, "A", "new-a", key))
+        # In memory: A refreshed, B left untouched.
+        self.assertEqual({"A": "new-a", "B": "keep-b"}, self.orch.tokens_for(rec.bottle_id))
+        # At rest: the whole set stays decryptable with the SAME env_var_secret,
+        # so a later reprovision restores the refreshed value (not the launch one).
+        self.orch._tokens.clear()
+        self.assertTrue(self.orch.reprovision_from_secret(rec.bottle_id, key))
+        self.assertEqual({"A": "new-a", "B": "keep-b"}, self.orch.tokens_for(rec.bottle_id))
+
+    def test_update_agent_secret_unknown_bottle_is_false(self) -> None:
+        self.assertFalse(
+            self.orch.update_agent_secret("ghost", "A", "v", new_env_var_secret())
+        )
+
     def test_reprovision_rejects_missing_rows_and_wrong_key(self) -> None:
         self.assertFalse(self.orch.reprovision_from_secret("missing", new_env_var_secret()))
         key = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE"