feat(reconcile): reprovision CA, git-gate, and egress tokens on gateway bring-up (PRD 0081)

On a gateway cold boot, reconcile every live agent VM against the fresh
gateway: push the new CA into each agent's trust store, re-provision
git-gate repos/creds from the persisted upstreams snapshot, and restore
egress tokens. Per-bottle failures are logged and skipped rather than
aborting the whole reconcile.

New: reconcile.py — attach_bottled_agents_to_gateway, _push_ca,
_reprovision_git_gate, _guest_ip_from_config.

New: git_gate/provision.py writes upstreams.json after key provisioning
so the bring-up reconcile can reconstruct the upstream table without the
manifest.

Wired into FirecrackerInfraService.ensure_running() cold-boot path;
base.py BottleBackend gets a no-op default. Old _reprovision_running_bottles
removed from consolidated_launch.py (superseded by reconcile.py). Tests
migrated and extended.

bot_bottle/backend/firecracker/consolidated_launch.py

@@ -26,22 +26,19 @@ The TAP slot allocation, rootfs build, and VM boot are the caller's job.
 
 from __future__ import annotations
 
-import json
 import subprocess
 from dataclasses import dataclass
 from pathlib import Path
 
 from ...egress import EgressPlan
 from ...git_gate import GitGatePlan
-from ...log import info
-from ...orchestrator.client import OrchestratorClient, OrchestratorClientError
+from ...orchestrator.client import OrchestratorClient
 from ...orchestrator.lifecycle import (
     OrchestratorStartError,  # re-exported so callers can catch it
 )
-from ...orchestrator.reprovision import reprovision_bottles
 from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
 from ..provision_bottle import deprovision_bottle, provision_bottle
-from . import cleanup, util
+from . import util
 from .gateway import FirecrackerGateway
 from .infra import FirecrackerInfraService
 
@@ -64,17 +61,6 @@ class LaunchContext:
     env_var_secret: str = ""  # encryption key injected into the agent's env
 
 
-def _guest_ip_from_config(config_path: Path) -> str:
-    """Read the kernel's configured guest IP from a Firecracker config."""
-    try:
-        config = json.loads(config_path.read_text())
-        args = config["boot-source"]["boot_args"]
-        ip_arg = next(part for part in args.split() if part.startswith("ip="))
-        return ip_arg.removeprefix("ip=").split(":", 1)[0]
-    except (OSError, ValueError, KeyError, TypeError, StopIteration):
-        return ""
-
-
 def persist_env_var_secret(private_key: Path, guest_ip: str, secret: str) -> None:
     """Mirror the exec-time key into guest tmpfs for restart recovery."""
     proc = subprocess.run(
@@ -89,29 +75,6 @@ def persist_env_var_secret(private_key: Path, guest_ip: str, secret: str) -> Non
         )
 
 
-def _reprovision_running_bottles(client: OrchestratorClient) -> None:
-    """Read keys from live agent VMs and restore the restarted gateway."""
-    try:
-        secrets_by_ip: dict[str, str] = {}
-        for run_dir in cleanup.live_run_dirs():
-            guest_ip = _guest_ip_from_config(run_dir / "config.json")
-            private_key = run_dir / "bottle_id_ed25519"
-            if not guest_ip or not private_key.is_file():
-                continue
-            proc = subprocess.run(
-                util.ssh_base_argv(private_key, guest_ip)
-                + [f"cat {_ENV_VAR_SECRET_PATH}"],
-                capture_output=True, text=True, check=False,
-            )
-            if proc.returncode == 0 and proc.stdout.strip():
-                secrets_by_ip[guest_ip] = proc.stdout.strip()
-        count = reprovision_bottles(client, secrets_by_ip)
-        if count:
-            info(f"reprovisioned egress tokens for {count} Firecracker bottle(s)")
-    except (OSError, OrchestratorClientError) as exc:
-        info(f"egress token reprovision skipped: {exc}")
-
-
 def launch_consolidated(
     egress_plan: EgressPlan,
     git_gate_plan: GitGatePlan,
@@ -126,7 +89,6 @@ def launch_consolidated(
     service = FirecrackerInfraService()
     url = service.ensure_running()
     client = OrchestratorClient(url)
-    _reprovision_running_bottles(client)
 
     # Read the gateway's provisioning transport + CA off the Gateway service.
     gateway = service.gateway()

bot_bottle/backend/firecracker/infra.py

@@ -17,6 +17,7 @@ from ...orchestrator.lifecycle import DEFAULT_STARTUP_TIMEOUT_SECONDS
 from . import infra_vm
 from .gateway import FirecrackerGateway
 from .orchestrator import FirecrackerOrchestrator
+from .reconcile import attach_bottled_agents_to_gateway
 
 
 class FirecrackerInfraService(InfraService):
@@ -67,9 +68,11 @@ class FirecrackerInfraService(InfraService):
             # holds the signing key) mints the role-scoped `gateway` JWT for the
             # gateway, which never sees the key (#469).
             orchestrator.ensure_running(startup_timeout=startup_timeout)
-            self.gateway().connect_to_orchestrator(
+            gateway = self.gateway()
+            gateway.connect_to_orchestrator(
                 orchestrator.gateway_url(), orchestrator.mint_gateway_token())
             infra_vm.record_booted_version(want)
+            attach_bottled_agents_to_gateway(url, gateway)
             return url
 
     def stop(self) -> None:

bot_bottle/backend/firecracker/reconcile.py

@@ -0,0 +1,198 @@
+"""Bring-up reconcile: re-attach all running agent VMs to a freshly-booted gateway.
+
+Called from the cold-boot branch of `FirecrackerInfraService.ensure_running()`
+after `gateway.connect_to_orchestrator()` completes. Restores the three
+gateway-dependent services for every live bottle:
+
+  - **CA** — push the current (freshly-minted) gateway CA to each agent's
+    trust store and run `update-ca-certificates`, so the agent trusts the
+    new CA on its next egress call.
+  - **git-gate** — re-provision each bottle's bare repos and per-repo creds
+    in the gateway, rebuilt from the persisted host-side state dir.
+  - **egress tokens** — read the agent's `ENV_VAR_SECRET` and feed
+    `reprovision_bottles` to restore the orchestrator's in-memory tokens.
+
+Per-bottle failures are caught, logged, and skipped so one unreachable VM
+does not block the others or the gateway coming up (PRD 0081).
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+from pathlib import Path
+
+from ...bottle_state import git_gate_state_dir
+from ...gateway import GatewayError
+from ...gateway.git_gate.render import GitGateUpstream
+from ...git_gate.plan import GitGatePlan
+from ...log import info
+from ...orchestrator.client import OrchestratorClient, OrchestratorClientError
+from ...orchestrator.reprovision import reprovision_bottles
+from ..provision_gateway import GatewayProvisionError, provision_git_gate
+from ..util import AGENT_CA_PATH
+from . import cleanup, util
+from .gateway import FirecrackerGateway
+
+# Where persist_env_var_secret writes the key on the agent VM (matches
+# consolidated_launch._ENV_VAR_SECRET_PATH — duplicated to avoid a
+# circular import through infra.py).
+_ENV_VAR_SECRET_PATH = "/run/bot-bottle/env-var-secret"
+
+
+def _guest_ip_from_config(config_path: Path) -> str:
+    """Read the agent VM's guest IP from its Firecracker config file."""
+    try:
+        config = json.loads(config_path.read_text())
+        args = config["boot-source"]["boot_args"]
+        ip_arg = next(p for p in args.split() if p.startswith("ip="))
+        return ip_arg.removeprefix("ip=").split(":", 1)[0]
+    except (OSError, ValueError, KeyError, TypeError, StopIteration):
+        return ""
+
+
+def attach_bottled_agents_to_gateway(
+    orchestrator_url: str, gateway: FirecrackerGateway,
+) -> None:
+    """Reconcile all live agent VMs against the freshly-booted gateway.
+
+    Fetches the new CA, lists registered bottles, then for each live run dir
+    pushes the CA, re-provisions git-gate, and restores egress tokens. Each
+    per-bottle step is wrapped so a failure is logged and skipped."""
+    try:
+        ca_pem = gateway.ca_cert_pem()
+    except GatewayError as exc:
+        info(f"bring-up reconcile: could not fetch gateway CA, skipping: {exc}")
+        return
+
+    client = OrchestratorClient(orchestrator_url)
+    try:
+        bottles = client.list_bottles()
+    except OrchestratorClientError as exc:
+        info(f"bring-up reconcile: could not list bottles, skipping: {exc}")
+        return
+
+    source_ip_to_bottle_id: dict[str, str] = {
+        b["source_ip"]: b["bottle_id"]
+        for b in bottles
+        if isinstance(b.get("source_ip"), str) and isinstance(b.get("bottle_id"), str)
+    }
+
+    transport = gateway.provisioning_transport()
+    secrets_by_ip: dict[str, str] = {}
+
+    for run_dir in cleanup.live_run_dirs():
+        slug = run_dir.name
+        guest_ip = _guest_ip_from_config(run_dir / "config.json")
+        private_key = run_dir / "bottle_id_ed25519"
+        if not guest_ip or not private_key.is_file():
+            continue
+
+        # CA: push the gateway's current certificate to the agent's trust store.
+        try:
+            _push_ca(private_key, guest_ip, ca_pem)
+        except Exception as exc:
+            info(f"bring-up reconcile: CA push to {slug!r} failed: {exc}")
+
+        # git-gate: re-provision repos + creds from the persisted state dir.
+        bottle_id = source_ip_to_bottle_id.get(guest_ip)
+        if bottle_id:
+            try:
+                _reprovision_git_gate(transport, bottle_id, slug)
+            except Exception as exc:
+                info(f"bring-up reconcile: git-gate for {slug!r} failed: {exc}")
+
+        # egress tokens: read ENV_VAR_SECRET from the agent's tmpfs.
+        proc = subprocess.run(
+            util.ssh_base_argv(private_key, guest_ip) + [f"cat {_ENV_VAR_SECRET_PATH}"],
+            capture_output=True, text=True, check=False,
+        )
+        if proc.returncode == 0 and proc.stdout.strip():
+            secrets_by_ip[guest_ip] = proc.stdout.strip()
+
+    # Restore in-memory egress tokens for all bottles that exposed a key.
+    if secrets_by_ip:
+        try:
+            count = reprovision_bottles(client, secrets_by_ip)
+            if count:
+                info(
+                    f"bring-up reconcile: restored egress tokens for {count} bottle(s)"
+                )
+        except OrchestratorClientError as exc:
+            info(f"bring-up reconcile: egress token restore failed: {exc}")
+
+
+def _push_ca(private_key: Path, guest_ip: str, ca_pem: str) -> None:
+    """SSH the gateway CA PEM into the agent VM and update its trust store.
+
+    Runs as the SSH root user (the agent VM's dropbear accepts root). Each
+    step uses check=True so a failure raises and the caller's except clause
+    logs and continues."""
+    ssh = util.ssh_base_argv(private_key, guest_ip)
+    # mkdir is idempotent; rootfs builds may not preserve the target dir.
+    subprocess.run(
+        ssh + ["mkdir -p /usr/local/share/ca-certificates"],
+        capture_output=True, check=True,
+    )
+    subprocess.run(
+        ssh + [f"cat > {AGENT_CA_PATH}"],
+        input=ca_pem, text=True, capture_output=True, check=True,
+    )
+    subprocess.run(
+        ssh + [f"chmod 644 {AGENT_CA_PATH} && update-ca-certificates"],
+        capture_output=True, check=True,
+    )
+
+
+def _reprovision_git_gate(
+    transport: object, bottle_id: str, slug: str,
+) -> None:
+    """Re-provision one bottle's git-gate repos and creds from its state dir.
+
+    Reads `upstreams.json` (written by `provision_git_gate_dynamic_keys` at
+    launch) to reconstruct the GitGateUpstream table, then calls
+    `provision_git_gate` which places the credential files and inits the bare
+    repos in the gateway. No-op when there is no `upstreams.json` (the bottle
+    has no git upstreams)."""
+    state_dir = git_gate_state_dir(slug)
+    upstreams_file = state_dir / "upstreams.json"
+    if not upstreams_file.exists():
+        return
+
+    raw = json.loads(upstreams_file.read_text())
+    upstreams: list[GitGateUpstream] = []
+    for u in raw:
+        name = u["name"]
+        # The key in the state dir is preferred: gitea deploy keys are written
+        # there by provision_git_gate_dynamic_keys; static keys keep their
+        # original manifest path.
+        key_in_state = state_dir / f"{name}-key"
+        identity_file = (
+            str(key_in_state) if key_in_state.is_file()
+            else u.get("identity_file", "")
+        )
+        known_hosts = state_dir / f"{name}-known_hosts"
+        upstreams.append(GitGateUpstream(
+            name=name,
+            upstream_url=u["upstream_url"],
+            upstream_host=u.get("upstream_host", ""),
+            upstream_port=u.get("upstream_port", ""),
+            identity_file=identity_file,
+            known_host_key=u.get("known_host_key", ""),
+            known_hosts_file=known_hosts if known_hosts.is_file() else Path(),
+        ))
+
+    if not upstreams:
+        return
+
+    plan = GitGatePlan(
+        slug=slug,
+        entrypoint_script=state_dir / "git_gate_entrypoint.sh",
+        hook_script=state_dir / "git_gate_pre_receive.sh",
+        access_hook_script=state_dir / "git_gate_access_hook.sh",
+        upstreams=tuple(upstreams),
+    )
+    provision_git_gate(transport, bottle_id, plan)  # type: ignore[arg-type]
+
+
+__all__ = ["attach_bottled_agents_to_gateway"]