feat(prd-0081): reprovision CA, git-gate, and egress tokens on gateway bring-up

On a Firecracker gateway cold boot, reconcile every live agent VM against
the fresh gateway: push the new mitmproxy CA into each agent's trust store,
re-provision git-gate repos/creds from the persisted upstreams snapshot,
and restore egress tokens. Per-bottle failures are logged and skipped so
one unreachable VM does not block the rest.

New modules / changes:
- backend/firecracker/reconcile.py: attach_bottled_agents_to_gateway,
  _push_ca, _reprovision_git_gate, _guest_ip_from_config
- git_gate/provision.py: write upstreams.json after key provisioning so
  the bring-up reconcile can reconstruct the upstream table without the
  manifest
- backend/firecracker/infra.py: call attach_bottled_agents_to_gateway in
  the cold-boot branch of ensure_running()
- backend/base.py: no-op default on BottleBackend
- backend/firecracker/consolidated_launch.py: remove superseded
  _reprovision_running_bottles / _guest_ip_from_config
- orchestrator: OrchestratorCore.update_agent_secret + POST
  /bottles/<id>/secret + client.update_agent_secret (single-secret
  in-place update, the reusable primitive from the 2026-07-26 hotfix)
- tests: 15 new tests in test_firecracker_reconcile.py; updated
  test_firecracker_infra.py cold-boot cases; stale FC reprovision tests
  removed from test_backend_secret_reprovision.py

Closes #516.

bot_bottle/backend/firecracker/consolidated_launch.py

@@ -26,22 +26,19 @@ The TAP slot allocation, rootfs build, and VM boot are the caller's job.
 
 from __future__ import annotations
 
-import json
 import subprocess
 from dataclasses import dataclass
 from pathlib import Path
 
 from ...egress import EgressPlan
 from ...git_gate import GitGatePlan
-from ...log import info
-from ...orchestrator.client import OrchestratorClient, OrchestratorClientError
+from ...orchestrator.client import OrchestratorClient
 from ...orchestrator.lifecycle import (
     OrchestratorStartError,  # re-exported so callers can catch it
 )
-from ...orchestrator.reprovision import reprovision_bottles
 from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
 from ..provision_bottle import deprovision_bottle, provision_bottle
-from . import cleanup, util
+from . import util
 from .gateway import FirecrackerGateway
 from .infra import FirecrackerInfraService
 
@@ -64,17 +61,6 @@ class LaunchContext:
     env_var_secret: str = ""  # encryption key injected into the agent's env
 
 
-def _guest_ip_from_config(config_path: Path) -> str:
-    """Read the kernel's configured guest IP from a Firecracker config."""
-    try:
-        config = json.loads(config_path.read_text())
-        args = config["boot-source"]["boot_args"]
-        ip_arg = next(part for part in args.split() if part.startswith("ip="))
-        return ip_arg.removeprefix("ip=").split(":", 1)[0]
-    except (OSError, ValueError, KeyError, TypeError, StopIteration):
-        return ""
-
-
 def persist_env_var_secret(private_key: Path, guest_ip: str, secret: str) -> None:
     """Mirror the exec-time key into guest tmpfs for restart recovery."""
     proc = subprocess.run(
@@ -89,29 +75,6 @@ def persist_env_var_secret(private_key: Path, guest_ip: str, secret: str) -> Non
         )
 
 
-def _reprovision_running_bottles(client: OrchestratorClient) -> None:
-    """Read keys from live agent VMs and restore the restarted gateway."""
-    try:
-        secrets_by_ip: dict[str, str] = {}
-        for run_dir in cleanup.live_run_dirs():
-            guest_ip = _guest_ip_from_config(run_dir / "config.json")
-            private_key = run_dir / "bottle_id_ed25519"
-            if not guest_ip or not private_key.is_file():
-                continue
-            proc = subprocess.run(
-                util.ssh_base_argv(private_key, guest_ip)
-                + [f"cat {_ENV_VAR_SECRET_PATH}"],
-                capture_output=True, text=True, check=False,
-            )
-            if proc.returncode == 0 and proc.stdout.strip():
-                secrets_by_ip[guest_ip] = proc.stdout.strip()
-        count = reprovision_bottles(client, secrets_by_ip)
-        if count:
-            info(f"reprovisioned egress tokens for {count} Firecracker bottle(s)")
-    except (OSError, OrchestratorClientError) as exc:
-        info(f"egress token reprovision skipped: {exc}")
-
-
 def launch_consolidated(
     egress_plan: EgressPlan,
     git_gate_plan: GitGatePlan,
@@ -126,7 +89,6 @@ def launch_consolidated(
     service = FirecrackerInfraService()
     url = service.ensure_running()
     client = OrchestratorClient(url)
-    _reprovision_running_bottles(client)
 
     # Read the gateway's provisioning transport + CA off the Gateway service.
     gateway = service.gateway()

bot_bottle/backend/firecracker/image_builder.py

@@ -19,7 +19,6 @@ from __future__ import annotations
 import fcntl
 import hashlib
 import os
-import re
 import shlex
 import shutil
 import subprocess
@@ -42,72 +41,19 @@ _BUILD_TIMEOUT_SECONDS = 900.0
 
 
 def _dockerfile_hash(dockerfile: Path) -> str:
-    """The Dockerfile's content hash. Agent Dockerfiles mostly COPY nothing
-    from the build context, so their text nearly determines the built image;
-    any files they *do* COPY are folded into `_rootfs_digest` (so a changed
-    input busts the cache) and shipped to the VM-side context by
-    `_send_build_context`."""
+    """The Dockerfile's content hash. The shipped agent Dockerfiles COPY
+    nothing from the build context (see .dockerignore), so their content fully
+    determines the built image; a Dockerfile that adds COPY will want the
+    context folded in here too."""
     return hashlib.sha256(dockerfile.read_bytes()).hexdigest()[:16]
 
 
-def _context_copy_sources(dockerfile: Path) -> list[str]:
-    """The build-context-relative paths a Dockerfile ``COPY``s in.
-
-    Agent Dockerfiles are meant to COPY nothing from the context (the VM-side
-    build ships only the Dockerfile), but one may pin an input by COPYing a
-    committed file — a checksum list, an npm lockfile. Return those source
-    paths so the builder can both ship them to the VM context and fold them
-    into the cache key. ``COPY --from=<stage>`` reads a build stage, not the
-    context, so it is excluded; the JSON/exec COPY form is unused by the
-    shipped images and is skipped rather than mis-parsed."""
-    joined = re.sub(r"\\\n", " ", dockerfile.read_text(encoding="utf-8"))
-    sources: list[str] = []
-    for line in joined.splitlines():
-        stripped = line.strip()
-        if not re.match(r"(?i)^COPY\s", stripped):
-            continue
-        tokens = stripped.split()[1:]
-        if any(t.startswith("--from=") for t in tokens):
-            continue
-        args = [t for t in tokens if not t.startswith("--")]
-        if len(args) < 2 or args[0].startswith("["):
-            continue
-        sources.extend(args[:-1])
-    return sources
-
-
-def _context_files(dockerfile: Path) -> list[tuple[str, Path]]:
-    """``(context-relative path, host path)`` for every existing file a
-    Dockerfile COPYs from the build root — globs expanded, sorted, de-duped.
-    Absolute or traversing (`..`) sources are dropped: the shipped context
-    only ever mirrors files under the build root."""
-    root = resources.build_root()
-    resolved: dict[str, Path] = {}
-    for src in _context_copy_sources(dockerfile):
-        if src.startswith("/") or ".." in Path(src).parts:
-            continue
-        if any(ch in src for ch in "*?["):
-            matches = [p for p in root.glob(src) if p.is_file()]
-        else:
-            candidate = root / src
-            if candidate.is_dir():
-                matches = [path for path in candidate.rglob("*") if path.is_file()]
-            else:
-                matches = [candidate] if candidate.is_file() else []
-        for path in matches:
-            resolved[str(path.relative_to(root))] = path
-    return sorted(resolved.items())
-
-
 def _rootfs_digest(dockerfile: Path) -> str:
-    """Cache key for the built AND boot-injected agent rootfs. Its inputs are
-    the Dockerfile (the image), the centralized build args, the guest init
-    injected into it (`util._GUEST_INIT`), and the content of any files the
-    Dockerfile COPYs from the build context. Folding the init in means a fix to
+    """Cache key for the built AND boot-injected agent rootfs. Two inputs
+    determine the on-disk rootfs: the Dockerfile (the image) and the guest init
+    injected into it (`util._GUEST_INIT`). Folding the init in means a fix to
     it — e.g. making /tmp world-writable — busts the cache instead of silently
-    reusing a stale rootfs built with the old init; folding the COPYed context
-    files in means a repinned input (e.g. a changed checksum list) rebuilds
-    rather than reusing a rootfs baked from the old bytes."""
+    reusing a stale rootfs built with the old init."""
     h = hashlib.sha256()
     h.update(_dockerfile_hash(dockerfile).encode())
     h.update(b"\0")
@@ -117,11 +63,6 @@ def _rootfs_digest(dockerfile: Path) -> str:
         h.update(value.encode())
         h.update(b"\0")
     h.update(util._GUEST_INIT.encode())
-    for rel, path in _context_files(dockerfile):
-        h.update(b"\0")
-        h.update(rel.encode())
-        h.update(b"\0")
-        h.update(path.read_bytes())
     return h.hexdigest()[:16]
 
 
@@ -131,46 +72,6 @@ def cached_agent_rootfs_dir(dockerfile: Path) -> Path | None:
     return base if (base / ".bb-ready").is_file() else None
 
 
-def _image_rootfs_digest(image: str) -> str:
-    h = hashlib.sha256()
-    h.update(image.encode())
-    h.update(b"\0")
-    h.update(util._GUEST_INIT.encode())
-    return h.hexdigest()[:16]
-
-
-def cached_agent_image_rootfs_dir(image: str) -> Path | None:
-    """Return a ready rootfs exported from an immutable OCI image."""
-    base = util.cache_dir() / "rootfs" / f"agent-image-{_image_rootfs_digest(image)}"
-    return base if (base / ".bb-ready").is_file() else None
-
-
-def acquire_agent_image_rootfs_dir(
-    image: str, *, smoke_test: tuple[str, ...] = (),
-) -> Path:
-    """Pull a digest-pinned image in the infra VM and export its rootfs."""
-    if "@sha256:" not in image:
-        die(f"prebuilt Firecracker agent image is not digest-pinned: {image}")
-    digest = _image_rootfs_digest(image)
-    base = util.cache_dir() / "rootfs" / f"agent-image-{digest}"
-    cached = cached_agent_image_rootfs_dir(image)
-    if cached is not None:
-        info(f"using cached agent rootfs {cached.name}")
-        return cached
-    with _build_lock():
-        if (base / ".bb-ready").is_file():
-            return base
-        staging = util.cache_dir() / "rootfs" / f".pulling-{digest}"
-        shutil.rmtree(staging, ignore_errors=True)
-        staging.mkdir(parents=True)
-        _pull_in_infra(image, staging, smoke_test, digest)
-        util.inject_guest_boot(staging)
-        (staging / ".bb-ready").write_text("ok\n")
-        shutil.rmtree(base, ignore_errors=True)
-        os.rename(staging, base)
-    return base
-
-
 def build_agent_rootfs_dir(
     dockerfile: Path, *, image_tag: str, smoke_test: tuple[str, ...] = (),
 ) -> Path:
@@ -253,7 +154,6 @@ def _build_in_infra(
         if prep.returncode != 0:
             die(f"preparing build dir in the infra VM failed: {prep.stderr.strip()}")
         _send_dockerfile(key, ip, dockerfile, ctx)
-        _send_build_context(key, ip, dockerfile, ctx)
         _buildah_build(
             key,
             ip,
@@ -267,43 +167,6 @@ def _build_in_infra(
         _cleanup()
 
 
-def _pull_in_infra(
-    image: str, base: Path, smoke_test: tuple[str, ...], digest: str,
-) -> None:
-    """Pull and export a published agent image inside the orchestrator VM."""
-    service = FirecrackerInfraService()
-    service.ensure_running()
-    key, ip = service.orchestrator().ssh_target()
-    tag = f"bot-bottle-agent-pull-{digest}"
-    smoke_ctr, export_ctr = f"{tag}-smoke", f"{tag}-export"
-    quoted_image = shlex.quote(image)
-
-    def cleanup() -> None:
-        _ssh(
-            key,
-            ip,
-            f"buildah rm {smoke_ctr} {export_ctr} >/dev/null 2>&1; "
-            f"buildah rmi {_STORE_FLAG} {tag} >/dev/null 2>&1",
-            timeout=60,
-        )
-
-    cleanup()
-    try:
-        result = _ssh(
-            key,
-            ip,
-            f"buildah pull {_STORE_FLAG} {quoted_image} && "
-            f"buildah tag {_STORE_FLAG} {quoted_image} {tag}",
-            timeout=_BUILD_TIMEOUT_SECONDS,
-        )
-        if result.returncode != 0:
-            die(f"pulling pinned agent image failed: {result.stderr.strip()}")
-        _smoke_test(key, ip, tag, smoke_ctr, smoke_test)
-        _stream_rootfs(key, ip, tag, export_ctr, base)
-    finally:
-        cleanup()
-
-
 def _ssh(private_key: Path, guest_ip: str, script: str,
          *, timeout: float = 60.0) -> subprocess.CompletedProcess[str]:
     return subprocess.run(
@@ -334,40 +197,6 @@ def _send_dockerfile(private_key: Path, guest_ip: str, dockerfile: Path, ctx: st
             f"{proc.stderr.decode(errors='replace').strip()}")
 
 
-def _send_build_context(private_key: Path, guest_ip: str, dockerfile: Path, ctx: str) -> None:
-    """Ship the files ``dockerfile`` COPYs from the build root into the infra
-    VM's ``{ctx}/ctx``, preserving their build-root-relative paths.
-
-    Usually a no-op — agent Dockerfiles COPY nothing — so `{ctx}/ctx` stays the
-    empty context the build otherwise runs against. It exists so a Dockerfile
-    that pins an input by COPYing a committed file (a checksum list, an npm
-    lockfile) still finds that file in the VM-side context. Streamed as a tar
-    so directories and multiple files land in one round trip."""
-    files = _context_files(dockerfile)
-    if not files:
-        return
-    root = resources.build_root()
-    rels = [rel for rel, _ in files]
-    tar = subprocess.Popen(
-        ["tar", "-C", str(root), "-cf", "-", "--", *rels],
-        stdout=subprocess.PIPE,
-    )
-    try:
-        proc = subprocess.run(
-            util.ssh_base_argv(private_key, guest_ip) + [f"tar -C {ctx}/ctx -xf -"],
-            stdin=tar.stdout, capture_output=True, timeout=120, check=False,
-        )
-    finally:
-        if tar.stdout is not None:
-            tar.stdout.close()
-        tar.wait()
-    if tar.returncode != 0:
-        die(f"packing the agent build context failed (tar exit {tar.returncode})")
-    if proc.returncode != 0:
-        die("sending the agent build context to the infra VM failed: "
-            f"{proc.stderr.decode(errors='replace').strip() or '<no stderr>'}")
-
-
 def _buildah_build(
     private_key: Path,
     guest_ip: str,

bot_bottle/backend/firecracker/infra.py

@@ -17,6 +17,7 @@ from ...orchestrator.lifecycle import DEFAULT_STARTUP_TIMEOUT_SECONDS
 from . import infra_vm
 from .gateway import FirecrackerGateway
 from .orchestrator import FirecrackerOrchestrator
+from .reconcile import attach_bottled_agents_to_gateway
 
 
 class FirecrackerInfraService(InfraService):
@@ -61,15 +62,17 @@ class FirecrackerInfraService(InfraService):
                 return url
             # Clear stale/hung/OUTDATED VMs holding either link before booting fresh.
             infra_vm.stop()
-            infra_vm.ensure_available()
+            infra_vm.ensure_built()
             # Orchestrator first — the gateway daemons reach the control plane at
             # startup. Each service owns its own boot; the orchestrator (which
             # holds the signing key) mints the role-scoped `gateway` JWT for the
             # gateway, which never sees the key (#469).
             orchestrator.ensure_running(startup_timeout=startup_timeout)
-            self.gateway().connect_to_orchestrator(
+            gateway = self.gateway()
+            gateway.connect_to_orchestrator(
                 orchestrator.gateway_url(), orchestrator.mint_gateway_token())
             infra_vm.record_booted_version(want)
+            attach_bottled_agents_to_gateway(url, gateway)
             return url
 
     def stop(self) -> None:

bot_bottle/backend/firecracker/infra_artifact.py

@@ -195,9 +195,7 @@ def _sha256_file(path: Path) -> str:
     return h.hexdigest()
 
 
-def ensure_artifact_gz(
-    version: str, *, role: str, expected_sha256: str | None = None,
-) -> Path:
+def ensure_artifact_gz(version: str, *, role: str) -> Path:
     """The verified, cached `rootfs.ext4.gz` for `role` at `version` —
     downloading it (and its `.sha256`) once, then reusing it. Fail-closed on a
     checksum mismatch: the partial is removed and we die rather than boot an
@@ -224,11 +222,6 @@ def ensure_artifact_gz(
         if not gz.is_file() or not sha.is_file():
             die(f"infra candidate bundle is incomplete: {root}")
         expected = sha.read_text().split()[0].strip().lower()
-        if expected_sha256 is not None and expected != expected_sha256:
-            die(
-                f"infra candidate packaged checksum mismatch ({role}) for {version}:\n"
-                f"    packaged {expected_sha256}\n    candidate {expected}"
-            )
         actual = _sha256_file(gz)
         if actual != expected:
             die(
@@ -242,13 +235,7 @@ def ensure_artifact_gz(
     gz = root / _GZ_NAME
     ok = root / ".verified"
     if gz.is_file() and ok.is_file():
-        actual = _sha256_file(gz)
-        recorded = ok.read_text(encoding="utf-8").strip()
-        wanted = expected_sha256 or recorded
-        if actual == recorded == wanted:
-            return gz
-        gz.unlink(missing_ok=True)
-        ok.unlink(missing_ok=True)
+        return gz
 
     info(f"pulling infra rootfs artifact {_package(role)}/{version}")
     _download(artifact_url(version, _GZ_NAME, role=role), gz)
@@ -256,14 +243,6 @@ def ensure_artifact_gz(
     _download(artifact_url(version, _SHA_NAME, role=role), sha)
 
     expected = sha.read_text().split()[0].strip().lower()
-    if expected_sha256 is not None and expected != expected_sha256:
-        gz.unlink(missing_ok=True)
-        sha.unlink(missing_ok=True)
-        die(
-            f"infra artifact published checksum mismatch ({role}) for {version}:\n"
-            f"    packaged  {expected_sha256}\n"
-            f"    published {expected}"
-        )
     actual = _sha256_file(gz)
     if actual != expected:
         gz.unlink(missing_ok=True)
@@ -274,22 +253,15 @@ def ensure_artifact_gz(
             f"    actual   {actual}\n"
             f"    refusing to boot an unverified rootfs."
         )
-    ok.write_text(actual + "\n")
+    ok.write_text("ok\n")
     return gz
 
 
-def materialize_ext4(
-    version: str,
-    dest: Path,
-    *,
-    role: str,
-    expected_sha256: str | None = None,
-) -> None:
+def materialize_ext4(version: str, dest: Path, *, role: str) -> None:
     """Ensure the verified `role` artifact is cached, then gunzip it to `dest` —
     a fresh, writable per-boot rootfs (the VM mutates it; the cached `.gz` stays
     pristine). Atomic via a `.part` sibling."""
-    gz = ensure_artifact_gz(
-        version, role=role, expected_sha256=expected_sha256)
+    gz = ensure_artifact_gz(version, role=role)
     tmp = dest.with_suffix(dest.suffix + ".part")
     info(f"expanding {role} infra rootfs -> {dest}")
     with gzip.open(gz, "rb") as src, open(tmp, "wb") as out:

bot_bottle/backend/firecracker/infra_vm.py

@@ -43,7 +43,6 @@ from pathlib import Path
 from typing import Generator
 
 from ... import resources
-from ... import release_manifest
 from ...log import die, info
 from ..docker import util as docker_mod
 from . import firecracker_vm, infra_artifact, netpool, util
@@ -109,18 +108,6 @@ def _role_version(role: str) -> str:
     return infra_artifact.infra_artifact_version(role_init(role), role)
 
 
-def _role_release(role: str) -> tuple[str, str | None]:
-    manifest = release_manifest.packaged_manifest()
-    if manifest is None or release_manifest.local_build_requested():
-        return _role_version(role), None
-    artifact = (
-        manifest.firecracker_orchestrator
-        if role == "orchestrator"
-        else manifest.firecracker_gateway
-    )
-    return artifact.version, artifact.sha256
-
-
 def ensure_built() -> None:
     """Ensure both infra rootfs artifacts are available before boot.
 
@@ -130,16 +117,11 @@ def ensure_built() -> None:
     `BOT_BOTTLE_INFRA_BUILD=local` instead builds the images from source with
     host Docker (the orchestrator-fc image is `FROM` the orchestrator image, so
     it must exist first) — for iterating on the Dockerfiles."""
-    if release_manifest.local_build_requested():
+    if infra_artifact.local_build_requested():
         build_infra_images_with_docker()
         return
     for role in infra_artifact.ROLES:
-        version, expected = _role_release(role)
-        infra_artifact.ensure_artifact_gz(
-            version, role=role, expected_sha256=expected)
-
-
-ensure_available = ensure_built
+        infra_artifact.ensure_artifact_gz(_role_version(role), role=role)
 
 
 def build_infra_images_with_docker() -> None:
@@ -232,9 +214,7 @@ def boot_vm(
     else:
         # Prebuilt artifact already carries the role's build slack; expand it to
         # a fresh writable rootfs for this boot.
-        version, expected = _role_release(role)
-        infra_artifact.materialize_ext4(
-            version, rootfs, role=role, expected_sha256=expected)
+        infra_artifact.materialize_ext4(_role_version(role), rootfs, role=role)
     private_key, pubkey = _stable_keypair()
 
     info(f"booting {role} VM on {slot.iface} (guest {slot.guest_ip})")
@@ -284,8 +264,7 @@ def _version_file() -> Path:
 def expected_version() -> str:
     """The combined marker for the running pair: both per-plane artifact
     versions, so a change to either rootfs dislodges the adopted pair."""
-    return " ".join(
-        f"{role}={_role_release(role)[0]}" for role in infra_artifact.ROLES)
+    return " ".join(f"{role}={_role_version(role)}" for role in infra_artifact.ROLES)
 
 
 def adoptable(key: Path, url: str, want: str) -> bool:

bot_bottle/backend/firecracker/launch.py

@@ -228,13 +228,8 @@ def build_or_load_agent_base(plan: FirecrackerBottlePlan) -> Path:
         info(f"resuming from committed rootfs {committed_tar}")
         return util.build_committed_rootfs_dir(committed_tar)
     dockerfile = Path(plan.dockerfile_path)
-    prebuilt = "@sha256:" in plan.image
     if plan.spec.image_policy == "cached":
-        cached = (
-            image_builder.cached_agent_image_rootfs_dir(plan.image)
-            if prebuilt
-            else image_builder.cached_agent_rootfs_dir(dockerfile)
-        )
+        cached = image_builder.cached_agent_rootfs_dir(dockerfile)
         if cached is None:
             die(
                 f"cached agent rootfs for {plan.image!r} not found; "
@@ -242,11 +237,6 @@ def build_or_load_agent_base(plan: FirecrackerBottlePlan) -> Path:
             )
         info(f"using cached agent rootfs {cached.name}")
         return cached
-    if prebuilt:
-        return image_builder.acquire_agent_image_rootfs_dir(
-            plan.image,
-            smoke_test=runtime_for(plan.agent_provider_template).smoke_test,
-        )
     return image_builder.build_agent_rootfs_dir(
         dockerfile,
         image_tag=plan.image,
@@ -263,11 +253,7 @@ def stale_checks(plan: FirecrackerBottlePlan) -> None:
     if committed and committed_tar.is_file():
         check_stale_path(f"agent rootfs {committed_tar}", committed_tar)
         return
-    cached = (
-        image_builder.cached_agent_image_rootfs_dir(plan.image)
-        if "@sha256:" in plan.image
-        else image_builder.cached_agent_rootfs_dir(Path(plan.dockerfile_path))
-    )
+    cached = image_builder.cached_agent_rootfs_dir(Path(plan.dockerfile_path))
     if cached is not None:
         check_stale_path(f"agent rootfs {cached}", cached / ".bb-ready")
 

bot_bottle/backend/firecracker/reconcile.py

@@ -0,0 +1,199 @@
+"""Bring-up reconcile: re-attach all running agent VMs to a freshly-booted gateway.
+
+Called from the cold-boot branch of `FirecrackerInfraService.ensure_running()`
+after `gateway.connect_to_orchestrator()` completes. Restores the three
+gateway-dependent services for every live bottle:
+
+  - **CA** — push the current (freshly-minted) gateway CA to each agent's
+    trust store and run `update-ca-certificates`, so the agent trusts the
+    new CA on its next egress call.
+  - **git-gate** — re-provision each bottle's bare repos and per-repo creds
+    in the gateway, rebuilt from the persisted host-side state dir.
+  - **egress tokens** — read the agent's `ENV_VAR_SECRET` and feed
+    `reprovision_bottles` to restore the orchestrator's in-memory tokens.
+
+Per-bottle failures are caught, logged, and skipped so one unreachable VM
+does not block the others or the gateway coming up (PRD 0081).
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+from pathlib import Path
+
+from ...bottle_state import git_gate_state_dir
+from ...gateway import GatewayError
+from ...gateway.git_gate.render import GitGateUpstream
+from ...git_gate.plan import GitGatePlan
+from ...log import info
+from ...orchestrator.client import OrchestratorClient, OrchestratorClientError
+from ...orchestrator.reprovision import reprovision_bottles
+from ..provision_gateway import provision_git_gate
+from ..util import AGENT_CA_PATH
+from . import cleanup, util
+from .gateway import FirecrackerGateway
+
+# Where persist_env_var_secret writes the key on the agent VM (matches
+# consolidated_launch._ENV_VAR_SECRET_PATH — duplicated to avoid a
+# circular import through infra.py).
+_ENV_VAR_SECRET_PATH = "/run/bot-bottle/env-var-secret"
+
+
+def _guest_ip_from_config(config_path: Path) -> str:
+    """Read the agent VM's guest IP from its Firecracker config file."""
+    try:
+        config = json.loads(config_path.read_text())
+        args = config["boot-source"]["boot_args"]
+        ip_arg = next(p for p in args.split() if p.startswith("ip="))
+        return ip_arg.removeprefix("ip=").split(":", 1)[0]
+    except (OSError, ValueError, KeyError, TypeError, StopIteration):
+        return ""
+
+
+def attach_bottled_agents_to_gateway(
+    orchestrator_url: str, gateway: FirecrackerGateway,
+) -> None:
+    """Reconcile all live agent VMs against the freshly-booted gateway.
+
+    Fetches the new CA, lists registered bottles, then for each live run dir
+    pushes the CA, re-provisions git-gate, and restores egress tokens. Each
+    per-bottle step is wrapped so a failure is logged and skipped."""
+    try:
+        ca_pem = gateway.ca_cert_pem()
+    except GatewayError as exc:
+        info(f"bring-up reconcile: could not fetch gateway CA, skipping: {exc}")
+        return
+
+    client = OrchestratorClient(orchestrator_url)
+    try:
+        bottles = client.list_bottles()
+    except OrchestratorClientError as exc:
+        info(f"bring-up reconcile: could not list bottles, skipping: {exc}")
+        return
+
+    source_ip_to_bottle_id: dict[str, str] = {}
+    for b in bottles:
+        source_ip = b.get("source_ip")
+        bottle_id = b.get("bottle_id")
+        if isinstance(source_ip, str) and isinstance(bottle_id, str):
+            source_ip_to_bottle_id[source_ip] = bottle_id
+
+    transport = gateway.provisioning_transport()
+    secrets_by_ip: dict[str, str] = {}
+
+    for run_dir in cleanup.live_run_dirs():
+        slug = run_dir.name
+        guest_ip = _guest_ip_from_config(run_dir / "config.json")
+        private_key = run_dir / "bottle_id_ed25519"
+        if not guest_ip or not private_key.is_file():
+            continue
+
+        # CA: push the gateway's current certificate to the agent's trust store.
+        try:
+            _push_ca(private_key, guest_ip, ca_pem)
+        except Exception as exc:
+            info(f"bring-up reconcile: CA push to {slug!r} failed: {exc}")
+
+        # git-gate: re-provision repos + creds from the persisted state dir.
+        bottle_id = source_ip_to_bottle_id.get(guest_ip)
+        if bottle_id:
+            try:
+                _reprovision_git_gate(transport, bottle_id, slug)
+            except Exception as exc:
+                info(f"bring-up reconcile: git-gate for {slug!r} failed: {exc}")
+
+        # egress tokens: read ENV_VAR_SECRET from the agent's tmpfs.
+        proc = subprocess.run(
+            util.ssh_base_argv(private_key, guest_ip) + [f"cat {_ENV_VAR_SECRET_PATH}"],
+            capture_output=True, text=True, check=False,
+        )
+        if proc.returncode == 0 and proc.stdout.strip():
+            secrets_by_ip[guest_ip] = proc.stdout.strip()
+
+    # Restore in-memory egress tokens for all bottles that exposed a key.
+    if secrets_by_ip:
+        try:
+            count = reprovision_bottles(client, secrets_by_ip)
+            if count:
+                info(
+                    f"bring-up reconcile: restored egress tokens for {count} bottle(s)"
+                )
+        except OrchestratorClientError as exc:
+            info(f"bring-up reconcile: egress token restore failed: {exc}")
+
+
+def _push_ca(private_key: Path, guest_ip: str, ca_pem: str) -> None:
+    """SSH the gateway CA PEM into the agent VM and update its trust store.
+
+    Runs as the SSH root user (the agent VM's dropbear accepts root). Each
+    step uses check=True so a failure raises and the caller's except clause
+    logs and continues."""
+    ssh = util.ssh_base_argv(private_key, guest_ip)
+    # mkdir is idempotent; rootfs builds may not preserve the target dir.
+    subprocess.run(
+        ssh + ["mkdir -p /usr/local/share/ca-certificates"],
+        capture_output=True, check=True,
+    )
+    subprocess.run(
+        ssh + [f"cat > {AGENT_CA_PATH}"],
+        input=ca_pem, text=True, capture_output=True, check=True,
+    )
+    subprocess.run(
+        ssh + [f"chmod 644 {AGENT_CA_PATH} && update-ca-certificates"],
+        capture_output=True, check=True,
+    )
+
+
+def _reprovision_git_gate(
+    transport: object, bottle_id: str, slug: str,
+) -> None:
+    """Re-provision one bottle's git-gate repos and creds from its state dir.
+
+    Reads `upstreams.json` (written by `provision_git_gate_dynamic_keys` at
+    launch) to reconstruct the GitGateUpstream table, then calls
+    `provision_git_gate` which places the credential files and inits the bare
+    repos in the gateway. No-op when there is no `upstreams.json` (the bottle
+    has no git upstreams)."""
+    state_dir = git_gate_state_dir(slug)
+    upstreams_file = state_dir / "upstreams.json"
+    if not upstreams_file.exists():
+        return
+
+    raw = json.loads(upstreams_file.read_text())
+    upstreams: list[GitGateUpstream] = []
+    for u in raw:
+        name = u["name"]
+        # The key in the state dir is preferred: gitea deploy keys are written
+        # there by provision_git_gate_dynamic_keys; static keys keep their
+        # original manifest path.
+        key_in_state = state_dir / f"{name}-key"
+        identity_file = (
+            str(key_in_state) if key_in_state.is_file()
+            else u.get("identity_file", "")
+        )
+        known_hosts = state_dir / f"{name}-known_hosts"
+        upstreams.append(GitGateUpstream(
+            name=name,
+            upstream_url=u["upstream_url"],
+            upstream_host=u.get("upstream_host", ""),
+            upstream_port=u.get("upstream_port", ""),
+            identity_file=identity_file,
+            known_host_key=u.get("known_host_key", ""),
+            known_hosts_file=known_hosts if known_hosts.is_file() else Path(),
+        ))
+
+    if not upstreams:
+        return
+
+    plan = GitGatePlan(
+        slug=slug,
+        entrypoint_script=state_dir / "git_gate_entrypoint.sh",
+        hook_script=state_dir / "git_gate_pre_receive.sh",
+        access_hook_script=state_dir / "git_gate_access_hook.sh",
+        upstreams=tuple(upstreams),
+    )
+    provision_git_gate(transport, bottle_id, plan)  # type: ignore[arg-type]
+
+
+__all__ = ["attach_bottled_agents_to_gateway"]

bot_bottle/backend/firecracker/setup.py

@@ -20,7 +20,6 @@ import subprocess
 import sys
 from pathlib import Path
 
-from ... import invocation
 from ... import resources
 from . import netpool
 from . import util
@@ -180,11 +179,9 @@ def _setup_systemd() -> None:
         f"sudo systemctl daemon-reload\n"
         f"sudo systemctl enable --now {netpool.SYSTEMD_UNIT}\n"
     )
-    # Absolute path, not `sudo bot-bottle`: sudo's secure_path drops
-    # ~/.local/bin, where both pipx and install.sh put the entry point.
     sys.stderr.write(
-        f"\n(Or re-run this as root to install it directly:\n"
-        f"    {invocation.sudo_command('backend', 'setup', '--backend=firecracker')})\n"
+        f"\n(Or re-run this as root to install it directly: "
+        f"sudo bot-bottle backend setup --backend=firecracker)\n"
     )