refactor(backend): base owns the gateway-attach flow, fail hard (0081 review)

Addresses review on #519 (@didericis 6143 + the fail-hard direction over the
codex skip).

Base owns the flow (6143 / ADR 0006). `attach_bottled_agents_to_gateway()` is
now a concrete method on `BottleBackend` that delegates to
`gateway_attach.reconcile_running_bottles`; backends override only three
primitives — `_gateway_attach_resources()`, `_running_bottles()`,
`_attach_bottle_to_gateway()`. The shared control flow + error policy live in
one place so a backend can't drift onto a bespoke loop or a silent skip. Keeps
the reconcile flow + `GatewayAttachResources` out of `backend/base.py` (the
size guardrail) in a dedicated `backend/gateway_attach.py`. New ADR 0006 records
the "shared behaviour in the base backend, subclasses override primitives"
theme.

Fail hard, never skip (the maintainer's direction over the codex review's
skip). Any attach failure now aborts bring-up instead of being logged and
skipped: a bottle that silently can't reach the fresh gateway (its egress just
starts failing TLS) is worse than a loud failure. Every bottle is attempted and
the failures are raised together (aggregate `InfraLaunchError`) so one bring-up
surfaces the full blast radius. Resource gathering (the CA fetch) also fails
hard. PRD 0081 goal + design updated to match (reversed from the earlier
"tolerate per-bottle failures" draft).

Bumps the base.py guardrail cap 580->600 for the new contract surface (the
delegator + 3 abstract primitives); the flow itself lives in gateway_attach.py.

Refs #516, #519.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

bot_bottle/backend/firecracker/backend.py

@@ -20,6 +20,7 @@ from ...git_gate import GitGatePlan
 from ...manifest import Manifest
 from ...supervisor.plan import SupervisePlan
 from .. import ActiveAgent, BottleBackend, BottleImages, BottleSpec
+from ..gateway_attach import GatewayAttachResources
 from . import cleanup as _cleanup
 from . import enumerate as _enumerate
 from . import launch as _launch
@@ -31,7 +32,7 @@ from .bottle_plan import FirecrackerBottlePlan
 
 
 class FirecrackerBottleBackend(
-    BottleBackend["FirecrackerBottlePlan", "FirecrackerBottleCleanupPlan"]
+    BottleBackend["FirecrackerBottlePlan", "FirecrackerBottleCleanupPlan", "Path"]
 ):
     name = "firecracker"
 
@@ -119,9 +120,18 @@ class FirecrackerBottleBackend(
     def enumerate_active(self) -> Sequence[ActiveAgent]:
         return _enumerate.enumerate_active()
 
-    def attach_bottled_agents_to_gateway(self) -> None:
-        from .infra_launch import attach_bottled_agents_to_gateway
-        attach_bottled_agents_to_gateway()
+    def _gateway_attach_resources(self) -> GatewayAttachResources:
+        from .gateway import FirecrackerGateway
+        return GatewayAttachResources(ca_pem=FirecrackerGateway().ca_cert_pem())
+
+    def _running_bottles(self) -> Sequence[Path]:
+        return _cleanup.live_run_dirs()
+
+    def _attach_bottle_to_gateway(
+        self, bottle: Path, resources: GatewayAttachResources,
+    ) -> None:
+        from .infra_launch import attach_ca_to_agent_vm
+        attach_ca_to_agent_vm(bottle, resources.ca_pem)
 
     def supervise_mcp_url(self, plan: FirecrackerBottlePlan) -> str:
         return plan.agent_supervise_url

bot_bottle/backend/firecracker/infra_launch.py

@@ -32,7 +32,6 @@ from dataclasses import dataclass
 from pathlib import Path
 
 from ...egress import EgressPlan
-from ...gateway import GatewayError
 from ...git_gate import GitGatePlan
 from ...log import info
 from ...orchestrator.client import OrchestratorClient, OrchestratorClientError
@@ -88,56 +87,42 @@ def persist_env_var_secret(private_key: Path, guest_ip: str, secret: str) -> Non
         )
 
 
-def _push_gateway_ca_to_agent(private_key: Path, guest_ip: str, ca_pem: str) -> None:
-    """Replace one running agent VM's trusted gateway CA with `ca_pem` and
-    rebuild its trust store over SSH. Unconditional install — there is one
+def attach_ca_to_agent_vm(run_dir: Path, ca_pem: str) -> None:
+    """(Re)attach one running agent VM (identified by its `run_dir`) to the
+    current gateway: replace its trusted gateway CA with `ca_pem` and rebuild
+    its trust store over SSH (PRD 0081). Unconditional install — there is one
     gateway, so no fingerprint match is needed. Bare-pipe input keeps the cert
-    off argv. Raises `InfraLaunchError` on failure."""
+    off argv.
+
+    Fail hard: raises `InfraLaunchError` (naming the bottle) if the run dir is
+    malformed or the push fails — the base reconcile aggregates and surfaces it
+    rather than leaving the agent silently unable to reach the gateway."""
+    guest_ip = _guest_ip_from_config(run_dir / "config.json")
+    private_key = run_dir / "bottle_id_ed25519"
+    if not guest_ip or not private_key.is_file():
+        raise InfraLaunchError(
+            f"{run_dir.name}: cannot resolve guest IP / SSH key to attach it "
+            f"to the gateway"
+        )
     install = (
         "umask 022; mkdir -p /usr/local/share/ca-certificates && "
         f"cat > {AGENT_CA_PATH} && chmod 644 {AGENT_CA_PATH} && "
         "update-ca-certificates"
     )
-    proc = subprocess.run(
-        util.ssh_base_argv(private_key, guest_ip) + [install],
-        input=ca_pem, capture_output=True, text=True, check=False,
-    )
+    try:
+        proc = subprocess.run(
+            util.ssh_base_argv(private_key, guest_ip) + [install],
+            input=ca_pem, capture_output=True, text=True, check=False,
+        )
+    except OSError as exc:
+        raise InfraLaunchError(f"{run_dir.name} ({guest_ip}): CA push failed: {exc}") from exc
     if proc.returncode != 0:
         raise InfraLaunchError(
-            f"CA push to {guest_ip} failed: "
+            f"{run_dir.name} ({guest_ip}): CA push failed: "
             f"{proc.stderr.strip() or '<no stderr>'}"
         )
 
 
-def attach_bottled_agents_to_gateway() -> None:
-    """Reconcile every running agent VM against the freshly-booted gateway
-    (PRD 0081). Replaces each agent's trusted CA with the current gateway CA —
-    the gateway rootfs is ephemeral, so a cold boot mints a fresh CA and this is
-    what distributes it (a gateway rebuild doubles as a free CA rotation, #510).
-
-    Per-bottle steps are best-effort: one unreachable or malformed VM is logged
-    and skipped so it never blocks the others or the gateway coming up."""
-    gateway = FirecrackerGateway()
-    try:
-        ca_pem = gateway.ca_cert_pem()
-    except GatewayError as exc:
-        info(f"bring-up reconcile skipped: gateway CA unavailable: {exc}")
-        return
-    reconciled = 0
-    for run_dir in cleanup.live_run_dirs():
-        guest_ip = _guest_ip_from_config(run_dir / "config.json")
-        private_key = run_dir / "bottle_id_ed25519"
-        if not guest_ip or not private_key.is_file():
-            continue
-        try:
-            _push_gateway_ca_to_agent(private_key, guest_ip, ca_pem)
-            reconciled += 1
-        except (OSError, InfraLaunchError) as exc:
-            info(f"CA reconcile skipped for {guest_ip}: {exc}")
-    if reconciled:
-        info(f"reconciled gateway CA into {reconciled} running Firecracker bottle(s)")
-
-
 def _reprovision_running_bottles(client: OrchestratorClient) -> None:
     """Read keys from live agent VMs and restore the restarted gateway."""
     try:
@@ -210,7 +195,7 @@ def deprovision_consolidated(
 __all__ = [
     "LaunchContext",
     "launch_consolidated",
-    "attach_bottled_agents_to_gateway",
+    "attach_ca_to_agent_vm",
     "deprovision_consolidated",
     "InfraLaunchError",
     "OrchestratorStartError",