feat(prd-0081): reprovision CA, git-gate, and egress tokens on gateway bring-up

On a Firecracker gateway cold boot, reconcile every live agent VM against the fresh gateway: push the new mitmproxy CA into each agent's trust store, re-provision git-gate repos/creds from the persisted upstreams snapshot, and restore egress tokens. Per-bottle failures are logged and skipped so one unreachable VM does not block the rest. New modules / changes: - backend/firecracker/reconcile.py: attach_bottled_agents_to_gateway, _push_ca, _reprovision_git_gate, _guest_ip_from_config - git_gate/provision.py: write upstreams.json after key provisioning so the bring-up reconcile can reconstruct the upstream table without the manifest - backend/firecracker/infra.py: call attach_bottled_agents_to_gateway in the cold-boot branch of ensure_running() - backend/base.py: no-op default on BottleBackend - backend/firecracker/consolidated_launch.py: remove superseded _reprovision_running_bottles / _guest_ip_from_config - orchestrator: OrchestratorCore.update_agent_secret + POST /bottles/<id>/secret + client.update_agent_secret (single-secret in-place update, the reusable primitive from the 2026-07-26 hotfix) - tests: 15 new tests in test_firecracker_reconcile.py; updated test_firecracker_infra.py cold-boot cases; stale FC reprovision tests removed from test_backend_secret_reprovision.py Closes #516.
test(release): cover publication failure paths
2026-07-28 01:50:15 +00:00 · 2026-07-27 17:00:50 -04:00 · 2026-07-27 17:00:50 -04:00 · 2026-07-27 17:00:50 -04:00 · 2026-07-27 17:00:50 -04:00 · 2026-07-27 17:00:50 -04:00
23 changed files with 869 additions and 1197 deletions
@@ -496,6 +496,20 @@ class BottleBackend(ABC, Generic[PlanT, CleanupT]):
        (macos-container) die with a pointer — the default here."""
        die(f"backend {self.name!r} has no orchestrator control plane")
    def attach_bottled_agents_to_gateway(self) -> None:
        """Reconcile all running bottles against the current gateway.
        Called when the gateway is (re)brought up (cold-boot path) to restore
        the three gateway-dependent services for every live bottle: CA trust,
        git-gate repos/creds, and egress tokens. Per-bottle failures must be
        logged and skipped — one unreachable agent must not block the rest.
        Default: no-op. Backends that run a gateway (Firecracker, docker,
        macOS) override this with a backend-native implementation that reaches
        running agents via their transport (SSH for Firecracker, exec/cp for
        docker and macOS). PRD 0081."""
        return
    @abstractmethod
    def prepare_cleanup(self) -> CleanupT:
        """Enumerate orphaned resources from previous bottles. No side
@@ -26,22 +26,19 @@ The TAP slot allocation, rootfs build, and VM boot are the caller's job.
 from __future__ import annotations
 import json
 import subprocess
 from dataclasses import dataclass
 from pathlib import Path
 from ...egress import EgressPlan
 from ...git_gate import GitGatePlan
-from ...log import info
+from ...orchestrator.client import OrchestratorClient
 from ...orchestrator.client import OrchestratorClient, OrchestratorClientError
 from ...orchestrator.lifecycle import (
    OrchestratorStartError,  # re-exported so callers can catch it
 )
 from ...orchestrator.reprovision import reprovision_bottles
 from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
 from ..provision_bottle import deprovision_bottle, provision_bottle
-from . import cleanup, util
+from . import util
 from .gateway import FirecrackerGateway
 from .infra import FirecrackerInfraService
@@ -64,17 +61,6 @@ class LaunchContext:
    env_var_secret: str = ""  # encryption key injected into the agent's env
 def _guest_ip_from_config(config_path: Path) -> str:
    """Read the kernel's configured guest IP from a Firecracker config."""
    try:
        config = json.loads(config_path.read_text())
        args = config["boot-source"]["boot_args"]
        ip_arg = next(part for part in args.split() if part.startswith("ip="))
        return ip_arg.removeprefix("ip=").split(":", 1)[0]
    except (OSError, ValueError, KeyError, TypeError, StopIteration):
        return ""
 def persist_env_var_secret(private_key: Path, guest_ip: str, secret: str) -> None:
    """Mirror the exec-time key into guest tmpfs for restart recovery."""
    proc = subprocess.run(
@@ -89,29 +75,6 @@ def persist_env_var_secret(private_key: Path, guest_ip: str, secret: str) -> Non
        )
 def _reprovision_running_bottles(client: OrchestratorClient) -> None:
    """Read keys from live agent VMs and restore the restarted gateway."""
    try:
        secrets_by_ip: dict[str, str] = {}
        for run_dir in cleanup.live_run_dirs():
            guest_ip = _guest_ip_from_config(run_dir / "config.json")
            private_key = run_dir / "bottle_id_ed25519"
            if not guest_ip or not private_key.is_file():
                continue
            proc = subprocess.run(
                util.ssh_base_argv(private_key, guest_ip)
                + [f"cat {_ENV_VAR_SECRET_PATH}"],
                capture_output=True, text=True, check=False,
            )
            if proc.returncode == 0 and proc.stdout.strip():
                secrets_by_ip[guest_ip] = proc.stdout.strip()
        count = reprovision_bottles(client, secrets_by_ip)
        if count:
            info(f"reprovisioned egress tokens for {count} Firecracker bottle(s)")
    except (OSError, OrchestratorClientError) as exc:
        info(f"egress token reprovision skipped: {exc}")
 def launch_consolidated(
    egress_plan: EgressPlan,
    git_gate_plan: GitGatePlan,
@@ -126,7 +89,6 @@ def launch_consolidated(
    service = FirecrackerInfraService()
    url = service.ensure_running()
    client = OrchestratorClient(url)
    _reprovision_running_bottles(client)
    # Read the gateway's provisioning transport + CA off the Gateway service.
    gateway = service.gateway()
@@ -17,6 +17,7 @@ from ...orchestrator.lifecycle import DEFAULT_STARTUP_TIMEOUT_SECONDS
 from . import infra_vm
 from .gateway import FirecrackerGateway
 from .orchestrator import FirecrackerOrchestrator
 from .reconcile import attach_bottled_agents_to_gateway
 class FirecrackerInfraService(InfraService):
@@ -67,9 +68,11 @@ class FirecrackerInfraService(InfraService):
            # holds the signing key) mints the role-scoped `gateway` JWT for the
            # gateway, which never sees the key (#469).
            orchestrator.ensure_running(startup_timeout=startup_timeout)
-            self.gateway().connect_to_orchestrator(
+            gateway = self.gateway()
            gateway.connect_to_orchestrator(
                orchestrator.gateway_url(), orchestrator.mint_gateway_token())
            infra_vm.record_booted_version(want)
            attach_bottled_agents_to_gateway(url, gateway)
            return url
    def stop(self) -> None:
@@ -0,0 +1,199 @@
 """Bring-up reconcile: re-attach all running agent VMs to a freshly-booted gateway.
 Called from the cold-boot branch of `FirecrackerInfraService.ensure_running()`
 after `gateway.connect_to_orchestrator()` completes. Restores the three
 gateway-dependent services for every live bottle:
  - **CA** — push the current (freshly-minted) gateway CA to each agent's
    trust store and run `update-ca-certificates`, so the agent trusts the
    new CA on its next egress call.
  - **git-gate** — re-provision each bottle's bare repos and per-repo creds
    in the gateway, rebuilt from the persisted host-side state dir.
  - **egress tokens** — read the agent's `ENV_VAR_SECRET` and feed
    `reprovision_bottles` to restore the orchestrator's in-memory tokens.
 Per-bottle failures are caught, logged, and skipped so one unreachable VM
 does not block the others or the gateway coming up (PRD 0081).
 """
 from __future__ import annotations
 import json
 import subprocess
 from pathlib import Path
 from ...bottle_state import git_gate_state_dir
 from ...gateway import GatewayError
 from ...gateway.git_gate.render import GitGateUpstream
 from ...git_gate.plan import GitGatePlan
 from ...log import info
 from ...orchestrator.client import OrchestratorClient, OrchestratorClientError
 from ...orchestrator.reprovision import reprovision_bottles
 from ..provision_gateway import provision_git_gate
 from ..util import AGENT_CA_PATH
 from . import cleanup, util
 from .gateway import FirecrackerGateway
 # Where persist_env_var_secret writes the key on the agent VM (matches
 # consolidated_launch._ENV_VAR_SECRET_PATH — duplicated to avoid a
 # circular import through infra.py).
 _ENV_VAR_SECRET_PATH = "/run/bot-bottle/env-var-secret"
 def _guest_ip_from_config(config_path: Path) -> str:
    """Read the agent VM's guest IP from its Firecracker config file."""
    try:
        config = json.loads(config_path.read_text())
        args = config["boot-source"]["boot_args"]
        ip_arg = next(p for p in args.split() if p.startswith("ip="))
        return ip_arg.removeprefix("ip=").split(":", 1)[0]
    except (OSError, ValueError, KeyError, TypeError, StopIteration):
        return ""
 def attach_bottled_agents_to_gateway(
    orchestrator_url: str, gateway: FirecrackerGateway,
 ) -> None:
    """Reconcile all live agent VMs against the freshly-booted gateway.
    Fetches the new CA, lists registered bottles, then for each live run dir
    pushes the CA, re-provisions git-gate, and restores egress tokens. Each
    per-bottle step is wrapped so a failure is logged and skipped."""
    try:
        ca_pem = gateway.ca_cert_pem()
    except GatewayError as exc:
        info(f"bring-up reconcile: could not fetch gateway CA, skipping: {exc}")
        return
    client = OrchestratorClient(orchestrator_url)
    try:
        bottles = client.list_bottles()
    except OrchestratorClientError as exc:
        info(f"bring-up reconcile: could not list bottles, skipping: {exc}")
        return
    source_ip_to_bottle_id: dict[str, str] = {}
    for b in bottles:
        source_ip = b.get("source_ip")
        bottle_id = b.get("bottle_id")
        if isinstance(source_ip, str) and isinstance(bottle_id, str):
            source_ip_to_bottle_id[source_ip] = bottle_id
    transport = gateway.provisioning_transport()
    secrets_by_ip: dict[str, str] = {}
    for run_dir in cleanup.live_run_dirs():
        slug = run_dir.name
        guest_ip = _guest_ip_from_config(run_dir / "config.json")
        private_key = run_dir / "bottle_id_ed25519"
        if not guest_ip or not private_key.is_file():
            continue
        # CA: push the gateway's current certificate to the agent's trust store.
        try:
            _push_ca(private_key, guest_ip, ca_pem)
        except Exception as exc:
            info(f"bring-up reconcile: CA push to {slug!r} failed: {exc}")
        # git-gate: re-provision repos + creds from the persisted state dir.
        bottle_id = source_ip_to_bottle_id.get(guest_ip)
        if bottle_id:
            try:
                _reprovision_git_gate(transport, bottle_id, slug)
            except Exception as exc:
                info(f"bring-up reconcile: git-gate for {slug!r} failed: {exc}")
        # egress tokens: read ENV_VAR_SECRET from the agent's tmpfs.
        proc = subprocess.run(
            util.ssh_base_argv(private_key, guest_ip) + [f"cat {_ENV_VAR_SECRET_PATH}"],
            capture_output=True, text=True, check=False,
        )
        if proc.returncode == 0 and proc.stdout.strip():
            secrets_by_ip[guest_ip] = proc.stdout.strip()
    # Restore in-memory egress tokens for all bottles that exposed a key.
    if secrets_by_ip:
        try:
            count = reprovision_bottles(client, secrets_by_ip)
            if count:
                info(
                    f"bring-up reconcile: restored egress tokens for {count} bottle(s)"
                )
        except OrchestratorClientError as exc:
            info(f"bring-up reconcile: egress token restore failed: {exc}")
 def _push_ca(private_key: Path, guest_ip: str, ca_pem: str) -> None:
    """SSH the gateway CA PEM into the agent VM and update its trust store.
    Runs as the SSH root user (the agent VM's dropbear accepts root). Each
    step uses check=True so a failure raises and the caller's except clause
    logs and continues."""
    ssh = util.ssh_base_argv(private_key, guest_ip)
    # mkdir is idempotent; rootfs builds may not preserve the target dir.
    subprocess.run(
        ssh + ["mkdir -p /usr/local/share/ca-certificates"],
        capture_output=True, check=True,
    )
    subprocess.run(
        ssh + [f"cat > {AGENT_CA_PATH}"],
        input=ca_pem, text=True, capture_output=True, check=True,
    )
    subprocess.run(
        ssh + [f"chmod 644 {AGENT_CA_PATH} && update-ca-certificates"],
        capture_output=True, check=True,
    )
 def _reprovision_git_gate(
    transport: object, bottle_id: str, slug: str,
 ) -> None:
    """Re-provision one bottle's git-gate repos and creds from its state dir.
    Reads `upstreams.json` (written by `provision_git_gate_dynamic_keys` at
    launch) to reconstruct the GitGateUpstream table, then calls
    `provision_git_gate` which places the credential files and inits the bare
    repos in the gateway. No-op when there is no `upstreams.json` (the bottle
    has no git upstreams)."""
    state_dir = git_gate_state_dir(slug)
    upstreams_file = state_dir / "upstreams.json"
    if not upstreams_file.exists():
        return
    raw = json.loads(upstreams_file.read_text())
    upstreams: list[GitGateUpstream] = []
    for u in raw:
        name = u["name"]
        # The key in the state dir is preferred: gitea deploy keys are written
        # there by provision_git_gate_dynamic_keys; static keys keep their
        # original manifest path.
        key_in_state = state_dir / f"{name}-key"
        identity_file = (
            str(key_in_state) if key_in_state.is_file()
            else u.get("identity_file", "")
        )
        known_hosts = state_dir / f"{name}-known_hosts"
        upstreams.append(GitGateUpstream(
            name=name,
            upstream_url=u["upstream_url"],
            upstream_host=u.get("upstream_host", ""),
            upstream_port=u.get("upstream_port", ""),
            identity_file=identity_file,
            known_host_key=u.get("known_host_key", ""),
            known_hosts_file=known_hosts if known_hosts.is_file() else Path(),
        ))
    if not upstreams:
        return
    plan = GitGatePlan(
        slug=slug,
        entrypoint_script=state_dir / "git_gate_entrypoint.sh",
        hook_script=state_dir / "git_gate_pre_receive.sh",
        access_hook_script=state_dir / "git_gate_access_hook.sh",
        upstreams=tuple(upstreams),
    )
    provision_git_gate(transport, bottle_id, plan)  # type: ignore[arg-type]
 __all__ = ["attach_bottled_agents_to_gateway"]
@@ -20,7 +20,6 @@ import subprocess
 import sys
 from pathlib import Path
 from ... import invocation
 from ... import resources
 from . import netpool
 from . import util
@@ -180,11 +179,9 @@ def _setup_systemd() -> None:
        f"sudo systemctl daemon-reload\n"
        f"sudo systemctl enable --now {netpool.SYSTEMD_UNIT}\n"
    )
    # Absolute path, not `sudo bot-bottle`: sudo's secure_path drops
    # ~/.local/bin, where both pipx and install.sh put the entry point.
    sys.stderr.write(
-        f"\n(Or re-run this as root to install it directly:\n"
+        f"\n(Or re-run this as root to install it directly: "
-        f"    {invocation.sudo_command('backend', 'setup', '--backend=firecracker')})\n"
+        f"sudo bot-bottle backend setup --backend=firecracker)\n"
    )
@@ -8,6 +8,7 @@ imported (`deploy_key_provisioner`) to keep its cost off the host path.
 from __future__ import annotations
 import json
 import os
 import dataclasses
 from pathlib import Path
@@ -138,7 +139,32 @@ def provision_git_gate_dynamic_keys(
            if upstream.name not in updated_names:
                updated.append(upstream)
-    return dataclasses.replace(plan, upstreams=tuple(updated))
+    final_plan = dataclasses.replace(plan, upstreams=tuple(updated))
    _write_upstreams_snapshot(stage_dir, final_plan.upstreams)
    return final_plan
 def _write_upstreams_snapshot(
    stage_dir: Path, upstreams: tuple[GitGateUpstream, ...]
 ) -> None:
    """Persist the fully-resolved upstream table to `upstreams.json` in
    `stage_dir` so the bring-up reconcile can re-provision git-gate without
    the manifest. Written after dynamic keys are provisioned so every
    identity_file is set."""
    data = [
        {
            "name": u.name,
            "upstream_url": u.upstream_url,
            "upstream_host": u.upstream_host,
            "upstream_port": u.upstream_port,
            "identity_file": u.identity_file,
            "known_host_key": u.known_host_key,
        }
        for u in upstreams
    ]
    snapshot = stage_dir / "upstreams.json"
    snapshot.write_text(json.dumps(data, indent=2))
    snapshot.chmod(0o600)
 __all__ = [
@@ -1,48 +0,0 @@
 """How to tell a user to re-run this CLI.
 `bot-bottle …` is the right thing to print for anything the user runs as
 themselves — it is on their PATH, since that is how they got here.
 Under `sudo` it is not. sudo replaces PATH with sudoers' `secure_path`
 (`/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin` on Debian and
 Ubuntu, similar elsewhere), which deliberately excludes user-writable
 directories. Both supported install paths put the entry point in one of those:
 pipx uses `~/.local/bin`, and `install.sh`'s venv fallback symlinks there too.
 So `sudo bot-bottle …` fails with "command not found" for exactly the users who
 followed the documented install, while working for anyone who happened to
 install system-wide — which is why it survives review so easily.
 Naming the absolute path sidesteps secure_path entirely.
 """
 from __future__ import annotations
 import os
 import shutil
 import sys
 def self_path() -> str:
    """Absolute path to this CLI's entry point.
    Falls back to the bare name when the entry point cannot be resolved (an
    unusual invocation such as `python -m`), because a slightly wrong hint is
    better than a traceback while reporting an unrelated problem.
    """
    argv0 = sys.argv[0] or "bot-bottle"
    resolved = shutil.which(argv0) or argv0
    if not os.path.isabs(resolved):
        if os.path.exists(resolved):
            resolved = os.path.abspath(resolved)
        else:
            return "bot-bottle"
    return resolved
 def sudo_command(*args: str) -> str:
    """A copy-pasteable `sudo …` invocation of this CLI.
    >>> sudo_command("backend", "setup", "--backend=firecracker")
    'sudo /home/u/.local/bin/bot-bottle backend setup --backend=firecracker'
    """
    return " ".join(["sudo", self_path(), *args])
@@ -50,6 +50,12 @@ class ReprovisionBody(_StrictModel):
    env_var_secret: StrictStr
 class SecretBody(_StrictModel):
    name: StrictStr
    value: StrictStr
    env_var_secret: StrictStr
 class ReconcileBody(_StrictModel):
    live_source_ips: list[StrictStr]
    grace_seconds: float | None = None
@@ -259,6 +265,21 @@ def create_app(orch: OrchestratorCore, *, signing_key: str) -> FastAPI:
            return {"reprovisioned": True}
        raise HTTPException(404, "no stored secrets for this bottle")
    @app.post("/bottles/{bottle_id}/secret")
    def update_secret(bottle_id: str, body: SecretBody) -> dict[str, object]:
        # Update ONE egress token for a running bottle in place — the
        # single-secret form of reprovision, for refreshing a short-lived host
        # credential (e.g. the Codex access token) without a relaunch. cli-only
        # (not in _GATEWAY_ROUTES): a bottle must never set its own tokens.
        if orch.update_agent_secret(
            bottle_id,
            _required(body.name, "name"),
            _required(body.value, "value"),
            _required(body.env_var_secret, "env_var_secret"),
        ):
            return {"updated": True}
        raise HTTPException(404, "no such bottle")
    @app.delete("/bottles/{bottle_id}")
    def teardown(bottle_id: str) -> dict[str, object]:
        if orch.teardown_bottle(bottle_id):
@@ -184,6 +184,27 @@ class OrchestratorClient:
            )
        return True
    def update_agent_secret(
        self, bottle_id: str, name: str, value: str, env_var_secret: str,
    ) -> bool:
        """Update ONE egress token for a running bottle in place
        (`POST /bottles/<id>/secret`) — the single-secret form of
        `reprovision_gateway`, for pushing a freshly-refreshed host credential
        into a bottle without a relaunch. Returns True on success, False when the
        orchestrator doesn't know the bottle (404)."""
        status, _ = self._request(
            "POST",
            f"/bottles/{bottle_id}/secret",
            {"name": name, "value": value, "env_var_secret": env_var_secret},
        )
        if status == 404:
            return False
        if not 200 <= status < 300:
            raise OrchestratorClientError(
                f"update_agent_secret {bottle_id}: HTTP {status}"
            )
        return True
    def teardown_bottle(self, bottle_id: str) -> bool:
        """Tear a bottle down (`DELETE /bottles/<id>`). False if the
        orchestrator didn't know it (404) — idempotent for cleanup paths."""
@@ -379,6 +379,30 @@ class OrchestratorCore:
        self._tokens[bottle_id] = decrypted
        return True
    def update_agent_secret(
        self, bottle_id: str, name: str, value: str, env_var_secret: str,
    ) -> bool:
        """Update ONE egress token for a known bottle in place — the
        single-secret form of ``reprovision_from_secret``.
        Sets the in-memory token AND upserts the single re-encrypted row under
        *env_var_secret* (the same key the rest of the rows are encrypted with, so
        the whole set stays decryptable by a later ``reprovision_from_secret``),
        leaving every other token untouched. Returns False if the bottle is
        unknown.
        Unlike ``reprovision_from_secret`` (which restores the values captured at
        launch), this pushes a *caller-supplied* value — used to refresh a
        short-lived host credential (e.g. the Codex access token) into a
        still-running bottle without a relaunch."""
        from .store.secret_store import encrypt_value
        if self.registry.get(bottle_id) is None:
            return False
        self._tokens.setdefault(bottle_id, {})[name] = value
        self.registry.store_agent_secret(
            bottle_id, name, encrypt_value(env_var_secret, value))
        return True
    # --- consolidated gateway ----------------------------------------------
    def gateway_status(self) -> dict[str, object]:
@@ -370,6 +370,30 @@ class RegistryStore(DbStore):
            )
        self._chmod()
    def store_agent_secret(
        self,
        bottle_id: str,
        key: str,
        encrypted_value: str,
        secret_type: str = "injected_env_var",
    ) -> None:
        """Upsert ONE encrypted secret row (env-var name → ciphertext) for
        *bottle_id*, leaving the bottle's other secrets untouched — the per-key
        counterpart of ``store_agent_secrets``' replace-all. Delete-then-insert
        because the table carries no unique constraint to `ON CONFLICT` against."""
        with self._connection() as conn:
            conn.execute(
                "DELETE FROM bottled_agent_secrets "
                "WHERE bottled_agent_id = ? AND key = ? AND type = ?",
                (bottle_id, key, secret_type),
            )
            conn.execute(
                "INSERT INTO bottled_agent_secrets "
                "(bottled_agent_id, key, value, type) VALUES (?, ?, ?, ?)",
                (bottle_id, key, encrypted_value, secret_type),
            )
        self._chmod()
    def get_agent_secrets(
        self,
        bottle_id: str,
@@ -0,0 +1,117 @@
 # PRD 0081: Reprovision gateway-dependent state on gateway bring-up
 - **Status:** Active
 - **Author:** claude
 - **Created:** 2026-07-26
 - **Issue:** #516
 ## Summary
 When the gateway is (re)built, reconcile every already-running bottle against the
 fresh gateway instead of persisting the gateway's state. On a gateway cold boot,
 the gateway reconciles all live bottles in one flow: **replace** each agent's
 trusted CA with the freshly-minted gateway CA, **re-provision** each bottle's
 git-gate repos + creds onto the gateway, and **restore** each bottle's egress
 tokens. One mechanism across all three services; the CA rotates for free on every
 bring-up.
 ## Problem
 A gateway rebuild/restart silently breaks every already-running bottle:
 - **CA (#510).** The gateway's mitmproxy mints a new CA on a fresh rootfs; agents
  still trust the old one, so egress fails TLS verification (`SSL certificate
  verification failed`).
 - **git-gate (#512).** Per-bottle bare repos (`/git/<id>`) + deploy creds
  (`/git-gate/creds/<id>`) live in the gateway's ephemeral rootfs; a rebuild wipes
  them and the agent 404s on fetch/push.
 Both are the same root cause: per-bottle gateway-dependent state is provisioned
 **once, at bottle launch**, and nothing restores it for already-running bottles
 when the gateway comes back. The existing launch-time reprovision
 (`reprovision_bottles`) restores **only** egress tokens, and only as a side effect
 of the *next* launch.
 Persisting the state (a host bind-mount / a per-VM data volume per service) was
 prototyped and rejected: it differs per service (a volume for the CA, another for
 git-gate, reprovision for tokens), it pins the CA static forever (no rotation),
 and it adds volume surface that `docker volume prune` / a wiped cache can silently
 destroy (the original #450 failure mode).
 ## Goals / Success Criteria
 - A gateway (re)boot restores **all** running bottles' gateway-dependent state
  with no manual step and no relaunch — the agent's next egress / fetch / push
  just works.
 - **One** reconcile flow covering CA, git-gate, and egress tokens, rather than a
  different mechanism per service.
 - The CA **rotates** on every gateway bring-up (no long-lived CA), distributed to
  running agents by the same reconcile.
 - Per-bottle failures are tolerated: one unreachable or malformed bottle does not
  block the others or the gateway coming up.
 - **All backends** (Firecracker, docker, macOS) reconcile through the *same*
  contract — an abstract method on the backend base class, so a new backend
  cannot forget to implement it and none drifts onto a bespoke mechanism.
 ## Non-goals
 - Deliberate mid-session CA rotation *without* a gateway restart — `rotate_ca`
  stays for that operator action.
 - Changing source-IP attribution, `/resolve`, or the plane split (#469).
 ## Design
 **The contract — `attach_bottled_agents_to_gateway()` on the backend ABC.**
 Reconciling running bottles against the current gateway is a backend
 responsibility (only the backend can enumerate its agents and reach them —
 firecracker over SSH, docker/macOS over `exec`/`cp`), so it is an
 `@abc.abstractmethod` on `BottleBackend` (`backend/base.py`). Every backend
 implements it; the host calls it whenever the gateway is (re)brought up. This is
 what makes the fix cross-backend by construction rather than a per-backend
 follow-up. It reprovisions **all** registered bottles' gateway-dependent state:
 CA, git-gate, and egress tokens.
 **Trigger — the gateway bring-up path.** The host calls
 `attach_bottled_agents_to_gateway()` only when the gateway was actually
 (re)brought up — the cold-boot branch of the infra bring-up (e.g.
 `FirecrackerInfraService.ensure_running` after it boots a fresh pair), never on an
 adopt of a healthy, current gateway (state intact). So it fires exactly when the
 gateway was (re)booted — including orchestrator restarts, since the pair boots
 together — and there is no bare-restart path that bypasses bring-up.
 **Per-backend implementation.** Each backend's `attach_bottled_agents_to_gateway`
 enumerates its live bottles and, for each, reconciles the three services against
 the current gateway. The firecracker implementation, once the gateway VM is up
 and its CA is available:
 1. Map each live bottle's guest IP → `bottle_id` from the orchestrator registry
   (`list_bottles`).
 2. Install the **current** shared git-gate hooks (`git_gate_render_hook` /
   `git_gate_render_access_hook`) into the fresh gateway once — rendered from
   code, never from a bottle's possibly-stale state dir.
 3. For each live agent VM (enumerated from its run dir):
   - **CA:** SSH the current gateway CA into the agent's trust store and run
     `update-ca-certificates` (unconditional replace — there is one gateway, so no
     fingerprint match is needed).
   - **git-gate:** rebuild the bottle's upstreams from its persisted git-gate
     state dir (deploy key, known_hosts, upstream URL) and re-init its bare repos
     + per-repo creds under `/git/<bottle_id>`.
   - **egress token:** read the agent's `ENV_VAR_SECRET` and feed
     `reprovision_bottles`, restoring the orchestrator's in-memory tokens.
 4. Every per-bottle step is wrapped so one failure is logged and skipped.
 **Retire the persistence prototype.** No CA data volume, no git-gate data volume
 (the abandoned PRs #511 / #513). The launch-time `_reprovision_running_bottles`
 call folds into this bring-up reconcile, so egress tokens are restored on the same
 cold-boot trigger (an adopt needs no restore — the orchestrator never restarted).
 **CA rotation.** Because the gateway rootfs is ephemeral, every cold boot mints a
 fresh CA; reconcile is what distributes it, so a routine gateway rebuild doubles
 as a CA rotation with zero extra machinery.
 ## Open questions
 - Docker's existing host-bind-mounted CA (`host_gateway_ca_dir`): once docker's
  `attach_bottled_agents_to_gateway` pushes the CA to running agents on bring-up,
  the bind-mount is redundant — drop it (so docker rotates like firecracker) or
  keep it as belt-and-suspenders? Leaning drop, for one behaviour across backends.
@@ -1,183 +0,0 @@
 # Testing a clean bot-bottle install on Linux
 How do you exercise `install.sh` the way a brand-new user would — on a
 pristine Linux environment you can throw away afterward — *without*
 polluting your daily-driver host, and across the several package-management
 regimes Linux fragments into? This is the Linux counterpart to
 [`testing-clean-install-on-macos.md`](testing-clean-install-on-macos.md);
 the conclusion is different because Linux gives us a boundary macOS doesn't.
 ## Summary
 On macOS the honest options were a throwaway user or a VM, and the throwaway
 user won on pragmatics (nested virtualization is gated to M3+). On Linux the
 calculus flips: a **disposable KVM virtual machine, booted from a distro
 cloud image and deleted per run, is both the cleanest boundary and the one
 that lets a single harness cover Ubuntu, Fedora, Arch, Alpine, and NixOS**.
 The host already requires KVM for the Firecracker backend, so the VM is cheap
 here.
 The harness lives at [`scripts/linux-install-test.sh`](../../scripts/linux-install-test.sh).
 Per run it caches one read-only base image, boots a throwaway copy-on-write
 overlay (`qemu-img create -b base`), installs the distro's prerequisites,
 pipes *this checkout's* `install.sh` into the guest exactly as `curl … | sh`
 would, asserts the CLI installed, and deletes the overlay — the Linux
 equivalent of `docker run --rm`, for a whole machine.
 ## Why a VM, not a container or a throwaway user
 | Mechanism | Why it's the wrong boundary here |
 |---|---|
 | **Container** (`docker run --rm`) | Shares the host kernel and ships a deliberately minimal userland — no systemd, a stubbed-out package manager story, and (crucially) it doesn't reproduce the *externally-managed Python* (PEP 668) that real desktop/server installs put in front of the user. It tests "does install.sh run in a container," not "does it run on a real distro." |
 | **Throwaway user** (`useradd`/`userdel`) | The macOS pick, but weaker on Linux: it reaches the real host, yet every system package it installs (python, pipx, git via `apt`/`dnf`/…) stays behind, and it can only ever test the *one* distro the host runs. The whole Linux-specific value is the cross-distro matrix. |
 | **Disposable KVM VM** (this harness) | A genuine kernel + userland + package-manager boundary that wipes to nothing on teardown, and swaps freely between distro cloud images. The one real cost — nested virtualization for the *backend* — doesn't apply, because we gate the installer, not the runtime (below). |
 ## Two variants: `test` (bare host) and `test-ready` (prepared host)
 `install.sh` never installs a backend, and never installs its own toolchain
 prerequisites (python3, git, pipx) — it installs the `bot-bottle` package and
 runs `doctor`, which *reports* what's missing
 ([`install.sh`](../../install.sh) header,
 [`bot_bottle/cli/commands/doctor.py`](../../bot_bottle/cli/commands/doctor.py)).
 That leaves two distinct things worth testing, split into two subcommands that
 mirror the macOS harness's `test` / `test-ready` convention (there the split is
 the backend service; here it is the toolchain the installer needs):
 - **`test`** — `install.sh` runs on the **bare cloud image**, prerequisites and
  all left as the vendor ships them. This exercises `install.sh`'s own
  prerequisite-guard logic — the entire first half of the script (python
  version gate, git-for-git-specs gate, pipx/pip PEP-668 handling).
 - **`test-ready`** — the harness installs python3 + git + pipx first (the
  `prereqs` step), then runs `install.sh`. This is the *prepared-host happy
  path*: does a clean install actually land and produce a working CLI?
 **Pass criteria differ by variant:**
 | Variant | PASS when |
 |---|---|
 | `test` | `install.sh` **either** installs cleanly (the image already carried enough) **or** declines with one of its own recognized, actionable prerequisite errors (missing python3/git, no usable pip, PEP 668). A crash or an *unrecognized* failure is a FAIL. |
 | `test-ready` | `install.sh` actually lands: the `bot-bottle` entry point is present and runs, and `doctor` reports a usable python and config without crashing. A graceful decline is no longer good enough. |
 Neither variant requires a green `doctor`: inside the VM there is no nested KVM
 or Docker, so **the backend is correctly reported not-ready** — install.sh does
 not install a backend and cannot regress one, and this harness does not
 provision the Docker backend. This is where Linux necessarily diverges from the
 macOS `test-ready`, which reaches the host backend; `BB_TEST_REQUIRE_BACKEND=1`
 makes readiness fatal anyway, for a nested-virt host that can satisfy it. The
 verdict instead classifies `doctor`'s output the way the macOS harness does — a
 `Traceback` is an install defect (fail), a missing `python`/`config` line is a
 fail, a not-ready backend is reported — so a genuine installer regression (a
 broken shim, an import error, a botched PATH) stays visible.
 `test-all` runs the full matrix — every distro × both variants — each cell in
 its own throwaway VM, and prints a per-cell PASS/FAIL summary.
 ## The distro matrix is the point
 Each distro exercises a different corner of the installer:
 | Distro | Cloud image | What it stresses |
 |---|---|---|
 | **Ubuntu** (noble) | `cloud-images.ubuntu.com` | The common case; `apt`'s `pipx`, externally-managed Python (PEP 668) → install.sh's pipx path. |
 | **Fedora** | Fedora Cloud Base Generic | `dnf` packaging, a different default Python, BSD-style checksum file. |
 | **Arch** | `geo.mirror.pkgbuild.com/images/latest` | Rolling / newest Python; `python-pipx`. |
 | **Alpine** | Alpine `nocloud_` (cloudinit) image | musl libc + BusyBox `sh` — the harshest POSIX-`sh` host for a `#!/bin/sh` installer. |
 | **NixOS** | locally built with `nixos-generators` | No FHS `~/.local` on PATH by default; `nix profile install` prereqs; pipx laying a self-contained venv on a non-FHS host. |
 ### Validation run (2026-07-27) — full green
 Full matrix on the delphi KVM host, QEMU 11.0.2, both variants × all five
 distros passing:
 | Distro | `test` (bare) | `test-ready` (prepared) |
 |---|---|---|
 | Ubuntu 24.04 | ✅ declines at git gate | ✅ installs, doctor python+config green |
 | Fedora 44 | ✅ declines at git gate | ✅ installs |
 | Arch (latest) | ✅ declines at git gate | ✅ installs |
 | Alpine 3.21 | ✅ declines at git gate | ✅ installs |
 | NixOS 24.11 | ✅ declines (no python3) | ✅ installs |
 The bare `test` sees `install.sh` decline soundly — exit 1 at the
 git-for-git-specs gate on the Debian/Fedora/Arch/Alpine images (they ship
 python3 but not git), and at the python3 gate on NixOS (no python3 on PATH) —
 and `test-ready` installs cleanly with `doctor` reporting a usable python and
 config (backends all not-ready, as expected in a plain VM).
 Getting to green surfaced and fixed a series of real defects:
 - **Fedora 41 was EOL/404** → bumped to 44.
 - The liveness probe used `bot-bottle --version`, which the CLI does not
  implement (unknown args die non-zero), so every *successful* install was
  misreported as failed → switched to `bot-bottle --help`.
 - **Alpine** needed three fixes: the `generic_` image ignores a NoCloud seed
  (switched to the `nocloud_` variant); OpenRC does not auto-start sshd after
  cloud-init injects the key (start it via `runcmd`); and Alpine's non-PAM
  sshd refuses pubkey auth for a cloud-init-*locked* account (give it a
  throwaway password). It also has no `sudo` by default (install it via
  cloud-init `packages:`).
 - **NixOS** publishes no downloadable cloud qcow2 (its cloud images are
  Hydra-built AMIs), so the harness builds one with `nixos-generators`
  ([`linux-install-test-nixos.nix`](../../scripts/linux-install-test-nixos.nix)):
  cloud-init for the key, flakes enabled, deliberately no python/git/pipx. The
  `test-ready` prereq install pins `nixpkgs/nixos-24.11` because the guest's
  default unstable registry builds pipx from source (and its test suite
  currently fails to build).
 - Two harness-hygiene bugs also fixed: `cmd_down` left `serial.log` behind
  (orphaned run dirs), and the teardown trap was armed after `cmd_up`, leaking
  a VM when `wait_for_ssh` timed out.
 In `test-ready` the harness installs `python3 + git + pipx` first on each
 distro (install.sh installs none of them), so all five drive the recommended
 pipx path. `test` then removes that scaffolding and lets each distro's bare
 image collide with install.sh's guards — on most cloud images python3 is
 present (cloud-init needs it) but git and pipx are not, so install.sh is
 expected to decline at the git-for-git-specs gate or the PEP-668 pip check with
 an actionable message. Both are legitimate, and the two variants together cover
 the whole first half of the installer as well as the happy path.
 ## What a clean install touches (the footprint that decides "wipeable")
 | Artifact | Location | In the guest's `$HOME`? | Survives VM teardown? |
 |---|---|---|---|
 | Config / state / db | `~/.bot-bottle/{agents,bottles,contrib,…}` ([`install.sh`](../../install.sh)) | ✅ | ❌ overlay deleted |
 | pipx venv + shim | `~/.local/pipx/venvs/bot-bottle`, shim in `~/.local/bin` | ✅ | ❌ overlay deleted |
 | pip `--user` fallback | `~/.local/lib` + `~/.local/bin` | ✅ | ❌ overlay deleted |
 | **Distro prerequisites** (python/git/pipx, `test-ready` only) | system paths via `apt`/`dnf`/`pacman`/`apk`/`nix profile` | ❌ | ❌ **overlay deleted** |
 Unlike the macOS throwaway user (whose Homebrew / Apple-Container / Rosetta
 footprint *survives*), **every row here dies with the overlay** — that is the
 VM's whole advantage. The cached base image is read-only backing and is the
 only thing that persists between runs, on purpose.
 ## Design notes baked into the harness
 - **User-mode networking** (`-netdev user,hostfwd=tcp:127.0.0.1:PORT-:22`):
  no root, no bridge, no host network state touched. Only SSH is forwarded.
 - **cloud-init seed ISO** (`cloud-localds`) injects an ephemeral SSH keypair
  and a passwordless-sudo login. The keypair is generated per run and deleted
  on teardown; the guest can't be logged into after it's gone.
 - **Copy-on-write overlay**: the cached base is never mutated, so a corrupt or
  interrupted run can't poison the cache; downloads land at `*.partial` and
  are renamed only after checksum verification.
 - **Checksums**: verified against each vendor's published sums file at
  download time (GNU `hash  file`, bare-hash, and Fedora's BSD
  `SHA256 (file) = hash` formats are all handled). Alpine ships `.sha512` only
  (this verifier is sha256) so it is skipped; NixOS is built locally, not
  downloaded, so there is nothing to verify.
 - **NixOS is built, not downloaded**: `ensure_base_image` runs
  `nixos-generate -f qcow` against
  [`linux-install-test-nixos.nix`](../../scripts/linux-install-test-nixos.nix)
  once and caches the result; the per-run seed/overlay flow is otherwise
  identical to the downloaded distros.
 - **`test-all`** runs every distro × both variants (`test` and `test-ready`),
  each cell in its own subshell on its own forwarded port, so one cell's
  failure (or teardown trap) can't abort the matrix; it prints a per-cell
  PASS/FAIL summary.
 ## Not wired into PR CI
 Like the macOS harness, the runtime is host-specific (needs `/dev/kvm`,
 `qemu`, and `cloud-localds`) and is not exercised by the Linux pull-request
 runner. It is validated statically (`bash -n`, `shellcheck`) and run by hand
 on a KVM-capable host. The cloud-image URLs in the `DISTRO` table are the one
 place to bump when a distro cuts a newer build.
@@ -1,24 +0,0 @@
 # NixOS image for scripts/linux-install-test.sh.
 #
 # NixOS publishes no downloadable cloud qcow2 (its cloud images are Hydra-built
 # AMIs), so the harness BUILDS this one with nixos-generators (-f qcow). It is
 # deliberately minimal — no python3/git/pipx — so the bare `test` variant is
 # genuinely under-provisioned and exercises install.sh's guards; `test-ready`
 # provisions them with `nix profile install` (hence flakes below).
 { lib, ... }:
 {
  # Consume the same NoCloud seed the other distros use: cloud-init injects the
  # per-run ephemeral SSH key for root. Leave networking to NixOS's default
  # dhcpcd (QEMU user-mode NAT) — enabling cloud-init's networkd here conflicts
  # with dhcpcd and can drop the guest's network.
  services.cloud-init.enable = true;
  services.cloud-init.network.enable = false;
  services.openssh.enable = true;
  services.openssh.settings.PermitRootLogin = lib.mkForce "prohibit-password";
  # Flakes so the `test-ready` prereq step can `nix profile install nixpkgs#...`.
  nix.settings.experimental-features = [ "nix-command" "flakes" ];
  system.stateVersion = "24.11";
 }
@@ -1,753 +0,0 @@
 #!/usr/bin/env bash
 # Clean-install test harness for the Linux path.
 #
 # Exercises install.sh the way a brand-new user would, inside a THROWAWAY
 # QEMU/KVM virtual machine that is booted from a distro cloud image and
 # deleted afterward. install.sh's entire footprint is user-home-local (the
 # pipx venv under ~/.local, the ~/.bot-bottle config dir, and a printed PATH
 # hint), so a fresh VM's fresh $HOME is the clean surface we want — and unlike
 # a throwaway user account, tearing the VM down also wipes any OS-level
 # prerequisites installed into it, so the reset is total. Full rationale in
 # docs/research/testing-clean-install-on-linux.md.
 #
 # Why a VM and not a container or a throwaway user: a container shares the
 # host kernel and cannot exercise a genuinely pristine OS (systemd, the distro
 # package manager, PEP 668 externally-managed Python) the way a real guest
 # does, and a throwaway user leaves every system package it installs behind.
 # The host already needs KVM for the Firecracker backend, so a per-run,
 # copy-on-write VM is cheap here: one cached base image, a throwaway overlay
 # per run (`qemu-img create -b base`), deleted on teardown — the Linux
 # equivalent of `docker run --rm`, but for a whole machine.
 #
 # Usage:
 #   ./scripts/linux-install-test.sh test        # up -> run -> verdict -> down (bare host)
 #   ./scripts/linux-install-test.sh test-ready  # ... with prerequisites installed first
 #   ./scripts/linux-install-test.sh test-all    # every distro × both variants, with a summary
 #   ./scripts/linux-install-test.sh up          # fetch base image, boot a fresh VM
 #   ./scripts/linux-install-test.sh prereqs     # install python3 + git + pipx in the VM
 #   ./scripts/linux-install-test.sh run         # pipe install.sh into the VM
 #   ./scripts/linux-install-test.sh status      # VM reachable? is the install sound?
 #   ./scripts/linux-install-test.sh down        # kill the VM, delete overlay + seed (the reset)
 #   ./scripts/linux-install-test.sh ssh         # open an interactive shell in the running VM
 #
 # TWO TEST VARIANTS, because "does install.sh handle an unprepared host" and
 # "does a prepared host get a clean install" are different questions (mirrors
 # the macOS harness's test / test-ready split):
 #
 #   test        A Linux system WITHOUT the prerequisites set up — the default
 #               state of a stock cloud image (python3 is usually present for
 #               cloud-init, but git and pipx are not). This exercises
 #               install.sh's own prerequisite-guard logic. It is SOUND — a
 #               PASS — when install.sh EITHER installs cleanly (the image
 #               already had enough) OR declines with one of its own recognized,
 #               actionable errors (missing python3/git, no usable pip, PEP 668
 #               externally-managed). A crash or an unrecognized failure fails.
 #
 #   test-ready  A Linux system WITH the prerequisites satisfied — the harness
 #               installs python3 + git + pipx first (see the DISTRO table),
 #               then runs install.sh. A graceful decline is no longer good
 #               enough here: the install MUST land, the entry point must run,
 #               and doctor must report a usable python and config without
 #               crashing.
 #
 # Neither variant requires a green backend. install.sh does not install a
 # backend and cannot regress one, and inside a plain VM there is no nested KVM
 # for Firecracker; the harness does not provision the Docker backend either. So
 # backend readiness is reported, not required (this is where Linux necessarily
 # diverges from the macOS test-ready, which reaches the host backend).
 # BB_TEST_REQUIRE_BACKEND=1 makes it fatal anyway, for a nested-virt host.
 #
 # Config via env:
 #   BB_TEST_DISTRO         ubuntu | fedora | arch | alpine | nixos (default: ubuntu)
 #   BB_TEST_SSH_PORT       host port forwarded to the guest's :22  (default: 2222)
 #   BB_TEST_CACHE_DIR      where base images are cached            (default: ~/.cache/bot-bottle-install-test)
 #   BB_TEST_RUN_DIR        per-run scratch (overlay, seed, key, …) (default: a mktemp dir)
 #   BB_TEST_MEM_MB         guest RAM                               (default: 2048)
 #   BB_TEST_CPUS           guest vCPUs                             (default: 2)
 #   BB_TEST_DISK           overlay virtual size                   (default: 12G)
 #   BB_TEST_BOOT_TIMEOUT   seconds to wait for SSH after boot      (default: 300)
 #   BB_TEST_KEEP           1 = the test cycles skip teardown, to poke at a failure
 #   BB_TEST_REQUIRE_BACKEND 1 = make a not-ready backend fatal (needs nested virt)
 #   BB_TEST_INSTALL_URL    curl this install.sh in the guest instead of piping the local checkout
 #   BB_TEST_SKIP_VERIFY    1 = skip base-image checksum verification (not recommended)
 #   BOT_BOTTLE_INSTALL_SPEC   passed through to install.sh (pip / git spec)
 #
 # Notes:
 #   * Needs /dev/kvm, qemu-system-x86_64, and cloud-localds (cloud-image-utils
 #     / cloud-utils); the nixos distro additionally needs nixos-generate. On the
 #     NixOS host:  nix shell nixpkgs#qemu nixpkgs#cloud-utils nixpkgs#nixos-generators
 #   * Networking is user-mode (`-netdev user,hostfwd`) so the harness needs no
 #     root, no bridge, and touches no host network state. Only SSH is forwarded.
 #   * The cloud-image URLs in the DISTRO table are the one place to bump when a
 #     distro cuts a new build; each is verified against the vendor's published
 #     checksum at download time (guarding against truncated/corrupt pulls).
 set -euo pipefail
 DISTRO="${BB_TEST_DISTRO:-ubuntu}"
 SSH_PORT="${BB_TEST_SSH_PORT:-2222}"
 CACHE_DIR="${BB_TEST_CACHE_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/bot-bottle-install-test}"
 MEM_MB="${BB_TEST_MEM_MB:-2048}"
 CPUS="${BB_TEST_CPUS:-2}"
 DISK="${BB_TEST_DISK:-12G}"
 BOOT_TIMEOUT="${BB_TEST_BOOT_TIMEOUT:-300}"
 _SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 _REPO_ROOT="$(cd "$_SCRIPT_DIR/.." && pwd)"
 # Per-run scratch. Persisted across sub-commands (up/prereqs/run/status/down)
 # via a marker file so `up` in one invocation and `down` in the next find the
 # same VM; the test cycles set RUN_DIR themselves and never write the marker.
 RUN_DIR="${BB_TEST_RUN_DIR:-}"
 # Set by the test cycles, which chain the steps and suppress the per-step "next
 # command" hints. _STEPS / _PASS_CLAIM / _REQUIRE_INSTALL are the per-variant
 # knobs the shared cycle and teardown read.
 IN_TEST=0
 _STEPS=4
 _PASS_CLAIM=""
 _REQUIRE_INSTALL=0
 # --- distro table ----------------------------------------------------
 # For each distro:  cloud-image URL | checksum-file URL | default SSH user |
 # prerequisite-install command (run in the guest; uses sudo when user != root).
 #
 # The prerequisite command installs python3 + git + pipx — install.sh installs
 # none of them — so `test-ready` (and the `prereqs` sub-command) drive
 # install.sh down its recommended pipx path. Bump the URLs here when a distro
 # publishes a newer build.
 declare -A IMAGE_URL SUM_URL SSH_USER PREREQ
 IMAGE_URL[ubuntu]="https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"
 SUM_URL[ubuntu]="https://cloud-images.ubuntu.com/noble/current/SHA256SUMS"
 SSH_USER[ubuntu]="ubuntu"
 PREREQ[ubuntu]="sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y python3 git pipx"
 IMAGE_URL[fedora]="https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-44-1.7.x86_64.qcow2"
 SUM_URL[fedora]="https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/x86_64/images/Fedora-Cloud-44-1.7-x86_64-CHECKSUM"
 SSH_USER[fedora]="fedora"
 PREREQ[fedora]="sudo dnf install -y python3 git pipx"
 IMAGE_URL[arch]="https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-cloudimg.qcow2"
 SUM_URL[arch]="https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-cloudimg.qcow2.SHA256"
 SSH_USER[arch]="arch"
 PREREQ[arch]="sudo pacman -Sy --noconfirm python git python-pipx"
 # Use the *nocloud_* Alpine variant, not generic_: the generic image probes
 # network datasources and ignores the local NoCloud seed, so cloud-init never
 # runs and the SSH key is never injected. (Only published at .0 patch levels.)
 IMAGE_URL[alpine]="https://dl-cdn.alpinelinux.org/alpine/v3.21/releases/cloud/nocloud_alpine-3.21.0-x86_64-bios-cloudinit-r0.qcow2"
 SUM_URL[alpine]=""   # Alpine cloud images ship .sha512 only; this verifier is sha256.
 SSH_USER[alpine]="alpine"
 PREREQ[alpine]="sudo apk add --no-cache python3 git pipx"
 # NixOS publishes no downloadable cloud qcow2 (its cloud images are Hydra-built
 # AMIs), so the harness BUILDS one with nixos-generators — see build_nixos_image
 # and linux-install-test-nixos.nix. It's externally-managed in its own way (no
 # FHS ~/.local on PATH by default); `nix profile install` provisions the
 # prerequisites into root's profile (the image enables flakes for this).
 IMAGE_URL[nixos]="nix:build"   # sentinel: ensure_base_image builds instead of downloading
 SUM_URL[nixos]=""
 SSH_USER[nixos]="root"
 # Pin to a stable release: the guest's default `nixpkgs` registry is unstable,
 # where pipx isn't in the binary cache and builds from source (its test suite
 # currently fails to build). nixos-24.11 has these cached as substitutes.
 PREREQ[nixos]="nix profile install nixpkgs/nixos-24.11#python3 nixpkgs/nixos-24.11#git nixpkgs/nixos-24.11#pipx"
 ALL_DISTROS=(ubuntu fedora arch alpine nixos)
 # --- guards ----------------------------------------------------------
 require_linux() {
    [ "$(uname -s)" = "Linux" ] \
        || { echo "error: this harness is Linux-only (uname is $(uname -s))" >&2; exit 1; }
 }
 require_kvm() {
    [ -e /dev/kvm ] && [ -r /dev/kvm ] && [ -w /dev/kvm ] \
        || { echo "error: /dev/kvm is missing or not accessible (add yourself to the 'kvm' group)" >&2; exit 1; }
 }
 require_tools() {
    local missing=()
    command -v qemu-system-x86_64 >/dev/null 2>&1 || missing+=(qemu-system-x86_64)
    command -v qemu-img          >/dev/null 2>&1 || missing+=(qemu-img)
    command -v cloud-localds     >/dev/null 2>&1 || missing+=(cloud-localds)
    command -v ssh               >/dev/null 2>&1 || missing+=(ssh)
    command -v curl              >/dev/null 2>&1 || missing+=(curl)
    if [ "${#missing[@]}" -ne 0 ]; then
        echo "error: missing required tools: ${missing[*]}" >&2
        echo "  on NixOS:  nix shell nixpkgs#qemu nixpkgs#cloud-utils nixpkgs#openssh nixpkgs#curl" >&2
        exit 1
    fi
 }
 known_distro() {
    [ -n "${IMAGE_URL[$DISTRO]:-}" ] \
        || { echo "error: unknown distro '$DISTRO' (known: ${ALL_DISTROS[*]})" >&2; exit 1; }
 }
 # --- run-dir bookkeeping ---------------------------------------------
 # The marker lets prereqs/run/status/down in separate invocations find the VM
 # that `up` started. The test cycles set RUN_DIR themselves and never write it.
 _marker() { echo "${TMPDIR:-/tmp}/bot-bottle-install-test.$DISTRO.run"; }
 _ensure_run_dir() {
    if [ -z "$RUN_DIR" ]; then
        RUN_DIR="$(mktemp -d "${TMPDIR:-/tmp}/bb-install-test.$DISTRO.XXXXXX")"
    fi
    mkdir -p "$RUN_DIR"
 }
 _load_run_dir() {
    if [ -z "$RUN_DIR" ] && [ -f "$(_marker)" ]; then
        RUN_DIR="$(cat "$(_marker)")"
    fi
    [ -n "$RUN_DIR" ] && [ -d "$RUN_DIR" ]
 }
 _ssh_key()  { echo "$RUN_DIR/id_ed25519"; }
 _overlay()  { echo "$RUN_DIR/overlay.qcow2"; }
 _seed()     { echo "$RUN_DIR/seed.iso"; }
 _pidfile()  { echo "$RUN_DIR/qemu.pid"; }
 _serial()   { echo "$RUN_DIR/serial.log"; }
 # --- ssh helpers -----------------------------------------------------
 _ssh_opts() {
    # No host-key pinning: the guest is thrown away every run.
    printf '%s\0' \
        -i "$(_ssh_key)" \
        -p "$SSH_PORT" \
        -o StrictHostKeyChecking=no \
        -o UserKnownHostsFile=/dev/null \
        -o LogLevel=ERROR \
        -o ConnectTimeout=8 \
        -o BatchMode=yes
 }
 guest() {
    local -a opts
    mapfile -d '' -t opts < <(_ssh_opts)
    ssh "${opts[@]}" "${SSH_USER[$DISTRO]}@127.0.0.1" "$@"
 }
 wait_for_ssh() {
    local deadline=$(( SECONDS + BOOT_TIMEOUT ))
    echo "== waiting for SSH on 127.0.0.1:$SSH_PORT (up to ${BOOT_TIMEOUT}s) =="
    while [ "$SECONDS" -lt "$deadline" ]; do
        if guest true 2>/dev/null; then
            echo "   guest is up"
            return 0
        fi
        # Bail early if QEMU has died — no point waiting out the timeout.
        if [ -f "$(_pidfile)" ] && ! kill -0 "$(cat "$(_pidfile)")" 2>/dev/null; then
            echo "error: QEMU exited before SSH came up; see $(_serial)" >&2
            return 1
        fi
        sleep 3
    done
    echo "error: timed out waiting for SSH; see $(_serial)" >&2
    return 1
 }
 # --- image cache -----------------------------------------------------
 _base_image() {
    # One cached file per distro. NixOS is built (not downloaded), so it has a
    # fixed cache name; the rest are keyed by the image's basename so a URL bump
    # lands as a new cache entry rather than a stale hit.
    if [ "$DISTRO" = nixos ]; then
        echo "$CACHE_DIR/nixos-built.qcow2"
        return
    fi
    local url="${IMAGE_URL[$DISTRO]}"
    echo "$CACHE_DIR/$DISTRO-$(basename "$url")"
 }
 # NixOS has no upstream cloud qcow2; build one with nixos-generators and copy it
 # out of the (immutable, GC-able) store into the cache.
 build_nixos_image() {
    local out; out="$(_base_image)"
    [ -f "$out" ] && { echo "== nixos base image cached: $out =="; return 0; }
    command -v nixos-generate >/dev/null 2>&1 || {
        echo "error: 'nixos-generate' is required to build the NixOS image" >&2
        echo "  run inside:  nix shell nixpkgs#nixos-generators nixpkgs#qemu nixpkgs#cloud-utils" >&2
        return 1
    }
    echo "== building NixOS cloud image with nixos-generators (first run is slow) =="
    local link="$CACHE_DIR/nixos-result"
    nixos-generate -f qcow --system x86_64-linux \
        -c "$_SCRIPT_DIR/linux-install-test-nixos.nix" -o "$link"
    cp -L "$link"/*.qcow2 "$out"
    rm -f "$link"
    echo "   built + cached: $out"
 }
 verify_checksum() {
    local file="$1" sums_url="${SUM_URL[$DISTRO]}" base
    base="$(basename "${IMAGE_URL[$DISTRO]}")"
    if [ "${BB_TEST_SKIP_VERIFY:-0}" = "1" ] || [ -z "$sums_url" ]; then
        echo "   checksum: SKIPPED (${sums_url:+set BB_TEST_SKIP_VERIFY=0 to enable}${sums_url:-no sums URL for $DISTRO})" >&2
        return 0
    fi
    local want
    # Vendors publish either "HASH  filename" tables or a bare "HASH" (or a
    # "SHA256 (file) = HASH" BSD line, e.g. Fedora). Cover all three.
    local sums; sums="$(curl -fsSL "$sums_url")"
    want="$(printf '%s\n' "$sums" | awk -v f="$base" '
        $0 ~ f && $1 ~ /^[0-9a-fA-F]{64}$/ { print $1; exit }        # GNU "hash  file"
        $1=="SHA256" && $0 ~ f { gsub(/[()]/,""); print $NF; exit }  # BSD "SHA256 (file) = hash"
    ')"
    [ -z "$want" ] && want="$(printf '%s\n' "$sums" | awk '/^[0-9a-fA-F]{64}$/ {print $1; exit}')"
    [ -n "$want" ] || { echo "error: could not find a sha256 for $base in $sums_url" >&2; return 1; }
    local got; got="$(sha256sum "$file" | awk '{print $1}')"
    if [ "$want" != "$got" ]; then
        echo "error: checksum mismatch for $base" >&2
        echo "  want $want" >&2
        echo "  got  $got" >&2
        return 1
    fi
    echo "   checksum: OK"
 }
 ensure_base_image() {
    mkdir -p "$CACHE_DIR"
    if [ "$DISTRO" = nixos ]; then
        build_nixos_image
        return
    fi
    local base; base="$(_base_image)"
    if [ -f "$base" ]; then
        echo "== base image cached: $base =="
        return 0
    fi
    echo "== downloading $DISTRO cloud image =="
    echo "   ${IMAGE_URL[$DISTRO]}"
    # Download to a temp name and rename on success so an interrupted pull
    # never poisons the cache with a truncated image.
    local tmp="$base.partial"
    curl -fSL --retry 3 -o "$tmp" "${IMAGE_URL[$DISTRO]}"
    verify_checksum "$tmp"
    mv "$tmp" "$base"
    echo "   cached: $base"
 }
 # --- cloud-init seed -------------------------------------------------
 make_seed() {
    ssh-keygen -t ed25519 -N '' -f "$(_ssh_key)" -q
    local pub; pub="$(cat "$(_ssh_key).pub")"
    local user="${SSH_USER[$DISTRO]}"
    local user_data="$RUN_DIR/user-data"
    if [ "$user" = "root" ]; then
        # NixOS' cloud-init lands the key straight on root; no sudo needed.
        cat > "$user_data" <<EOF
 #cloud-config
 ssh_authorized_keys:
  - $pub
 EOF
    elif [ "$DISTRO" = alpine ]; then
        # Alpine needs three things the systemd distros don't: cloud-init locks
        # the account (lock_passwd), but Alpine's non-PAM sshd then refuses
        # pubkey auth for a locked account — so give it a throwaway password;
        # and OpenRC does not auto-start sshd after the key is injected, so
        # start it via runcmd.
        cat > "$user_data" <<EOF
 #cloud-config
 users:
  - name: $user
    sudo: ALL=(ALL) NOPASSWD:ALL
    shell: /bin/sh
    lock_passwd: false
    ssh_authorized_keys:
      - $pub
 chpasswd:
  expire: false
  list: |
    $user:bbtest
 packages:
  - sudo
 runcmd:
  - [ sh, -c, "rc-service sshd start 2>/dev/null || true" ]
 EOF
    else
        cat > "$user_data" <<EOF
 #cloud-config
 users:
  - name: $user
    sudo: ALL=(ALL) NOPASSWD:ALL
    shell: /bin/sh
    lock_passwd: true
    ssh_authorized_keys:
      - $pub
 EOF
    fi
    # NoCloud wants a meta-data with an instance-id, or cloud-init may not treat
    # the seed as a new instance (the Alpine nocloud image is strict about this).
    printf 'instance-id: bbtest-%s\nlocal-hostname: bbtest-%s\n' "$DISTRO" "$DISTRO" \
        > "$RUN_DIR/meta-data"
    cloud-localds "$(_seed)" "$user_data" "$RUN_DIR/meta-data"
 }
 # --- commands --------------------------------------------------------
 cmd_up() {
    require_linux; require_kvm; require_tools; known_distro
    _ensure_run_dir
    ensure_base_image
    # Throwaway copy-on-write overlay: the cached base is read-only backing,
    # all guest writes land in the overlay, and `down` deletes it. Resize so
    # pipx + a git build have headroom (cloud-init grows the rootfs to fit).
    qemu-img create -q -f qcow2 -F qcow2 -b "$(_base_image)" "$(_overlay)" "$DISK"
    make_seed
    echo "== booting $DISTRO VM (mem=${MEM_MB}M cpus=$CPUS, ssh -> :$SSH_PORT) =="
    qemu-system-x86_64 \
        -machine accel=kvm -cpu host -smp "$CPUS" -m "$MEM_MB" \
        -display none -daemonize \
        -pidfile "$(_pidfile)" \
        -serial "file:$(_serial)" \
        -drive "file=$(_overlay),if=virtio,format=qcow2" \
        -drive "file=$(_seed),if=virtio,format=raw" \
        -netdev "user,id=n0,hostfwd=tcp:127.0.0.1:$SSH_PORT-:22" \
        -device virtio-net-pci,netdev=n0
    # Only publish the marker (so a later prereqs/run/down finds this VM) when
    # we aren't inside a test cycle, which manages its own RUN_DIR + teardown.
    [ "$IN_TEST" = 1 ] || echo "$RUN_DIR" > "$(_marker)"
    wait_for_ssh
    if [ "$IN_TEST" != 1 ]; then
        echo "== VM is up. Prepare it with: BB_TEST_DISTRO=$DISTRO $0 prereqs (or go straight to 'run') =="
    fi
 }
 # Install install.sh's toolchain prerequisites (python3 + git + pipx) into the
 # running VM. This is what separates `test-ready` from `test`, and it is a
 # distinct sub-command so a manual up/prereqs/run/down cycle is possible.
 cmd_prereqs() {
    require_linux
    _load_run_dir || { echo "error: no running VM for $DISTRO; run '$0 up' first" >&2; return 1; }
    echo "== installing prerequisites (python3 + git + pipx) on $DISTRO =="
    # Runs via the guest login shell; PREREQ is a client-side table value.
    guest "${PREREQ[$DISTRO]}"
 }
 cmd_run() {
    require_linux
    _load_run_dir || { echo "error: no running VM for $DISTRO; run '$0 up' first" >&2; return 1; }
    local spec_env=""
    [ -n "${BOT_BOTTLE_INSTALL_SPEC:-}" ] \
        && spec_env="BOT_BOTTLE_INSTALL_SPEC='$BOT_BOTTLE_INSTALL_SPEC' "
    echo "== installing bot-bottle as ${SSH_USER[$DISTRO]} =="
    # Capture install.sh's exit code and full output rather than aborting on
    # non-zero: on a bare host a clean prerequisite *decline* is sound, so the
    # verdict step — not set -e — decides the outcome.
    local rc
    set +e
    if [ -n "${BB_TEST_INSTALL_URL:-}" ]; then
        guest "curl -fsSL '$BB_TEST_INSTALL_URL' | ${spec_env}sh" 2>&1 | tee "$RUN_DIR/install.log"
    else
        # Feed THIS checkout's install.sh in over stdin — the same `curl … | sh`
        # shape a real user runs, and nothing is staged in the guest to leak.
        guest "${spec_env}sh -s" < "$_REPO_ROOT/install.sh" 2>&1 | tee "$RUN_DIR/install.log"
    fi
    rc="${PIPESTATUS[0]}"
    set -e
    printf '%s\n' "$rc" > "$RUN_DIR/install.rc"
    echo "== install.sh exited $rc =="
    [ "$IN_TEST" = 1 ] \
        || echo "== verdict anytime with: BB_TEST_DISTRO=$DISTRO $0 status =="
 }
 # Quietly report whether a runnable bot-bottle entry point exists for the
 # guest user, checking the pipx/pip locations install.sh may leave off PATH.
 entry_point_runnable() {
    # shellcheck disable=SC2016  # expand in the GUEST shell.
    guest '
        for bb in "$HOME/.local/bin/bot-bottle" "$HOME/.bot-bottle/venv/bin/bot-bottle" "$(command -v bot-bottle 2>/dev/null)"; do
            [ -n "$bb" ] && [ -x "$bb" ] || continue
            # --help exits 0 before any DB/migration/network work; it is the
            # cheapest proof the package imports and the shim runs. (bot-bottle
            # has no --version: an unknown arg would die non-zero.)
            "$bb" --help >/dev/null 2>&1 && exit 0
        done
        exit 1
    ' >/dev/null 2>&1
 }
 # `bot-bottle doctor` in the guest, classified. doctor's own exit code
 # conflates "is the install sound" with "is a backend ready to run a bottle" —
 # and inside a plain VM no backend can be ready (no nested KVM/Docker), so the
 # raw exit code is non-zero by design. This separates the two: an unhandled
 # traceback, or a missing python/config line, is an install defect and fails;
 # a not-ready backend is reported, not fatal (unless BB_TEST_REQUIRE_BACKEND=1,
 # for a nested-virt host that can actually satisfy it).
 doctor_in_guest() {
    local out rc=0 bad=0
    out="$(mktemp "${TMPDIR:-/tmp}/bb-doctor.XXXXXX")"
    # shellcheck disable=SC2016  # $HOME/$bb must expand in the GUEST shell.
    guest '
        for bb in bot-bottle "$HOME/.local/bin/bot-bottle" "$HOME/.bot-bottle/venv/bin/bot-bottle"; do
            if command -v "$bb" >/dev/null 2>&1; then
                case "$bb" in
                    bot-bottle) : ;;
                    *) echo "  (not on PATH — running $bb directly, as install.sh advises)" ;;
                esac
                exec "$bb" doctor
            fi
        done
        echo "  no bot-bottle entry point found for this user" >&2
        exit 1
    ' >"$out" 2>&1 || rc=$?
    cat "$out"
    # An unhandled exception is always an install/product defect, never an
    # environment fact — doctor's non-zero exit alone would not distinguish it.
    if grep -q 'Traceback (most recent call last)' "$out"; then
        echo "  doctor crashed (traceback above) — a defect, not a missing prerequisite" >&2
        bad=1
    fi
    grep -qE '^ok: +python:' "$out" \
        || { echo "  doctor never reported a usable python" >&2; bad=1; }
    grep -qE '^ok: +config:' "$out" \
        || { echo "  doctor never reported a usable config dir" >&2; bad=1; }
    if [ "$rc" -ne 0 ] && ! grep -qE '^(fail|warn): +backend' "$out"; then
        echo "  doctor failed for something other than backend readiness" >&2
        bad=1
    fi
    local backend_ready=1
    grep -qE '^fail: +backend' "$out" && backend_ready=0
    rm -f "$out"
    [ "$bad" -eq 0 ] || return 1
    if [ "${BB_TEST_REQUIRE_BACKEND:-0}" = "1" ] && [ "$backend_ready" -eq 0 ]; then
        echo "  backend is not ready and BB_TEST_REQUIRE_BACKEND=1 — failing" >&2
        return 1
    fi
    if [ "$backend_ready" -eq 1 ]; then
        echo "  doctor: install sound; a backend is ready"
    else
        echo "  doctor: install sound; no backend ready (expected in a plain VM — install gate only)"
    fi
    return 0
 }
 # The prerequisite-decline messages install.sh prints via die(). On a bare host
 # ANY of these means install.sh correctly refused rather than half-installing —
 # a sound outcome for `test`.
 PREREQ_ERR_RE='is required but was not found|or newer is required|git is required to install from|neither pipx nor a usable|externally managed \(PEP 668\)|is not on PATH'
 # The verdict: is the install sound? Reads install.sh's captured exit code and
 # output (from cmd_run) plus the guest's resulting state.
 #   - installed & runnable  -> the verdict is doctor's soundness classification.
 #   - no entry point, but a recognized prerequisite decline, and declines are
 #     allowed (bare `test`, _REQUIRE_INSTALL=0) -> sound.
 #   - anything else -> not sound.
 install_verdict() {
    local rc=""
    [ -f "$RUN_DIR/install.rc" ] && rc="$(cat "$RUN_DIR/install.rc")"
    if [ "${rc:-1}" = 0 ] && entry_point_runnable; then
        echo "doctor (in guest):"
        doctor_in_guest
        return $?
    fi
    if [ "${_REQUIRE_INSTALL:-0}" != "1" ] \
            && [ -n "$rc" ] && [ "$rc" != 0 ] \
            && [ -f "$RUN_DIR/install.log" ] \
            && grep -Eiq "$PREREQ_ERR_RE" "$RUN_DIR/install.log"; then
        echo "  install.sh declined with an actionable prerequisite error (rc=$rc)"
        echo "  — sound on a bare host; run 'test-ready' (or 'prereqs') to install."
        return 0
    fi
    if [ "${_REQUIRE_INSTALL:-0}" = "1" ]; then
        echo "  prerequisites were provisioned, but install.sh left no runnable entry point (rc=${rc:-?})" >&2
    else
        echo "  install.sh neither installed nor gave a recognized prerequisite error (rc=${rc:-?})" >&2
    fi
    return 1
 }
 cmd_status() {
    require_linux
    if ! _load_run_dir; then
        echo "vm:      no running VM for $DISTRO"
        return 0
    fi
    if [ -f "$(_pidfile)" ] && kill -0 "$(cat "$(_pidfile)")" 2>/dev/null; then
        echo "vm:      $DISTRO running (pid $(cat "$(_pidfile)"), ssh :$SSH_PORT)"
    else
        echo "vm:      $DISTRO run-dir present but QEMU not alive"
        return 1
    fi
    if install_verdict; then
        echo "OK[$DISTRO]: the install is sound"
        return 0
    fi
    echo "FAIL[$DISTRO]: the install is not sound (see above)" >&2
    return 1
 }
 cmd_down() {
    require_linux
    if ! _load_run_dir; then
        echo "$DISTRO: nothing running"
        return 0
    fi
    if [ -f "$(_pidfile)" ]; then
        local pid; pid="$(cat "$(_pidfile)")"
        if kill -0 "$pid" 2>/dev/null; then
            kill "$pid" 2>/dev/null || true
            for _ in 1 2 3 4 5; do kill -0 "$pid" 2>/dev/null || break; sleep 1; done
            kill -9 "$pid" 2>/dev/null || true
        fi
    fi
    # Deleting the overlay + seed is the reset; the read-only base stays cached.
    rm -f "$(_overlay)" "$(_seed)" "$(_ssh_key)" "$(_ssh_key).pub" \
          "$RUN_DIR/user-data" "$RUN_DIR/meta-data" \
          "$RUN_DIR/install.rc" "$RUN_DIR/install.log" "$(_serial)"
    # Only remove a scratch dir we created (leave a user-provided one alone).
    [ -n "${BB_TEST_RUN_DIR:-}" ] || rmdir "$RUN_DIR" 2>/dev/null || true
    rm -f "$(_marker)"
    echo "removed $DISTRO VM and its overlay — install surface is clean."
 }
 cmd_ssh() {
    require_linux
    _load_run_dir || { echo "error: no running VM for $DISTRO" >&2; return 1; }
    local -a opts
    mapfile -d '' -t opts < <(_ssh_opts)
    exec ssh -t "${opts[@]}" "${SSH_USER[$DISTRO]}@127.0.0.1"
 }
 # Teardown half of the test cycles, armed the moment the VM exists so a failure
 # or a Ctrl-C still leaves nothing running.
 _test_teardown() {
    local rc=$?
    trap - EXIT INT TERM
    if [ "${BB_TEST_KEEP:-0}" = "1" ]; then
        echo
        echo "== [$_STEPS/$_STEPS] down: SKIPPED (BB_TEST_KEEP=1) =="
        echo "   VM still up; remove with: BB_TEST_DISTRO=$DISTRO $0 down"
        echo "   ssh in with: BB_TEST_RUN_DIR=$RUN_DIR BB_TEST_DISTRO=$DISTRO $0 ssh"
        exit "$rc"
    fi
    echo
    echo "== [$_STEPS/$_STEPS] down =="
    cmd_down || rc=1
    if [ "$rc" -eq 0 ]; then
        echo
        echo "PASS[$DISTRO]: $_PASS_CLAIM"
    else
        echo
        echo "FAIL[$DISTRO]: see above (the VM was torn down regardless)." >&2
    fi
    exit "$rc"
 }
 # The two variants differ only in whether the prerequisites get installed
 # before install.sh runs, which is exactly the question each one asks:
 #
 #   test        a bare host — assert install.sh is SOUND (installs cleanly, or
 #               declines with an actionable prerequisite error).
 #   test-ready  prerequisites satisfied — assert install.sh actually LANDS.
 _test_cycle() {
    local with_prereqs="$1"
    require_linux; require_kvm; require_tools; known_distro
    IN_TEST=1
    RUN_DIR="$(mktemp -d "${TMPDIR:-/tmp}/bb-install-test.$DISTRO.XXXXXX")"
    # Arm teardown BEFORE cmd_up: its wait_for_ssh can fail after QEMU is
    # already running (e.g. a guest that never opens SSH), and without the trap
    # in place that would leak the VM.
    trap _test_teardown EXIT INT TERM
    echo "== [1/$_STEPS] up ($DISTRO) =="
    cmd_up
    local step=2
    if [ "$with_prereqs" = 1 ]; then
        echo
        echo "== [$step/$_STEPS] prereqs =="
        cmd_prereqs || { echo "error: could not install prerequisites (see above)." >&2; return 1; }
        step=$(( step + 1 ))
    fi
    echo
    echo "== [$step/$_STEPS] run =="
    cmd_run
    step=$(( step + 1 ))
    echo
    echo "== [$step/$_STEPS] verdict =="
    # install.sh exits 0 even when doctor reports unmet prerequisites, so the
    # install succeeding is not the verdict — this is.
    cmd_status || {
        echo "error: the install is not sound for $DISTRO (see above)." >&2
        echo "  re-run with BB_TEST_KEEP=1 to keep the VM and dig in." >&2
        return 1
    }
 }
 cmd_test() {
    _STEPS=4
    _PASS_CLAIM="on a bare $DISTRO host, install.sh behaves soundly."
    _test_cycle 0
 }
 cmd_test_ready() {
    _STEPS=5
    _PASS_CLAIM="a $DISTRO host with prerequisites satisfied installs bot-bottle cleanly."
    # Prerequisites are provisioned, so a graceful decline is no longer an
    # acceptable outcome — the install must actually land.
    _REQUIRE_INSTALL=1
    _test_cycle 1
 }
 cmd_test_all() {
    require_linux; require_kvm; require_tools
    local -a passed=() failed=()
    local port="$SSH_PORT"
    local script
    script="$_SCRIPT_DIR/$(basename "${BASH_SOURCE[0]}")"
    # Every distro × both variants. Each cell gets its own forwarded port so a
    # leftover from a prior cell can't collide, and its own subshell so one
    # cell's failure (or teardown trap) doesn't abort the matrix.
    for sub in test test-ready; do
        for d in "${ALL_DISTROS[@]}"; do
            echo
            echo "########################################################"
            echo "# $d  ($sub)"
            echo "########################################################"
            if ( DISTRO="$d" SSH_PORT="$port" \
                    BB_TEST_DISTRO="$d" BB_TEST_SSH_PORT="$port" \
                    bash "$script" "$sub" ); then
                passed+=("$d/$sub")
            else
                failed+=("$d/$sub")
            fi
            port=$(( port + 1 ))
        done
    done
    echo
    echo "== matrix summary =="
    echo "  PASS: ${passed[*]:-(none)}"
    echo "  FAIL: ${failed[*]:-(none)}"
    [ "${#failed[@]}" -eq 0 ]
 }
 case "${1:-}" in
    test)       cmd_test ;;
    test-ready) cmd_test_ready ;;
    test-all)   cmd_test_all ;;
    up)         cmd_up ;;
    prereqs)    cmd_prereqs ;;
    run)        cmd_run ;;
    status)     cmd_status ;;
    down)       cmd_down ;;
    ssh)        cmd_ssh ;;
    *) echo "usage: $0 {test|test-ready|test-all|up|prereqs|run|status|down|ssh}  (distro via BB_TEST_DISTRO)" >&2; exit 2 ;;
 esac
@@ -2,9 +2,7 @@
 from __future__ import annotations
 import json
 import subprocess
 import tempfile
 import unittest
 from pathlib import Path
 from types import SimpleNamespace
@@ -107,21 +105,6 @@ class TestDockerReprovision(unittest.TestCase):
 class TestFirecrackerReprovision(unittest.TestCase):
    def _run_dir(self, root: Path, ip: str = "10.243.0.3") -> Path:
        run_dir = root / "demo"
        run_dir.mkdir()
        (run_dir / "bottle_id_ed25519").write_text("key")
        (run_dir / "config.json").write_text(json.dumps({
            "boot-source": {"boot_args": f"root=/dev/vda ip={ip}::gw:mask::eth0:off"}
        }))
        return run_dir
    def test_extracts_guest_ip_from_config(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            run_dir = self._run_dir(Path(tmp))
            self.assertEqual("10.243.0.3", fc._guest_ip_from_config(run_dir / "config.json"))
            self.assertEqual("", fc._guest_ip_from_config(run_dir / "missing.json"))
    def test_persists_key_over_stdin_not_argv(self) -> None:
        with patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
             patch.object(fc.subprocess, "run", return_value=_proc()) as run:
@@ -135,29 +118,6 @@ class TestFirecrackerReprovision(unittest.TestCase):
            with self.assertRaisesRegex(fc.ConsolidatedLaunchError, "denied"):
                fc.persist_env_var_secret(Path("/key"), "10.0.0.1", "secret")
    def test_reads_live_vm_key_and_reprovisions(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            run_dir = self._run_dir(Path(tmp))
            client = Mock()
            with patch.object(fc.cleanup, "live_run_dirs", return_value=(run_dir,)), \
                 patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
                 patch.object(fc.subprocess, "run", return_value=_proc(stdout="secret\n")), \
                 patch.object(fc, "reprovision_bottles", return_value=1) as restore, \
                 patch.object(fc, "info"):
                fc._reprovision_running_bottles(client)
        restore.assert_called_once_with(client, {"10.243.0.3": "secret"})
    def test_unreadable_vm_is_skipped(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            run_dir = self._run_dir(Path(tmp))
            client = Mock()
            with patch.object(fc.cleanup, "live_run_dirs", return_value=(run_dir,)), \
                 patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
                 patch.object(fc.subprocess, "run", return_value=_proc(1)), \
                 patch.object(fc, "reprovision_bottles", return_value=0) as restore:
                fc._reprovision_running_bottles(client)
        restore.assert_called_once_with(client, {})
 if __name__ == "__main__":
    unittest.main()
@@ -21,6 +21,7 @@ from bot_bottle.backend.firecracker.orchestrator import FirecrackerOrchestrator
 # there (not at their home modules).
 _ORCH_CLS = "bot_bottle.backend.firecracker.infra.FirecrackerOrchestrator"
 _GW_CLS = "bot_bottle.backend.firecracker.infra.FirecrackerGateway"
 _RECONCILE = "bot_bottle.backend.firecracker.infra.attach_bottled_agents_to_gateway"
 class TestAccessors(unittest.TestCase):
@@ -53,10 +54,13 @@ class TestEnsureRunning(unittest.TestCase):
                    patch.object(infra_vm, "_health_ok", return_value=True), \
                    patch.object(infra_vm, "_pidfile_alive", return_value=True), \
                    patch.object(infra_vm, "stop") as stop, \
-                    patch.object(infra_vm, "ensure_built") as built:
+                    patch.object(infra_vm, "ensure_built") as built, \
                    patch(_RECONCILE) as reconcile:
                url = FirecrackerInfraService().ensure_running()
            stop.assert_not_called()
            built.assert_not_called()
            # Adopt path: no reconcile — gateway state is intact.
            reconcile.assert_not_called()
            ip = infra_vm.netpool.orch_slot().guest_ip
            self.assertEqual(f"http://{ip}:{infra_vm.ORCHESTRATOR_PORT}", url)
@@ -73,7 +77,8 @@ class TestEnsureRunning(unittest.TestCase):
                    patch.object(infra_vm, "_pidfile_alive", return_value=True), \
                    patch.object(infra_vm, "stop") as stop, \
                    patch.object(infra_vm, "ensure_built"), \
-                    patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls:
+                    patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls, \
                    patch(_RECONCILE) as reconcile:
                orch_cls.return_value.gateway_url.return_value = "http://10.243.255.1:8099"
                orch_cls.return_value.mint_gateway_token.return_value = "gw.jwt"
                FirecrackerInfraService().ensure_running()
@@ -85,6 +90,8 @@ class TestEnsureRunning(unittest.TestCase):
                "http://10.243.255.1:8099", "gw.jwt")
            # The fresh boot records the current version for the next launcher.
            self.assertEqual("v-current\n", (d / "booted-version").read_text())
            # Cold boot: reconcile fires after gateway is up.
            reconcile.assert_called_once()
    def test_reboots_when_gateway_dead(self):
        # Orchestrator healthy + marker current, but the gateway VM is gone ->
@@ -99,11 +106,13 @@ class TestEnsureRunning(unittest.TestCase):
                    patch.object(infra_vm, "_pidfile_alive", return_value=False), \
                    patch.object(infra_vm, "stop") as stop, \
                    patch.object(infra_vm, "ensure_built"), \
-                    patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls:
+                    patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls, \
                    patch(_RECONCILE) as reconcile:
                FirecrackerInfraService().ensure_running()
            stop.assert_called_once()
            orch_cls.return_value.ensure_running.assert_called_once()
            gw_cls.return_value.connect_to_orchestrator.assert_called_once()
            reconcile.assert_called_once()
    def test_boots_both_when_no_running_pair(self):
        with tempfile.TemporaryDirectory() as td:
@@ -112,7 +121,8 @@ class TestEnsureRunning(unittest.TestCase):
                    patch.object(infra_vm, "_health_ok", return_value=False), \
                    patch.object(infra_vm, "stop") as stop, \
                    patch.object(infra_vm, "ensure_built") as built, \
-                    patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls:
+                    patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls, \
                    patch(_RECONCILE) as reconcile:
                FirecrackerInfraService().ensure_running()
        stop.assert_called_once()      # clear stale VMs first
        built.assert_called_once()
@@ -120,6 +130,27 @@ class TestEnsureRunning(unittest.TestCase):
        # reach the control plane at startup).
        orch_cls.return_value.ensure_running.assert_called_once()
        gw_cls.return_value.connect_to_orchestrator.assert_called_once()
        # Cold boot: reconcile fires to restore CA, git-gate, and egress tokens.
        reconcile.assert_called_once()
    def test_reconcile_receives_url_and_fresh_gateway(self):
        # reconcile gets the orchestrator URL and the gateway service (not the
        # class). Verify the args to ensure it can reach the right endpoints.
        with tempfile.TemporaryDirectory() as td:
            with patch.object(infra_vm, "_infra_dir", return_value=Path(td)), \
                    patch.object(infra_vm, "expected_version", return_value="v-current"), \
                    patch.object(infra_vm, "_health_ok", return_value=False), \
                    patch.object(infra_vm, "stop"), \
                    patch.object(infra_vm, "ensure_built"), \
                    patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls, \
                    patch(_RECONCILE) as reconcile:
                orch_cls.return_value.url.return_value = "http://10.243.255.1:8099"
                FirecrackerInfraService().ensure_running()
        call_args = reconcile.call_args
        # First arg: the orchestrator's host URL.
        self.assertEqual("http://10.243.255.1:8099", call_args.args[0])
        # Second arg: the gateway service instance (not the class).
        self.assertIs(gw_cls.return_value, call_args.args[1])
 if __name__ == "__main__":
@@ -0,0 +1,280 @@
 """Unit: bring-up reconcile — CA push, git-gate reprovision, egress tokens.
 Covers attach_bottled_agents_to_gateway (reconcile.py): each per-bottle step
 fires with correct args, failures on one bottle don't block others, and the
 egress-token restore path works end-to-end. Does NOT spin up VMs or real SSH.
 """
 from __future__ import annotations
 import json
 import tempfile
 import unittest
 from pathlib import Path
 from subprocess import CalledProcessError, CompletedProcess
 from unittest.mock import MagicMock, patch
 _RECONCILE = "bot_bottle.backend.firecracker.reconcile"
 class TestGuestIpFromConfig(unittest.TestCase):
    def test_reads_ip_from_boot_args(self) -> None:
        from bot_bottle.backend.firecracker.reconcile import _guest_ip_from_config
        with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
            json.dump({
                "boot-source": {"boot_args": "console=ttyS0 ip=10.243.0.5:10.243.0.6:255.255.255.254"},
            }, f)
            name = f.name
        self.assertEqual("10.243.0.5", _guest_ip_from_config(Path(name)))
    def test_returns_empty_on_missing_file(self) -> None:
        from bot_bottle.backend.firecracker.reconcile import _guest_ip_from_config
        self.assertEqual("", _guest_ip_from_config(Path("/nonexistent/config.json")))
    def test_returns_empty_on_bad_json(self) -> None:
        from bot_bottle.backend.firecracker.reconcile import _guest_ip_from_config
        with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
            f.write("not-json")
            name = f.name
        self.assertEqual("", _guest_ip_from_config(Path(name)))
 class TestPushCa(unittest.TestCase):
    def test_runs_three_ssh_commands_in_order(self) -> None:
        from bot_bottle.backend.firecracker.reconcile import _push_ca
        with tempfile.TemporaryDirectory() as td:
            key = Path(td) / "key"
            key.write_text("k")
            with patch(f"{_RECONCILE}.subprocess.run") as run:
                run.return_value = CompletedProcess([], 0)
                _push_ca(key, "10.0.0.1", "-----BEGIN CERTIFICATE-----\n")
        calls = run.call_args_list
        self.assertEqual(3, len(calls))
        # Each call's argv is a list; the remote command is the last element.
        self.assertIn("mkdir", calls[0].args[0][-1])
        self.assertIn("cat >", calls[1].args[0][-1])
        self.assertIn("update-ca-certificates", calls[2].args[0][-1])
    def test_passes_ca_pem_via_stdin(self) -> None:
        from bot_bottle.backend.firecracker.reconcile import _push_ca
        with tempfile.TemporaryDirectory() as td:
            key = Path(td) / "key"
            key.write_text("k")
            with patch(f"{_RECONCILE}.subprocess.run") as run:
                run.return_value = CompletedProcess([], 0)
                _push_ca(key, "10.0.0.1", "MY-CA-PEM")
        cat_call = run.call_args_list[1]
        self.assertEqual("MY-CA-PEM", cat_call.kwargs.get("input"))
    def test_raises_on_ssh_failure(self) -> None:
        from bot_bottle.backend.firecracker.reconcile import _push_ca
        with tempfile.TemporaryDirectory() as td:
            key = Path(td) / "key"
            key.write_text("k")
            with patch(f"{_RECONCILE}.subprocess.run") as run:
                run.side_effect = CalledProcessError(1, "ssh")
                with self.assertRaises(CalledProcessError):
                    _push_ca(key, "10.0.0.1", "pem")
 class TestReprovisionGitGate(unittest.TestCase):
    def _make_state_dir(self, upstreams: list[dict[str, str]]) -> Path:
        d = Path(tempfile.mkdtemp())
        (d / "upstreams.json").write_text(json.dumps(upstreams))
        (d / "git_gate_pre_receive.sh").write_text("#!/bin/sh")
        (d / "git_gate_access_hook.sh").write_text("#!/bin/sh")
        (d / "git_gate_entrypoint.sh").write_text("#!/bin/sh")
        return d
    def test_no_op_when_no_upstreams_json(self) -> None:
        from bot_bottle.backend.firecracker.reconcile import _reprovision_git_gate
        transport = MagicMock()
        with tempfile.TemporaryDirectory() as td:
            # The state dir exists but has no upstreams.json
            with patch(f"{_RECONCILE}.git_gate_state_dir", return_value=Path(td)):
                _reprovision_git_gate(transport, "bottle-123", "my-slug")
        transport.exec.assert_not_called()
    def test_calls_provision_git_gate_with_upstreams(self) -> None:
        from bot_bottle.backend.firecracker.reconcile import _reprovision_git_gate
        transport = MagicMock()
        upstreams = [{
            "name": "bot-bottle",
            "upstream_url": "ssh://git@gitea.example/org/bot-bottle.git",
            "upstream_host": "gitea.example",
            "upstream_port": "22",
            "identity_file": "/home/node/.ssh/id_ed25519",
            "known_host_key": "ssh-ed25519 AAAA...",
        }]
        state_dir = self._make_state_dir(upstreams)
        # Write the key file so identity_file falls back to state dir path.
        (state_dir / "bot-bottle-key").write_bytes(b"PRIVATE")
        try:
            with patch(f"{_RECONCILE}.git_gate_state_dir", return_value=state_dir), \
                    patch(f"{_RECONCILE}.provision_git_gate") as prov:
                _reprovision_git_gate(transport, "bottle-abc", "my-slug")
            prov.assert_called_once()
            _, bottle_id, plan = prov.call_args.args
            self.assertEqual("bottle-abc", bottle_id)
            self.assertEqual(1, len(plan.upstreams))
            self.assertEqual("bot-bottle", plan.upstreams[0].name)
            # Key in state dir takes priority over identity_file in JSON.
            self.assertEqual(str(state_dir / "bot-bottle-key"), plan.upstreams[0].identity_file)
        finally:
            import shutil; shutil.rmtree(state_dir, ignore_errors=True)
    def test_falls_back_to_manifest_identity_file_when_no_key_in_state(self) -> None:
        from bot_bottle.backend.firecracker.reconcile import _reprovision_git_gate
        transport = MagicMock()
        upstreams = [{
            "name": "repo",
            "upstream_url": "ssh://git@github.com/org/repo.git",
            "upstream_host": "github.com",
            "upstream_port": "22",
            "identity_file": "/home/node/.ssh/id_ed25519",
            "known_host_key": "",
        }]
        state_dir = self._make_state_dir(upstreams)
        # No `repo-key` in state dir — static manifest path should be used.
        try:
            with patch(f"{_RECONCILE}.git_gate_state_dir", return_value=state_dir), \
                    patch(f"{_RECONCILE}.provision_git_gate") as prov:
                _reprovision_git_gate(transport, "bottle-xyz", "my-slug")
            prov.assert_called_once()
            _, _, plan = prov.call_args.args
            self.assertEqual("/home/node/.ssh/id_ed25519", plan.upstreams[0].identity_file)
        finally:
            import shutil; shutil.rmtree(state_dir, ignore_errors=True)
 class TestAttachBottledAgentsToGateway(unittest.TestCase):
    """Integration-level: the full reconcile loop against mocked SSH + gateway."""
    def _make_run_dir(self, tmp: Path, slug: str, guest_ip: str) -> Path:
        run_dir = tmp / slug
        run_dir.mkdir()
        key = run_dir / "bottle_id_ed25519"
        key.write_text("PRIVATE")
        cfg = {
            "boot-source": {
                "boot_args": f"console=ttyS0 ip={guest_ip}:{guest_ip}:255.255.255.254",
            }
        }
        (run_dir / "config.json").write_text(json.dumps(cfg))
        return run_dir
    def _make_gateway(self, ca_pem: str = "-----BEGIN CERTIFICATE-----\n") -> MagicMock:
        gw = MagicMock()
        gw.ca_cert_pem.return_value = ca_pem
        gw.provisioning_transport.return_value = MagicMock()
        return gw
    def test_skips_all_when_ca_fetch_fails(self) -> None:
        from bot_bottle.backend.firecracker.gateway import FirecrackerGateway
        from bot_bottle.backend.firecracker.reconcile import attach_bottled_agents_to_gateway
        from bot_bottle.gateway import GatewayError
        gw = MagicMock(spec=FirecrackerGateway)
        gw.ca_cert_pem.side_effect = GatewayError("timeout")
        with patch(f"{_RECONCILE}.cleanup.live_run_dirs", return_value=()):
            attach_bottled_agents_to_gateway("http://orch:8099", gw)
        # Nothing else is called when CA fetch fails.
        gw.provisioning_transport.assert_not_called()
    def test_ca_push_called_per_live_bottle(self) -> None:
        from bot_bottle.backend.firecracker.reconcile import attach_bottled_agents_to_gateway
        gw = self._make_gateway()
        with tempfile.TemporaryDirectory() as td:
            tmp = Path(td)
            rd1 = self._make_run_dir(tmp, "agent-abc12", "10.243.0.5")
            rd2 = self._make_run_dir(tmp, "agent-xyz99", "10.243.0.7")
            with patch(f"{_RECONCILE}.cleanup.live_run_dirs", return_value=(rd1, rd2)), \
                    patch(f"{_RECONCILE}.OrchestratorClient") as client_cls, \
                    patch(f"{_RECONCILE}._push_ca") as push_ca, \
                    patch(f"{_RECONCILE}._reprovision_git_gate"), \
                    patch(f"{_RECONCILE}.subprocess.run",
                          return_value=CompletedProcess([], 1)):
                client_cls.return_value.list_bottles.return_value = []
                attach_bottled_agents_to_gateway("http://orch:8099", gw)
        # CA pushed to both bottles.
        self.assertEqual(2, push_ca.call_count)
    def test_per_bottle_ca_failure_does_not_block_others(self) -> None:
        from bot_bottle.backend.firecracker.reconcile import attach_bottled_agents_to_gateway
        gw = self._make_gateway()
        with tempfile.TemporaryDirectory() as td:
            tmp = Path(td)
            rd1 = self._make_run_dir(tmp, "agent-fail1", "10.243.0.5")
            rd2 = self._make_run_dir(tmp, "agent-ok99", "10.243.0.7")
            with patch(f"{_RECONCILE}.cleanup.live_run_dirs", return_value=(rd1, rd2)), \
                    patch(f"{_RECONCILE}.OrchestratorClient") as client_cls, \
                    patch(f"{_RECONCILE}._push_ca",
                          side_effect=[CalledProcessError(1, "ssh"), None]) as push_ca, \
                    patch(f"{_RECONCILE}._reprovision_git_gate"), \
                    patch(f"{_RECONCILE}.subprocess.run",
                          return_value=CompletedProcess([], 1)):
                client_cls.return_value.list_bottles.return_value = []
                # Must not raise even though the first bottle's CA push fails.
                attach_bottled_agents_to_gateway("http://orch:8099", gw)
        # Both bottles were attempted.
        self.assertEqual(2, push_ca.call_count)
    def test_egress_token_restored_for_bottles_with_secret(self) -> None:
        from bot_bottle.backend.firecracker.reconcile import attach_bottled_agents_to_gateway
        gw = self._make_gateway()
        with tempfile.TemporaryDirectory() as td:
            tmp = Path(td)
            rd = self._make_run_dir(tmp, "agent-tok11", "10.243.0.5")
            bottles = [{"bottle_id": "bid-001", "source_ip": "10.243.0.5"}]
            with patch(f"{_RECONCILE}.cleanup.live_run_dirs", return_value=(rd,)), \
                    patch(f"{_RECONCILE}.OrchestratorClient") as client_cls, \
                    patch(f"{_RECONCILE}._push_ca"), \
                    patch(f"{_RECONCILE}._reprovision_git_gate"), \
                    patch(f"{_RECONCILE}.reprovision_bottles") as reprov, \
                    patch(f"{_RECONCILE}.subprocess.run",
                          return_value=CompletedProcess([], 0, stdout="secret-key\n")):
                client_cls.return_value.list_bottles.return_value = bottles
                reprov.return_value = 1
                attach_bottled_agents_to_gateway("http://orch:8099", gw)
        reprov.assert_called_once()
        _, secrets = reprov.call_args.args
        self.assertIn("10.243.0.5", secrets)
        self.assertEqual("secret-key", secrets["10.243.0.5"])
    def test_git_gate_reprovision_called_with_bottle_id(self) -> None:
        from bot_bottle.backend.firecracker.reconcile import attach_bottled_agents_to_gateway
        gw = self._make_gateway()
        with tempfile.TemporaryDirectory() as td:
            tmp = Path(td)
            rd = self._make_run_dir(tmp, "agent-git11", "10.243.0.5")
            bottles = [{"bottle_id": "bid-git", "source_ip": "10.243.0.5"}]
            with patch(f"{_RECONCILE}.cleanup.live_run_dirs", return_value=(rd,)), \
                    patch(f"{_RECONCILE}.OrchestratorClient") as client_cls, \
                    patch(f"{_RECONCILE}._push_ca"), \
                    patch(f"{_RECONCILE}._reprovision_git_gate") as reprov_gw, \
                    patch(f"{_RECONCILE}.subprocess.run",
                          return_value=CompletedProcess([], 1)):
                client_cls.return_value.list_bottles.return_value = bottles
                attach_bottled_agents_to_gateway("http://orch:8099", gw)
        reprov_gw.assert_called_once()
        _, bottle_id_arg, slug_arg = reprov_gw.call_args.args
        self.assertEqual("bid-git", bottle_id_arg)
        self.assertEqual("agent-git11", slug_arg)
    def test_run_dir_without_key_file_is_skipped(self) -> None:
        from bot_bottle.backend.firecracker.reconcile import attach_bottled_agents_to_gateway
        gw = self._make_gateway()
        with tempfile.TemporaryDirectory() as td:
            tmp = Path(td)
            rd = self._make_run_dir(tmp, "agent-nokey", "10.243.0.5")
            (rd / "bottle_id_ed25519").unlink()  # remove the key
            with patch(f"{_RECONCILE}.cleanup.live_run_dirs", return_value=(rd,)), \
                    patch(f"{_RECONCILE}.OrchestratorClient") as client_cls, \
                    patch(f"{_RECONCILE}._push_ca") as push_ca, \
                    patch(f"{_RECONCILE}._reprovision_git_gate"):
                client_cls.return_value.list_bottles.return_value = []
                attach_bottled_agents_to_gateway("http://orch:8099", gw)
        push_ca.assert_not_called()
 if __name__ == "__main__":
    unittest.main()
@@ -1,98 +0,0 @@
 """Unit: how the CLI tells users to re-run it under sudo.
 `sudo bot-bottle …` is wrong for the users who followed the documented
 install: sudo's secure_path excludes ~/.local/bin, where both pipx and
 install.sh put the entry point. These lock in the absolute-path form.
 """
 from __future__ import annotations
 import contextlib
 import io
 import os
 import tempfile
 import unittest
 from pathlib import Path
 from unittest import mock
 from bot_bottle import invocation
 class TestSelfPath(unittest.TestCase):
    def test_absolute_argv0_is_used_as_is(self):
        with mock.patch.object(invocation.sys, "argv", ["/opt/venv/bin/bot-bottle"]):
            self.assertEqual("/opt/venv/bin/bot-bottle", invocation.self_path())
    def test_bare_name_is_resolved_through_path(self):
        # The case that matters: invoked as `bot-bottle`, installed in a
        # directory sudo would drop.
        with tempfile.TemporaryDirectory() as d:
            entry = Path(d, "bot-bottle")
            entry.write_text("#!/bin/sh\n")
            entry.chmod(0o755)
            with mock.patch.object(invocation.sys, "argv", ["bot-bottle"]), \
                 mock.patch.dict(os.environ, {"PATH": d}):
                self.assertEqual(str(entry), invocation.self_path())
    def test_relative_path_is_made_absolute(self):
        with tempfile.TemporaryDirectory() as d:
            entry = Path(d, "bot-bottle")
            entry.write_text("#!/bin/sh\n")
            entry.chmod(0o755)
            cwd = os.getcwd()
            try:
                os.chdir(d)
                with mock.patch.object(invocation.sys, "argv", ["./bot-bottle"]):
                    self.assertTrue(os.path.isabs(invocation.self_path()))
            finally:
                os.chdir(cwd)
    def test_unresolvable_entry_point_falls_back_to_the_name(self):
        # `python -m`-style invocation, or an argv[0] that no longer exists.
        # A slightly wrong hint beats a traceback raised while reporting some
        # unrelated problem.
        with mock.patch.object(invocation.sys, "argv", ["/nonexistent/gone"]), \
             mock.patch.object(invocation.shutil, "which", return_value=None):
            self.assertEqual("/nonexistent/gone", invocation.self_path())
        with mock.patch.object(invocation.sys, "argv", [""]), \
             mock.patch.object(invocation.shutil, "which", return_value=None):
            self.assertEqual("bot-bottle", invocation.self_path())
 class TestSudoCommand(unittest.TestCase):
    def test_names_an_absolute_path_not_the_bare_command(self):
        with mock.patch.object(invocation, "self_path",
                               return_value="/home/u/.local/bin/bot-bottle"):
            cmd = invocation.sudo_command("backend", "setup", "--backend=firecracker")
        self.assertEqual(
            "sudo /home/u/.local/bin/bot-bottle backend setup --backend=firecracker",
            cmd,
        )
        # The regression this exists to prevent.
        self.assertNotIn("sudo bot-bottle", cmd)
 class TestFirecrackerSetupUsesIt(unittest.TestCase):
    def test_root_reinvocation_hint_names_an_absolute_path(self):
        # The message that prompted all this. Drive the real code path rather
        # than scanning the source, which would also match the comment
        # explaining why the bare form is wrong.
        from bot_bottle.backend.firecracker import setup as fc_setup
        err, out = io.StringIO(), io.StringIO()
        with mock.patch.object(fc_setup.os, "geteuid", return_value=501), \
             mock.patch.object(fc_setup.invocation, "self_path",
                               return_value="/home/u/.local/bin/bot-bottle"), \
             contextlib.redirect_stderr(err), contextlib.redirect_stdout(out):
            fc_setup._setup_systemd()
        printed = err.getvalue()
        self.assertIn(
            "sudo /home/u/.local/bin/bot-bottle backend setup --backend=firecracker",
            printed,
        )
        self.assertNotIn("sudo bot-bottle", printed)
 if __name__ == "__main__":
    unittest.main()
@@ -151,6 +151,31 @@ class TestReprovisionGateway(unittest.TestCase):
                self.c.reprovision_gateway("b1", "key")
 class TestUpdateAgentSecret(unittest.TestCase):
    def setUp(self) -> None:
        self.c = OrchestratorClient("http://orch:8080")
    def test_success_posts_single_secret(self) -> None:
        with patch(_URLOPEN, return_value=_resp(200, {"updated": True})) as opened:
            self.assertTrue(self.c.update_agent_secret("b1", "A", "fresh", "key"))
        request = opened.call_args.args[0]
        self.assertEqual("POST", request.get_method())
        self.assertTrue(request.full_url.endswith("/bottles/b1/secret"))
        self.assertEqual(
            {"name": "A", "value": "fresh", "env_var_secret": "key"},
            json.loads(request.data),
        )
    def test_unknown_bottle_is_false(self) -> None:
        with patch(_URLOPEN, side_effect=_http_error(404)):
            self.assertFalse(self.c.update_agent_secret("b1", "A", "v", "key"))
    def test_other_status_raises(self) -> None:
        with patch(_URLOPEN, side_effect=_http_error(400)):
            with self.assertRaises(OrchestratorClientError):
                self.c.update_agent_secret("b1", "A", "v", "key")
 class TestHealthAndPolicy(unittest.TestCase):
    def setUp(self) -> None:
        self.c = OrchestratorClient("http://orch:8080")
@@ -199,6 +199,20 @@ class TestAgentSecrets(unittest.TestCase):
        got = self.store.get_agent_secrets("bottle-1")
        self.assertEqual({"K": "new", "K2": "v2"}, got)
    def test_store_agent_secret_upserts_one_key_leaving_others(self) -> None:
        self.store.store_agent_secrets("bottle-1", {"K": "v1", "K2": "v2"})
        self.store.store_agent_secret("bottle-1", "K", "v1-new")  # update existing
        self.store.store_agent_secret("bottle-1", "K3", "v3")     # insert new
        self.assertEqual(
            {"K": "v1-new", "K2": "v2", "K3": "v3"},
            self.store.get_agent_secrets("bottle-1"),
        )
    def test_store_agent_secret_does_not_duplicate_rows(self) -> None:
        self.store.store_agent_secret("bottle-1", "K", "v1")
        self.store.store_agent_secret("bottle-1", "K", "v2")
        self.assertEqual({"K": "v2"}, self.store.get_agent_secrets("bottle-1"))
    def test_delete_removes_secrets(self) -> None:
        self.store.store_agent_secrets("bottle-1", {"K": "v"})
        self.store.delete_agent_secrets("bottle-1")
@@ -125,6 +125,47 @@ class TestDispatch(unittest.TestCase):
            {"EGRESS_TOKEN_0": "upstream-secret"}, self.orch.tokens_for(bottle_id),
        )
    def test_update_agent_secret_in_place(self) -> None:
        key = base64.urlsafe_b64encode(b"unit-test-key").rstrip(b"=").decode()
        status, payload = dispatch(
            self.orch, "POST", "/bottles", _body({
                "source_ip": "10.243.0.21",
                "tokens": {"A": "old", "B": "keep"},
                "env_var_secret": key,
            }),
        )
        self.assertEqual(201, status)
        bottle_id = payload["bottle_id"]
        assert isinstance(bottle_id, str)
        status, response = dispatch(
            self.orch, "POST", f"/bottles/{bottle_id}/secret",
            _body({"name": "A", "value": "fresh", "env_var_secret": key}),
        )
        self.assertEqual((200, {"updated": True}), (status, response))
        self.assertEqual({"A": "fresh", "B": "keep"}, self.orch.tokens_for(bottle_id))
    def test_update_agent_secret_validates_and_unknown_bottle(self) -> None:
        status, _ = dispatch(self.orch, "POST", "/bottles/b1/secret", b"not-json")
        self.assertEqual(400, status)
        status, _ = dispatch(
            self.orch, "POST", "/bottles/b1/secret", _body({"name": "A"}),
        )
        self.assertEqual(400, status)  # value + env_var_secret missing
        status, _ = dispatch(
            self.orch, "POST", "/bottles/ghost/secret",
            _body({"name": "A", "value": "v", "env_var_secret": "k"}),
        )
        self.assertEqual(404, status)
    def test_update_agent_secret_is_cli_only(self) -> None:
        # A data-plane (`gateway`) caller must not set a bottle's tokens.
        status, _ = dispatch(
            self.orch, "POST", "/bottles/b1/secret",
            _body({"name": "A", "value": "v", "env_var_secret": "k"}),
            role=ROLE_GATEWAY,
        )
        self.assertEqual(403, status)
    def test_reprovision_validates_request_and_missing_rows(self) -> None:
        status, _ = dispatch(
            self.orch, "POST", "/bottles/b1/reprovision_gateway", b"not-json",
@@ -103,6 +103,25 @@ class TestOrchestrator(unittest.TestCase):
        self.assertTrue(self.orch.reprovision_from_secret(rec.bottle_id, key))
        self.assertEqual({"EGRESS_TOKEN_0": "secret"}, self.orch.tokens_for(rec.bottle_id))
    def test_update_agent_secret_refreshes_one_token_in_place(self) -> None:
        key = new_env_var_secret()
        rec = self.orch.launch_bottle(
            "10.243.0.20", tokens={"A": "old-a", "B": "keep-b"}, env_var_secret=key,
        )
        self.assertTrue(self.orch.update_agent_secret(rec.bottle_id, "A", "new-a", key))
        # In memory: A refreshed, B left untouched.
        self.assertEqual({"A": "new-a", "B": "keep-b"}, self.orch.tokens_for(rec.bottle_id))
        # At rest: the whole set stays decryptable with the SAME env_var_secret,
        # so a later reprovision restores the refreshed value (not the launch one).
        self.orch._tokens.clear()
        self.assertTrue(self.orch.reprovision_from_secret(rec.bottle_id, key))
        self.assertEqual({"A": "new-a", "B": "keep-b"}, self.orch.tokens_for(rec.bottle_id))
    def test_update_agent_secret_unknown_bottle_is_false(self) -> None:
        self.assertFalse(
            self.orch.update_agent_secret("ghost", "A", "v", new_env_var_secret())
        )
    def test_reprovision_rejects_missing_rows_and_wrong_key(self) -> None:
        self.assertFalse(self.orch.reprovision_from_secret("missing", new_env_var_secret()))
        key = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE"
Author	SHA1	Message	Date
didericis-claude	dee0121e8d	feat(prd-0081): reprovision CA, git-gate, and egress tokens on gateway bring-up prd-number-check / require-numbered-prds (pull_request) Successful in 13s Details tracker-policy-pr / check-pr (pull_request) Successful in 7s Details test / image-input-builds (pull_request) Successful in 41s Details test / unit (pull_request) Successful in 51s Details test / integration-docker (pull_request) Successful in 58s Details test / coverage (pull_request) Successful in 41s Details test / image-input-builds (push) Successful in 48s Details test / unit (push) Successful in 56s Details lint / lint (push) Successful in 1m2s Details Update Quality Badges / update-badges (push) Successful in 1m4s Details test / integration-docker (push) Failing after 2m53s Details test / coverage (push) Has been skipped Details On a Firecracker gateway cold boot, reconcile every live agent VM against the fresh gateway: push the new mitmproxy CA into each agent's trust store, re-provision git-gate repos/creds from the persisted upstreams snapshot, and restore egress tokens. Per-bottle failures are logged and skipped so one unreachable VM does not block the rest. New modules / changes: - backend/firecracker/reconcile.py: attach_bottled_agents_to_gateway, _push_ca, _reprovision_git_gate, _guest_ip_from_config - git_gate/provision.py: write upstreams.json after key provisioning so the bring-up reconcile can reconstruct the upstream table without the manifest - backend/firecracker/infra.py: call attach_bottled_agents_to_gateway in the cold-boot branch of ensure_running() - backend/base.py: no-op default on BottleBackend - backend/firecracker/consolidated_launch.py: remove superseded _reprovision_running_bottles / _guest_ip_from_config - orchestrator: OrchestratorCore.update_agent_secret + POST /bottles/<id>/secret + client.update_agent_secret (single-secret in-place update, the reusable primitive from the 2026-07-26 hotfix) - tests: 15 new tests in test_firecracker_reconcile.py; updated test_firecracker_infra.py cold-boot cases; stale FC reprovision tests removed from test_backend_secret_reprovision.py Closes #516.	2026-07-28 01:50:15 +00:00
didericis-codex	6f997bf118	test(release): cover publication failure paths test / unit (push) Successful in 1m0s Details test / image-input-builds (push) Successful in 58s Details test / integration-docker (push) Successful in 1m3s Details Update Quality Badges / update-badges (push) Successful in 1m6s Details test / coverage (push) Successful in 14s Details lint / lint (push) Successful in 3m47s Details	2026-07-27 17:00:50 -04:00
didericis-codex	2fb08fc976	ci: rerun release flow checks	2026-07-27 17:00:50 -04:00
didericis-codex	509eb73497	docs(prd): finalize complete release flow	2026-07-27 17:00:50 -04:00
didericis-codex	df9c4ca59b	feat(release): automate artifact and tag flows	2026-07-27 17:00:50 -04:00
didericis-codex	6ffa8d8843	feat(firecracker): pull pinned agent images	2026-07-27 17:00:50 -04:00
didericis-codex	e3d17dbc07	feat(release): publish qualified channel pointers	2026-07-27 17:00:50 -04:00
didericis-codex	1b86847f0d	feat(installer): select immutable release bundles	2026-07-27 17:00:50 -04:00
didericis-codex	9dee92406e	feat(release): publish immutable commit bundles	2026-07-27 17:00:50 -04:00
didericis-codex	309ec34f29	feat(release): pin first-party agent images	2026-07-27 17:00:50 -04:00
didericis-codex	9eab5be2c4	feat(release): define commit-addressed bundle index	2026-07-27 17:00:50 -04:00
didericis-codex	b3834ebf66	docs(prd): define complete release flow	2026-07-27 17:00:50 -04:00
didericis-codex	e2857f1eeb	fix(release): publish only verified wheel	2026-07-27 17:00:50 -04:00
didericis-codex	fd529a98b5	test(infra): cover malformed release manifests	2026-07-27 17:00:50 -04:00
didericis-codex	49b407437c	docs(prd): activate packaged infrastructure artifacts	2026-07-27 17:00:50 -04:00
didericis-codex	8315e4192a	feat(infra): pull artifacts pinned by package release	2026-07-27 17:00:50 -04:00
didericis-codex	938df8513f	docs(prd): define packaged infrastructure artifacts	2026-07-27 17:00:50 -04:00
didericis-codex	ec38458c94	fix(firecracker): include copied context directories prd-number-check / require-numbered-prds (pull_request) Successful in 9s Details tracker-policy-pr / check-pr (pull_request) Successful in 9s Details test / image-input-builds (pull_request) Successful in 39s Details test / unit (pull_request) Successful in 50s Details test / integration-docker (pull_request) Successful in 1m0s Details test / coverage (pull_request) Successful in 17s Details test / image-input-builds (push) Successful in 43s Details lint / lint (push) Successful in 1m0s Details test / integration-docker (push) Successful in 1m4s Details test / unit (push) Successful in 3m4s Details test / coverage (push) Successful in 18s Details Update Quality Badges / update-badges (push) Failing after 10m3s Details	2026-07-27 17:53:48 +00:00
didericis	f6df85a8cd	fix(fc): ship agent-image build context files to the infra VM test / integration-docker (pull_request) Has been cancelled Details prd-number-check / require-numbered-prds (pull_request) Successful in 11s Details lint / lint (push) Successful in 59s Details test / unit (pull_request) Successful in 1m0s Details test / coverage (pull_request) Has been skipped Details test / image-input-builds (pull_request) Successful in 1m1s Details tracker-policy-pr / check-pr (pull_request) Successful in 17s Details Agent images build with buildah inside the infra VM, and `_build_in_infra` shipped only the Dockerfile into an empty `{ctx}/ctx` context — on the documented assumption that agent Dockerfiles COPY nothing from the context. The "pin & verify image inputs" work broke that: the codex Dockerfile COPYs `codex-package_SHA256SUMS` and the claude/pi Dockerfiles COPY their npm `package.json`/`package-lock.json`, so any fresh agent image build fails with `COPY ...: no such file or directory` (existing agents only survive on a rootfs cached from before the change). Parse each Dockerfile's context COPY sources, tar those committed files from the build root into the VM-side `{ctx}/ctx`, and fold their content into the rootfs cache digest so a repinned input rebuilds instead of reusing stale bytes. `COPY --from=<stage>` and absolute/traversing sources are excluded. Closes #538. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>	2026-07-27 13:18:15 -04:00