feat(prd-0081): reprovision CA, git-gate, and egress tokens on gateway bring-up

On a Firecracker gateway cold boot, reconcile every live agent VM against
the fresh gateway: push the new mitmproxy CA into each agent's trust store,
re-provision git-gate repos/creds from the persisted upstreams snapshot,
and restore egress tokens. Per-bottle failures are logged and skipped so
one unreachable VM does not block the rest.

New modules / changes:
- backend/firecracker/reconcile.py: attach_bottled_agents_to_gateway,
  _push_ca, _reprovision_git_gate, _guest_ip_from_config
- git_gate/provision.py: write upstreams.json after key provisioning so
  the bring-up reconcile can reconstruct the upstream table without the
  manifest
- backend/firecracker/infra.py: call attach_bottled_agents_to_gateway in
  the cold-boot branch of ensure_running()
- backend/base.py: no-op default on BottleBackend
- backend/firecracker/consolidated_launch.py: remove superseded
  _reprovision_running_bottles / _guest_ip_from_config
- orchestrator: OrchestratorCore.update_agent_secret + POST
  /bottles/<id>/secret + client.update_agent_secret (single-secret
  in-place update, the reusable primitive from the 2026-07-26 hotfix)
- tests: 15 new tests in test_firecracker_reconcile.py; updated
  test_firecracker_infra.py cold-boot cases; stale FC reprovision tests
  removed from test_backend_secret_reprovision.py

Closes #516.

.gitea/workflows/pre-release-test.yml

@@ -19,19 +19,6 @@ name: pre-release-test
 
 on:
   workflow_dispatch:
-    inputs:
-      ref:
-        description: Revision to qualify
-        required: false
-        default: main
-  workflow_call:
-    inputs:
-      ref:
-        required: true
-        type: string
-    secrets:
-      BOT_BOTTLE_INFRA_ARTIFACT_TOKEN:
-        required: true
 
 jobs:
   unit:
@@ -39,8 +26,6 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-        with:
-          ref: ${{ inputs.ref }}
 
       # No actions/setup-python: the runner image already ships Python 3.12,
       # and older act_runner engines mishandle setup-python's PATH (coverage
@@ -81,8 +66,6 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-        with:
-          ref: ${{ inputs.ref }}
 
       # No actions/setup-python (see the note in the `unit` job); the
       # container's system Python 3.12 runs the stdlib test suite directly.
@@ -155,11 +138,10 @@ jobs:
   # the old build-infra → integration-firecracker + coverage chain incurred.
   integration-firecracker:
     runs-on: [self-hosted, kvm]
+    if: github.event_name == 'workflow_dispatch'
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-        with:
-          ref: ${{ inputs.ref }}
 
       - name: Preflight — Firecracker host is ready
         run: |
@@ -230,14 +212,13 @@ jobs:
   # Python >=3.11 with `coverage` importable on the launchd service PATH.
   integration-macos:
     runs-on: [self-hosted, macos]
+    if: github.event_name == 'workflow_dispatch'
     concurrency:
       group: integration-macos-infra
       cancel-in-progress: false
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-        with:
-          ref: ${{ inputs.ref }}
 
       # Fail loudly if the backend this job promises isn't actually usable,
       # rather than letting every test silently `unittest.skip` and the job go
@@ -309,7 +290,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          ref: ${{ inputs.ref }}
           fetch-depth: 0
 
       - name: Install coverage
@@ -360,8 +340,6 @@ jobs:
     steps:
       - name: Checkout the tested revision
         uses: actions/checkout@v4
-        with:
-          ref: ${{ inputs.ref }}
 
       - name: Download the tested rootfs
         uses: actions/download-artifact@v3

.gitea/workflows/publish-artifacts.yml

@@ -1,189 +0,0 @@
-name: publish-artifacts
-
-on:
-  workflow_dispatch:
-    inputs:
-      ref:
-        description: Branch, tag, or commit to publish
-        required: true
-        default: main
-  workflow_call:
-    inputs:
-      ref:
-        required: true
-        type: string
-    outputs:
-      source_commit:
-        description: Published immutable source commit
-        value: ${{ jobs.resolve.outputs.sha }}
-    secrets:
-      BOT_BOTTLE_RELEASE_TOKEN:
-        required: true
-      BOT_BOTTLE_INFRA_ARTIFACT_TOKEN:
-        required: true
-
-concurrency:
-  group: publish-artifacts-${{ inputs.ref }}
-  cancel-in-progress: false
-
-jobs:
-  resolve:
-    runs-on: ubuntu-latest
-    outputs:
-      sha: ${{ steps.revision.outputs.sha }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ inputs.ref }}
-          fetch-depth: 0
-      - id: revision
-        name: Resolve the immutable source revision
-        run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
-
-  oci:
-    needs: resolve
-    runs-on: ubuntu-latest
-    outputs:
-      orchestrator: ${{ steps.images.outputs.orchestrator }}
-      gateway: ${{ steps.images.outputs.gateway }}
-      agent_claude: ${{ steps.images.outputs.agent_claude }}
-      agent_codex: ${{ steps.images.outputs.agent_codex }}
-      agent_pi: ${{ steps.images.outputs.agent_pi }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ needs.resolve.outputs.sha }}
-      - name: Log in to the package registry
-        uses: docker/login-action@v3
-        with:
-          registry: gitea.dideric.is
-          username: ${{ gitea.actor }}
-          password: ${{ secrets.BOT_BOTTLE_RELEASE_TOKEN }}
-      - name: Set up multi-architecture builds
-        uses: docker/setup-buildx-action@v3
-      - id: images
-        name: Build, smoke-test, and publish OCI images
-        run: |
-          set -euo pipefail
-          python_base=$(python3 -c \
-            'import json; print(json.load(open("image-build-args.json"))["PYTHON_BASE_IMAGE"])')
-          node_base=$(python3 -c \
-            'import json; print(json.load(open("image-build-args.json"))["NODE_BASE_IMAGE"])')
-          sha='${{ needs.resolve.outputs.sha }}'
-          registry=gitea.dideric.is/didericis
-
-          build_image() {
-            key=$1
-            name=$2
-            dockerfile=$3
-            build_arg=$4
-            smoke=$5
-            local_ref="bot-bottle-${name}:candidate"
-            remote_ref="${registry}/bot-bottle-${name}:commit-${sha}"
-            docker build --build-arg "$build_arg" -t "$local_ref" \
-              -f "$dockerfile" .
-            smoke_command=$(printf "$smoke" "$local_ref")
-            sh -c "docker run --rm $smoke_command"
-            docker buildx build --platform linux/amd64,linux/arm64 \
-              --build-arg "$build_arg" --push --iidfile "${key}.iid" \
-              -t "$remote_ref" -f "$dockerfile" .
-            digest=$(cat "${key}.iid")
-            case "$digest" in sha256:*) ;; *) exit 1 ;; esac
-            echo "${key}=${registry}/bot-bottle-${name}@${digest}" \
-              >> "$GITHUB_OUTPUT"
-          }
-
-          build_image orchestrator orchestrator Dockerfile.orchestrator \
-            "PYTHON_BASE_IMAGE=$python_base" \
-            "--entrypoint python3 %s -c 'import bot_bottle.orchestrator'"
-          build_image gateway gateway Dockerfile.gateway \
-            "PYTHON_BASE_IMAGE=$python_base" \
-            "--entrypoint mitmdump %s --version"
-          build_image agent_claude claude bot_bottle/contrib/claude/Dockerfile \
-            "NODE_BASE_IMAGE=$node_base" "%s claude --version"
-          build_image agent_codex codex bot_bottle/contrib/codex/Dockerfile \
-            "NODE_BASE_IMAGE=$node_base" "%s codex --version"
-          build_image agent_pi pi bot_bottle/contrib/pi/Dockerfile \
-            "NODE_BASE_IMAGE=$node_base" "%s pi --version"
-
-  firecracker:
-    needs: resolve
-    runs-on: [self-hosted, kvm]
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ needs.resolve.outputs.sha }}
-      - name: Build and smoke-test Firecracker artifacts
-        env:
-          BOT_BOTTLE_FC_DROPBEAR: /var/cache/bot-bottle-fc/dropbear
-        run: |
-          python3 cli.py backend status --backend=firecracker
-          python3 -m bot_bottle.backend.firecracker.publish_infra \
-            --output infra-candidate --reuse-published
-          BOT_BOTTLE_INFRA_ARTIFACT_DIR="$PWD/infra-candidate" \
-            python3 -m unittest discover -t . -s tests/integration -v
-      - name: Publish tested Firecracker artifacts
-        env:
-          BOT_BOTTLE_INFRA_ARTIFACT_TOKEN: ${{ secrets.BOT_BOTTLE_INFRA_ARTIFACT_TOKEN }}
-          BOT_BOTTLE_FC_DROPBEAR: /var/cache/bot-bottle-fc/dropbear
-        run: |
-          python3 -m bot_bottle.backend.firecracker.publish_infra \
-            --publish-dir infra-candidate
-      - name: Record Firecracker artifact identities
-        run: |
-          cp infra-candidate/orchestrator/version.txt fc-orchestrator-version
-          cp infra-candidate/orchestrator/rootfs.ext4.gz.sha256 fc-orchestrator-sha
-          cp infra-candidate/gateway/version.txt fc-gateway-version
-          cp infra-candidate/gateway/rootfs.ext4.gz.sha256 fc-gateway-sha
-      - uses: actions/upload-artifact@v3
-        with:
-          name: firecracker-identities
-          path: |
-            fc-orchestrator-version
-            fc-orchestrator-sha
-            fc-gateway-version
-            fc-gateway-sha
-
-  package:
-    needs: [resolve, oci, firecracker]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ needs.resolve.outputs.sha }}
-      - uses: actions/download-artifact@v3
-        with:
-          name: firecracker-identities
-      - name: Build the wheel with the published identities
-        run: |
-          python3 -m pip install --break-system-packages build
-          python3 scripts/generate_release_manifest.py \
-            --source-commit '${{ needs.resolve.outputs.sha }}' \
-            --orchestrator-image '${{ needs.oci.outputs.orchestrator }}' \
-            --gateway-image '${{ needs.oci.outputs.gateway }}' \
-            --agent-claude-image '${{ needs.oci.outputs.agent_claude }}' \
-            --agent-codex-image '${{ needs.oci.outputs.agent_codex }}' \
-            --agent-pi-image '${{ needs.oci.outputs.agent_pi }}' \
-            --firecracker-orchestrator-version "$(cat fc-orchestrator-version)" \
-            --firecracker-orchestrator-sha256 "$(awk '{print $1}' fc-orchestrator-sha)" \
-            --firecracker-gateway-version "$(cat fc-gateway-version)" \
-            --firecracker-gateway-sha256 "$(awk '{print $1}' fc-gateway-sha)" \
-            --output release-manifest.json
-          BOT_BOTTLE_RELEASE_MANIFEST="$PWD/release-manifest.json" \
-            python3 -m build --wheel
-      - name: Verify and publish the immutable commit bundle
-        env:
-          BOT_BOTTLE_RELEASE_TOKEN: ${{ secrets.BOT_BOTTLE_RELEASE_TOKEN }}
-        run: |
-          wheel=$(find dist -maxdepth 1 -name '*.whl' -type f)
-          python3 -m venv verify-venv
-          verify-venv/bin/pip install --no-deps "$wheel"
-          verify-venv/bin/python -c \
-            'from bot_bottle.release_manifest import load_manifest; load_manifest()'
-          python3 scripts/generate_release_bundle.py \
-            --manifest release-manifest.json \
-            --wheel "$wheel" \
-            --workflow-run \
-              "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
-            --published-at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
-            --output bundle-index.json --publish

.gitea/workflows/release.yml

@@ -1,93 +0,0 @@
-name: release
-
-on:
-  push:
-    tags:
-      - "v*"
-
-concurrency:
-  group: release-${{ github.ref_name }}
-  cancel-in-progress: false
-
-jobs:
-  validate:
-    runs-on: ubuntu-latest
-    outputs:
-      sha: ${{ steps.release.outputs.sha }}
-      tag: ${{ steps.release.outputs.tag }}
-      channel: ${{ steps.release.outputs.channel }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - id: release
-        name: Validate tag shape and promotion branch
-        run: |
-          set -euo pipefail
-          tag=${GITHUB_REF#refs/tags/}
-          channel=$(python3 -c \
-            'import sys; from bot_bottle.release_qualification import release_channel; print(release_channel(sys.argv[1]))' \
-            "$tag")
-          branch=$channel
-          git fetch origin "$branch"
-          sha=$(git rev-list -n 1 "$tag")
-          git merge-base --is-ancestor "$sha" "origin/$branch" || {
-            echo "$tag is not reachable from protected $branch" >&2
-            exit 1
-          }
-          echo "sha=$sha" >> "$GITHUB_OUTPUT"
-          echo "tag=$tag" >> "$GITHUB_OUTPUT"
-          echo "channel=$channel" >> "$GITHUB_OUTPUT"
-
-  qualify:
-    needs: validate
-    uses: ./.gitea/workflows/pre-release-test.yml
-    with:
-      ref: ${{ needs.validate.outputs.sha }}
-    secrets:
-      BOT_BOTTLE_INFRA_ARTIFACT_TOKEN: ${{ secrets.BOT_BOTTLE_INFRA_ARTIFACT_TOKEN }}
-
-  artifacts:
-    needs: [validate, qualify]
-    uses: ./.gitea/workflows/publish-artifacts.yml
-    with:
-      ref: ${{ needs.validate.outputs.sha }}
-    secrets:
-      BOT_BOTTLE_RELEASE_TOKEN: ${{ secrets.BOT_BOTTLE_RELEASE_TOKEN }}
-      BOT_BOTTLE_INFRA_ARTIFACT_TOKEN: ${{ secrets.BOT_BOTTLE_INFRA_ARTIFACT_TOKEN }}
-
-  promote:
-    needs: [validate, artifacts]
-    runs-on: ubuntu-latest
-    concurrency:
-      group: release-channel-${{ needs.validate.outputs.channel }}
-      cancel-in-progress: false
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ needs.validate.outputs.sha }}
-      - name: Verify install from the published bundle
-        env:
-          BOT_BOTTLE_REF: ${{ needs.validate.outputs.sha }}
-        run: |
-          sh install.sh
-          "$HOME/.local/bin/bot-bottle" --help
-      - name: Publish release qualification and advance channel
-        env:
-          BOT_BOTTLE_RELEASE_TOKEN: ${{ secrets.BOT_BOTTLE_RELEASE_TOKEN }}
-        run: |
-          python3 scripts/publish_release_qualification.py \
-            --tag '${{ needs.validate.outputs.tag }}' \
-            --source-commit '${{ needs.validate.outputs.sha }}' \
-            --workflow-run \
-              "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
-            --qualified-at "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
-      - name: Create the Gitea release
-        env:
-          GITEA_TOKEN: ${{ secrets.BOT_BOTTLE_RELEASE_TOKEN }}
-        run: |
-          curl --fail-with-body --silent --show-error \
-            -H "Authorization: token $GITEA_TOKEN" \
-            -H "Content-Type: application/json" \
-            -d "{\"tag_name\":\"${{ needs.validate.outputs.tag }}\",\"name\":\"${{ needs.validate.outputs.tag }}\",\"target_commitish\":\"${{ needs.validate.outputs.sha }}\"}" \
-            "${GITHUB_SERVER_URL}/api/v1/repos/${GITHUB_REPOSITORY}/releases"

README.md

@@ -75,15 +75,7 @@ When the agent exits, `cli.py` tears down every gateway and both networks; nothi
 curl -fsSL https://gitea.dideric.is/didericis/bot-bottle/raw/branch/main/install.sh | sh
 ```
 
-The installer is a bootstrapper: it finds a suitable Python, resolves the
+The installer is a bootstrapper: it finds a suitable Python, installs bot-bottle with `pipx` (falling back to `pip --user`), creates `~/.bot-bottle`, and runs `bot-bottle doctor`. It is idempotent and never uses `sudo`. Python-native users can skip it entirely with `pipx install bot-bottle` or `uv tool install bot-bottle`.
-latest qualified production bundle, verifies its wheel checksum, installs with
-`pipx` (falling back to a private venv), creates `~/.bot-bottle`, and runs
-`bot-bottle doctor`. It is idempotent and never uses `sudo`.
-
-Select staging with `BOT_BOTTLE_CHANNEL=staging`, an exact qualified release
-with `BOT_BOTTLE_VERSION=vX.Y.Z[-rc.N]`, or an exact published commit with
-`BOT_BOTTLE_REF=<40-character-sha>`. Commit installs print an explicit warning
-because snapshots may not have passed release qualification.
 
 ### Requirements
 
@@ -91,8 +83,7 @@ because snapshots may not have passed release qualification.
 
 **No `pipx` required.** If `pipx` is present the installer uses it and stays out of the way. If it isn't, bot-bottle installs into a private venv at `~/.bot-bottle/venv` (override with `BOT_BOTTLE_VENV`) and symlinks the entry point into `~/.local/bin`. There is deliberately no `pip install --user` path: Homebrew, python.org and Debian/Ubuntu interpreters are all externally managed (PEP 668), which blocks `--user` outright — so on a Mac it is never the fallback it appears to be. A venv is exempt from PEP 668, and `venv` is stdlib, so unlike `pipx` there is nothing to bootstrap first.
 
-**`git`** is needed only when `BOT_BOTTLE_INSTALL_SPEC` explicitly selects a
+**`git`**, because the default install spec is a `git+` URL. Set `BOT_BOTTLE_INSTALL_SPEC` to a wheel path or index name to avoid it.
-`git+` URL. The normal published-wheel path does not require it.
 
 **A backend**, which the installer deliberately does *not* install for you — `doctor` reports what's missing afterwards. On compatible macOS hosts, the default backend requires Apple's `container` CLI and does not require Docker. The Firecracker backend (Linux) requires Docker on the host for the gateway plus the `firecracker` binary and KVM. The legacy Docker backend requires Docker. Claude bottles also need a long-lived Claude Code OAuth token (`claude setup-token`) exported as `BOT_BOTTLE_CLAUDE_OAUTH_TOKEN`.
 
@@ -200,19 +191,9 @@ BOT_BOTTLE_BACKEND=firecracker bot-bottle start <agent>
 > **CI:** Firecracker integration runs in the manually dispatched `.gitea/workflows/pre-release-test.yml` on a self-hosted runner labelled `kvm`; privileged KVM hosts never execute unreviewed PR code automatically. Provision it like a normal Firecracker host: `firecracker` on `PATH`, `/dev/kvm`, the cached guest kernel and static dropbear, and the persistent TAP/nft pool. The required pull-request workflow runs unit plus the complete Docker integration suite on `ubuntu-latest`; see `docs/ci.md`.
 
 ```sh
-bot-bottle start <agent>   # prepares the selected agent image, then attaches
+bot-bottle start <agent>   # builds the image on first run, drops you into claude
 ```
 
-Packaged releases carry immutable orchestrator, gateway, and first-party agent
-image identities. Docker and Apple Container pull the package-selected OCI
-digests; Firecracker pulls the matching checksum-verified rootfs artifacts.
-Each verified wheel and its identities are published as an immutable generic
-package keyed by the full source commit; `bundle-index.json` is uploaded last
-and is the completeness marker. A source checkout uses local builds for
-development. Set `BOT_BOTTLE_INFRA_BUILD=local` to make that development
-override explicit when diagnosing a packaged release; production never falls
-back to a build after an artifact pull fails.
-
 ## Manifest
 
 Bottles and agents are Markdown files with YAML frontmatter under `~/.bot-bottle/`. The Markdown body is the system prompt. Bottles live in `~/.bot-bottle/bottles/`; agents may also be shipped by a repo at `<repo>/.bot-bottle/agents/<name>.md`.

bot_bottle/backend/base.py

@@ -496,6 +496,20 @@ class BottleBackend(ABC, Generic[PlanT, CleanupT]):
         (macos-container) die with a pointer — the default here."""
         die(f"backend {self.name!r} has no orchestrator control plane")
 
+    def attach_bottled_agents_to_gateway(self) -> None:
+        """Reconcile all running bottles against the current gateway.
+
+        Called when the gateway is (re)brought up (cold-boot path) to restore
+        the three gateway-dependent services for every live bottle: CA trust,
+        git-gate repos/creds, and egress tokens. Per-bottle failures must be
+        logged and skipped — one unreachable agent must not block the rest.
+
+        Default: no-op. Backends that run a gateway (Firecracker, docker,
+        macOS) override this with a backend-native implementation that reaches
+        running agents via their transport (SSH for Firecracker, exec/cp for
+        docker and macOS). PRD 0081."""
+        return
+
     @abstractmethod
     def prepare_cleanup(self) -> CleanupT:
         """Enumerate orphaned resources from previous bottles. No side

bot_bottle/backend/docker/gateway.py

@@ -89,10 +89,6 @@ class DockerGateway(Gateway):
         when no dockerfile is configured (a pre-pulled image). BOT_BOTTLE_NO_CACHE
         forces a full rebuild (parity with `start --no-cache`)."""
         if self._dockerfile is None:
-            proc = run_docker(["docker", "pull", self.image_ref])
-            if proc.returncode != 0:
-                raise GatewayError(
-                    f"gateway image pull failed: {proc.stderr.strip()}")
             return
         context = self._build_context or resources.build_root()
         argv = ["docker", "build", "-t", self.image_ref,

bot_bottle/backend/docker/infra.py

@@ -34,7 +34,6 @@ from .orchestrator import (
     ORCHESTRATOR_NETWORK,
 )
 from ... import resources
-from ... import release_manifest
 from ...gateway import (
     GATEWAY_IMAGE,
     GATEWAY_NAME,
@@ -91,14 +90,6 @@ class DockerInfraService(InfraService):
         self._orchestrator_name = orchestrator_name
         self._orchestrator_label = orchestrator_label
         self._gateway_name = gateway_name
-        resolved_orchestrator, orchestrator_local = release_manifest.oci_image(
-            "orchestrator", orchestrator_image)
-        resolved_gateway, gateway_local = release_manifest.oci_image(
-            "gateway", gateway_image)
-        self.orchestrator_image = resolved_orchestrator
-        self.gateway_image = resolved_gateway
-        self._orchestrator_local = orchestrator_local
-        self._gateway_local = gateway_local
 
     def orchestrator(self) -> DockerOrchestrator:
         """The control-plane service. Cheap to reconstruct — `ensure_built`
@@ -113,8 +104,6 @@ class DockerInfraService(InfraService):
             repo_root=self._repo_root,
             host_root=self._host_root,
             root_mount_source=self._root_mount_source,
-            dockerfile=(
-                "Dockerfile.orchestrator" if self._orchestrator_local else None),
         )
 
     def gateway(self) -> DockerGateway:
@@ -130,7 +119,6 @@ class DockerInfraService(InfraService):
             control_network=self.control_network,
             build_context=self._repo_root,
             ca_mount_source=self._gateway_ca_mount_source,
-            dockerfile="Dockerfile.gateway" if self._gateway_local else None,
         )
 
     def ensure_running(
@@ -144,8 +132,8 @@ class DockerInfraService(InfraService):
         gateway = self.gateway()
         # Build both images (cache-aware; a no-op when nothing changed) before
         # bringing either plane up.
-        orchestrator.ensure_available()
+        orchestrator.ensure_built()
-        gateway.ensure_available()
+        gateway.ensure_built()
 
         orchestrator.ensure_running(startup_timeout=startup_timeout)
 

bot_bottle/backend/docker/launch.py

@@ -84,15 +84,6 @@ def build_or_load_images(plan: DockerBottlePlan) -> BottleImages:
             )
         info(f"using cached agent image {plan.image!r}")
         return BottleImages(agent=plan.image)
-    if "@sha256:" in plan.image:
-        pulled = docker_mod.run_docker(["docker", "pull", plan.image])
-        if pulled.returncode != 0:
-            die(f"pulling packaged agent image {plan.image!r} failed: "
-                f"{pulled.stderr.strip()}")
-        docker_mod.verify_agent_image(
-            plan.image, runtime_for(plan.agent_provider_template).smoke_test,
-        )
-        return BottleImages(agent=plan.image)
     docker_mod.build_image(plan.image, str(resources.build_root()), dockerfile=plan.dockerfile_path)
     docker_mod.verify_agent_image(
         plan.image, runtime_for(plan.agent_provider_template).smoke_test,

bot_bottle/backend/docker/orchestrator.py

@@ -125,10 +125,6 @@ class DockerOrchestrator(Orchestrator):
         no-op when nothing changed). No-op when no dockerfile is configured (a
         pre-pulled image). BOT_BOTTLE_NO_CACHE forces a full rebuild."""
         if self._dockerfile is None:
-            proc = run_docker(["docker", "pull", self.image_ref])
-            if proc.returncode != 0:
-                raise GatewayError(
-                    f"orchestrator image pull failed: {proc.stderr.strip()}")
             return
         argv = ["docker", "build", "-t", self.image_ref,
                 "-f", str(self._repo_root / self._dockerfile),

bot_bottle/backend/firecracker/consolidated_launch.py

@@ -26,22 +26,19 @@ The TAP slot allocation, rootfs build, and VM boot are the caller's job.
 
 from __future__ import annotations
 
-import json
 import subprocess
 from dataclasses import dataclass
 from pathlib import Path
 
 from ...egress import EgressPlan
 from ...git_gate import GitGatePlan
-from ...log import info
+from ...orchestrator.client import OrchestratorClient
-from ...orchestrator.client import OrchestratorClient, OrchestratorClientError
 from ...orchestrator.lifecycle import (
     OrchestratorStartError,  # re-exported so callers can catch it
 )
-from ...orchestrator.reprovision import reprovision_bottles
 from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
 from ..provision_bottle import deprovision_bottle, provision_bottle
-from . import cleanup, util
+from . import util
 from .gateway import FirecrackerGateway
 from .infra import FirecrackerInfraService
 
@@ -64,17 +61,6 @@ class LaunchContext:
     env_var_secret: str = ""  # encryption key injected into the agent's env
 
 
-def _guest_ip_from_config(config_path: Path) -> str:
-    """Read the kernel's configured guest IP from a Firecracker config."""
-    try:
-        config = json.loads(config_path.read_text())
-        args = config["boot-source"]["boot_args"]
-        ip_arg = next(part for part in args.split() if part.startswith("ip="))
-        return ip_arg.removeprefix("ip=").split(":", 1)[0]
-    except (OSError, ValueError, KeyError, TypeError, StopIteration):
-        return ""
-
-
 def persist_env_var_secret(private_key: Path, guest_ip: str, secret: str) -> None:
     """Mirror the exec-time key into guest tmpfs for restart recovery."""
     proc = subprocess.run(
@@ -89,29 +75,6 @@ def persist_env_var_secret(private_key: Path, guest_ip: str, secret: str) -> Non
         )
 
 
-def _reprovision_running_bottles(client: OrchestratorClient) -> None:
-    """Read keys from live agent VMs and restore the restarted gateway."""
-    try:
-        secrets_by_ip: dict[str, str] = {}
-        for run_dir in cleanup.live_run_dirs():
-            guest_ip = _guest_ip_from_config(run_dir / "config.json")
-            private_key = run_dir / "bottle_id_ed25519"
-            if not guest_ip or not private_key.is_file():
-                continue
-            proc = subprocess.run(
-                util.ssh_base_argv(private_key, guest_ip)
-                + [f"cat {_ENV_VAR_SECRET_PATH}"],
-                capture_output=True, text=True, check=False,
-            )
-            if proc.returncode == 0 and proc.stdout.strip():
-                secrets_by_ip[guest_ip] = proc.stdout.strip()
-        count = reprovision_bottles(client, secrets_by_ip)
-        if count:
-            info(f"reprovisioned egress tokens for {count} Firecracker bottle(s)")
-    except (OSError, OrchestratorClientError) as exc:
-        info(f"egress token reprovision skipped: {exc}")
-
-
 def launch_consolidated(
     egress_plan: EgressPlan,
     git_gate_plan: GitGatePlan,
@@ -126,7 +89,6 @@ def launch_consolidated(
     service = FirecrackerInfraService()
     url = service.ensure_running()
     client = OrchestratorClient(url)
-    _reprovision_running_bottles(client)
 
     # Read the gateway's provisioning transport + CA off the Gateway service.
     gateway = service.gateway()

bot_bottle/backend/firecracker/image_builder.py

@@ -19,7 +19,6 @@ from __future__ import annotations
 import fcntl
 import hashlib
 import os
-import re
 import shlex
 import shutil
 import subprocess
@@ -42,72 +41,19 @@ _BUILD_TIMEOUT_SECONDS = 900.0
 
 
 def _dockerfile_hash(dockerfile: Path) -> str:
-    """The Dockerfile's content hash. Agent Dockerfiles mostly COPY nothing
+    """The Dockerfile's content hash. The shipped agent Dockerfiles COPY
-    from the build context, so their text nearly determines the built image;
+    nothing from the build context (see .dockerignore), so their content fully
-    any files they *do* COPY are folded into `_rootfs_digest` (so a changed
+    determines the built image; a Dockerfile that adds COPY will want the
-    input busts the cache) and shipped to the VM-side context by
+    context folded in here too."""
-    `_send_build_context`."""
     return hashlib.sha256(dockerfile.read_bytes()).hexdigest()[:16]
 
 
-def _context_copy_sources(dockerfile: Path) -> list[str]:
-    """The build-context-relative paths a Dockerfile ``COPY``s in.
-
-    Agent Dockerfiles are meant to COPY nothing from the context (the VM-side
-    build ships only the Dockerfile), but one may pin an input by COPYing a
-    committed file — a checksum list, an npm lockfile. Return those source
-    paths so the builder can both ship them to the VM context and fold them
-    into the cache key. ``COPY --from=<stage>`` reads a build stage, not the
-    context, so it is excluded; the JSON/exec COPY form is unused by the
-    shipped images and is skipped rather than mis-parsed."""
-    joined = re.sub(r"\\\n", " ", dockerfile.read_text(encoding="utf-8"))
-    sources: list[str] = []
-    for line in joined.splitlines():
-        stripped = line.strip()
-        if not re.match(r"(?i)^COPY\s", stripped):
-            continue
-        tokens = stripped.split()[1:]
-        if any(t.startswith("--from=") for t in tokens):
-            continue
-        args = [t for t in tokens if not t.startswith("--")]
-        if len(args) < 2 or args[0].startswith("["):
-            continue
-        sources.extend(args[:-1])
-    return sources
-
-
-def _context_files(dockerfile: Path) -> list[tuple[str, Path]]:
-    """``(context-relative path, host path)`` for every existing file a
-    Dockerfile COPYs from the build root — globs expanded, sorted, de-duped.
-    Absolute or traversing (`..`) sources are dropped: the shipped context
-    only ever mirrors files under the build root."""
-    root = resources.build_root()
-    resolved: dict[str, Path] = {}
-    for src in _context_copy_sources(dockerfile):
-        if src.startswith("/") or ".." in Path(src).parts:
-            continue
-        if any(ch in src for ch in "*?["):
-            matches = [p for p in root.glob(src) if p.is_file()]
-        else:
-            candidate = root / src
-            if candidate.is_dir():
-                matches = [path for path in candidate.rglob("*") if path.is_file()]
-            else:
-                matches = [candidate] if candidate.is_file() else []
-        for path in matches:
-            resolved[str(path.relative_to(root))] = path
-    return sorted(resolved.items())
-
-
 def _rootfs_digest(dockerfile: Path) -> str:
-    """Cache key for the built AND boot-injected agent rootfs. Its inputs are
+    """Cache key for the built AND boot-injected agent rootfs. Two inputs
-    the Dockerfile (the image), the centralized build args, the guest init
+    determine the on-disk rootfs: the Dockerfile (the image) and the guest init
-    injected into it (`util._GUEST_INIT`), and the content of any files the
+    injected into it (`util._GUEST_INIT`). Folding the init in means a fix to
-    Dockerfile COPYs from the build context. Folding the init in means a fix to
     it — e.g. making /tmp world-writable — busts the cache instead of silently
-    reusing a stale rootfs built with the old init; folding the COPYed context
+    reusing a stale rootfs built with the old init."""
-    files in means a repinned input (e.g. a changed checksum list) rebuilds
-    rather than reusing a rootfs baked from the old bytes."""
     h = hashlib.sha256()
     h.update(_dockerfile_hash(dockerfile).encode())
     h.update(b"\0")
@@ -117,11 +63,6 @@ def _rootfs_digest(dockerfile: Path) -> str:
         h.update(value.encode())
         h.update(b"\0")
     h.update(util._GUEST_INIT.encode())
-    for rel, path in _context_files(dockerfile):
-        h.update(b"\0")
-        h.update(rel.encode())
-        h.update(b"\0")
-        h.update(path.read_bytes())
     return h.hexdigest()[:16]
 
 
@@ -131,46 +72,6 @@ def cached_agent_rootfs_dir(dockerfile: Path) -> Path | None:
     return base if (base / ".bb-ready").is_file() else None
 
 
-def _image_rootfs_digest(image: str) -> str:
-    h = hashlib.sha256()
-    h.update(image.encode())
-    h.update(b"\0")
-    h.update(util._GUEST_INIT.encode())
-    return h.hexdigest()[:16]
-
-
-def cached_agent_image_rootfs_dir(image: str) -> Path | None:
-    """Return a ready rootfs exported from an immutable OCI image."""
-    base = util.cache_dir() / "rootfs" / f"agent-image-{_image_rootfs_digest(image)}"
-    return base if (base / ".bb-ready").is_file() else None
-
-
-def acquire_agent_image_rootfs_dir(
-    image: str, *, smoke_test: tuple[str, ...] = (),
-) -> Path:
-    """Pull a digest-pinned image in the infra VM and export its rootfs."""
-    if "@sha256:" not in image:
-        die(f"prebuilt Firecracker agent image is not digest-pinned: {image}")
-    digest = _image_rootfs_digest(image)
-    base = util.cache_dir() / "rootfs" / f"agent-image-{digest}"
-    cached = cached_agent_image_rootfs_dir(image)
-    if cached is not None:
-        info(f"using cached agent rootfs {cached.name}")
-        return cached
-    with _build_lock():
-        if (base / ".bb-ready").is_file():
-            return base
-        staging = util.cache_dir() / "rootfs" / f".pulling-{digest}"
-        shutil.rmtree(staging, ignore_errors=True)
-        staging.mkdir(parents=True)
-        _pull_in_infra(image, staging, smoke_test, digest)
-        util.inject_guest_boot(staging)
-        (staging / ".bb-ready").write_text("ok\n")
-        shutil.rmtree(base, ignore_errors=True)
-        os.rename(staging, base)
-    return base
-
-
 def build_agent_rootfs_dir(
     dockerfile: Path, *, image_tag: str, smoke_test: tuple[str, ...] = (),
 ) -> Path:
@@ -253,7 +154,6 @@ def _build_in_infra(
         if prep.returncode != 0:
             die(f"preparing build dir in the infra VM failed: {prep.stderr.strip()}")
         _send_dockerfile(key, ip, dockerfile, ctx)
-        _send_build_context(key, ip, dockerfile, ctx)
         _buildah_build(
             key,
             ip,
@@ -267,43 +167,6 @@ def _build_in_infra(
         _cleanup()
 
 
-def _pull_in_infra(
-    image: str, base: Path, smoke_test: tuple[str, ...], digest: str,
-) -> None:
-    """Pull and export a published agent image inside the orchestrator VM."""
-    service = FirecrackerInfraService()
-    service.ensure_running()
-    key, ip = service.orchestrator().ssh_target()
-    tag = f"bot-bottle-agent-pull-{digest}"
-    smoke_ctr, export_ctr = f"{tag}-smoke", f"{tag}-export"
-    quoted_image = shlex.quote(image)
-
-    def cleanup() -> None:
-        _ssh(
-            key,
-            ip,
-            f"buildah rm {smoke_ctr} {export_ctr} >/dev/null 2>&1; "
-            f"buildah rmi {_STORE_FLAG} {tag} >/dev/null 2>&1",
-            timeout=60,
-        )
-
-    cleanup()
-    try:
-        result = _ssh(
-            key,
-            ip,
-            f"buildah pull {_STORE_FLAG} {quoted_image} && "
-            f"buildah tag {_STORE_FLAG} {quoted_image} {tag}",
-            timeout=_BUILD_TIMEOUT_SECONDS,
-        )
-        if result.returncode != 0:
-            die(f"pulling pinned agent image failed: {result.stderr.strip()}")
-        _smoke_test(key, ip, tag, smoke_ctr, smoke_test)
-        _stream_rootfs(key, ip, tag, export_ctr, base)
-    finally:
-        cleanup()
-
-
 def _ssh(private_key: Path, guest_ip: str, script: str,
          *, timeout: float = 60.0) -> subprocess.CompletedProcess[str]:
     return subprocess.run(
@@ -334,40 +197,6 @@ def _send_dockerfile(private_key: Path, guest_ip: str, dockerfile: Path, ctx: st
             f"{proc.stderr.decode(errors='replace').strip()}")
 
 
-def _send_build_context(private_key: Path, guest_ip: str, dockerfile: Path, ctx: str) -> None:
-    """Ship the files ``dockerfile`` COPYs from the build root into the infra
-    VM's ``{ctx}/ctx``, preserving their build-root-relative paths.
-
-    Usually a no-op — agent Dockerfiles COPY nothing — so `{ctx}/ctx` stays the
-    empty context the build otherwise runs against. It exists so a Dockerfile
-    that pins an input by COPYing a committed file (a checksum list, an npm
-    lockfile) still finds that file in the VM-side context. Streamed as a tar
-    so directories and multiple files land in one round trip."""
-    files = _context_files(dockerfile)
-    if not files:
-        return
-    root = resources.build_root()
-    rels = [rel for rel, _ in files]
-    tar = subprocess.Popen(
-        ["tar", "-C", str(root), "-cf", "-", "--", *rels],
-        stdout=subprocess.PIPE,
-    )
-    try:
-        proc = subprocess.run(
-            util.ssh_base_argv(private_key, guest_ip) + [f"tar -C {ctx}/ctx -xf -"],
-            stdin=tar.stdout, capture_output=True, timeout=120, check=False,
-        )
-    finally:
-        if tar.stdout is not None:
-            tar.stdout.close()
-        tar.wait()
-    if tar.returncode != 0:
-        die(f"packing the agent build context failed (tar exit {tar.returncode})")
-    if proc.returncode != 0:
-        die("sending the agent build context to the infra VM failed: "
-            f"{proc.stderr.decode(errors='replace').strip() or '<no stderr>'}")
-
-
 def _buildah_build(
     private_key: Path,
     guest_ip: str,

bot_bottle/backend/firecracker/infra.py

@@ -17,6 +17,7 @@ from ...orchestrator.lifecycle import DEFAULT_STARTUP_TIMEOUT_SECONDS
 from . import infra_vm
 from .gateway import FirecrackerGateway
 from .orchestrator import FirecrackerOrchestrator
+from .reconcile import attach_bottled_agents_to_gateway
 
 
 class FirecrackerInfraService(InfraService):
@@ -61,15 +62,17 @@ class FirecrackerInfraService(InfraService):
                 return url
             # Clear stale/hung/OUTDATED VMs holding either link before booting fresh.
             infra_vm.stop()
-            infra_vm.ensure_available()
+            infra_vm.ensure_built()
             # Orchestrator first — the gateway daemons reach the control plane at
             # startup. Each service owns its own boot; the orchestrator (which
             # holds the signing key) mints the role-scoped `gateway` JWT for the
             # gateway, which never sees the key (#469).
             orchestrator.ensure_running(startup_timeout=startup_timeout)
-            self.gateway().connect_to_orchestrator(
+            gateway = self.gateway()
+            gateway.connect_to_orchestrator(
                 orchestrator.gateway_url(), orchestrator.mint_gateway_token())
             infra_vm.record_booted_version(want)
+            attach_bottled_agents_to_gateway(url, gateway)
             return url
 
     def stop(self) -> None:

bot_bottle/backend/firecracker/infra_artifact.py

@@ -195,9 +195,7 @@ def _sha256_file(path: Path) -> str:
     return h.hexdigest()
 
 
-def ensure_artifact_gz(
+def ensure_artifact_gz(version: str, *, role: str) -> Path:
-    version: str, *, role: str, expected_sha256: str | None = None,
-) -> Path:
     """The verified, cached `rootfs.ext4.gz` for `role` at `version` —
     downloading it (and its `.sha256`) once, then reusing it. Fail-closed on a
     checksum mismatch: the partial is removed and we die rather than boot an
@@ -224,11 +222,6 @@ def ensure_artifact_gz(
         if not gz.is_file() or not sha.is_file():
             die(f"infra candidate bundle is incomplete: {root}")
         expected = sha.read_text().split()[0].strip().lower()
-        if expected_sha256 is not None and expected != expected_sha256:
-            die(
-                f"infra candidate packaged checksum mismatch ({role}) for {version}:\n"
-                f"    packaged {expected_sha256}\n    candidate {expected}"
-            )
         actual = _sha256_file(gz)
         if actual != expected:
             die(
@@ -242,13 +235,7 @@ def ensure_artifact_gz(
     gz = root / _GZ_NAME
     ok = root / ".verified"
     if gz.is_file() and ok.is_file():
-        actual = _sha256_file(gz)
+        return gz
-        recorded = ok.read_text(encoding="utf-8").strip()
-        wanted = expected_sha256 or recorded
-        if actual == recorded == wanted:
-            return gz
-        gz.unlink(missing_ok=True)
-        ok.unlink(missing_ok=True)
 
     info(f"pulling infra rootfs artifact {_package(role)}/{version}")
     _download(artifact_url(version, _GZ_NAME, role=role), gz)
@@ -256,14 +243,6 @@ def ensure_artifact_gz(
     _download(artifact_url(version, _SHA_NAME, role=role), sha)
 
     expected = sha.read_text().split()[0].strip().lower()
-    if expected_sha256 is not None and expected != expected_sha256:
-        gz.unlink(missing_ok=True)
-        sha.unlink(missing_ok=True)
-        die(
-            f"infra artifact published checksum mismatch ({role}) for {version}:\n"
-            f"    packaged  {expected_sha256}\n"
-            f"    published {expected}"
-        )
     actual = _sha256_file(gz)
     if actual != expected:
         gz.unlink(missing_ok=True)
@@ -274,22 +253,15 @@ def ensure_artifact_gz(
             f"    actual   {actual}\n"
             f"    refusing to boot an unverified rootfs."
         )
-    ok.write_text(actual + "\n")
+    ok.write_text("ok\n")
     return gz
 
 
-def materialize_ext4(
+def materialize_ext4(version: str, dest: Path, *, role: str) -> None:
-    version: str,
-    dest: Path,
-    *,
-    role: str,
-    expected_sha256: str | None = None,
-) -> None:
     """Ensure the verified `role` artifact is cached, then gunzip it to `dest` —
     a fresh, writable per-boot rootfs (the VM mutates it; the cached `.gz` stays
     pristine). Atomic via a `.part` sibling."""
-    gz = ensure_artifact_gz(
+    gz = ensure_artifact_gz(version, role=role)
-        version, role=role, expected_sha256=expected_sha256)
     tmp = dest.with_suffix(dest.suffix + ".part")
     info(f"expanding {role} infra rootfs -> {dest}")
     with gzip.open(gz, "rb") as src, open(tmp, "wb") as out:

bot_bottle/backend/firecracker/infra_vm.py

@@ -43,7 +43,6 @@ from pathlib import Path
 from typing import Generator
 
 from ... import resources
-from ... import release_manifest
 from ...log import die, info
 from ..docker import util as docker_mod
 from . import firecracker_vm, infra_artifact, netpool, util
@@ -109,18 +108,6 @@ def _role_version(role: str) -> str:
     return infra_artifact.infra_artifact_version(role_init(role), role)
 
 
-def _role_release(role: str) -> tuple[str, str | None]:
-    manifest = release_manifest.packaged_manifest()
-    if manifest is None or release_manifest.local_build_requested():
-        return _role_version(role), None
-    artifact = (
-        manifest.firecracker_orchestrator
-        if role == "orchestrator"
-        else manifest.firecracker_gateway
-    )
-    return artifact.version, artifact.sha256
-
-
 def ensure_built() -> None:
     """Ensure both infra rootfs artifacts are available before boot.
 
@@ -130,16 +117,11 @@ def ensure_built() -> None:
     `BOT_BOTTLE_INFRA_BUILD=local` instead builds the images from source with
     host Docker (the orchestrator-fc image is `FROM` the orchestrator image, so
     it must exist first) — for iterating on the Dockerfiles."""
-    if release_manifest.local_build_requested():
+    if infra_artifact.local_build_requested():
         build_infra_images_with_docker()
         return
     for role in infra_artifact.ROLES:
-        version, expected = _role_release(role)
+        infra_artifact.ensure_artifact_gz(_role_version(role), role=role)
-        infra_artifact.ensure_artifact_gz(
-            version, role=role, expected_sha256=expected)
-
-
-ensure_available = ensure_built
 
 
 def build_infra_images_with_docker() -> None:
@@ -232,9 +214,7 @@ def boot_vm(
     else:
         # Prebuilt artifact already carries the role's build slack; expand it to
         # a fresh writable rootfs for this boot.
-        version, expected = _role_release(role)
+        infra_artifact.materialize_ext4(_role_version(role), rootfs, role=role)
-        infra_artifact.materialize_ext4(
-            version, rootfs, role=role, expected_sha256=expected)
     private_key, pubkey = _stable_keypair()
 
     info(f"booting {role} VM on {slot.iface} (guest {slot.guest_ip})")
@@ -284,8 +264,7 @@ def _version_file() -> Path:
 def expected_version() -> str:
     """The combined marker for the running pair: both per-plane artifact
     versions, so a change to either rootfs dislodges the adopted pair."""
-    return " ".join(
+    return " ".join(f"{role}={_role_version(role)}" for role in infra_artifact.ROLES)
-        f"{role}={_role_release(role)[0]}" for role in infra_artifact.ROLES)
 
 
 def adoptable(key: Path, url: str, want: str) -> bool:

bot_bottle/backend/firecracker/launch.py

@@ -228,13 +228,8 @@ def build_or_load_agent_base(plan: FirecrackerBottlePlan) -> Path:
         info(f"resuming from committed rootfs {committed_tar}")
         return util.build_committed_rootfs_dir(committed_tar)
     dockerfile = Path(plan.dockerfile_path)
-    prebuilt = "@sha256:" in plan.image
     if plan.spec.image_policy == "cached":
-        cached = (
+        cached = image_builder.cached_agent_rootfs_dir(dockerfile)
-            image_builder.cached_agent_image_rootfs_dir(plan.image)
-            if prebuilt
-            else image_builder.cached_agent_rootfs_dir(dockerfile)
-        )
         if cached is None:
             die(
                 f"cached agent rootfs for {plan.image!r} not found; "
@@ -242,11 +237,6 @@ def build_or_load_agent_base(plan: FirecrackerBottlePlan) -> Path:
             )
         info(f"using cached agent rootfs {cached.name}")
         return cached
-    if prebuilt:
-        return image_builder.acquire_agent_image_rootfs_dir(
-            plan.image,
-            smoke_test=runtime_for(plan.agent_provider_template).smoke_test,
-        )
     return image_builder.build_agent_rootfs_dir(
         dockerfile,
         image_tag=plan.image,
@@ -263,11 +253,7 @@ def stale_checks(plan: FirecrackerBottlePlan) -> None:
     if committed and committed_tar.is_file():
         check_stale_path(f"agent rootfs {committed_tar}", committed_tar)
         return
-    cached = (
+    cached = image_builder.cached_agent_rootfs_dir(Path(plan.dockerfile_path))
-        image_builder.cached_agent_image_rootfs_dir(plan.image)
-        if "@sha256:" in plan.image
-        else image_builder.cached_agent_rootfs_dir(Path(plan.dockerfile_path))
-    )
     if cached is not None:
         check_stale_path(f"agent rootfs {cached}", cached / ".bb-ready")
 

bot_bottle/backend/firecracker/reconcile.py

@@ -0,0 +1,199 @@
+"""Bring-up reconcile: re-attach all running agent VMs to a freshly-booted gateway.
+
+Called from the cold-boot branch of `FirecrackerInfraService.ensure_running()`
+after `gateway.connect_to_orchestrator()` completes. Restores the three
+gateway-dependent services for every live bottle:
+
+  - **CA** — push the current (freshly-minted) gateway CA to each agent's
+    trust store and run `update-ca-certificates`, so the agent trusts the
+    new CA on its next egress call.
+  - **git-gate** — re-provision each bottle's bare repos and per-repo creds
+    in the gateway, rebuilt from the persisted host-side state dir.
+  - **egress tokens** — read the agent's `ENV_VAR_SECRET` and feed
+    `reprovision_bottles` to restore the orchestrator's in-memory tokens.
+
+Per-bottle failures are caught, logged, and skipped so one unreachable VM
+does not block the others or the gateway coming up (PRD 0081).
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+from pathlib import Path
+
+from ...bottle_state import git_gate_state_dir
+from ...gateway import GatewayError
+from ...gateway.git_gate.render import GitGateUpstream
+from ...git_gate.plan import GitGatePlan
+from ...log import info
+from ...orchestrator.client import OrchestratorClient, OrchestratorClientError
+from ...orchestrator.reprovision import reprovision_bottles
+from ..provision_gateway import provision_git_gate
+from ..util import AGENT_CA_PATH
+from . import cleanup, util
+from .gateway import FirecrackerGateway
+
+# Where persist_env_var_secret writes the key on the agent VM (matches
+# consolidated_launch._ENV_VAR_SECRET_PATH — duplicated to avoid a
+# circular import through infra.py).
+_ENV_VAR_SECRET_PATH = "/run/bot-bottle/env-var-secret"
+
+
+def _guest_ip_from_config(config_path: Path) -> str:
+    """Read the agent VM's guest IP from its Firecracker config file."""
+    try:
+        config = json.loads(config_path.read_text())
+        args = config["boot-source"]["boot_args"]
+        ip_arg = next(p for p in args.split() if p.startswith("ip="))
+        return ip_arg.removeprefix("ip=").split(":", 1)[0]
+    except (OSError, ValueError, KeyError, TypeError, StopIteration):
+        return ""
+
+
+def attach_bottled_agents_to_gateway(
+    orchestrator_url: str, gateway: FirecrackerGateway,
+) -> None:
+    """Reconcile all live agent VMs against the freshly-booted gateway.
+
+    Fetches the new CA, lists registered bottles, then for each live run dir
+    pushes the CA, re-provisions git-gate, and restores egress tokens. Each
+    per-bottle step is wrapped so a failure is logged and skipped."""
+    try:
+        ca_pem = gateway.ca_cert_pem()
+    except GatewayError as exc:
+        info(f"bring-up reconcile: could not fetch gateway CA, skipping: {exc}")
+        return
+
+    client = OrchestratorClient(orchestrator_url)
+    try:
+        bottles = client.list_bottles()
+    except OrchestratorClientError as exc:
+        info(f"bring-up reconcile: could not list bottles, skipping: {exc}")
+        return
+
+    source_ip_to_bottle_id: dict[str, str] = {}
+    for b in bottles:
+        source_ip = b.get("source_ip")
+        bottle_id = b.get("bottle_id")
+        if isinstance(source_ip, str) and isinstance(bottle_id, str):
+            source_ip_to_bottle_id[source_ip] = bottle_id
+
+    transport = gateway.provisioning_transport()
+    secrets_by_ip: dict[str, str] = {}
+
+    for run_dir in cleanup.live_run_dirs():
+        slug = run_dir.name
+        guest_ip = _guest_ip_from_config(run_dir / "config.json")
+        private_key = run_dir / "bottle_id_ed25519"
+        if not guest_ip or not private_key.is_file():
+            continue
+
+        # CA: push the gateway's current certificate to the agent's trust store.
+        try:
+            _push_ca(private_key, guest_ip, ca_pem)
+        except Exception as exc:
+            info(f"bring-up reconcile: CA push to {slug!r} failed: {exc}")
+
+        # git-gate: re-provision repos + creds from the persisted state dir.
+        bottle_id = source_ip_to_bottle_id.get(guest_ip)
+        if bottle_id:
+            try:
+                _reprovision_git_gate(transport, bottle_id, slug)
+            except Exception as exc:
+                info(f"bring-up reconcile: git-gate for {slug!r} failed: {exc}")
+
+        # egress tokens: read ENV_VAR_SECRET from the agent's tmpfs.
+        proc = subprocess.run(
+            util.ssh_base_argv(private_key, guest_ip) + [f"cat {_ENV_VAR_SECRET_PATH}"],
+            capture_output=True, text=True, check=False,
+        )
+        if proc.returncode == 0 and proc.stdout.strip():
+            secrets_by_ip[guest_ip] = proc.stdout.strip()
+
+    # Restore in-memory egress tokens for all bottles that exposed a key.
+    if secrets_by_ip:
+        try:
+            count = reprovision_bottles(client, secrets_by_ip)
+            if count:
+                info(
+                    f"bring-up reconcile: restored egress tokens for {count} bottle(s)"
+                )
+        except OrchestratorClientError as exc:
+            info(f"bring-up reconcile: egress token restore failed: {exc}")
+
+
+def _push_ca(private_key: Path, guest_ip: str, ca_pem: str) -> None:
+    """SSH the gateway CA PEM into the agent VM and update its trust store.
+
+    Runs as the SSH root user (the agent VM's dropbear accepts root). Each
+    step uses check=True so a failure raises and the caller's except clause
+    logs and continues."""
+    ssh = util.ssh_base_argv(private_key, guest_ip)
+    # mkdir is idempotent; rootfs builds may not preserve the target dir.
+    subprocess.run(
+        ssh + ["mkdir -p /usr/local/share/ca-certificates"],
+        capture_output=True, check=True,
+    )
+    subprocess.run(
+        ssh + [f"cat > {AGENT_CA_PATH}"],
+        input=ca_pem, text=True, capture_output=True, check=True,
+    )
+    subprocess.run(
+        ssh + [f"chmod 644 {AGENT_CA_PATH} && update-ca-certificates"],
+        capture_output=True, check=True,
+    )
+
+
+def _reprovision_git_gate(
+    transport: object, bottle_id: str, slug: str,
+) -> None:
+    """Re-provision one bottle's git-gate repos and creds from its state dir.
+
+    Reads `upstreams.json` (written by `provision_git_gate_dynamic_keys` at
+    launch) to reconstruct the GitGateUpstream table, then calls
+    `provision_git_gate` which places the credential files and inits the bare
+    repos in the gateway. No-op when there is no `upstreams.json` (the bottle
+    has no git upstreams)."""
+    state_dir = git_gate_state_dir(slug)
+    upstreams_file = state_dir / "upstreams.json"
+    if not upstreams_file.exists():
+        return
+
+    raw = json.loads(upstreams_file.read_text())
+    upstreams: list[GitGateUpstream] = []
+    for u in raw:
+        name = u["name"]
+        # The key in the state dir is preferred: gitea deploy keys are written
+        # there by provision_git_gate_dynamic_keys; static keys keep their
+        # original manifest path.
+        key_in_state = state_dir / f"{name}-key"
+        identity_file = (
+            str(key_in_state) if key_in_state.is_file()
+            else u.get("identity_file", "")
+        )
+        known_hosts = state_dir / f"{name}-known_hosts"
+        upstreams.append(GitGateUpstream(
+            name=name,
+            upstream_url=u["upstream_url"],
+            upstream_host=u.get("upstream_host", ""),
+            upstream_port=u.get("upstream_port", ""),
+            identity_file=identity_file,
+            known_host_key=u.get("known_host_key", ""),
+            known_hosts_file=known_hosts if known_hosts.is_file() else Path(),
+        ))
+
+    if not upstreams:
+        return
+
+    plan = GitGatePlan(
+        slug=slug,
+        entrypoint_script=state_dir / "git_gate_entrypoint.sh",
+        hook_script=state_dir / "git_gate_pre_receive.sh",
+        access_hook_script=state_dir / "git_gate_access_hook.sh",
+        upstreams=tuple(upstreams),
+    )
+    provision_git_gate(transport, bottle_id, plan)  # type: ignore[arg-type]
+
+
+__all__ = ["attach_bottled_agents_to_gateway"]

bot_bottle/backend/firecracker/setup.py

@@ -20,7 +20,6 @@ import subprocess
 import sys
 from pathlib import Path
 
-from ... import invocation
 from ... import resources
 from . import netpool
 from . import util
@@ -180,11 +179,9 @@ def _setup_systemd() -> None:
         f"sudo systemctl daemon-reload\n"
         f"sudo systemctl enable --now {netpool.SYSTEMD_UNIT}\n"
     )
-    # Absolute path, not `sudo bot-bottle`: sudo's secure_path drops
-    # ~/.local/bin, where both pipx and install.sh put the entry point.
     sys.stderr.write(
-        f"\n(Or re-run this as root to install it directly:\n"
+        f"\n(Or re-run this as root to install it directly: "
-        f"    {invocation.sudo_command('backend', 'setup', '--backend=firecracker')})\n"
+        f"sudo bot-bottle backend setup --backend=firecracker)\n"
     )
 
 

bot_bottle/backend/macos_container/gateway.py

@@ -84,7 +84,6 @@ class MacosGateway(Gateway):
         egress_network: str = GATEWAY_EGRESS_NETWORK,
         control_network: str = CONTROL_NETWORK,
         repo_root: Path | None = None,
-        local_build: bool = True,
     ) -> None:
         self.image_ref = image_ref
         self.name = name
@@ -94,7 +93,6 @@ class MacosGateway(Gateway):
         # Build context: the repo root in a checkout, a staged copy from the
         # installed wheel otherwise (bot_bottle.resources).
         self._repo_root = repo_root if repo_root is not None else resources.build_root()
-        self._local_build = local_build
         # Set by `connect_to_orchestrator`: the URL the daemons resolve policy
         # against + the pre-minted `gateway` token they present. The gateway
         # never mints, so it never holds the signing key (#469).
@@ -103,11 +101,8 @@ class MacosGateway(Gateway):
 
     def ensure_built(self) -> None:
         """Build the data-plane image from `Dockerfile.gateway`."""
-        if self._local_build:
+        container_mod.build_image(
-            container_mod.build_image(
+            self.image_ref, str(self._repo_root), dockerfile="Dockerfile.gateway")
-                self.image_ref, str(self._repo_root), dockerfile="Dockerfile.gateway")
-        else:
-            container_mod.pull_image(self.image_ref)
 
     def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None:
         """Bind the gateway to this orchestrator and (re)start it, dual-homed on

bot_bottle/backend/macos_container/infra.py

@@ -25,7 +25,6 @@ from __future__ import annotations
 from pathlib import Path
 
 from ... import resources
-from ... import release_manifest
 from ...orchestrator.lifecycle import (
     DEFAULT_PORT,
     DEFAULT_STARTUP_TIMEOUT_SECONDS,
@@ -87,14 +86,6 @@ class MacosInfraService(InfraService):
         self._orchestrator_name = orchestrator_name
         self._gateway_name = gateway_name
         self._db_volume = db_volume
-        resolved_orchestrator, orchestrator_local = release_manifest.oci_image(
-            "orchestrator", orchestrator_image)
-        resolved_gateway, gateway_local = release_manifest.oci_image(
-            "gateway", gateway_image)
-        self.orchestrator_image = resolved_orchestrator
-        self.gateway_image = resolved_gateway
-        self._orchestrator_local = orchestrator_local
-        self._gateway_local = gateway_local
 
     def orchestrator(self) -> MacosOrchestrator:
         """The control-plane service on the host-only control network. Cheap to
@@ -107,7 +98,6 @@ class MacosInfraService(InfraService):
             control_network=self.control_network,
             repo_root=self._repo_root,
             db_volume=self._db_volume,
-            local_build=self._orchestrator_local,
         )
 
     def gateway(self) -> MacosGateway:
@@ -122,7 +112,6 @@ class MacosInfraService(InfraService):
             egress_network=self.egress_network,
             control_network=self.control_network,
             repo_root=self._repo_root,
-            local_build=self._gateway_local,
         )
 
     def ensure_running(
@@ -138,8 +127,8 @@ class MacosInfraService(InfraService):
 
         orchestrator = self.orchestrator()
         gateway = self.gateway()
-        orchestrator.ensure_available()
+        orchestrator.ensure_built()
-        gateway.ensure_available()
+        gateway.ensure_built()
 
         orchestrator.ensure_running(startup_timeout=startup_timeout)
 

bot_bottle/backend/macos_container/launch.py

@@ -38,7 +38,6 @@ import subprocess
 from contextlib import ExitStack, contextmanager
 from typing import Callable, Generator
 
-from ...agent_provider import runtime_for
 from ...bottle_state import (
     egress_state_dir,
     git_gate_state_dir,
@@ -94,14 +93,7 @@ def _agent_image(plan: MacosContainerBottlePlan) -> str:
             )
         info(f"using cached agent image {plan.image!r}")
         return plan.image
-    if "@sha256:" in plan.image:
+    container_mod.build_image(plan.image, str(resources.build_root()), dockerfile=plan.dockerfile_path)
-        container_mod.pull_image(plan.image)
-        container_mod.verify_agent_image(
-            plan.image, runtime_for(plan.agent_provider_template).smoke_test,
-        )
-        return plan.image
-    container_mod.build_image(
-        plan.image, str(resources.build_root()), dockerfile=plan.dockerfile_path)
     return plan.image
 
 

bot_bottle/backend/macos_container/orchestrator.py

@@ -63,7 +63,6 @@ class MacosOrchestrator(Orchestrator):
         control_network: str = CONTROL_NETWORK,
         repo_root: Path | None = None,
         db_volume: str = ORCHESTRATOR_DB_VOLUME,
-        local_build: bool = True,
     ) -> None:
         self.image_ref = image_ref
         self.name = name
@@ -74,7 +73,6 @@ class MacosOrchestrator(Orchestrator):
         # staged copy from the installed wheel otherwise (bot_bottle.resources).
         self._repo_root = repo_root if repo_root is not None else resources.build_root()
         self._db_volume = db_volume
-        self._local_build = local_build
 
     def url(self) -> str:
         """The orchestrator's control-network address (host CLI + registration),
@@ -116,12 +114,8 @@ class MacosOrchestrator(Orchestrator):
         """Build the control-plane image. The source is bind-mounted so a code
         change takes effect without a rebuild; the image still carries the
         package for its entrypoint."""
-        if self._local_build:
+        container_mod.build_image(
-            container_mod.build_image(
+            self.image_ref, str(self._repo_root), dockerfile="Dockerfile.orchestrator")
-                self.image_ref, str(self._repo_root),
-                dockerfile="Dockerfile.orchestrator")
-        else:
-            container_mod.pull_image(self.image_ref)
 
     def ensure_running(
         self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
@@ -152,6 +146,11 @@ class MacosOrchestrator(Orchestrator):
             "--dns", container_mod.dns_server(),
             # Container-only DB volume: exactly one kernel writes bot-bottle.db.
             "--volume", f"{self._db_volume}:{_DB_ROOT_IN_CONTAINER}",
+            # Live control-plane source (a code change takes effect on relaunch).
+            "--mount",
+            container_mod.bind_mount_spec(
+                str(self._repo_root), _SRC_IN_CONTAINER, readonly=True),
+            "--env", f"PYTHONPATH={_SRC_IN_CONTAINER}",
             "--env", f"BOT_BOTTLE_ROOT={_DB_ROOT_IN_CONTAINER}",
             # Detect a real control-plane code change and recreate.
             "--env", f"BOT_BOTTLE_SOURCE_HASH={current_hash}",
@@ -162,14 +161,6 @@ class MacosOrchestrator(Orchestrator):
             # Dockerfile.orchestrator ENTRYPOINT is `-m bot_bottle.orchestrator`.
             "--host", "0.0.0.0", "--port", str(self.port), "--broker", "stub",
         ]
-        if self._local_build:
-            image_index = argv.index(self.image_ref)
-            argv[image_index:image_index] = [
-                "--mount",
-                container_mod.bind_mount_spec(
-                    str(self._repo_root), _SRC_IN_CONTAINER, readonly=True),
-                "--env", f"PYTHONPATH={_SRC_IN_CONTAINER}",
-            ]
         result = container_mod.run_container_argv(
             argv, env={**os.environ, ORCHESTRATOR_TOKEN_ENV: _signing_key})
         if result.returncode != 0:

bot_bottle/backend/macos_container/util.py

@@ -101,19 +101,6 @@ def build_image(
     subprocess.run(args, check=True)
 
 
-def pull_image(ref: str) -> None:
-    """Acquire an immutable OCI reference through Apple Container."""
-    result = subprocess.run(
-        [_CONTAINER, "image", "pull", ref],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
-    if result.returncode != 0:
-        detail = (result.stderr or result.stdout or "").strip()
-        raise RuntimeError(f"container image pull failed for {ref}: {detail}")
-
-
 def verify_agent_image(image: str, argv: tuple[str, ...]) -> None:
     """Run `argv` inside a throwaway container of a freshly built agent
     image and die loudly if it fails, instead of shipping an image

bot_bottle/backend/preparation.py

@@ -9,7 +9,7 @@ backend-specific final resolution.
 from __future__ import annotations
 
 import os
-from dataclasses import dataclass, replace
+from dataclasses import dataclass
 from typing import TYPE_CHECKING, Protocol
 
 from ..agent_provider import AgentProvisionPlan, build_agent_provision_plan, get_provider
@@ -17,7 +17,6 @@ from ..egress import EgressPlan
 from ..env import ResolvedEnv, resolve_env
 from ..git_gate import GitGate, GitGatePlan
 from ..manifest import Manifest
-from ..release_manifest import oci_image
 from ..supervisor.plan import SupervisePlan
 from ..workspace import workspace_plan
 from .resolve_common import (
@@ -113,11 +112,6 @@ class BottlePreparationPlanner:
             color=spec.color,
             provider_settings=provider_config.settings,
         )
-        if not provider_config.dockerfile:
-            image, local = oci_image(
-                f"agent_{provider_config.template}", provision.image)
-            if not local:
-                provision = replace(provision, image=image)
         provision = merge_provision_env_vars(provision)
         return PreparedBottle(
             manifest=manifest,

bot_bottle/gateway/__init__.py

@@ -115,12 +115,10 @@ class Gateway(abc.ABC):
 
     name: str
 
-    def ensure_available(self) -> None:
-        """Acquire the exact image/rootfs selected for this application."""
-        self.ensure_built()
-
     def ensure_built(self) -> None:
-        """Local-build implementation hook retained for backend compatibility."""
+        """Ensure the gateway's image / rootfs exists, building it if needed.
+        Default: nothing to build (e.g. a stub or a pre-pulled image). Call
+        before `connect_to_orchestrator`."""
         return
 
     @abc.abstractmethod

bot_bottle/git_gate/provision.py

@@ -8,6 +8,7 @@ imported (`deploy_key_provisioner`) to keep its cost off the host path.
 
 from __future__ import annotations
 
+import json
 import os
 import dataclasses
 from pathlib import Path
@@ -138,7 +139,32 @@ def provision_git_gate_dynamic_keys(
             if upstream.name not in updated_names:
                 updated.append(upstream)
 
-    return dataclasses.replace(plan, upstreams=tuple(updated))
+    final_plan = dataclasses.replace(plan, upstreams=tuple(updated))
+    _write_upstreams_snapshot(stage_dir, final_plan.upstreams)
+    return final_plan
+
+
+def _write_upstreams_snapshot(
+    stage_dir: Path, upstreams: tuple[GitGateUpstream, ...]
+) -> None:
+    """Persist the fully-resolved upstream table to `upstreams.json` in
+    `stage_dir` so the bring-up reconcile can re-provision git-gate without
+    the manifest. Written after dynamic keys are provisioned so every
+    identity_file is set."""
+    data = [
+        {
+            "name": u.name,
+            "upstream_url": u.upstream_url,
+            "upstream_host": u.upstream_host,
+            "upstream_port": u.upstream_port,
+            "identity_file": u.identity_file,
+            "known_host_key": u.known_host_key,
+        }
+        for u in upstreams
+    ]
+    snapshot = stage_dir / "upstreams.json"
+    snapshot.write_text(json.dumps(data, indent=2))
+    snapshot.chmod(0o600)
 
 
 __all__ = [

bot_bottle/invocation.py

@@ -1,48 +0,0 @@
-"""How to tell a user to re-run this CLI.
-
-`bot-bottle …` is the right thing to print for anything the user runs as
-themselves — it is on their PATH, since that is how they got here.
-
-Under `sudo` it is not. sudo replaces PATH with sudoers' `secure_path`
-(`/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin` on Debian and
-Ubuntu, similar elsewhere), which deliberately excludes user-writable
-directories. Both supported install paths put the entry point in one of those:
-pipx uses `~/.local/bin`, and `install.sh`'s venv fallback symlinks there too.
-So `sudo bot-bottle …` fails with "command not found" for exactly the users who
-followed the documented install, while working for anyone who happened to
-install system-wide — which is why it survives review so easily.
-
-Naming the absolute path sidesteps secure_path entirely.
-"""
-
-from __future__ import annotations
-
-import os
-import shutil
-import sys
-
-
-def self_path() -> str:
-    """Absolute path to this CLI's entry point.
-
-    Falls back to the bare name when the entry point cannot be resolved (an
-    unusual invocation such as `python -m`), because a slightly wrong hint is
-    better than a traceback while reporting an unrelated problem.
-    """
-    argv0 = sys.argv[0] or "bot-bottle"
-    resolved = shutil.which(argv0) or argv0
-    if not os.path.isabs(resolved):
-        if os.path.exists(resolved):
-            resolved = os.path.abspath(resolved)
-        else:
-            return "bot-bottle"
-    return resolved
-
-
-def sudo_command(*args: str) -> str:
-    """A copy-pasteable `sudo …` invocation of this CLI.
-
-    >>> sudo_command("backend", "setup", "--backend=firecracker")
-    'sudo /home/u/.local/bin/bot-bottle backend setup --backend=firecracker'
-    """
-    return " ".join(["sudo", self_path(), *args])

bot_bottle/orchestrator/api.py

@@ -50,6 +50,12 @@ class ReprovisionBody(_StrictModel):
     env_var_secret: StrictStr
 
 
+class SecretBody(_StrictModel):
+    name: StrictStr
+    value: StrictStr
+    env_var_secret: StrictStr
+
+
 class ReconcileBody(_StrictModel):
     live_source_ips: list[StrictStr]
     grace_seconds: float | None = None
@@ -259,6 +265,21 @@ def create_app(orch: OrchestratorCore, *, signing_key: str) -> FastAPI:
             return {"reprovisioned": True}
         raise HTTPException(404, "no stored secrets for this bottle")
 
+    @app.post("/bottles/{bottle_id}/secret")
+    def update_secret(bottle_id: str, body: SecretBody) -> dict[str, object]:
+        # Update ONE egress token for a running bottle in place — the
+        # single-secret form of reprovision, for refreshing a short-lived host
+        # credential (e.g. the Codex access token) without a relaunch. cli-only
+        # (not in _GATEWAY_ROUTES): a bottle must never set its own tokens.
+        if orch.update_agent_secret(
+            bottle_id,
+            _required(body.name, "name"),
+            _required(body.value, "value"),
+            _required(body.env_var_secret, "env_var_secret"),
+        ):
+            return {"updated": True}
+        raise HTTPException(404, "no such bottle")
+
     @app.delete("/bottles/{bottle_id}")
     def teardown(bottle_id: str) -> dict[str, object]:
         if orch.teardown_bottle(bottle_id):

bot_bottle/orchestrator/client.py

@@ -184,6 +184,27 @@ class OrchestratorClient:
             )
         return True
 
+    def update_agent_secret(
+        self, bottle_id: str, name: str, value: str, env_var_secret: str,
+    ) -> bool:
+        """Update ONE egress token for a running bottle in place
+        (`POST /bottles/<id>/secret`) — the single-secret form of
+        `reprovision_gateway`, for pushing a freshly-refreshed host credential
+        into a bottle without a relaunch. Returns True on success, False when the
+        orchestrator doesn't know the bottle (404)."""
+        status, _ = self._request(
+            "POST",
+            f"/bottles/{bottle_id}/secret",
+            {"name": name, "value": value, "env_var_secret": env_var_secret},
+        )
+        if status == 404:
+            return False
+        if not 200 <= status < 300:
+            raise OrchestratorClientError(
+                f"update_agent_secret {bottle_id}: HTTP {status}"
+            )
+        return True
+
     def teardown_bottle(self, bottle_id: str) -> bool:
         """Tear a bottle down (`DELETE /bottles/<id>`). False if the
         orchestrator didn't know it (404) — idempotent for cleanup paths."""

bot_bottle/orchestrator/lifecycle.py

@@ -63,12 +63,10 @@ class Orchestrator(abc.ABC):
     # orchestrator never starts without its signing key.
     provisioning: ControlPlaneProvisioning = ControlPlaneProvisioning()
 
-    def ensure_available(self) -> None:
-        """Acquire the exact image/rootfs selected for this application."""
-        self.ensure_built()
-
     def ensure_built(self) -> None:
-        """Local-build implementation hook retained for backend compatibility."""
+        """Ensure the orchestrator's image / rootfs exists, building it if
+        needed. Default: nothing to build (e.g. a pre-pulled image). Call before
+        `ensure_running`."""
         return
 
     @abc.abstractmethod

bot_bottle/orchestrator/service.py

@@ -379,6 +379,30 @@ class OrchestratorCore:
         self._tokens[bottle_id] = decrypted
         return True
 
+    def update_agent_secret(
+        self, bottle_id: str, name: str, value: str, env_var_secret: str,
+    ) -> bool:
+        """Update ONE egress token for a known bottle in place — the
+        single-secret form of ``reprovision_from_secret``.
+
+        Sets the in-memory token AND upserts the single re-encrypted row under
+        *env_var_secret* (the same key the rest of the rows are encrypted with, so
+        the whole set stays decryptable by a later ``reprovision_from_secret``),
+        leaving every other token untouched. Returns False if the bottle is
+        unknown.
+
+        Unlike ``reprovision_from_secret`` (which restores the values captured at
+        launch), this pushes a *caller-supplied* value — used to refresh a
+        short-lived host credential (e.g. the Codex access token) into a
+        still-running bottle without a relaunch."""
+        from .store.secret_store import encrypt_value
+        if self.registry.get(bottle_id) is None:
+            return False
+        self._tokens.setdefault(bottle_id, {})[name] = value
+        self.registry.store_agent_secret(
+            bottle_id, name, encrypt_value(env_var_secret, value))
+        return True
+
     # --- consolidated gateway ----------------------------------------------
 
     def gateway_status(self) -> dict[str, object]:

bot_bottle/orchestrator/store/registry_store.py

@@ -370,6 +370,30 @@ class RegistryStore(DbStore):
             )
         self._chmod()
 
+    def store_agent_secret(
+        self,
+        bottle_id: str,
+        key: str,
+        encrypted_value: str,
+        secret_type: str = "injected_env_var",
+    ) -> None:
+        """Upsert ONE encrypted secret row (env-var name → ciphertext) for
+        *bottle_id*, leaving the bottle's other secrets untouched — the per-key
+        counterpart of ``store_agent_secrets``' replace-all. Delete-then-insert
+        because the table carries no unique constraint to `ON CONFLICT` against."""
+        with self._connection() as conn:
+            conn.execute(
+                "DELETE FROM bottled_agent_secrets "
+                "WHERE bottled_agent_id = ? AND key = ? AND type = ?",
+                (bottle_id, key, secret_type),
+            )
+            conn.execute(
+                "INSERT INTO bottled_agent_secrets "
+                "(bottled_agent_id, key, value, type) VALUES (?, ?, ?, ?)",
+                (bottle_id, key, encrypted_value, secret_type),
+            )
+        self._chmod()
+
     def get_agent_secrets(
         self,
         bottle_id: str,

bot_bottle/release-manifest.json

@@ -1 +0,0 @@
-{"schema": 1, "development": true}

bot_bottle/release_bundle.py

@@ -1,127 +0,0 @@
-"""External, commit-addressed release bundle metadata."""
-
-from __future__ import annotations
-
-import hashlib
-import json
-import re
-from pathlib import Path
-from typing import Any
-
-from .release_manifest import ReleaseManifestError, parse_manifest
-
-_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$")
-_SHA_RE = re.compile(r"^[0-9a-f]{64}$")
-
-
-def sha256_file(path: Path) -> str:
-    digest = hashlib.sha256()
-    with path.open("rb") as stream:
-        for chunk in iter(lambda: stream.read(1 << 20), b""):
-            digest.update(chunk)
-    return digest.hexdigest()
-
-
-def build_bundle_index(
-    manifest_data: Any,
-    *,
-    wheel: Path,
-    wheel_url: str,
-    workflow_run: str,
-    published_at: str,
-) -> dict[str, Any]:
-    """Build and validate the immutable external index for one source commit."""
-    manifest = parse_manifest(manifest_data)
-    if not wheel.is_file() or wheel.suffix != ".whl":
-        raise ReleaseManifestError("release bundle wheel must be one .whl file")
-    if not wheel_url.startswith("https://") or not wheel_url.endswith(wheel.name):
-        raise ReleaseManifestError(
-            "release bundle wheel URL must be HTTPS and end with the wheel filename")
-    if not workflow_run.strip() or not published_at.strip():
-        raise ReleaseManifestError(
-            "release bundle provenance requires workflow_run and published_at")
-    return {
-        "schema": 1,
-        "source_commit": manifest.source_commit,
-        "wheel": {
-            "filename": wheel.name,
-            "url": wheel_url,
-            "sha256": sha256_file(wheel),
-        },
-        "oci": {
-            "orchestrator": manifest.orchestrator_image,
-            "gateway": manifest.gateway_image,
-            **{
-                f"agent_{role}": reference
-                for role, reference in manifest.agent_images.items()
-            },
-        },
-        "firecracker": {
-            "orchestrator": {
-                "version": manifest.firecracker_orchestrator.version,
-                "sha256": manifest.firecracker_orchestrator.sha256,
-            },
-            "gateway": {
-                "version": manifest.firecracker_gateway.version,
-                "sha256": manifest.firecracker_gateway.sha256,
-            },
-        },
-        "provenance": {
-            "workflow_run": workflow_run.strip(),
-            "published_at": published_at.strip(),
-        },
-        "qualifications": [],
-    }
-
-
-def parse_bundle_index(data: Any) -> dict[str, Any]:
-    """Validate an index before publication or installer consumption."""
-    if not isinstance(data, dict) or data.get("schema") != 1:
-        raise ReleaseManifestError("release bundle index must use schema 1")
-    commit = data.get("source_commit")
-    if not isinstance(commit, str) or _COMMIT_RE.fullmatch(commit) is None:
-        raise ReleaseManifestError(
-            "release bundle source_commit must be 40 lowercase hex")
-    wheel = data.get("wheel")
-    if not isinstance(wheel, dict):
-        raise ReleaseManifestError("release bundle wheel must be an object")
-    filename, url, digest = (
-        wheel.get("filename"), wheel.get("url"), wheel.get("sha256"))
-    if not isinstance(filename, str) or not filename.endswith(".whl"):
-        raise ReleaseManifestError("release bundle wheel filename must end in .whl")
-    if not isinstance(url, str) or not url.startswith("https://") or not url.endswith(filename):
-        raise ReleaseManifestError(
-            "release bundle wheel URL must be HTTPS and match its filename")
-    if not isinstance(digest, str) or _SHA_RE.fullmatch(digest) is None:
-        raise ReleaseManifestError(
-            "release bundle wheel sha256 must be 64 lowercase hex")
-    manifest_data = {
-        "schema": 1,
-        "source_commit": commit,
-        "oci": data.get("oci"),
-        "firecracker": data.get("firecracker"),
-    }
-    parse_manifest(manifest_data)
-    provenance = data.get("provenance")
-    if not isinstance(provenance, dict) or not all(
-        isinstance(provenance.get(key), str) and provenance[key].strip()
-        for key in ("workflow_run", "published_at")
-    ):
-        raise ReleaseManifestError("release bundle provenance is incomplete")
-    qualifications = data.get("qualifications")
-    if not isinstance(qualifications, list):
-        raise ReleaseManifestError("release bundle qualifications must be an array")
-    return data
-
-
-def canonical_bytes(data: Any) -> bytes:
-    parse_bundle_index(data)
-    return (json.dumps(data, indent=2, sort_keys=True) + "\n").encode("utf-8")
-
-
-__all__ = [
-    "build_bundle_index",
-    "canonical_bytes",
-    "parse_bundle_index",
-    "sha256_file",
-]

bot_bottle/release_manifest.py

@@ -1,154 +0,0 @@
-"""Immutable infrastructure identities assigned when bot-bottle is packaged."""
-
-from __future__ import annotations
-
-import json
-import os
-import re
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any
-
-from . import resources
-
-_MANIFEST_NAME = "release-manifest.json"
-_OCI_RE = re.compile(r"^[^@\s]+@sha256:([0-9a-f]{64})$")
-_SHA_RE = re.compile(r"^[0-9a-f]{64}$")
-_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$")
-
-
-class ReleaseManifestError(RuntimeError):
-    """The installed package has no usable infrastructure release identity."""
-
-
-@dataclass(frozen=True)
-class FirecrackerArtifact:
-    version: str
-    sha256: str
-
-
-@dataclass(frozen=True)
-class ReleaseManifest:
-    source_commit: str
-    orchestrator_image: str
-    gateway_image: str
-    agent_images: dict[str, str]
-    firecracker_orchestrator: FirecrackerArtifact
-    firecracker_gateway: FirecrackerArtifact
-
-
-def manifest_path() -> Path:
-    override = os.environ.get("BOT_BOTTLE_RELEASE_MANIFEST", "").strip()
-    return Path(override) if override else Path(__file__).resolve().parent / _MANIFEST_NAME
-
-
-def _artifact(value: Any, role: str) -> FirecrackerArtifact:
-    if not isinstance(value, dict):
-        raise ReleaseManifestError(f"release manifest firecracker.{role} must be an object")
-    version = value.get("version")
-    sha256 = value.get("sha256")
-    if not isinstance(version, str) or not version.strip():
-        raise ReleaseManifestError(
-            f"release manifest firecracker.{role}.version must be non-empty")
-    if not isinstance(sha256, str) or _SHA_RE.fullmatch(sha256) is None:
-        raise ReleaseManifestError(
-            f"release manifest firecracker.{role}.sha256 must be 64 lowercase hex")
-    return FirecrackerArtifact(version.strip(), sha256)
-
-
-def parse_manifest(data: Any) -> ReleaseManifest:
-    if not isinstance(data, dict) or data.get("schema") != 1:
-        raise ReleaseManifestError("release manifest must use schema 1")
-    commit = data.get("source_commit")
-    if not isinstance(commit, str) or _COMMIT_RE.fullmatch(commit) is None:
-        raise ReleaseManifestError(
-            "release manifest source_commit must be 40 lowercase hex")
-    oci = data.get("oci")
-    if not isinstance(oci, dict):
-        raise ReleaseManifestError("release manifest oci must be an object")
-    images: dict[str, str] = {}
-    for role in ("orchestrator", "gateway", "agent_claude", "agent_codex", "agent_pi"):
-        ref = oci.get(role)
-        if not isinstance(ref, str) or _OCI_RE.fullmatch(ref) is None:
-            raise ReleaseManifestError(
-                f"release manifest oci.{role} must be digest-pinned")
-        images[role] = ref
-    firecracker = data.get("firecracker")
-    if not isinstance(firecracker, dict):
-        raise ReleaseManifestError("release manifest firecracker must be an object")
-    return ReleaseManifest(
-        source_commit=commit,
-        orchestrator_image=images["orchestrator"],
-        gateway_image=images["gateway"],
-        agent_images={
-            role: images[f"agent_{role}"] for role in ("claude", "codex", "pi")
-        },
-        firecracker_orchestrator=_artifact(
-            firecracker.get("orchestrator"), "orchestrator"),
-        firecracker_gateway=_artifact(firecracker.get("gateway"), "gateway"),
-    )
-
-
-def load_manifest() -> ReleaseManifest:
-    path = manifest_path()
-    try:
-        data = json.loads(path.read_text(encoding="utf-8"))
-    except (OSError, json.JSONDecodeError) as exc:
-        raise ReleaseManifestError(
-            f"bot-bottle was packaged without valid infrastructure pins "
-            f"({path}): {exc}") from exc
-    return parse_manifest(data)
-
-
-def packaged_manifest() -> ReleaseManifest | None:
-    """Release identity, or ``None`` for checkouts/ad-hoc development wheels."""
-    if resources.is_source_checkout() and not os.environ.get(
-        "BOT_BOTTLE_RELEASE_MANIFEST"
-    ):
-        return None
-    path = manifest_path()
-    try:
-        raw = json.loads(path.read_text(encoding="utf-8"))
-    except (OSError, json.JSONDecodeError):
-        return load_manifest()  # raise the detailed, fail-closed error
-    if isinstance(raw, dict) and raw.get("development") is True:
-        return None
-    return load_manifest()
-
-
-def local_build_requested() -> bool:
-    return os.environ.get("BOT_BOTTLE_INFRA_BUILD", "").strip().lower() == "local"
-
-
-def oci_image(role: str, local_default: str) -> tuple[str, bool]:
-    """Return ``(reference, should_build_locally)`` for a fixed OCI service."""
-    override_name = f"BOT_BOTTLE_{role.upper()}_IMAGE"
-    override = os.environ.get(override_name, "").strip()
-    manifest = packaged_manifest()
-    local = local_build_requested() or manifest is None
-    if override:
-        if not local and _OCI_RE.fullmatch(override) is None:
-            raise ReleaseManifestError(
-                f"{override_name} must be digest-pinned outside local build mode")
-        return override, local
-    if local:
-        return local_default, True
-    assert manifest is not None
-    return (
-        manifest.orchestrator_image if role == "orchestrator"
-        else manifest.gateway_image if role == "gateway"
-        else manifest.agent_images[role.removeprefix("agent_")],
-        False,
-    )
-
-
-__all__ = [
-    "FirecrackerArtifact",
-    "ReleaseManifest",
-    "ReleaseManifestError",
-    "load_manifest",
-    "local_build_requested",
-    "oci_image",
-    "packaged_manifest",
-    "parse_manifest",
-]

bot_bottle/release_publish.py

@@ -1,148 +0,0 @@
-"""Durable publication of immutable, commit-addressed release bundles."""
-
-from __future__ import annotations
-
-import hashlib
-import os
-import urllib.error
-import urllib.request
-from pathlib import Path
-
-from .release_bundle import canonical_bytes, parse_bundle_index, sha256_file
-from .release_manifest import ReleaseManifestError
-
-HTTP_TIMEOUT_SECONDS = 60.0
-INDEX_NAME = "bundle-index.json"
-PACKAGE_NAME = "bot-bottle-builds"
-
-
-def config() -> tuple[str, str, str]:
-    """Return generic-package base URL, owner, and write token."""
-    base = os.environ.get(
-        "BOT_BOTTLE_RELEASE_BASE", "https://gitea.dideric.is").rstrip("/")
-    owner = os.environ.get("BOT_BOTTLE_RELEASE_OWNER", "didericis").strip()
-    token = os.environ.get("BOT_BOTTLE_RELEASE_TOKEN", "")
-    return base, owner, token
-
-
-def bundle_url(source_commit: str, filename: str) -> str:
-    base, owner, _ = config()
-    return (
-        f"{base}/api/packages/{owner}/generic/{PACKAGE_NAME}/"
-        f"{source_commit}/{filename}"
-    )
-
-
-def request(url: str, *, method: str = "GET", data: object | None = None,
-            length: int | None = None) -> urllib.request.Request:
-    result = urllib.request.Request(url, data=data, method=method)  # type: ignore[arg-type]
-    _, _, token = config()
-    if token:
-        result.add_header("Authorization", f"token {token}")
-    if length is not None:
-        result.add_header("Content-Length", str(length))
-    result.add_header("Content-Type", "application/octet-stream")
-    return result
-
-
-def remote_bytes(url: str) -> bytes | None:
-    try:
-        with urllib.request.urlopen(
-            request(url), timeout=HTTP_TIMEOUT_SECONDS,
-        ) as response:
-            return response.read()
-    except urllib.error.HTTPError as exc:
-        if exc.code == 404:
-            return None
-        raise ReleaseManifestError(
-            f"checking release bundle failed (HTTP {exc.code}): {url}") from exc
-    except urllib.error.URLError as exc:
-        raise ReleaseManifestError(
-            f"release registry unreachable: {url} ({exc.reason})") from exc
-
-
-def remote_sha256(url: str) -> str | None:
-    try:
-        with urllib.request.urlopen(
-            request(url), timeout=HTTP_TIMEOUT_SECONDS,
-        ) as response:
-            digest = hashlib.sha256()
-            for chunk in iter(lambda: response.read(1 << 20), b""):
-                digest.update(chunk)
-            return digest.hexdigest()
-    except urllib.error.HTTPError as exc:
-        if exc.code == 404:
-            return None
-        raise ReleaseManifestError(
-            f"checking release artifact failed (HTTP {exc.code}): {url}") from exc
-    except urllib.error.URLError as exc:
-        raise ReleaseManifestError(
-            f"release registry unreachable: {url} ({exc.reason})") from exc
-
-
-def upload(url: str, source: bytes | Path) -> None:
-    handle = None
-    if isinstance(source, Path):
-        handle = source.open("rb")
-        data: object = handle
-        length = source.stat().st_size
-    else:
-        data = source
-        length = len(source)
-    try:
-        with urllib.request.urlopen(
-            request(url, method="PUT", data=data, length=length),
-            timeout=HTTP_TIMEOUT_SECONDS,
-        ):
-            pass
-    except (urllib.error.HTTPError, urllib.error.URLError) as exc:
-        raise ReleaseManifestError(
-            f"publishing release artifact failed: {url} ({exc})") from exc
-    finally:
-        if handle is not None:
-            handle.close()
-
-
-def publish_bundle(index: dict[str, object], wheel: Path) -> str:
-    """Publish wheel first and index last, or verify an existing exact bundle."""
-    parse_bundle_index(index)
-    source_commit = str(index["source_commit"])
-    wheel_data = index["wheel"]
-    assert isinstance(wheel_data, dict)
-    filename = str(wheel_data["filename"])
-    expected_sha = str(wheel_data["sha256"])
-    if wheel.name != filename or sha256_file(wheel) != expected_sha:
-        raise ReleaseManifestError(
-            "release bundle wheel does not match its index")
-
-    wheel_url = bundle_url(source_commit, filename)
-    index_url = bundle_url(source_commit, INDEX_NAME)
-    if wheel_data["url"] != wheel_url:
-        raise ReleaseManifestError(
-            "release bundle wheel URL does not match its commit coordinate")
-
-    wanted_index = canonical_bytes(index)
-    existing_index = remote_bytes(index_url)
-    existing_wheel_sha = remote_sha256(wheel_url)
-    if existing_index is not None:
-        if existing_index != wanted_index or existing_wheel_sha != expected_sha:
-            raise ReleaseManifestError(
-                f"immutable release bundle {source_commit} already differs")
-        return index_url
-    if existing_wheel_sha is not None:
-        raise ReleaseManifestError(
-            f"partial release bundle {source_commit} exists without its index; "
-            "administrative cleanup is required")
-
-    upload(wheel_url, wheel)
-    upload(index_url, wanted_index)
-    return index_url
-
-
-__all__ = [
-    "INDEX_NAME",
-    "PACKAGE_NAME",
-    "bundle_url",
-    "config",
-    "publish_bundle",
-]

bot_bottle/release_qualification.py

@@ -1,197 +0,0 @@
-"""Validated release and channel pointers for qualified commit bundles."""
-
-from __future__ import annotations
-
-import json
-import re
-import urllib.error
-import urllib.request
-from typing import Any
-
-from .release_bundle import parse_bundle_index
-from .release_manifest import ReleaseManifestError
-from .release_publish import (
-    HTTP_TIMEOUT_SECONDS,
-    bundle_url,
-    config,
-    remote_bytes,
-    request,
-    upload,
-)
-
-_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$")
-_STABLE_TAG_RE = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$")
-_STAGING_TAG_RE = re.compile(r"^v(\d+)\.(\d+)\.(\d+)-rc\.(\d+)$")
-
-
-def release_channel(tag: str) -> str:
-    """Return the only channel eligible for *tag*."""
-    if _STABLE_TAG_RE.fullmatch(tag):
-        return "production"
-    if _STAGING_TAG_RE.fullmatch(tag):
-        return "staging"
-    raise ReleaseManifestError(
-        "release tag must be vX.Y.Z or vX.Y.Z-rc.N")
-
-
-def release_version(tag: str) -> tuple[int, int, int, int]:
-    """Return a monotonically comparable version tuple."""
-    stable = _STABLE_TAG_RE.fullmatch(tag)
-    if stable:
-        return (
-            int(stable.group(1)),
-            int(stable.group(2)),
-            int(stable.group(3)),
-            1 << 30,
-        )
-    staging = _STAGING_TAG_RE.fullmatch(tag)
-    if staging:
-        return (
-            int(staging.group(1)),
-            int(staging.group(2)),
-            int(staging.group(3)),
-            int(staging.group(4)),
-        )
-    raise ReleaseManifestError(
-        "release tag must be vX.Y.Z or vX.Y.Z-rc.N")
-
-
-def pointer_url(package: str, version: str, filename: str) -> str:
-    base, owner, _ = config()
-    return f"{base}/api/packages/{owner}/generic/{package}/{version}/{filename}"
-
-
-def build_release_pointer(
-    *,
-    tag: str,
-    source_commit: str,
-    workflow_run: str,
-    qualified_at: str,
-) -> dict[str, Any]:
-    """Build a qualified pointer to one immutable commit bundle."""
-    channel = release_channel(tag)
-    if _COMMIT_RE.fullmatch(source_commit) is None:
-        raise ReleaseManifestError(
-            "release source_commit must be 40 lowercase hex")
-    if not workflow_run.strip() or not qualified_at.strip():
-        raise ReleaseManifestError(
-            "release qualification provenance is incomplete")
-    return {
-        "schema": 1,
-        "qualified": True,
-        "tag": tag,
-        "channel": channel,
-        "source_commit": source_commit,
-        "bundle_index_url": bundle_url(source_commit, "bundle-index.json"),
-        "qualification": {
-            "workflow_run": workflow_run.strip(),
-            "qualified_at": qualified_at.strip(),
-        },
-    }
-
-
-def parse_release_pointer(data: Any) -> dict[str, Any]:
-    """Validate a release or channel pointer."""
-    if not isinstance(data, dict) or data.get("schema") != 1:
-        raise ReleaseManifestError("release pointer must use schema 1")
-    if data.get("qualified") is not True:
-        raise ReleaseManifestError("release pointer must be qualified")
-    tag = data.get("tag")
-    channel = data.get("channel")
-    commit = data.get("source_commit")
-    if not isinstance(tag, str) or release_channel(tag) != channel:
-        raise ReleaseManifestError("release pointer tag and channel disagree")
-    if not isinstance(commit, str) or _COMMIT_RE.fullmatch(commit) is None:
-        raise ReleaseManifestError("release pointer source commit is invalid")
-    if data.get("bundle_index_url") != bundle_url(commit, "bundle-index.json"):
-        raise ReleaseManifestError("release pointer bundle URL is invalid")
-    qualification = data.get("qualification")
-    if not isinstance(qualification, dict) or not all(
-        isinstance(qualification.get(key), str) and qualification[key].strip()
-        for key in ("workflow_run", "qualified_at")
-    ):
-        raise ReleaseManifestError("release qualification is incomplete")
-    return data
-
-
-def canonical_pointer(data: Any) -> bytes:
-    parse_release_pointer(data)
-    return (json.dumps(data, indent=2, sort_keys=True) + "\n").encode()
-
-
-def delete(url: str) -> None:
-    try:
-        with urllib.request.urlopen(
-            request(url, method="DELETE"), timeout=HTTP_TIMEOUT_SECONDS,
-        ):
-            pass
-    except urllib.error.HTTPError as exc:
-        if exc.code != 404:
-            raise ReleaseManifestError(
-                f"replacing release channel failed (HTTP {exc.code}): {url}"
-            ) from exc
-    except urllib.error.URLError as exc:
-        raise ReleaseManifestError(
-            f"release registry unreachable: {url} ({exc.reason})") from exc
-
-
-def publish_qualification(
-    pointer: dict[str, Any], *, allow_rollback: bool = False,
-) -> tuple[str, str]:
-    """Publish an immutable tag pointer and advance its channel."""
-    parse_release_pointer(pointer)
-    tag = str(pointer["tag"])
-    channel = str(pointer["channel"])
-    wanted = canonical_pointer(pointer)
-    release_url = pointer_url(
-        "bot-bottle-releases", tag, "release.json")
-    channel_url = pointer_url(
-        "bot-bottle-channels", channel, "channel.json")
-
-    bundle = remote_bytes(str(pointer["bundle_index_url"]))
-    if bundle is None:
-        raise ReleaseManifestError(
-            f"commit bundle does not exist for {pointer['source_commit']}")
-    try:
-        bundle_data = parse_bundle_index(json.loads(bundle))
-    except (json.JSONDecodeError, UnicodeDecodeError) as exc:
-        raise ReleaseManifestError("commit bundle index is malformed") from exc
-    if bundle_data["source_commit"] != pointer["source_commit"]:
-        raise ReleaseManifestError(
-            "commit bundle index does not match the qualified source commit")
-    existing_release = remote_bytes(release_url)
-    if existing_release is not None and existing_release != wanted:
-        raise ReleaseManifestError(
-            f"immutable release pointer {tag} already differs")
-    if existing_release is None:
-        upload(release_url, wanted)
-
-    existing_channel = remote_bytes(channel_url)
-    if existing_channel == wanted:
-        return release_url, channel_url
-    if existing_channel is not None:
-        try:
-            current = parse_release_pointer(json.loads(existing_channel))
-        except (json.JSONDecodeError, UnicodeDecodeError) as exc:
-            raise ReleaseManifestError(
-                f"existing {channel} channel pointer is malformed") from exc
-        if (
-            not allow_rollback
-            and release_version(tag) <= release_version(str(current["tag"]))
-        ):
-            raise ReleaseManifestError(
-                f"{channel} channel update is not monotonic; "
-                "use the explicit rollback operation")
-        delete(channel_url)
-    upload(channel_url, wanted)
-    return release_url, channel_url
-
-
-__all__ = [
-    "build_release_pointer",
-    "canonical_pointer",
-    "parse_release_pointer",
-    "publish_qualification",
-    "release_channel",
-    "release_version",
-]

docs/ci.md

@@ -45,23 +45,6 @@ report; macOS reports advisory coverage in its own job. The macOS infra
 container is a singleton, so its job uses a concurrency group and always tears
 the service down.
 
-## Artifact publication and promotion
-
-[`publish-artifacts.yml`](../.gitea/workflows/publish-artifacts.yml) is the
-manual deployment-test boundary. Dispatch it with any branch, tag, or commit;
-it resolves that input once, publishes the orchestrator, gateway, Claude,
-Codex, Pi, and Firecracker artifacts, then publishes only the wheel verified
-against those identities. Install the reported commit with
-`BOT_BOTTLE_REF=<sha> sh install.sh`; the installer warns that snapshots may
-not have completed release qualification.
-
-[`release.yml`](../.gitea/workflows/release.yml) handles promotion tags.
-`vX.Y.Z-rc.N` must be reachable from `staging`; `vX.Y.Z` must be reachable
-from `production`. It runs the complete pre-release workflow, ensures that the
-same commit bundle is published, verifies a clean install, creates the
-qualified release pointer, and advances the matching channel. Promotion
-reuses immutable bundle bytes and never rebuilds them.
-
 ## Scheduled canary
 
 [`.gitea/workflows/canaries.yml`](../.gitea/workflows/canaries.yml) runs weekly

docs/prds/0081-reprovision-gateway-state-on-bringup.md

@@ -0,0 +1,117 @@
+# PRD 0081: Reprovision gateway-dependent state on gateway bring-up
+
+- **Status:** Active
+- **Author:** claude
+- **Created:** 2026-07-26
+- **Issue:** #516
+
+## Summary
+
+When the gateway is (re)built, reconcile every already-running bottle against the
+fresh gateway instead of persisting the gateway's state. On a gateway cold boot,
+the gateway reconciles all live bottles in one flow: **replace** each agent's
+trusted CA with the freshly-minted gateway CA, **re-provision** each bottle's
+git-gate repos + creds onto the gateway, and **restore** each bottle's egress
+tokens. One mechanism across all three services; the CA rotates for free on every
+bring-up.
+
+## Problem
+
+A gateway rebuild/restart silently breaks every already-running bottle:
+
+- **CA (#510).** The gateway's mitmproxy mints a new CA on a fresh rootfs; agents
+  still trust the old one, so egress fails TLS verification (`SSL certificate
+  verification failed`).
+- **git-gate (#512).** Per-bottle bare repos (`/git/<id>`) + deploy creds
+  (`/git-gate/creds/<id>`) live in the gateway's ephemeral rootfs; a rebuild wipes
+  them and the agent 404s on fetch/push.
+
+Both are the same root cause: per-bottle gateway-dependent state is provisioned
+**once, at bottle launch**, and nothing restores it for already-running bottles
+when the gateway comes back. The existing launch-time reprovision
+(`reprovision_bottles`) restores **only** egress tokens, and only as a side effect
+of the *next* launch.
+
+Persisting the state (a host bind-mount / a per-VM data volume per service) was
+prototyped and rejected: it differs per service (a volume for the CA, another for
+git-gate, reprovision for tokens), it pins the CA static forever (no rotation),
+and it adds volume surface that `docker volume prune` / a wiped cache can silently
+destroy (the original #450 failure mode).
+
+## Goals / Success Criteria
+
+- A gateway (re)boot restores **all** running bottles' gateway-dependent state
+  with no manual step and no relaunch — the agent's next egress / fetch / push
+  just works.
+- **One** reconcile flow covering CA, git-gate, and egress tokens, rather than a
+  different mechanism per service.
+- The CA **rotates** on every gateway bring-up (no long-lived CA), distributed to
+  running agents by the same reconcile.
+- Per-bottle failures are tolerated: one unreachable or malformed bottle does not
+  block the others or the gateway coming up.
+- **All backends** (Firecracker, docker, macOS) reconcile through the *same*
+  contract — an abstract method on the backend base class, so a new backend
+  cannot forget to implement it and none drifts onto a bespoke mechanism.
+
+## Non-goals
+
+- Deliberate mid-session CA rotation *without* a gateway restart — `rotate_ca`
+  stays for that operator action.
+- Changing source-IP attribution, `/resolve`, or the plane split (#469).
+
+## Design
+
+**The contract — `attach_bottled_agents_to_gateway()` on the backend ABC.**
+Reconciling running bottles against the current gateway is a backend
+responsibility (only the backend can enumerate its agents and reach them —
+firecracker over SSH, docker/macOS over `exec`/`cp`), so it is an
+`@abc.abstractmethod` on `BottleBackend` (`backend/base.py`). Every backend
+implements it; the host calls it whenever the gateway is (re)brought up. This is
+what makes the fix cross-backend by construction rather than a per-backend
+follow-up. It reprovisions **all** registered bottles' gateway-dependent state:
+CA, git-gate, and egress tokens.
+
+**Trigger — the gateway bring-up path.** The host calls
+`attach_bottled_agents_to_gateway()` only when the gateway was actually
+(re)brought up — the cold-boot branch of the infra bring-up (e.g.
+`FirecrackerInfraService.ensure_running` after it boots a fresh pair), never on an
+adopt of a healthy, current gateway (state intact). So it fires exactly when the
+gateway was (re)booted — including orchestrator restarts, since the pair boots
+together — and there is no bare-restart path that bypasses bring-up.
+
+**Per-backend implementation.** Each backend's `attach_bottled_agents_to_gateway`
+enumerates its live bottles and, for each, reconciles the three services against
+the current gateway. The firecracker implementation, once the gateway VM is up
+and its CA is available:
+
+1. Map each live bottle's guest IP → `bottle_id` from the orchestrator registry
+   (`list_bottles`).
+2. Install the **current** shared git-gate hooks (`git_gate_render_hook` /
+   `git_gate_render_access_hook`) into the fresh gateway once — rendered from
+   code, never from a bottle's possibly-stale state dir.
+3. For each live agent VM (enumerated from its run dir):
+   - **CA:** SSH the current gateway CA into the agent's trust store and run
+     `update-ca-certificates` (unconditional replace — there is one gateway, so no
+     fingerprint match is needed).
+   - **git-gate:** rebuild the bottle's upstreams from its persisted git-gate
+     state dir (deploy key, known_hosts, upstream URL) and re-init its bare repos
+     + per-repo creds under `/git/<bottle_id>`.
+   - **egress token:** read the agent's `ENV_VAR_SECRET` and feed
+     `reprovision_bottles`, restoring the orchestrator's in-memory tokens.
+4. Every per-bottle step is wrapped so one failure is logged and skipped.
+
+**Retire the persistence prototype.** No CA data volume, no git-gate data volume
+(the abandoned PRs #511 / #513). The launch-time `_reprovision_running_bottles`
+call folds into this bring-up reconcile, so egress tokens are restored on the same
+cold-boot trigger (an adopt needs no restore — the orchestrator never restarted).
+
+**CA rotation.** Because the gateway rootfs is ephemeral, every cold boot mints a
+fresh CA; reconcile is what distributes it, so a routine gateway rebuild doubles
+as a CA rotation with zero extra machinery.
+
+## Open questions
+
+- Docker's existing host-bind-mounted CA (`host_gateway_ca_dir`): once docker's
+  `attach_bottled_agents_to_gateway` pushes the CA to running agents on bring-up,
+  the bind-mount is redundant — drop it (so docker rotates like firecracker) or
+  keep it as belt-and-suspenders? Leaning drop, for one behaviour across backends.

docs/prds/0083-packaged-infra-artifacts.md

@@ -1,374 +0,0 @@
-# PRD 0083: Packaged infrastructure artifacts
-
-- **Status:** Active
-- **Author:** Codex
-- **Created:** 2026-07-27
-- **Issue:** #536
-
-## Summary
-
-Publish the orchestrator and gateway infrastructure before packaging
-bot-bottle, record their immutable identities in the application package, and
-make every production backend acquire those exact artifacts instead of
-building infrastructure during bottle startup. Docker and Apple Container pull
-digest-pinned OCI images; Firecracker pulls checksum-verified rootfs artifacts
-selected by the same packaged release manifest.
-
-## Problem
-
-Starting a bottle currently invokes an infrastructure build on Docker and
-Apple Container even when a healthy orchestrator is already running. Both
-infra composers call `ensure_built()` before their health/currentness checks,
-and both implementations unconditionally invoke their OCI builder. Layer
-caching can make this less expensive, but startup still depends on a working
-builder, build context, base-image registry, and package installation network.
-
-The Apple Container orchestrator also bind-mounts the installed source over the
-package baked into its image. A pinned image alone would therefore not pin the
-code that actually runs.
-
-Firecracker already downloads fixed rootfs artifacts, but it derives their
-versions from the package contents at launch. There is no package-level record
-that says which already-published infrastructure release belongs to an
-installed bot-bottle release.
-
-The result is three different delivery contracts:
-
-- Docker and Apple Container build mutable `:latest` images on the launch host.
-- Firecracker computes an artifact version locally and downloads it.
-- An installed wheel contains Dockerfiles specifically so production startup
-  can rebuild its own infrastructure.
-
-That makes startup slower and less reliable, prevents a packaged application
-from identifying the infrastructure it was tested with, and weakens the
-supply-chain boundary between release production and runtime.
-
-## Goals / Success criteria
-
-- A packaged bot-bottle release contains one validated release manifest with
-  immutable orchestrator and gateway identities.
-- Docker and Apple Container acquire and run the manifest's digest-pinned OCI
-  images without invoking `docker build` or `container build`.
-- The shipped Claude, Codex, and Pi agent images are built, smoke-tested,
-  published, and selected by digest as part of the same commit bundle.
-- Firecracker acquires the manifest's versioned, checksum-verified
-  orchestrator and gateway rootfs artifacts rather than deriving versions at
-  launch.
-- All three backends expose the same backend-neutral `ensure_available`
-  lifecycle contract.
-- A healthy running orchestrator is retained only when its recorded release
-  identity matches the installed package.
-- Apple Container runs the package baked into the orchestrator image; no host
-  source overlays production code.
-- Missing, mutable, malformed, or internally inconsistent release metadata
-  fails closed before infrastructure starts.
-- Source-checkout development retains an explicit local-build mode.
-- Publishing produces the infrastructure first and the installable Python
-  package last, so the package cannot name artifacts that were not published.
-- Any branch, tag, or commit can be resolved once to a full source SHA and
-  published as one immutable, commit-addressed artifact bundle.
-- The installer defaults to the production channel, can select staging or an
-  exact commit, and warns for commit snapshots that have not been qualified as
-  a release.
-- Staging and production releases promote the same tested artifact bytes;
-  promotion never rebuilds them.
-- A release is published only after the complete pre-release suite and a clean
-  install from the published bundle succeed.
-
-## Non-goals
-
-- Prebuilding user-supplied Dockerfiles. Custom agent images continue to build
-  through the backend-specific agent-image paths.
-- Installing Docker, Apple Container, Firecracker, or other host prerequisites.
-- Replacing Gitea's OCI or generic-package registries.
-- Adding image signing or a transparency log. Immutable digests and existing
-  Firecracker SHA-256 verification are the integrity boundary for this version.
-- Forcing a network download when an immutable, verified artifact is already
-  cached locally. "Pull" means acquire the packaged identity, with safe cache
-  reuse.
-- Supporting live source overlays in packaged production installs.
-- Defining the project's version-numbering policy beyond stable
-  `vX.Y.Z` and staging `vX.Y.Z-rc.N` tag shapes.
-
-## Design
-
-### Release manifest
-
-`bot_bottle/release-manifest.json` is generated during package production and
-shipped as package data:
-
-```json
-{
-  "schema": 1,
-  "source_commit": "<40 lowercase hex>",
-  "oci": {
-    "orchestrator": "gitea.dideric.is/didericis/bot-bottle-orchestrator@sha256:<64 hex>",
-    "gateway": "gitea.dideric.is/didericis/bot-bottle-gateway@sha256:<64 hex>",
-    "agent_claude": "gitea.dideric.is/didericis/bot-bottle-claude@sha256:<64 hex>",
-    "agent_codex": "gitea.dideric.is/didericis/bot-bottle-codex@sha256:<64 hex>",
-    "agent_pi": "gitea.dideric.is/didericis/bot-bottle-pi@sha256:<64 hex>"
-  },
-  "firecracker": {
-    "orchestrator": {
-      "version": "<artifact version>",
-      "sha256": "<rootfs.ext4.gz sha256>"
-    },
-    "gateway": {
-      "version": "<artifact version>",
-      "sha256": "<rootfs.ext4.gz sha256>"
-    }
-  }
-}
-```
-
-The runtime loader accepts only schema 1, digest-qualified OCI references,
-non-empty Firecracker versions, 64-character lowercase SHA-256 values, and a
-full source commit. It returns immutable value objects rather than passing raw
-dictionaries through backend code.
-
-The repository carries no pretend production pins. A source checkout or ad-hoc
-Git wheel carries a `development: true` marker and selects local infrastructure
-builds, preserving the contributor and current `install.sh` paths. The release
-packaging workflow must replace that marker with the validated schema above;
-it refuses mutable or incomplete release inputs.
-
-### Packaging order
-
-The release job operates on one tested commit:
-
-1. Build and smoke-test the multi-architecture orchestrator, gateway, Claude,
-   Codex, and Pi OCI images.
-2. Push them and resolve their registry manifest digests.
-3. Build Firecracker's orchestrator and gateway rootfs artifacts from the same
-   source and pinned OCI bases.
-4. Publish both generic-package artifacts and retain their versions and
-   compressed-file checksums.
-5. Generate `release-manifest.json` from those published identities.
-6. Build the wheel with that manifest. Do not produce a source distribution:
-   rebuilding an sdist outside this release boundary could replace the pins
-   with the development manifest.
-7. Install the wheel in a clean environment and assert that every manifest
-   identity is valid and retrievable.
-8. Publish only that verified wheel.
-
-The build hook receives the manifest as a file input. It must not query a
-registry or choose versions itself: package builds stay deterministic and
-network-independent once their release inputs exist.
-
-### Commit-addressed bundle index
-
-Every publication resolves its requested branch, tag, or commit to one
-40-character source SHA before doing any work. The publisher writes an
-immutable external bundle index under that SHA. The index contains:
-
-- schema version and source SHA;
-- wheel URL and SHA-256;
-- orchestrator and gateway OCI digest references;
-- Claude, Codex, and Pi agent OCI digest references;
-- Firecracker package versions and compressed-file SHA-256 values;
-- producing workflow/run identity and publication timestamp; and
-- producing workflow provenance. Qualification is recorded separately in an
-  immutable tag pointer so the commit bundle itself never changes.
-
-The embedded wheel manifest contains the runtime subset of the same identities.
-Publication fails if the embedded manifest and external index differ.
-
-Bundle coordinates are immutable. Re-running publication for a SHA verifies
-and reuses a byte-identical complete bundle; it never replaces one artifact or
-mixes outputs from different runs. A partial existing bundle is an error and
-requires an explicit administrative cleanup before retrying.
-
-Commit bundles are durable release inputs, not expiring CI artifacts. The
-project may garbage-collect unqualified snapshots only under a documented
-retention policy; qualified staging and production bundles are retained.
-
-### Branches, tags, and promotion
-
-The protected promotion topology is:
-
-```text
-feature -> main -> staging -> production
-                       |           |
-                 vX.Y.Z-rc.N     vX.Y.Z
-```
-
-Changes reach `staging` and `production` through promotion pull requests; those
-branches do not accept direct development commits. A staging tag must point to
-a commit reachable from `staging` and uses `vX.Y.Z-rc.N`. A production tag must
-point to a commit reachable from `production` and uses `vX.Y.Z`.
-
-Channels are small, mutable pointers to immutable qualified bundles:
-
-- `production` points to the newest promoted stable tag;
-- `staging` points to the newest promoted release-candidate tag; and
-- `main` is not a release channel; its commits are installable snapshots.
-
-Channel updates are serialized and monotonic. Moving a channel to an older
-release requires a distinct, explicit rollback operation that records the
-reason and target. Deleting or moving a Git tag does not mutate a published
-bundle or channel silently.
-
-Promotion reuses the exact bundle selected by source SHA. It does not rebuild
-the wheel, OCI images, or Firecracker artifacts. Thus production runs the same
-bytes qualified in staging.
-
-### Publication workflows
-
-`publish-artifacts` is manually dispatchable for any branch, tag, or commit.
-It resolves the input to a source SHA, checks out that SHA, builds and publishes
-all infrastructure artifacts, generates the bundle index and embedded
-manifest, builds the wheel, installs the wheel in a clean environment, and
-publishes the verified wheel. It does not create a release or update staging or
-production. Its output is an unqualified snapshot and publication is
-serialized per source SHA.
-
-After required CI succeeds, `main` may invoke the same publication operation
-automatically. Manual publication remains available so a branch can be tested
-before merge. Automatic and manual runs converge on the same idempotent bundle.
-
-The release workflow is triggered by an eligible staging or production tag:
-
-1. Resolve and validate the tag, source SHA, tag shape, and branch reachability.
-2. Run the complete pre-release suite against that SHA, including Docker,
-   Firecracker, and Apple Container.
-3. After the suite passes, ensure the immutable commit bundle exists. Build it
-   then if absent; otherwise verify every existing identity and checksum.
-4. Install using the published installer and bundle index on clean supported
-   hosts, and run the post-install smoke test against the acquired artifacts.
-5. Publish an immutable qualified tag pointer, create the Gitea release, and
-   advance the matching channel pointer.
-
-No release artifact or channel update occurs before qualification succeeds.
-Jobs use per-SHA and per-channel concurrency locks so concurrent dispatches
-cannot publish twice or race a promotion.
-
-### Installer selection and warnings
-
-The installer installs the latest `production` channel by default. It accepts:
-
-- `BOT_BOTTLE_CHANNEL=production` or `staging` to select the latest release in
-  that channel;
-- `BOT_BOTTLE_REF=<40-character-sha>` to install an exact commit bundle; and
-- `BOT_BOTTLE_VERSION=<tag>` to install an exact qualified release.
-
-Selectors are mutually exclusive. Branch names are accepted by the publication
-workflow but not by the installer because they are mutable; callers resolve
-and publish them first, then install the reported SHA.
-
-An exact-commit install prints a prominent warning that the snapshot may not
-have passed release qualification, even if the same SHA is later associated
-with a tag. A tag or channel install suppresses that warning only when its
-validated pointer is marked qualified and selects the same immutable bundle
-SHA.
-
-The installer downloads the wheel named by the external index, verifies its
-SHA-256 before invoking pipx or pip, and reports the selected channel/tag,
-source SHA, and bundle identity. It never builds a Git checkout for a packaged
-install and never trusts a mutable channel response without resolving it to an
-immutable bundle index.
-
-### Backend-neutral lifecycle
-
-Rename the host-side control-plane lifecycle operation from `ensure_built()` to
-`ensure_available()`. The production meaning is "make the package-selected
-artifact locally available and verified." Infrastructure composers call it
-before starting either plane.
-
-The gateway receives the same contract so startup does not continue to rebuild
-the adjacent half of the fixed infrastructure pair.
-
-A source checkout is itself a development boundary and selects the existing
-builders. `BOT_BOTTLE_INFRA_BUILD=local` provides the same explicit override
-for release debugging. Both paths deliberately bypass release metadata.
-Packaged production startup never silently falls back from a failed pull to a
-local build.
-
-### Docker
-
-The Docker backend runs `docker pull` with each digest-qualified manifest
-reference. Docker may reuse its content-addressed local store. Containers are
-created from the immutable reference, and the running container's image ID is
-compared with the locally resolved ID for currentness.
-
-The orchestrator source-hash label is replaced by a release-identity label.
-The label is diagnostic; the image ID comparison remains authoritative.
-
-### Apple Container
-
-The Apple Container backend pulls the same multi-architecture OCI references.
-Because Apple Container's accepted syntax for pull, tag, inspect, and run is
-not identical to Docker's, one utility owns normalization from a digest
-reference to the local runnable name. Integration coverage on the macOS runner
-guards that adapter.
-
-Production orchestrator launch removes the build-root bind mount, `PYTHONPATH`,
-and `BOT_BOTTLE_SOURCE_HASH`. It runs only the code baked into the pulled
-image. Currentness is based on the packaged release identity/image ID.
-
-### Firecracker
-
-The publishing tool may continue computing content-derived artifact versions;
-that is a producer concern. Runtime launch instead reads the version and
-expected compressed checksum from the release manifest.
-
-The artifact cache validates the packaged expected checksum even when a
-previous `.verified` marker exists. The marker records the checksum it
-validated, rather than an unqualified `ok`, so changing package metadata cannot
-adopt an artifact verified against a different expectation.
-
-The existing candidate-directory and explicit local-build paths remain for CI
-and development. Candidate metadata must match the packaged identity unless
-the caller selected local development mode.
-
-### Overrides and failure behavior
-
-`BOT_BOTTLE_ORCHESTRATOR_IMAGE` and `BOT_BOTTLE_GATEWAY_IMAGE` remain useful
-for testing but production accepts them only when they are digest-qualified.
-Mutable tag overrides require explicit local-build mode.
-
-Artifact acquisition errors identify the packaged reference and backend. No
-backend catches an acquisition failure and rebuilds from local source.
-
-## Testing strategy
-
-- Unit-test manifest parsing, immutability validation, missing-manifest errors,
-  package-data inclusion, and deterministic build-hook injection.
-- Assert Docker and Apple production preparation pulls and never builds.
-- Assert explicit local mode retains existing build behavior.
-- Assert the Apple orchestrator command has no source bind mount or
-  `PYTHONPATH`.
-- Assert running-image mismatches recreate infrastructure while matching,
-  healthy instances are retained.
-- Assert Firecracker uses packaged versions/checksums and rejects mismatches in
-  downloaded, cached, and candidate artifacts.
-- Extend wheel-install coverage to inspect the installed manifest.
-- Run the full unit and Docker integration gates; run advisory Firecracker and
-  macOS integration before publishing a release.
-- Test ref resolution, tag-shape and branch-reachability enforcement, bundle
-  idempotency, partial-publication rejection, and per-SHA concurrency.
-- Test installer channel, tag, and exact-commit selection; checksum rejection;
-  mutually exclusive selectors; and snapshot warning behavior.
-- Test that release qualification happens before publication and that a failed
-  suite or clean-install smoke test cannot create a release or move a channel.
-- Test promotion reuses the qualified bundle byte-for-byte and that monotonic
-  channel movement rejects an accidental rollback.
-
-## Rollout
-
-Land runtime, installer, bundle publication, and release qualification support
-together, but do not move the production installer away from its current
-source-install path until an end-to-end staging release has passed on Docker,
-Firecracker, and Apple Container.
-
-Create and protect `staging` and `production`, then promote one commit from
-`main` through both branches. Publish its commit bundle, qualify an RC tag,
-perform clean installs through the staging channel, promote the same SHA, and
-qualify a stable tag. Until that first stable qualification exists, the
-installer's production default fails closed because there is no production
-pointer. The stable promotion must demonstrate that every production artifact
-identity matches the staging-qualified bundle.
-
-Existing source checkouts use explicit local mode. Existing mutable local
-images are ignored by production acquisition and may be pruned independently.
-Snapshot retention and rollback procedures must be documented before automatic
-publication from every successful `main` commit is enabled.

docs/research/testing-clean-install-on-linux.md

@@ -1,183 +0,0 @@
-# Testing a clean bot-bottle install on Linux
-
-How do you exercise `install.sh` the way a brand-new user would — on a
-pristine Linux environment you can throw away afterward — *without*
-polluting your daily-driver host, and across the several package-management
-regimes Linux fragments into? This is the Linux counterpart to
-[`testing-clean-install-on-macos.md`](testing-clean-install-on-macos.md);
-the conclusion is different because Linux gives us a boundary macOS doesn't.
-
-## Summary
-
-On macOS the honest options were a throwaway user or a VM, and the throwaway
-user won on pragmatics (nested virtualization is gated to M3+). On Linux the
-calculus flips: a **disposable KVM virtual machine, booted from a distro
-cloud image and deleted per run, is both the cleanest boundary and the one
-that lets a single harness cover Ubuntu, Fedora, Arch, Alpine, and NixOS**.
-The host already requires KVM for the Firecracker backend, so the VM is cheap
-here.
-
-The harness lives at [`scripts/linux-install-test.sh`](../../scripts/linux-install-test.sh).
-Per run it caches one read-only base image, boots a throwaway copy-on-write
-overlay (`qemu-img create -b base`), installs the distro's prerequisites,
-pipes *this checkout's* `install.sh` into the guest exactly as `curl … | sh`
-would, asserts the CLI installed, and deletes the overlay — the Linux
-equivalent of `docker run --rm`, for a whole machine.
-
-## Why a VM, not a container or a throwaway user
-
-| Mechanism | Why it's the wrong boundary here |
-|---|---|
-| **Container** (`docker run --rm`) | Shares the host kernel and ships a deliberately minimal userland — no systemd, a stubbed-out package manager story, and (crucially) it doesn't reproduce the *externally-managed Python* (PEP 668) that real desktop/server installs put in front of the user. It tests "does install.sh run in a container," not "does it run on a real distro." |
-| **Throwaway user** (`useradd`/`userdel`) | The macOS pick, but weaker on Linux: it reaches the real host, yet every system package it installs (python, pipx, git via `apt`/`dnf`/…) stays behind, and it can only ever test the *one* distro the host runs. The whole Linux-specific value is the cross-distro matrix. |
-| **Disposable KVM VM** (this harness) | A genuine kernel + userland + package-manager boundary that wipes to nothing on teardown, and swaps freely between distro cloud images. The one real cost — nested virtualization for the *backend* — doesn't apply, because we gate the installer, not the runtime (below). |
-
-## Two variants: `test` (bare host) and `test-ready` (prepared host)
-
-`install.sh` never installs a backend, and never installs its own toolchain
-prerequisites (python3, git, pipx) — it installs the `bot-bottle` package and
-runs `doctor`, which *reports* what's missing
-([`install.sh`](../../install.sh) header,
-[`bot_bottle/cli/commands/doctor.py`](../../bot_bottle/cli/commands/doctor.py)).
-That leaves two distinct things worth testing, split into two subcommands that
-mirror the macOS harness's `test` / `test-ready` convention (there the split is
-the backend service; here it is the toolchain the installer needs):
-
-- **`test`** — `install.sh` runs on the **bare cloud image**, prerequisites and
-  all left as the vendor ships them. This exercises `install.sh`'s own
-  prerequisite-guard logic — the entire first half of the script (python
-  version gate, git-for-git-specs gate, pipx/pip PEP-668 handling).
-- **`test-ready`** — the harness installs python3 + git + pipx first (the
-  `prereqs` step), then runs `install.sh`. This is the *prepared-host happy
-  path*: does a clean install actually land and produce a working CLI?
-
-**Pass criteria differ by variant:**
-
-| Variant | PASS when |
-|---|---|
-| `test` | `install.sh` **either** installs cleanly (the image already carried enough) **or** declines with one of its own recognized, actionable prerequisite errors (missing python3/git, no usable pip, PEP 668). A crash or an *unrecognized* failure is a FAIL. |
-| `test-ready` | `install.sh` actually lands: the `bot-bottle` entry point is present and runs, and `doctor` reports a usable python and config without crashing. A graceful decline is no longer good enough. |
-
-Neither variant requires a green `doctor`: inside the VM there is no nested KVM
-or Docker, so **the backend is correctly reported not-ready** — install.sh does
-not install a backend and cannot regress one, and this harness does not
-provision the Docker backend. This is where Linux necessarily diverges from the
-macOS `test-ready`, which reaches the host backend; `BB_TEST_REQUIRE_BACKEND=1`
-makes readiness fatal anyway, for a nested-virt host that can satisfy it. The
-verdict instead classifies `doctor`'s output the way the macOS harness does — a
-`Traceback` is an install defect (fail), a missing `python`/`config` line is a
-fail, a not-ready backend is reported — so a genuine installer regression (a
-broken shim, an import error, a botched PATH) stays visible.
-
-`test-all` runs the full matrix — every distro × both variants — each cell in
-its own throwaway VM, and prints a per-cell PASS/FAIL summary.
-
-## The distro matrix is the point
-
-Each distro exercises a different corner of the installer:
-
-| Distro | Cloud image | What it stresses |
-|---|---|---|
-| **Ubuntu** (noble) | `cloud-images.ubuntu.com` | The common case; `apt`'s `pipx`, externally-managed Python (PEP 668) → install.sh's pipx path. |
-| **Fedora** | Fedora Cloud Base Generic | `dnf` packaging, a different default Python, BSD-style checksum file. |
-| **Arch** | `geo.mirror.pkgbuild.com/images/latest` | Rolling / newest Python; `python-pipx`. |
-| **Alpine** | Alpine `nocloud_` (cloudinit) image | musl libc + BusyBox `sh` — the harshest POSIX-`sh` host for a `#!/bin/sh` installer. |
-| **NixOS** | locally built with `nixos-generators` | No FHS `~/.local` on PATH by default; `nix profile install` prereqs; pipx laying a self-contained venv on a non-FHS host. |
-
-### Validation run (2026-07-27) — full green
-
-Full matrix on the delphi KVM host, QEMU 11.0.2, both variants × all five
-distros passing:
-
-| Distro | `test` (bare) | `test-ready` (prepared) |
-|---|---|---|
-| Ubuntu 24.04 | ✅ declines at git gate | ✅ installs, doctor python+config green |
-| Fedora 44 | ✅ declines at git gate | ✅ installs |
-| Arch (latest) | ✅ declines at git gate | ✅ installs |
-| Alpine 3.21 | ✅ declines at git gate | ✅ installs |
-| NixOS 24.11 | ✅ declines (no python3) | ✅ installs |
-
-The bare `test` sees `install.sh` decline soundly — exit 1 at the
-git-for-git-specs gate on the Debian/Fedora/Arch/Alpine images (they ship
-python3 but not git), and at the python3 gate on NixOS (no python3 on PATH) —
-and `test-ready` installs cleanly with `doctor` reporting a usable python and
-config (backends all not-ready, as expected in a plain VM).
-
-Getting to green surfaced and fixed a series of real defects:
-
-- **Fedora 41 was EOL/404** → bumped to 44.
-- The liveness probe used `bot-bottle --version`, which the CLI does not
-  implement (unknown args die non-zero), so every *successful* install was
-  misreported as failed → switched to `bot-bottle --help`.
-- **Alpine** needed three fixes: the `generic_` image ignores a NoCloud seed
-  (switched to the `nocloud_` variant); OpenRC does not auto-start sshd after
-  cloud-init injects the key (start it via `runcmd`); and Alpine's non-PAM
-  sshd refuses pubkey auth for a cloud-init-*locked* account (give it a
-  throwaway password). It also has no `sudo` by default (install it via
-  cloud-init `packages:`).
-- **NixOS** publishes no downloadable cloud qcow2 (its cloud images are
-  Hydra-built AMIs), so the harness builds one with `nixos-generators`
-  ([`linux-install-test-nixos.nix`](../../scripts/linux-install-test-nixos.nix)):
-  cloud-init for the key, flakes enabled, deliberately no python/git/pipx. The
-  `test-ready` prereq install pins `nixpkgs/nixos-24.11` because the guest's
-  default unstable registry builds pipx from source (and its test suite
-  currently fails to build).
-- Two harness-hygiene bugs also fixed: `cmd_down` left `serial.log` behind
-  (orphaned run dirs), and the teardown trap was armed after `cmd_up`, leaking
-  a VM when `wait_for_ssh` timed out.
-
-In `test-ready` the harness installs `python3 + git + pipx` first on each
-distro (install.sh installs none of them), so all five drive the recommended
-pipx path. `test` then removes that scaffolding and lets each distro's bare
-image collide with install.sh's guards — on most cloud images python3 is
-present (cloud-init needs it) but git and pipx are not, so install.sh is
-expected to decline at the git-for-git-specs gate or the PEP-668 pip check with
-an actionable message. Both are legitimate, and the two variants together cover
-the whole first half of the installer as well as the happy path.
-
-## What a clean install touches (the footprint that decides "wipeable")
-
-| Artifact | Location | In the guest's `$HOME`? | Survives VM teardown? |
-|---|---|---|---|
-| Config / state / db | `~/.bot-bottle/{agents,bottles,contrib,…}` ([`install.sh`](../../install.sh)) | ✅ | ❌ overlay deleted |
-| pipx venv + shim | `~/.local/pipx/venvs/bot-bottle`, shim in `~/.local/bin` | ✅ | ❌ overlay deleted |
-| pip `--user` fallback | `~/.local/lib` + `~/.local/bin` | ✅ | ❌ overlay deleted |
-| **Distro prerequisites** (python/git/pipx, `test-ready` only) | system paths via `apt`/`dnf`/`pacman`/`apk`/`nix profile` | ❌ | ❌ **overlay deleted** |
-
-Unlike the macOS throwaway user (whose Homebrew / Apple-Container / Rosetta
-footprint *survives*), **every row here dies with the overlay** — that is the
-VM's whole advantage. The cached base image is read-only backing and is the
-only thing that persists between runs, on purpose.
-
-## Design notes baked into the harness
-
-- **User-mode networking** (`-netdev user,hostfwd=tcp:127.0.0.1:PORT-:22`):
-  no root, no bridge, no host network state touched. Only SSH is forwarded.
-- **cloud-init seed ISO** (`cloud-localds`) injects an ephemeral SSH keypair
-  and a passwordless-sudo login. The keypair is generated per run and deleted
-  on teardown; the guest can't be logged into after it's gone.
-- **Copy-on-write overlay**: the cached base is never mutated, so a corrupt or
-  interrupted run can't poison the cache; downloads land at `*.partial` and
-  are renamed only after checksum verification.
-- **Checksums**: verified against each vendor's published sums file at
-  download time (GNU `hash  file`, bare-hash, and Fedora's BSD
-  `SHA256 (file) = hash` formats are all handled). Alpine ships `.sha512` only
-  (this verifier is sha256) so it is skipped; NixOS is built locally, not
-  downloaded, so there is nothing to verify.
-- **NixOS is built, not downloaded**: `ensure_base_image` runs
-  `nixos-generate -f qcow` against
-  [`linux-install-test-nixos.nix`](../../scripts/linux-install-test-nixos.nix)
-  once and caches the result; the per-run seed/overlay flow is otherwise
-  identical to the downloaded distros.
-- **`test-all`** runs every distro × both variants (`test` and `test-ready`),
-  each cell in its own subshell on its own forwarded port, so one cell's
-  failure (or teardown trap) can't abort the matrix; it prints a per-cell
-  PASS/FAIL summary.
-
-## Not wired into PR CI
-
-Like the macOS harness, the runtime is host-specific (needs `/dev/kvm`,
-`qemu`, and `cloud-localds`) and is not exercised by the Linux pull-request
-runner. It is validated statically (`bash -n`, `shellcheck`) and run by hand
-on a KVM-capable host. The cloud-image URLs in the `DISTRO` table are the one
-place to bump when a distro cuts a newer build.

install.sh

@@ -17,13 +17,10 @@
 # Env:
 #   BOT_BOTTLE_PYTHON        interpreter to install with (skips the search)
 #   BOT_BOTTLE_INSTALL_SPEC  pip/git spec to install instead of the default
-#   BOT_BOTTLE_CHANNEL       production (default) or staging
-#   BOT_BOTTLE_VERSION       exact qualified vX.Y.Z[-rc.N] release
-#   BOT_BOTTLE_REF           exact 40-character commit snapshot (warns untested)
 #   BOT_BOTTLE_VENV          where the non-pipx install lives (~/.bot-bottle/venv)
 set -eu
 
-PACKAGE_SPEC="${BOT_BOTTLE_INSTALL_SPEC:-}"
+PACKAGE_SPEC="${BOT_BOTTLE_INSTALL_SPEC:-git+https://gitea.dideric.is/didericis/bot-bottle.git}"
 VENV="${BOT_BOTTLE_VENV:-${HOME}/.bot-bottle/venv}"
 MIN_PYTHON_MAJOR=3
 MIN_PYTHON_MINOR=11
@@ -129,113 +126,9 @@ if [ "${PYTHON}" != "${path_python}" ]; then
     fi
 fi
 
-# --- resolve an immutable published wheel -----------------------------------
+# Installing a `git+` spec (the default) shells out to git under the hood,
-
+# whether via pipx or pip. Fail early with a clear message rather than deep
-if [ -z "${PACKAGE_SPEC}" ]; then
+# inside the installer's output.
-    selectors=0
-    [ -n "${BOT_BOTTLE_REF:-}" ] && selectors=$((selectors + 1))
-    [ -n "${BOT_BOTTLE_VERSION:-}" ] && selectors=$((selectors + 1))
-    [ -n "${BOT_BOTTLE_CHANNEL:-}" ] && selectors=$((selectors + 1))
-    [ "${selectors}" -le 1 ] || die \
-        "BOT_BOTTLE_REF, BOT_BOTTLE_VERSION, and BOT_BOTTLE_CHANNEL are mutually exclusive."
-
-    downloads="${HOME}/.bot-bottle/downloads"
-    mkdir -p "${downloads}"
-    PACKAGE_SPEC="$("${PYTHON}" - \
-        "${BOT_BOTTLE_REF:-}" \
-        "${BOT_BOTTLE_VERSION:-}" \
-        "${BOT_BOTTLE_CHANNEL:-production}" \
-        "${downloads}" <<'PY'
-import hashlib
-import json
-import re
-import sys
-import urllib.request
-from pathlib import Path
-
-ref, version, channel, download_root = sys.argv[1:]
-base = "https://gitea.dideric.is/api/packages/didericis/generic"
-
-def fetch_json(url):
-    with urllib.request.urlopen(url, timeout=30) as response:
-        return json.load(response)
-
-if ref:
-    if re.fullmatch(r"[0-9a-f]{40}", ref) is None:
-        raise SystemExit("bot-bottle install: error: BOT_BOTTLE_REF must be 40 lowercase hex")
-    index_url = f"{base}/bot-bottle-builds/{ref}/bundle-index.json"
-    print(
-        "bot-bottle install: WARNING: installing an exact commit snapshot; "
-        "it may not have passed release qualification.",
-        file=sys.stderr,
-    )
-    selector = f"commit {ref}"
-else:
-    if version:
-        if re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+(?:-rc\.[0-9]+)?", version) is None:
-            raise SystemExit("bot-bottle install: error: invalid BOT_BOTTLE_VERSION")
-        pointer_url = f"{base}/bot-bottle-releases/{version}/release.json"
-        selector = f"release {version}"
-    else:
-        if channel not in ("production", "staging"):
-            raise SystemExit(
-                "bot-bottle install: error: BOT_BOTTLE_CHANNEL must be production or staging")
-        pointer_url = f"{base}/bot-bottle-channels/{channel}/channel.json"
-        selector = f"{channel} channel"
-    pointer = fetch_json(pointer_url)
-    if pointer.get("schema") != 1 or pointer.get("qualified") is not True:
-        raise SystemExit("bot-bottle install: error: release pointer is not qualified")
-    index_url = pointer.get("bundle_index_url", "")
-    if not isinstance(index_url, str) or not index_url.startswith("https://"):
-        raise SystemExit("bot-bottle install: error: release pointer has no immutable bundle")
-
-index = fetch_json(index_url)
-source_commit = index.get("source_commit", "")
-if re.fullmatch(r"[0-9a-f]{40}", source_commit) is None:
-    raise SystemExit("bot-bottle install: error: bundle has an invalid source commit")
-if ref and source_commit != ref:
-    raise SystemExit("bot-bottle install: error: bundle does not match BOT_BOTTLE_REF")
-wheel = index.get("wheel")
-if not isinstance(wheel, dict):
-    raise SystemExit("bot-bottle install: error: bundle has no wheel")
-filename, url, expected = (
-    wheel.get("filename"), wheel.get("url"), wheel.get("sha256"))
-if (
-    not isinstance(filename, str)
-    or re.fullmatch(r"[A-Za-z0-9_.-]+\.whl", filename) is None
-    or not isinstance(url, str)
-    or not url.startswith("https://")
-    or not url.endswith("/" + filename)
-    or not isinstance(expected, str)
-    or re.fullmatch(r"[0-9a-f]{64}", expected) is None
-):
-    raise SystemExit("bot-bottle install: error: bundle wheel metadata is invalid")
-
-destination = Path(download_root) / source_commit / filename
-destination.parent.mkdir(parents=True, exist_ok=True)
-temporary = destination.with_suffix(destination.suffix + ".part")
-digest = hashlib.sha256()
-with urllib.request.urlopen(url, timeout=60) as response, temporary.open("wb") as output:
-    while chunk := response.read(1 << 20):
-        digest.update(chunk)
-        output.write(chunk)
-if digest.hexdigest() != expected:
-    temporary.unlink(missing_ok=True)
-    raise SystemExit("bot-bottle install: error: downloaded wheel checksum mismatch")
-temporary.replace(destination)
-print(
-    f"bot-bottle install: selected {selector} "
-    f"(source {source_commit}, sha256 {expected[:16]}…)",
-    file=sys.stderr,
-)
-print(destination)
-PY
-    )"
-fi
-
-# An explicit `git+` override shells out to git under the hood, whether via
-# pipx or pip. Fail early with a clear message rather than deep inside the
-# installer's output.
 case "${PACKAGE_SPEC}" in
     git+*|*.git)
         command -v git >/dev/null 2>&1 || die \

pyproject.toml

@@ -36,7 +36,6 @@ include = ["bot_bottle*"]
 # files shipped under bot_bottle/; test_pyproject.py asserts they exist.
 [tool.setuptools.package-data]
 bot_bottle = [
-    "release-manifest.json",
     "gateway/egress/entrypoint.sh",
     "contrib/claude/Dockerfile",
     "contrib/claude/package.json",

scripts/generate_release_bundle.py

@@ -1,39 +0,0 @@
-"""Generate the external commit-addressed release bundle index."""
-
-from __future__ import annotations
-
-import argparse
-import json
-from pathlib import Path
-
-from bot_bottle.release_bundle import build_bundle_index, canonical_bytes
-from bot_bottle.release_publish import bundle_url, publish_bundle
-
-
-def main(argv: list[str] | None = None) -> int:
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--manifest", required=True, type=Path)
-    parser.add_argument("--wheel", required=True, type=Path)
-    parser.add_argument("--wheel-url")
-    parser.add_argument("--workflow-run", required=True)
-    parser.add_argument("--published-at", required=True)
-    parser.add_argument("--output", required=True, type=Path)
-    parser.add_argument("--publish", action="store_true")
-    args = parser.parse_args(argv)
-    manifest = json.loads(args.manifest.read_text(encoding="utf-8"))
-    index = build_bundle_index(
-        manifest,
-        wheel=args.wheel,
-        wheel_url=args.wheel_url or bundle_url(
-            manifest["source_commit"], args.wheel.name),
-        workflow_run=args.workflow_run,
-        published_at=args.published_at,
-    )
-    args.output.write_bytes(canonical_bytes(index))
-    if args.publish:
-        publish_bundle(index, args.wheel)
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

scripts/generate_release_manifest.py

@@ -1,55 +0,0 @@
-"""Generate the validated infrastructure manifest consumed by package builds."""
-
-from __future__ import annotations
-
-import argparse
-import json
-from pathlib import Path
-
-from bot_bottle.release_manifest import parse_manifest
-
-
-def parser() -> argparse.ArgumentParser:
-    result = argparse.ArgumentParser()
-    result.add_argument("--source-commit", required=True)
-    result.add_argument("--orchestrator-image", required=True)
-    result.add_argument("--gateway-image", required=True)
-    for provider in ("claude", "codex", "pi"):
-        result.add_argument(f"--agent-{provider}-image", required=True)
-    for role in ("orchestrator", "gateway"):
-        result.add_argument(f"--firecracker-{role}-version", required=True)
-        result.add_argument(f"--firecracker-{role}-sha256", required=True)
-    result.add_argument("--output", type=Path, required=True)
-    return result
-
-
-def main(argv: list[str] | None = None) -> int:
-    args = parser().parse_args(argv)
-    data = {
-        "schema": 1,
-        "source_commit": args.source_commit,
-        "oci": {
-            "orchestrator": args.orchestrator_image,
-            "gateway": args.gateway_image,
-            **{
-                f"agent_{provider}": getattr(args, f"agent_{provider}_image")
-                for provider in ("claude", "codex", "pi")
-            },
-        },
-        "firecracker": {
-            role: {
-                "version": getattr(args, f"firecracker_{role}_version"),
-                "sha256": getattr(args, f"firecracker_{role}_sha256"),
-            }
-            for role in ("orchestrator", "gateway")
-        },
-    }
-    parse_manifest(data)
-    args.output.parent.mkdir(parents=True, exist_ok=True)
-    args.output.write_text(
-        json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

scripts/linux-install-test-nixos.nix

@@ -1,24 +0,0 @@
-# NixOS image for scripts/linux-install-test.sh.
-#
-# NixOS publishes no downloadable cloud qcow2 (its cloud images are Hydra-built
-# AMIs), so the harness BUILDS this one with nixos-generators (-f qcow). It is
-# deliberately minimal — no python3/git/pipx — so the bare `test` variant is
-# genuinely under-provisioned and exercises install.sh's guards; `test-ready`
-# provisions them with `nix profile install` (hence flakes below).
-{ lib, ... }:
-{
-  # Consume the same NoCloud seed the other distros use: cloud-init injects the
-  # per-run ephemeral SSH key for root. Leave networking to NixOS's default
-  # dhcpcd (QEMU user-mode NAT) — enabling cloud-init's networkd here conflicts
-  # with dhcpcd and can drop the guest's network.
-  services.cloud-init.enable = true;
-  services.cloud-init.network.enable = false;
-
-  services.openssh.enable = true;
-  services.openssh.settings.PermitRootLogin = lib.mkForce "prohibit-password";
-
-  # Flakes so the `test-ready` prereq step can `nix profile install nixpkgs#...`.
-  nix.settings.experimental-features = [ "nix-command" "flakes" ];
-
-  system.stateVersion = "24.11";
-}

scripts/linux-install-test.sh

@@ -1,753 +0,0 @@
-#!/usr/bin/env bash
-# Clean-install test harness for the Linux path.
-#
-# Exercises install.sh the way a brand-new user would, inside a THROWAWAY
-# QEMU/KVM virtual machine that is booted from a distro cloud image and
-# deleted afterward. install.sh's entire footprint is user-home-local (the
-# pipx venv under ~/.local, the ~/.bot-bottle config dir, and a printed PATH
-# hint), so a fresh VM's fresh $HOME is the clean surface we want — and unlike
-# a throwaway user account, tearing the VM down also wipes any OS-level
-# prerequisites installed into it, so the reset is total. Full rationale in
-# docs/research/testing-clean-install-on-linux.md.
-#
-# Why a VM and not a container or a throwaway user: a container shares the
-# host kernel and cannot exercise a genuinely pristine OS (systemd, the distro
-# package manager, PEP 668 externally-managed Python) the way a real guest
-# does, and a throwaway user leaves every system package it installs behind.
-# The host already needs KVM for the Firecracker backend, so a per-run,
-# copy-on-write VM is cheap here: one cached base image, a throwaway overlay
-# per run (`qemu-img create -b base`), deleted on teardown — the Linux
-# equivalent of `docker run --rm`, but for a whole machine.
-#
-# Usage:
-#   ./scripts/linux-install-test.sh test        # up -> run -> verdict -> down (bare host)
-#   ./scripts/linux-install-test.sh test-ready  # ... with prerequisites installed first
-#   ./scripts/linux-install-test.sh test-all    # every distro × both variants, with a summary
-#   ./scripts/linux-install-test.sh up          # fetch base image, boot a fresh VM
-#   ./scripts/linux-install-test.sh prereqs     # install python3 + git + pipx in the VM
-#   ./scripts/linux-install-test.sh run         # pipe install.sh into the VM
-#   ./scripts/linux-install-test.sh status      # VM reachable? is the install sound?
-#   ./scripts/linux-install-test.sh down        # kill the VM, delete overlay + seed (the reset)
-#   ./scripts/linux-install-test.sh ssh         # open an interactive shell in the running VM
-#
-# TWO TEST VARIANTS, because "does install.sh handle an unprepared host" and
-# "does a prepared host get a clean install" are different questions (mirrors
-# the macOS harness's test / test-ready split):
-#
-#   test        A Linux system WITHOUT the prerequisites set up — the default
-#               state of a stock cloud image (python3 is usually present for
-#               cloud-init, but git and pipx are not). This exercises
-#               install.sh's own prerequisite-guard logic. It is SOUND — a
-#               PASS — when install.sh EITHER installs cleanly (the image
-#               already had enough) OR declines with one of its own recognized,
-#               actionable errors (missing python3/git, no usable pip, PEP 668
-#               externally-managed). A crash or an unrecognized failure fails.
-#
-#   test-ready  A Linux system WITH the prerequisites satisfied — the harness
-#               installs python3 + git + pipx first (see the DISTRO table),
-#               then runs install.sh. A graceful decline is no longer good
-#               enough here: the install MUST land, the entry point must run,
-#               and doctor must report a usable python and config without
-#               crashing.
-#
-# Neither variant requires a green backend. install.sh does not install a
-# backend and cannot regress one, and inside a plain VM there is no nested KVM
-# for Firecracker; the harness does not provision the Docker backend either. So
-# backend readiness is reported, not required (this is where Linux necessarily
-# diverges from the macOS test-ready, which reaches the host backend).
-# BB_TEST_REQUIRE_BACKEND=1 makes it fatal anyway, for a nested-virt host.
-#
-# Config via env:
-#   BB_TEST_DISTRO         ubuntu | fedora | arch | alpine | nixos (default: ubuntu)
-#   BB_TEST_SSH_PORT       host port forwarded to the guest's :22  (default: 2222)
-#   BB_TEST_CACHE_DIR      where base images are cached            (default: ~/.cache/bot-bottle-install-test)
-#   BB_TEST_RUN_DIR        per-run scratch (overlay, seed, key, …) (default: a mktemp dir)
-#   BB_TEST_MEM_MB         guest RAM                               (default: 2048)
-#   BB_TEST_CPUS           guest vCPUs                             (default: 2)
-#   BB_TEST_DISK           overlay virtual size                   (default: 12G)
-#   BB_TEST_BOOT_TIMEOUT   seconds to wait for SSH after boot      (default: 300)
-#   BB_TEST_KEEP           1 = the test cycles skip teardown, to poke at a failure
-#   BB_TEST_REQUIRE_BACKEND 1 = make a not-ready backend fatal (needs nested virt)
-#   BB_TEST_INSTALL_URL    curl this install.sh in the guest instead of piping the local checkout
-#   BB_TEST_SKIP_VERIFY    1 = skip base-image checksum verification (not recommended)
-#   BOT_BOTTLE_INSTALL_SPEC   passed through to install.sh (pip / git spec)
-#
-# Notes:
-#   * Needs /dev/kvm, qemu-system-x86_64, and cloud-localds (cloud-image-utils
-#     / cloud-utils); the nixos distro additionally needs nixos-generate. On the
-#     NixOS host:  nix shell nixpkgs#qemu nixpkgs#cloud-utils nixpkgs#nixos-generators
-#   * Networking is user-mode (`-netdev user,hostfwd`) so the harness needs no
-#     root, no bridge, and touches no host network state. Only SSH is forwarded.
-#   * The cloud-image URLs in the DISTRO table are the one place to bump when a
-#     distro cuts a new build; each is verified against the vendor's published
-#     checksum at download time (guarding against truncated/corrupt pulls).
-
-set -euo pipefail
-
-DISTRO="${BB_TEST_DISTRO:-ubuntu}"
-SSH_PORT="${BB_TEST_SSH_PORT:-2222}"
-CACHE_DIR="${BB_TEST_CACHE_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/bot-bottle-install-test}"
-MEM_MB="${BB_TEST_MEM_MB:-2048}"
-CPUS="${BB_TEST_CPUS:-2}"
-DISK="${BB_TEST_DISK:-12G}"
-BOOT_TIMEOUT="${BB_TEST_BOOT_TIMEOUT:-300}"
-
-_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-_REPO_ROOT="$(cd "$_SCRIPT_DIR/.." && pwd)"
-
-# Per-run scratch. Persisted across sub-commands (up/prereqs/run/status/down)
-# via a marker file so `up` in one invocation and `down` in the next find the
-# same VM; the test cycles set RUN_DIR themselves and never write the marker.
-RUN_DIR="${BB_TEST_RUN_DIR:-}"
-
-# Set by the test cycles, which chain the steps and suppress the per-step "next
-# command" hints. _STEPS / _PASS_CLAIM / _REQUIRE_INSTALL are the per-variant
-# knobs the shared cycle and teardown read.
-IN_TEST=0
-_STEPS=4
-_PASS_CLAIM=""
-_REQUIRE_INSTALL=0
-
-# --- distro table ----------------------------------------------------
-# For each distro:  cloud-image URL | checksum-file URL | default SSH user |
-# prerequisite-install command (run in the guest; uses sudo when user != root).
-#
-# The prerequisite command installs python3 + git + pipx — install.sh installs
-# none of them — so `test-ready` (and the `prereqs` sub-command) drive
-# install.sh down its recommended pipx path. Bump the URLs here when a distro
-# publishes a newer build.
-declare -A IMAGE_URL SUM_URL SSH_USER PREREQ
-
-IMAGE_URL[ubuntu]="https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"
-SUM_URL[ubuntu]="https://cloud-images.ubuntu.com/noble/current/SHA256SUMS"
-SSH_USER[ubuntu]="ubuntu"
-PREREQ[ubuntu]="sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y python3 git pipx"
-
-IMAGE_URL[fedora]="https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-44-1.7.x86_64.qcow2"
-SUM_URL[fedora]="https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/x86_64/images/Fedora-Cloud-44-1.7-x86_64-CHECKSUM"
-SSH_USER[fedora]="fedora"
-PREREQ[fedora]="sudo dnf install -y python3 git pipx"
-
-IMAGE_URL[arch]="https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-cloudimg.qcow2"
-SUM_URL[arch]="https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-cloudimg.qcow2.SHA256"
-SSH_USER[arch]="arch"
-PREREQ[arch]="sudo pacman -Sy --noconfirm python git python-pipx"
-
-# Use the *nocloud_* Alpine variant, not generic_: the generic image probes
-# network datasources and ignores the local NoCloud seed, so cloud-init never
-# runs and the SSH key is never injected. (Only published at .0 patch levels.)
-IMAGE_URL[alpine]="https://dl-cdn.alpinelinux.org/alpine/v3.21/releases/cloud/nocloud_alpine-3.21.0-x86_64-bios-cloudinit-r0.qcow2"
-SUM_URL[alpine]=""   # Alpine cloud images ship .sha512 only; this verifier is sha256.
-SSH_USER[alpine]="alpine"
-PREREQ[alpine]="sudo apk add --no-cache python3 git pipx"
-
-# NixOS publishes no downloadable cloud qcow2 (its cloud images are Hydra-built
-# AMIs), so the harness BUILDS one with nixos-generators — see build_nixos_image
-# and linux-install-test-nixos.nix. It's externally-managed in its own way (no
-# FHS ~/.local on PATH by default); `nix profile install` provisions the
-# prerequisites into root's profile (the image enables flakes for this).
-IMAGE_URL[nixos]="nix:build"   # sentinel: ensure_base_image builds instead of downloading
-SUM_URL[nixos]=""
-SSH_USER[nixos]="root"
-# Pin to a stable release: the guest's default `nixpkgs` registry is unstable,
-# where pipx isn't in the binary cache and builds from source (its test suite
-# currently fails to build). nixos-24.11 has these cached as substitutes.
-PREREQ[nixos]="nix profile install nixpkgs/nixos-24.11#python3 nixpkgs/nixos-24.11#git nixpkgs/nixos-24.11#pipx"
-
-ALL_DISTROS=(ubuntu fedora arch alpine nixos)
-
-# --- guards ----------------------------------------------------------
-require_linux() {
-    [ "$(uname -s)" = "Linux" ] \
-        || { echo "error: this harness is Linux-only (uname is $(uname -s))" >&2; exit 1; }
-}
-
-require_kvm() {
-    [ -e /dev/kvm ] && [ -r /dev/kvm ] && [ -w /dev/kvm ] \
-        || { echo "error: /dev/kvm is missing or not accessible (add yourself to the 'kvm' group)" >&2; exit 1; }
-}
-
-require_tools() {
-    local missing=()
-    command -v qemu-system-x86_64 >/dev/null 2>&1 || missing+=(qemu-system-x86_64)
-    command -v qemu-img          >/dev/null 2>&1 || missing+=(qemu-img)
-    command -v cloud-localds     >/dev/null 2>&1 || missing+=(cloud-localds)
-    command -v ssh               >/dev/null 2>&1 || missing+=(ssh)
-    command -v curl              >/dev/null 2>&1 || missing+=(curl)
-    if [ "${#missing[@]}" -ne 0 ]; then
-        echo "error: missing required tools: ${missing[*]}" >&2
-        echo "  on NixOS:  nix shell nixpkgs#qemu nixpkgs#cloud-utils nixpkgs#openssh nixpkgs#curl" >&2
-        exit 1
-    fi
-}
-
-known_distro() {
-    [ -n "${IMAGE_URL[$DISTRO]:-}" ] \
-        || { echo "error: unknown distro '$DISTRO' (known: ${ALL_DISTROS[*]})" >&2; exit 1; }
-}
-
-# --- run-dir bookkeeping ---------------------------------------------
-# The marker lets prereqs/run/status/down in separate invocations find the VM
-# that `up` started. The test cycles set RUN_DIR themselves and never write it.
-_marker() { echo "${TMPDIR:-/tmp}/bot-bottle-install-test.$DISTRO.run"; }
-
-_ensure_run_dir() {
-    if [ -z "$RUN_DIR" ]; then
-        RUN_DIR="$(mktemp -d "${TMPDIR:-/tmp}/bb-install-test.$DISTRO.XXXXXX")"
-    fi
-    mkdir -p "$RUN_DIR"
-}
-
-_load_run_dir() {
-    if [ -z "$RUN_DIR" ] && [ -f "$(_marker)" ]; then
-        RUN_DIR="$(cat "$(_marker)")"
-    fi
-    [ -n "$RUN_DIR" ] && [ -d "$RUN_DIR" ]
-}
-
-_ssh_key()  { echo "$RUN_DIR/id_ed25519"; }
-_overlay()  { echo "$RUN_DIR/overlay.qcow2"; }
-_seed()     { echo "$RUN_DIR/seed.iso"; }
-_pidfile()  { echo "$RUN_DIR/qemu.pid"; }
-_serial()   { echo "$RUN_DIR/serial.log"; }
-
-# --- ssh helpers -----------------------------------------------------
-_ssh_opts() {
-    # No host-key pinning: the guest is thrown away every run.
-    printf '%s\0' \
-        -i "$(_ssh_key)" \
-        -p "$SSH_PORT" \
-        -o StrictHostKeyChecking=no \
-        -o UserKnownHostsFile=/dev/null \
-        -o LogLevel=ERROR \
-        -o ConnectTimeout=8 \
-        -o BatchMode=yes
-}
-
-guest() {
-    local -a opts
-    mapfile -d '' -t opts < <(_ssh_opts)
-    ssh "${opts[@]}" "${SSH_USER[$DISTRO]}@127.0.0.1" "$@"
-}
-
-wait_for_ssh() {
-    local deadline=$(( SECONDS + BOOT_TIMEOUT ))
-    echo "== waiting for SSH on 127.0.0.1:$SSH_PORT (up to ${BOOT_TIMEOUT}s) =="
-    while [ "$SECONDS" -lt "$deadline" ]; do
-        if guest true 2>/dev/null; then
-            echo "   guest is up"
-            return 0
-        fi
-        # Bail early if QEMU has died — no point waiting out the timeout.
-        if [ -f "$(_pidfile)" ] && ! kill -0 "$(cat "$(_pidfile)")" 2>/dev/null; then
-            echo "error: QEMU exited before SSH came up; see $(_serial)" >&2
-            return 1
-        fi
-        sleep 3
-    done
-    echo "error: timed out waiting for SSH; see $(_serial)" >&2
-    return 1
-}
-
-# --- image cache -----------------------------------------------------
-_base_image() {
-    # One cached file per distro. NixOS is built (not downloaded), so it has a
-    # fixed cache name; the rest are keyed by the image's basename so a URL bump
-    # lands as a new cache entry rather than a stale hit.
-    if [ "$DISTRO" = nixos ]; then
-        echo "$CACHE_DIR/nixos-built.qcow2"
-        return
-    fi
-    local url="${IMAGE_URL[$DISTRO]}"
-    echo "$CACHE_DIR/$DISTRO-$(basename "$url")"
-}
-
-# NixOS has no upstream cloud qcow2; build one with nixos-generators and copy it
-# out of the (immutable, GC-able) store into the cache.
-build_nixos_image() {
-    local out; out="$(_base_image)"
-    [ -f "$out" ] && { echo "== nixos base image cached: $out =="; return 0; }
-    command -v nixos-generate >/dev/null 2>&1 || {
-        echo "error: 'nixos-generate' is required to build the NixOS image" >&2
-        echo "  run inside:  nix shell nixpkgs#nixos-generators nixpkgs#qemu nixpkgs#cloud-utils" >&2
-        return 1
-    }
-    echo "== building NixOS cloud image with nixos-generators (first run is slow) =="
-    local link="$CACHE_DIR/nixos-result"
-    nixos-generate -f qcow --system x86_64-linux \
-        -c "$_SCRIPT_DIR/linux-install-test-nixos.nix" -o "$link"
-    cp -L "$link"/*.qcow2 "$out"
-    rm -f "$link"
-    echo "   built + cached: $out"
-}
-
-verify_checksum() {
-    local file="$1" sums_url="${SUM_URL[$DISTRO]}" base
-    base="$(basename "${IMAGE_URL[$DISTRO]}")"
-    if [ "${BB_TEST_SKIP_VERIFY:-0}" = "1" ] || [ -z "$sums_url" ]; then
-        echo "   checksum: SKIPPED (${sums_url:+set BB_TEST_SKIP_VERIFY=0 to enable}${sums_url:-no sums URL for $DISTRO})" >&2
-        return 0
-    fi
-    local want
-    # Vendors publish either "HASH  filename" tables or a bare "HASH" (or a
-    # "SHA256 (file) = HASH" BSD line, e.g. Fedora). Cover all three.
-    local sums; sums="$(curl -fsSL "$sums_url")"
-    want="$(printf '%s\n' "$sums" | awk -v f="$base" '
-        $0 ~ f && $1 ~ /^[0-9a-fA-F]{64}$/ { print $1; exit }        # GNU "hash  file"
-        $1=="SHA256" && $0 ~ f { gsub(/[()]/,""); print $NF; exit }  # BSD "SHA256 (file) = hash"
-    ')"
-    [ -z "$want" ] && want="$(printf '%s\n' "$sums" | awk '/^[0-9a-fA-F]{64}$/ {print $1; exit}')"
-    [ -n "$want" ] || { echo "error: could not find a sha256 for $base in $sums_url" >&2; return 1; }
-    local got; got="$(sha256sum "$file" | awk '{print $1}')"
-    if [ "$want" != "$got" ]; then
-        echo "error: checksum mismatch for $base" >&2
-        echo "  want $want" >&2
-        echo "  got  $got" >&2
-        return 1
-    fi
-    echo "   checksum: OK"
-}
-
-ensure_base_image() {
-    mkdir -p "$CACHE_DIR"
-    if [ "$DISTRO" = nixos ]; then
-        build_nixos_image
-        return
-    fi
-    local base; base="$(_base_image)"
-    if [ -f "$base" ]; then
-        echo "== base image cached: $base =="
-        return 0
-    fi
-    echo "== downloading $DISTRO cloud image =="
-    echo "   ${IMAGE_URL[$DISTRO]}"
-    # Download to a temp name and rename on success so an interrupted pull
-    # never poisons the cache with a truncated image.
-    local tmp="$base.partial"
-    curl -fSL --retry 3 -o "$tmp" "${IMAGE_URL[$DISTRO]}"
-    verify_checksum "$tmp"
-    mv "$tmp" "$base"
-    echo "   cached: $base"
-}
-
-# --- cloud-init seed -------------------------------------------------
-make_seed() {
-    ssh-keygen -t ed25519 -N '' -f "$(_ssh_key)" -q
-    local pub; pub="$(cat "$(_ssh_key).pub")"
-    local user="${SSH_USER[$DISTRO]}"
-    local user_data="$RUN_DIR/user-data"
-
-    if [ "$user" = "root" ]; then
-        # NixOS' cloud-init lands the key straight on root; no sudo needed.
-        cat > "$user_data" <<EOF
-#cloud-config
-ssh_authorized_keys:
-  - $pub
-EOF
-    elif [ "$DISTRO" = alpine ]; then
-        # Alpine needs three things the systemd distros don't: cloud-init locks
-        # the account (lock_passwd), but Alpine's non-PAM sshd then refuses
-        # pubkey auth for a locked account — so give it a throwaway password;
-        # and OpenRC does not auto-start sshd after the key is injected, so
-        # start it via runcmd.
-        cat > "$user_data" <<EOF
-#cloud-config
-users:
-  - name: $user
-    sudo: ALL=(ALL) NOPASSWD:ALL
-    shell: /bin/sh
-    lock_passwd: false
-    ssh_authorized_keys:
-      - $pub
-chpasswd:
-  expire: false
-  list: |
-    $user:bbtest
-packages:
-  - sudo
-runcmd:
-  - [ sh, -c, "rc-service sshd start 2>/dev/null || true" ]
-EOF
-    else
-        cat > "$user_data" <<EOF
-#cloud-config
-users:
-  - name: $user
-    sudo: ALL=(ALL) NOPASSWD:ALL
-    shell: /bin/sh
-    lock_passwd: true
-    ssh_authorized_keys:
-      - $pub
-EOF
-    fi
-    # NoCloud wants a meta-data with an instance-id, or cloud-init may not treat
-    # the seed as a new instance (the Alpine nocloud image is strict about this).
-    printf 'instance-id: bbtest-%s\nlocal-hostname: bbtest-%s\n' "$DISTRO" "$DISTRO" \
-        > "$RUN_DIR/meta-data"
-    cloud-localds "$(_seed)" "$user_data" "$RUN_DIR/meta-data"
-}
-
-# --- commands --------------------------------------------------------
-cmd_up() {
-    require_linux; require_kvm; require_tools; known_distro
-    _ensure_run_dir
-    ensure_base_image
-
-    # Throwaway copy-on-write overlay: the cached base is read-only backing,
-    # all guest writes land in the overlay, and `down` deletes it. Resize so
-    # pipx + a git build have headroom (cloud-init grows the rootfs to fit).
-    qemu-img create -q -f qcow2 -F qcow2 -b "$(_base_image)" "$(_overlay)" "$DISK"
-    make_seed
-
-    echo "== booting $DISTRO VM (mem=${MEM_MB}M cpus=$CPUS, ssh -> :$SSH_PORT) =="
-    qemu-system-x86_64 \
-        -machine accel=kvm -cpu host -smp "$CPUS" -m "$MEM_MB" \
-        -display none -daemonize \
-        -pidfile "$(_pidfile)" \
-        -serial "file:$(_serial)" \
-        -drive "file=$(_overlay),if=virtio,format=qcow2" \
-        -drive "file=$(_seed),if=virtio,format=raw" \
-        -netdev "user,id=n0,hostfwd=tcp:127.0.0.1:$SSH_PORT-:22" \
-        -device virtio-net-pci,netdev=n0
-
-    # Only publish the marker (so a later prereqs/run/down finds this VM) when
-    # we aren't inside a test cycle, which manages its own RUN_DIR + teardown.
-    [ "$IN_TEST" = 1 ] || echo "$RUN_DIR" > "$(_marker)"
-
-    wait_for_ssh
-    if [ "$IN_TEST" != 1 ]; then
-        echo "== VM is up. Prepare it with: BB_TEST_DISTRO=$DISTRO $0 prereqs (or go straight to 'run') =="
-    fi
-}
-
-# Install install.sh's toolchain prerequisites (python3 + git + pipx) into the
-# running VM. This is what separates `test-ready` from `test`, and it is a
-# distinct sub-command so a manual up/prereqs/run/down cycle is possible.
-cmd_prereqs() {
-    require_linux
-    _load_run_dir || { echo "error: no running VM for $DISTRO; run '$0 up' first" >&2; return 1; }
-    echo "== installing prerequisites (python3 + git + pipx) on $DISTRO =="
-    # Runs via the guest login shell; PREREQ is a client-side table value.
-    guest "${PREREQ[$DISTRO]}"
-}
-
-cmd_run() {
-    require_linux
-    _load_run_dir || { echo "error: no running VM for $DISTRO; run '$0 up' first" >&2; return 1; }
-
-    local spec_env=""
-    [ -n "${BOT_BOTTLE_INSTALL_SPEC:-}" ] \
-        && spec_env="BOT_BOTTLE_INSTALL_SPEC='$BOT_BOTTLE_INSTALL_SPEC' "
-
-    echo "== installing bot-bottle as ${SSH_USER[$DISTRO]} =="
-    # Capture install.sh's exit code and full output rather than aborting on
-    # non-zero: on a bare host a clean prerequisite *decline* is sound, so the
-    # verdict step — not set -e — decides the outcome.
-    local rc
-    set +e
-    if [ -n "${BB_TEST_INSTALL_URL:-}" ]; then
-        guest "curl -fsSL '$BB_TEST_INSTALL_URL' | ${spec_env}sh" 2>&1 | tee "$RUN_DIR/install.log"
-    else
-        # Feed THIS checkout's install.sh in over stdin — the same `curl … | sh`
-        # shape a real user runs, and nothing is staged in the guest to leak.
-        guest "${spec_env}sh -s" < "$_REPO_ROOT/install.sh" 2>&1 | tee "$RUN_DIR/install.log"
-    fi
-    rc="${PIPESTATUS[0]}"
-    set -e
-    printf '%s\n' "$rc" > "$RUN_DIR/install.rc"
-    echo "== install.sh exited $rc =="
-    [ "$IN_TEST" = 1 ] \
-        || echo "== verdict anytime with: BB_TEST_DISTRO=$DISTRO $0 status =="
-}
-
-# Quietly report whether a runnable bot-bottle entry point exists for the
-# guest user, checking the pipx/pip locations install.sh may leave off PATH.
-entry_point_runnable() {
-    # shellcheck disable=SC2016  # expand in the GUEST shell.
-    guest '
-        for bb in "$HOME/.local/bin/bot-bottle" "$HOME/.bot-bottle/venv/bin/bot-bottle" "$(command -v bot-bottle 2>/dev/null)"; do
-            [ -n "$bb" ] && [ -x "$bb" ] || continue
-            # --help exits 0 before any DB/migration/network work; it is the
-            # cheapest proof the package imports and the shim runs. (bot-bottle
-            # has no --version: an unknown arg would die non-zero.)
-            "$bb" --help >/dev/null 2>&1 && exit 0
-        done
-        exit 1
-    ' >/dev/null 2>&1
-}
-
-# `bot-bottle doctor` in the guest, classified. doctor's own exit code
-# conflates "is the install sound" with "is a backend ready to run a bottle" —
-# and inside a plain VM no backend can be ready (no nested KVM/Docker), so the
-# raw exit code is non-zero by design. This separates the two: an unhandled
-# traceback, or a missing python/config line, is an install defect and fails;
-# a not-ready backend is reported, not fatal (unless BB_TEST_REQUIRE_BACKEND=1,
-# for a nested-virt host that can actually satisfy it).
-doctor_in_guest() {
-    local out rc=0 bad=0
-    out="$(mktemp "${TMPDIR:-/tmp}/bb-doctor.XXXXXX")"
-    # shellcheck disable=SC2016  # $HOME/$bb must expand in the GUEST shell.
-    guest '
-        for bb in bot-bottle "$HOME/.local/bin/bot-bottle" "$HOME/.bot-bottle/venv/bin/bot-bottle"; do
-            if command -v "$bb" >/dev/null 2>&1; then
-                case "$bb" in
-                    bot-bottle) : ;;
-                    *) echo "  (not on PATH — running $bb directly, as install.sh advises)" ;;
-                esac
-                exec "$bb" doctor
-            fi
-        done
-        echo "  no bot-bottle entry point found for this user" >&2
-        exit 1
-    ' >"$out" 2>&1 || rc=$?
-    cat "$out"
-
-    # An unhandled exception is always an install/product defect, never an
-    # environment fact — doctor's non-zero exit alone would not distinguish it.
-    if grep -q 'Traceback (most recent call last)' "$out"; then
-        echo "  doctor crashed (traceback above) — a defect, not a missing prerequisite" >&2
-        bad=1
-    fi
-    grep -qE '^ok: +python:' "$out" \
-        || { echo "  doctor never reported a usable python" >&2; bad=1; }
-    grep -qE '^ok: +config:' "$out" \
-        || { echo "  doctor never reported a usable config dir" >&2; bad=1; }
-    if [ "$rc" -ne 0 ] && ! grep -qE '^(fail|warn): +backend' "$out"; then
-        echo "  doctor failed for something other than backend readiness" >&2
-        bad=1
-    fi
-
-    local backend_ready=1
-    grep -qE '^fail: +backend' "$out" && backend_ready=0
-    rm -f "$out"
-
-    [ "$bad" -eq 0 ] || return 1
-    if [ "${BB_TEST_REQUIRE_BACKEND:-0}" = "1" ] && [ "$backend_ready" -eq 0 ]; then
-        echo "  backend is not ready and BB_TEST_REQUIRE_BACKEND=1 — failing" >&2
-        return 1
-    fi
-    if [ "$backend_ready" -eq 1 ]; then
-        echo "  doctor: install sound; a backend is ready"
-    else
-        echo "  doctor: install sound; no backend ready (expected in a plain VM — install gate only)"
-    fi
-    return 0
-}
-
-# The prerequisite-decline messages install.sh prints via die(). On a bare host
-# ANY of these means install.sh correctly refused rather than half-installing —
-# a sound outcome for `test`.
-PREREQ_ERR_RE='is required but was not found|or newer is required|git is required to install from|neither pipx nor a usable|externally managed \(PEP 668\)|is not on PATH'
-
-# The verdict: is the install sound? Reads install.sh's captured exit code and
-# output (from cmd_run) plus the guest's resulting state.
-#   - installed & runnable  -> the verdict is doctor's soundness classification.
-#   - no entry point, but a recognized prerequisite decline, and declines are
-#     allowed (bare `test`, _REQUIRE_INSTALL=0) -> sound.
-#   - anything else -> not sound.
-install_verdict() {
-    local rc=""
-    [ -f "$RUN_DIR/install.rc" ] && rc="$(cat "$RUN_DIR/install.rc")"
-
-    if [ "${rc:-1}" = 0 ] && entry_point_runnable; then
-        echo "doctor (in guest):"
-        doctor_in_guest
-        return $?
-    fi
-
-    if [ "${_REQUIRE_INSTALL:-0}" != "1" ] \
-            && [ -n "$rc" ] && [ "$rc" != 0 ] \
-            && [ -f "$RUN_DIR/install.log" ] \
-            && grep -Eiq "$PREREQ_ERR_RE" "$RUN_DIR/install.log"; then
-        echo "  install.sh declined with an actionable prerequisite error (rc=$rc)"
-        echo "  — sound on a bare host; run 'test-ready' (or 'prereqs') to install."
-        return 0
-    fi
-
-    if [ "${_REQUIRE_INSTALL:-0}" = "1" ]; then
-        echo "  prerequisites were provisioned, but install.sh left no runnable entry point (rc=${rc:-?})" >&2
-    else
-        echo "  install.sh neither installed nor gave a recognized prerequisite error (rc=${rc:-?})" >&2
-    fi
-    return 1
-}
-
-cmd_status() {
-    require_linux
-    if ! _load_run_dir; then
-        echo "vm:      no running VM for $DISTRO"
-        return 0
-    fi
-    if [ -f "$(_pidfile)" ] && kill -0 "$(cat "$(_pidfile)")" 2>/dev/null; then
-        echo "vm:      $DISTRO running (pid $(cat "$(_pidfile)"), ssh :$SSH_PORT)"
-    else
-        echo "vm:      $DISTRO run-dir present but QEMU not alive"
-        return 1
-    fi
-    if install_verdict; then
-        echo "OK[$DISTRO]: the install is sound"
-        return 0
-    fi
-    echo "FAIL[$DISTRO]: the install is not sound (see above)" >&2
-    return 1
-}
-
-cmd_down() {
-    require_linux
-    if ! _load_run_dir; then
-        echo "$DISTRO: nothing running"
-        return 0
-    fi
-    if [ -f "$(_pidfile)" ]; then
-        local pid; pid="$(cat "$(_pidfile)")"
-        if kill -0 "$pid" 2>/dev/null; then
-            kill "$pid" 2>/dev/null || true
-            for _ in 1 2 3 4 5; do kill -0 "$pid" 2>/dev/null || break; sleep 1; done
-            kill -9 "$pid" 2>/dev/null || true
-        fi
-    fi
-    # Deleting the overlay + seed is the reset; the read-only base stays cached.
-    rm -f "$(_overlay)" "$(_seed)" "$(_ssh_key)" "$(_ssh_key).pub" \
-          "$RUN_DIR/user-data" "$RUN_DIR/meta-data" \
-          "$RUN_DIR/install.rc" "$RUN_DIR/install.log" "$(_serial)"
-    # Only remove a scratch dir we created (leave a user-provided one alone).
-    [ -n "${BB_TEST_RUN_DIR:-}" ] || rmdir "$RUN_DIR" 2>/dev/null || true
-    rm -f "$(_marker)"
-    echo "removed $DISTRO VM and its overlay — install surface is clean."
-}
-
-cmd_ssh() {
-    require_linux
-    _load_run_dir || { echo "error: no running VM for $DISTRO" >&2; return 1; }
-    local -a opts
-    mapfile -d '' -t opts < <(_ssh_opts)
-    exec ssh -t "${opts[@]}" "${SSH_USER[$DISTRO]}@127.0.0.1"
-}
-
-# Teardown half of the test cycles, armed the moment the VM exists so a failure
-# or a Ctrl-C still leaves nothing running.
-_test_teardown() {
-    local rc=$?
-    trap - EXIT INT TERM
-    if [ "${BB_TEST_KEEP:-0}" = "1" ]; then
-        echo
-        echo "== [$_STEPS/$_STEPS] down: SKIPPED (BB_TEST_KEEP=1) =="
-        echo "   VM still up; remove with: BB_TEST_DISTRO=$DISTRO $0 down"
-        echo "   ssh in with: BB_TEST_RUN_DIR=$RUN_DIR BB_TEST_DISTRO=$DISTRO $0 ssh"
-        exit "$rc"
-    fi
-    echo
-    echo "== [$_STEPS/$_STEPS] down =="
-    cmd_down || rc=1
-    if [ "$rc" -eq 0 ]; then
-        echo
-        echo "PASS[$DISTRO]: $_PASS_CLAIM"
-    else
-        echo
-        echo "FAIL[$DISTRO]: see above (the VM was torn down regardless)." >&2
-    fi
-    exit "$rc"
-}
-
-# The two variants differ only in whether the prerequisites get installed
-# before install.sh runs, which is exactly the question each one asks:
-#
-#   test        a bare host — assert install.sh is SOUND (installs cleanly, or
-#               declines with an actionable prerequisite error).
-#   test-ready  prerequisites satisfied — assert install.sh actually LANDS.
-_test_cycle() {
-    local with_prereqs="$1"
-    require_linux; require_kvm; require_tools; known_distro
-    IN_TEST=1
-    RUN_DIR="$(mktemp -d "${TMPDIR:-/tmp}/bb-install-test.$DISTRO.XXXXXX")"
-
-    # Arm teardown BEFORE cmd_up: its wait_for_ssh can fail after QEMU is
-    # already running (e.g. a guest that never opens SSH), and without the trap
-    # in place that would leak the VM.
-    trap _test_teardown EXIT INT TERM
-    echo "== [1/$_STEPS] up ($DISTRO) =="
-    cmd_up
-
-    local step=2
-    if [ "$with_prereqs" = 1 ]; then
-        echo
-        echo "== [$step/$_STEPS] prereqs =="
-        cmd_prereqs || { echo "error: could not install prerequisites (see above)." >&2; return 1; }
-        step=$(( step + 1 ))
-    fi
-
-    echo
-    echo "== [$step/$_STEPS] run =="
-    cmd_run
-    step=$(( step + 1 ))
-
-    echo
-    echo "== [$step/$_STEPS] verdict =="
-    # install.sh exits 0 even when doctor reports unmet prerequisites, so the
-    # install succeeding is not the verdict — this is.
-    cmd_status || {
-        echo "error: the install is not sound for $DISTRO (see above)." >&2
-        echo "  re-run with BB_TEST_KEEP=1 to keep the VM and dig in." >&2
-        return 1
-    }
-}
-
-cmd_test() {
-    _STEPS=4
-    _PASS_CLAIM="on a bare $DISTRO host, install.sh behaves soundly."
-    _test_cycle 0
-}
-
-cmd_test_ready() {
-    _STEPS=5
-    _PASS_CLAIM="a $DISTRO host with prerequisites satisfied installs bot-bottle cleanly."
-    # Prerequisites are provisioned, so a graceful decline is no longer an
-    # acceptable outcome — the install must actually land.
-    _REQUIRE_INSTALL=1
-    _test_cycle 1
-}
-
-cmd_test_all() {
-    require_linux; require_kvm; require_tools
-    local -a passed=() failed=()
-    local port="$SSH_PORT"
-    local script
-    script="$_SCRIPT_DIR/$(basename "${BASH_SOURCE[0]}")"
-    # Every distro × both variants. Each cell gets its own forwarded port so a
-    # leftover from a prior cell can't collide, and its own subshell so one
-    # cell's failure (or teardown trap) doesn't abort the matrix.
-    for sub in test test-ready; do
-        for d in "${ALL_DISTROS[@]}"; do
-            echo
-            echo "########################################################"
-            echo "# $d  ($sub)"
-            echo "########################################################"
-            if ( DISTRO="$d" SSH_PORT="$port" \
-                    BB_TEST_DISTRO="$d" BB_TEST_SSH_PORT="$port" \
-                    bash "$script" "$sub" ); then
-                passed+=("$d/$sub")
-            else
-                failed+=("$d/$sub")
-            fi
-            port=$(( port + 1 ))
-        done
-    done
-    echo
-    echo "== matrix summary =="
-    echo "  PASS: ${passed[*]:-(none)}"
-    echo "  FAIL: ${failed[*]:-(none)}"
-    [ "${#failed[@]}" -eq 0 ]
-}
-
-case "${1:-}" in
-    test)       cmd_test ;;
-    test-ready) cmd_test_ready ;;
-    test-all)   cmd_test_all ;;
-    up)         cmd_up ;;
-    prereqs)    cmd_prereqs ;;
-    run)        cmd_run ;;
-    status)     cmd_status ;;
-    down)       cmd_down ;;
-    ssh)        cmd_ssh ;;
-    *) echo "usage: $0 {test|test-ready|test-all|up|prereqs|run|status|down|ssh}  (distro via BB_TEST_DISTRO)" >&2; exit 2 ;;
-esac

scripts/publish_release_qualification.py

@@ -1,42 +0,0 @@
-#!/usr/bin/env python3
-"""Publish a qualified release and advance its matching channel."""
-
-from __future__ import annotations
-
-import argparse
-import json
-from pathlib import Path
-
-from bot_bottle.release_qualification import (
-    build_release_pointer,
-    canonical_pointer,
-    publish_qualification,
-)
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--tag", required=True)
-    parser.add_argument("--source-commit", required=True)
-    parser.add_argument("--workflow-run", required=True)
-    parser.add_argument("--qualified-at", required=True)
-    parser.add_argument("--output", type=Path, default=Path("release.json"))
-    parser.add_argument("--allow-rollback", action="store_true")
-    args = parser.parse_args()
-    pointer = build_release_pointer(
-        tag=args.tag,
-        source_commit=args.source_commit,
-        workflow_run=args.workflow_run,
-        qualified_at=args.qualified_at,
-    )
-    args.output.write_bytes(canonical_pointer(pointer))
-    release_url, channel_url = publish_qualification(
-        pointer, allow_rollback=args.allow_rollback)
-    print(json.dumps({
-        "release": release_url,
-        "channel": channel_url,
-    }, sort_keys=True))
-
-
-if __name__ == "__main__":
-    main()

setup.py

@@ -10,8 +10,6 @@ Kept in sync with ``bot_bottle.resources.BUNDLED_RESOURCES`` — the
 
 from __future__ import annotations
 
-import json
-import os
 import shutil
 from pathlib import Path
 
@@ -46,18 +44,6 @@ class _BundleResources(build_py):
             dst = pkg_resources / rel
             dst.parent.mkdir(parents=True, exist_ok=True)
             shutil.copy2(src, dst)
-        manifest_dest = Path(self.build_lib) / "bot_bottle" / "release-manifest.json"
-        manifest_input = os.environ.get("BOT_BOTTLE_RELEASE_MANIFEST", "").strip()
-        if manifest_input:
-            shutil.copy2(manifest_input, manifest_dest)
-        else:
-            # Wheels built ad hoc retain the contributor/install.sh local-build
-            # behavior. The release workflow always replaces this marker with
-            # validated immutable artifact identities.
-            manifest_dest.write_text(
-                json.dumps({"schema": 1, "development": True}) + "\n",
-                encoding="utf-8",
-            )
 
 
 setup(cmdclass={"build_py": _BundleResources})

tests/unit/test_backend_secret_reprovision.py

@@ -2,9 +2,7 @@
 
 from __future__ import annotations
 
-import json
 import subprocess
-import tempfile
 import unittest
 from pathlib import Path
 from types import SimpleNamespace
@@ -107,21 +105,6 @@ class TestDockerReprovision(unittest.TestCase):
 
 
 class TestFirecrackerReprovision(unittest.TestCase):
-    def _run_dir(self, root: Path, ip: str = "10.243.0.3") -> Path:
-        run_dir = root / "demo"
-        run_dir.mkdir()
-        (run_dir / "bottle_id_ed25519").write_text("key")
-        (run_dir / "config.json").write_text(json.dumps({
-            "boot-source": {"boot_args": f"root=/dev/vda ip={ip}::gw:mask::eth0:off"}
-        }))
-        return run_dir
-
-    def test_extracts_guest_ip_from_config(self) -> None:
-        with tempfile.TemporaryDirectory() as tmp:
-            run_dir = self._run_dir(Path(tmp))
-            self.assertEqual("10.243.0.3", fc._guest_ip_from_config(run_dir / "config.json"))
-            self.assertEqual("", fc._guest_ip_from_config(run_dir / "missing.json"))
-
     def test_persists_key_over_stdin_not_argv(self) -> None:
         with patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
              patch.object(fc.subprocess, "run", return_value=_proc()) as run:
@@ -135,29 +118,6 @@ class TestFirecrackerReprovision(unittest.TestCase):
             with self.assertRaisesRegex(fc.ConsolidatedLaunchError, "denied"):
                 fc.persist_env_var_secret(Path("/key"), "10.0.0.1", "secret")
 
-    def test_reads_live_vm_key_and_reprovisions(self) -> None:
-        with tempfile.TemporaryDirectory() as tmp:
-            run_dir = self._run_dir(Path(tmp))
-            client = Mock()
-            with patch.object(fc.cleanup, "live_run_dirs", return_value=(run_dir,)), \
-                 patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
-                 patch.object(fc.subprocess, "run", return_value=_proc(stdout="secret\n")), \
-                 patch.object(fc, "reprovision_bottles", return_value=1) as restore, \
-                 patch.object(fc, "info"):
-                fc._reprovision_running_bottles(client)
-        restore.assert_called_once_with(client, {"10.243.0.3": "secret"})
-
-    def test_unreadable_vm_is_skipped(self) -> None:
-        with tempfile.TemporaryDirectory() as tmp:
-            run_dir = self._run_dir(Path(tmp))
-            client = Mock()
-            with patch.object(fc.cleanup, "live_run_dirs", return_value=(run_dir,)), \
-                 patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
-                 patch.object(fc.subprocess, "run", return_value=_proc(1)), \
-                 patch.object(fc, "reprovision_bottles", return_value=0) as restore:
-                fc._reprovision_running_bottles(client)
-        restore.assert_called_once_with(client, {})
-
 
 if __name__ == "__main__":
     unittest.main()

tests/unit/test_docker_infra.py

@@ -46,8 +46,8 @@ class TestDockerInfraService(unittest.TestCase):
         url = self.svc.ensure_running()
         self.assertEqual("http://127.0.0.1:8099", url)
         # Both images built; the control plane comes up before the gateway.
-        self.orch.ensure_available.assert_called_once()
+        self.orch.ensure_built.assert_called_once()
-        self.gw.ensure_available.assert_called_once()
+        self.gw.ensure_built.assert_called_once()
         self.orch.ensure_running.assert_called_once()
 
     def test_ensure_running_connects_gateway_with_minted_token(self) -> None:
@@ -76,19 +76,6 @@ class TestDockerInfraService(unittest.TestCase):
         self.assertEqual("registry-volume", svc.orchestrator()._root_mount_source)
         self.assertEqual("ca-volume", svc.gateway()._ca_mount_source)
 
-    def test_packaged_images_disable_local_infra_dockerfiles(self) -> None:
-        refs = [
-            ("registry/orchestrator@sha256:" + "a" * 64, False),
-            ("registry/gateway@sha256:" + "b" * 64, False),
-        ]
-        with patch(
-            "bot_bottle.backend.docker.infra.release_manifest.oci_image",
-            side_effect=refs,
-        ):
-            svc = DockerInfraService()
-        self.assertIsNone(svc.orchestrator()._dockerfile)
-        self.assertIsNone(svc.gateway()._dockerfile)
-
     def test_host_path_and_named_root_mount_are_mutually_exclusive(self) -> None:
         with self.assertRaisesRegex(ValueError, "host_root or root_mount_source"):
             DockerInfraService(

tests/unit/test_docker_orchestrator.py

@@ -231,12 +231,11 @@ class TestDockerOrchestrator(unittest.TestCase):
             with self.assertRaises(GatewayError):
                 self.orch.ensure_built()
 
-    def test_ensure_built_pulls_without_dockerfile(self) -> None:
+    def test_ensure_built_noop_without_dockerfile(self) -> None:
-        orch = DockerOrchestrator("registry/orchestrator@sha256:" + "a" * 64,
+        orch = DockerOrchestrator(dockerfile=None)
-                                  dockerfile=None)
+        with patch(_RUN) as run:
-        with patch(_RUN, return_value=_proc()) as run:
             orch.ensure_built()
-        run.assert_called_once_with(["docker", "pull", orch.image_ref])
+        run.assert_not_called()
 
     def test_is_running_reads_docker_ps(self) -> None:
         with patch(_RUN, return_value=_proc(stdout=ORCHESTRATOR_NAME)):

tests/unit/test_firecracker_image_builder.py

@@ -7,7 +7,6 @@ the smoke-test no-op — the logic that must hold without a VM.
 
 from __future__ import annotations
 
-import subprocess
 import tempfile
 import unittest
 from pathlib import Path
@@ -91,120 +90,6 @@ class TestBuildAgentRootfsDir(unittest.TestCase):
             second = image_builder._rootfs_digest(self.dockerfile)
         self.assertNotEqual(first, second)
 
-    def test_pinned_image_is_pulled_and_cached(self):
-        image = f"registry.example/agent@sha256:{'a' * 64}"
-        with patch.object(image_builder.util, "cache_dir", return_value=self.cache), \
-             patch.object(image_builder, "_pull_in_infra") as pull, \
-             patch.object(image_builder.util, "inject_guest_boot"):
-            first = image_builder.acquire_agent_image_rootfs_dir(image)
-            second = image_builder.acquire_agent_image_rootfs_dir(image)
-        pull.assert_called_once()
-        self.assertEqual(first, second)
-
-    def test_mutable_image_cannot_use_prebuilt_path(self):
-        with self.assertRaises(SystemExit):
-            image_builder.acquire_agent_image_rootfs_dir("agent:latest")
-
-
-class TestBuildContext(unittest.TestCase):
-    """The COPY-source parsing + context shipping that lets a Dockerfile pin an
-    input by COPYing a committed file (codex checksum list, claude/pi npm
-    lockfiles) through the otherwise-empty VM-side build context."""
-
-    def setUp(self):
-        self._tmp = tempfile.TemporaryDirectory()
-        self.root = Path(self._tmp.name)
-        self.addCleanup(self._tmp.cleanup)
-        codex = self.root / "bot_bottle" / "contrib" / "codex"
-        codex.mkdir(parents=True)
-        self.sums = codex / "codex-package_SHA256SUMS"
-        self.sums.write_text("aaaa  codex-package-x86_64-unknown-linux-musl.tar.gz\n")
-        self.dockerfile = self.root / "Dockerfile"
-        self.dockerfile.write_text(
-            "FROM node:22-slim\n"
-            "COPY --chown=node:node "
-            "bot_bottle/contrib/codex/codex-package_SHA256SUMS /tmp/x\n"
-        )
-
-    def _patch_root(self):
-        return patch.object(
-            image_builder.resources, "build_root", return_value=self.root)
-
-    def test_copy_sources_parses_flags_and_continuations(self):
-        df = self.root / "Multi"
-        df.write_text(
-            "FROM x\n"
-            "COPY a/one.json \\\n     a/two.json /dest/\n"
-            "COPY --from=builder /built /built\n"          # excluded: build stage
-            "COPY --chown=n:n b/three /dest\n"
-        )
-        self.assertEqual(
-            image_builder._context_copy_sources(df),
-            ["a/one.json", "a/two.json", "b/three"],
-        )
-
-    def test_context_files_resolves_existing_under_root(self):
-        with self._patch_root():
-            files = image_builder._context_files(self.dockerfile)
-        self.assertEqual(
-            [rel for rel, _ in files],
-            ["bot_bottle/contrib/codex/codex-package_SHA256SUMS"],
-        )
-
-    def test_context_files_recurses_into_directory_sources(self):
-        nested = self.root / "assets" / "nested"
-        nested.mkdir(parents=True)
-        (self.root / "assets" / "one").write_text("one")
-        (nested / "two").write_text("two")
-        df = self.root / "Directory"
-        df.write_text("FROM x\nCOPY assets /opt/assets\n")
-        with self._patch_root():
-            files = image_builder._context_files(df)
-        self.assertEqual(
-            [rel for rel, _ in files],
-            ["assets/nested/two", "assets/one"],
-        )
-
-    def test_context_files_drops_missing_and_traversal(self):
-        df = self.root / "Bad"
-        df.write_text("FROM x\nCOPY ../escape /d\nCOPY does/not/exist /d\n")
-        with self._patch_root():
-            self.assertEqual(image_builder._context_files(df), [])
-
-    def test_rootfs_digest_tracks_context_file_content(self):
-        with self._patch_root(), \
-             patch.object(image_builder.util, "cache_dir", return_value=self.root):
-            first = image_builder._rootfs_digest(self.dockerfile)
-            self.sums.write_text("bbbb  codex-package-x86_64-unknown-linux-musl.tar.gz\n")
-            second = image_builder._rootfs_digest(self.dockerfile)
-        self.assertNotEqual(first, second)
-
-    def test_send_build_context_noop_without_copy(self):
-        df = self.root / "None"
-        df.write_text("FROM x\n")
-        with self._patch_root(), \
-             patch.object(image_builder.subprocess, "Popen") as popen:
-            image_builder._send_build_context(Path("/k"), "10.0.0.1", df, "/tmp/c")
-        popen.assert_not_called()
-
-    def test_send_build_context_streams_copied_files_to_vm(self):
-        completed = subprocess.CompletedProcess([], 0, stdout=b"", stderr=b"")
-        with self._patch_root(), \
-             patch.object(image_builder.subprocess, "Popen") as popen, \
-             patch.object(image_builder.subprocess, "run",
-                          return_value=completed) as run:
-            popen.return_value.stdout = None
-            popen.return_value.returncode = 0
-            popen.return_value.wait.return_value = 0
-            image_builder._send_build_context(
-                Path("/k"), "10.0.0.1", self.dockerfile, "/tmp/c")
-        tar_argv = popen.call_args.args[0]
-        self.assertEqual(tar_argv[:3], ["tar", "-C", str(self.root)])
-        self.assertIn("--", tar_argv)
-        self.assertIn(
-            "bot_bottle/contrib/codex/codex-package_SHA256SUMS", tar_argv)
-        self.assertIn("tar -C /tmp/c/ctx -xf -", run.call_args.args[0][-1])
-
 
 class TestSmokeTest(unittest.TestCase):
     def test_buildah_receives_centralized_image_build_args(self):
@@ -229,6 +114,7 @@ class TestSmokeTest(unittest.TestCase):
         ssh.assert_not_called()
 
     def test_failed_smoke_dies(self):
+        import subprocess
         result = subprocess.CompletedProcess([], 1, stdout="broken", stderr="")
         with patch.object(image_builder, "_ssh", return_value=result), \
              self.assertRaises(SystemExit):

tests/unit/test_firecracker_infra.py

@@ -21,6 +21,7 @@ from bot_bottle.backend.firecracker.orchestrator import FirecrackerOrchestrator
 # there (not at their home modules).
 _ORCH_CLS = "bot_bottle.backend.firecracker.infra.FirecrackerOrchestrator"
 _GW_CLS = "bot_bottle.backend.firecracker.infra.FirecrackerGateway"
+_RECONCILE = "bot_bottle.backend.firecracker.infra.attach_bottled_agents_to_gateway"
 
 
 class TestAccessors(unittest.TestCase):
@@ -53,10 +54,13 @@ class TestEnsureRunning(unittest.TestCase):
                     patch.object(infra_vm, "_health_ok", return_value=True), \
                     patch.object(infra_vm, "_pidfile_alive", return_value=True), \
                     patch.object(infra_vm, "stop") as stop, \
-                    patch.object(infra_vm, "ensure_available") as built:
+                    patch.object(infra_vm, "ensure_built") as built, \
+                    patch(_RECONCILE) as reconcile:
                 url = FirecrackerInfraService().ensure_running()
             stop.assert_not_called()
             built.assert_not_called()
+            # Adopt path: no reconcile — gateway state is intact.
+            reconcile.assert_not_called()
             ip = infra_vm.netpool.orch_slot().guest_ip
             self.assertEqual(f"http://{ip}:{infra_vm.ORCHESTRATOR_PORT}", url)
 
@@ -72,8 +76,9 @@ class TestEnsureRunning(unittest.TestCase):
                     patch.object(infra_vm, "_health_ok", return_value=True), \
                     patch.object(infra_vm, "_pidfile_alive", return_value=True), \
                     patch.object(infra_vm, "stop") as stop, \
-                    patch.object(infra_vm, "ensure_available"), \
+                    patch.object(infra_vm, "ensure_built"), \
-                    patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls:
+                    patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls, \
+                    patch(_RECONCILE) as reconcile:
                 orch_cls.return_value.gateway_url.return_value = "http://10.243.255.1:8099"
                 orch_cls.return_value.mint_gateway_token.return_value = "gw.jwt"
                 FirecrackerInfraService().ensure_running()
@@ -85,6 +90,8 @@ class TestEnsureRunning(unittest.TestCase):
                 "http://10.243.255.1:8099", "gw.jwt")
             # The fresh boot records the current version for the next launcher.
             self.assertEqual("v-current\n", (d / "booted-version").read_text())
+            # Cold boot: reconcile fires after gateway is up.
+            reconcile.assert_called_once()
 
     def test_reboots_when_gateway_dead(self):
         # Orchestrator healthy + marker current, but the gateway VM is gone ->
@@ -98,12 +105,14 @@ class TestEnsureRunning(unittest.TestCase):
                     patch.object(infra_vm, "_health_ok", return_value=True), \
                     patch.object(infra_vm, "_pidfile_alive", return_value=False), \
                     patch.object(infra_vm, "stop") as stop, \
-                    patch.object(infra_vm, "ensure_available"), \
+                    patch.object(infra_vm, "ensure_built"), \
-                    patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls:
+                    patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls, \
+                    patch(_RECONCILE) as reconcile:
                 FirecrackerInfraService().ensure_running()
             stop.assert_called_once()
             orch_cls.return_value.ensure_running.assert_called_once()
             gw_cls.return_value.connect_to_orchestrator.assert_called_once()
+            reconcile.assert_called_once()
 
     def test_boots_both_when_no_running_pair(self):
         with tempfile.TemporaryDirectory() as td:
@@ -111,8 +120,9 @@ class TestEnsureRunning(unittest.TestCase):
                     patch.object(infra_vm, "expected_version", return_value="v-current"), \
                     patch.object(infra_vm, "_health_ok", return_value=False), \
                     patch.object(infra_vm, "stop") as stop, \
-                    patch.object(infra_vm, "ensure_available") as built, \
+                    patch.object(infra_vm, "ensure_built") as built, \
-                    patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls:
+                    patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls, \
+                    patch(_RECONCILE) as reconcile:
                 FirecrackerInfraService().ensure_running()
         stop.assert_called_once()      # clear stale VMs first
         built.assert_called_once()
@@ -120,6 +130,27 @@ class TestEnsureRunning(unittest.TestCase):
         # reach the control plane at startup).
         orch_cls.return_value.ensure_running.assert_called_once()
         gw_cls.return_value.connect_to_orchestrator.assert_called_once()
+        # Cold boot: reconcile fires to restore CA, git-gate, and egress tokens.
+        reconcile.assert_called_once()
+
+    def test_reconcile_receives_url_and_fresh_gateway(self):
+        # reconcile gets the orchestrator URL and the gateway service (not the
+        # class). Verify the args to ensure it can reach the right endpoints.
+        with tempfile.TemporaryDirectory() as td:
+            with patch.object(infra_vm, "_infra_dir", return_value=Path(td)), \
+                    patch.object(infra_vm, "expected_version", return_value="v-current"), \
+                    patch.object(infra_vm, "_health_ok", return_value=False), \
+                    patch.object(infra_vm, "stop"), \
+                    patch.object(infra_vm, "ensure_built"), \
+                    patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls, \
+                    patch(_RECONCILE) as reconcile:
+                orch_cls.return_value.url.return_value = "http://10.243.255.1:8099"
+                FirecrackerInfraService().ensure_running()
+        call_args = reconcile.call_args
+        # First arg: the orchestrator's host URL.
+        self.assertEqual("http://10.243.255.1:8099", call_args.args[0])
+        # Second arg: the gateway service instance (not the class).
+        self.assertIs(gw_cls.return_value, call_args.args[1])
 
 
 if __name__ == "__main__":

tests/unit/test_firecracker_reconcile.py

@@ -0,0 +1,280 @@
+"""Unit: bring-up reconcile — CA push, git-gate reprovision, egress tokens.
+
+Covers attach_bottled_agents_to_gateway (reconcile.py): each per-bottle step
+fires with correct args, failures on one bottle don't block others, and the
+egress-token restore path works end-to-end. Does NOT spin up VMs or real SSH.
+"""
+
+from __future__ import annotations
+
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from subprocess import CalledProcessError, CompletedProcess
+from unittest.mock import MagicMock, patch
+
+_RECONCILE = "bot_bottle.backend.firecracker.reconcile"
+
+
+class TestGuestIpFromConfig(unittest.TestCase):
+    def test_reads_ip_from_boot_args(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import _guest_ip_from_config
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
+            json.dump({
+                "boot-source": {"boot_args": "console=ttyS0 ip=10.243.0.5:10.243.0.6:255.255.255.254"},
+            }, f)
+            name = f.name
+        self.assertEqual("10.243.0.5", _guest_ip_from_config(Path(name)))
+
+    def test_returns_empty_on_missing_file(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import _guest_ip_from_config
+        self.assertEqual("", _guest_ip_from_config(Path("/nonexistent/config.json")))
+
+    def test_returns_empty_on_bad_json(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import _guest_ip_from_config
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
+            f.write("not-json")
+            name = f.name
+        self.assertEqual("", _guest_ip_from_config(Path(name)))
+
+
+class TestPushCa(unittest.TestCase):
+    def test_runs_three_ssh_commands_in_order(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import _push_ca
+        with tempfile.TemporaryDirectory() as td:
+            key = Path(td) / "key"
+            key.write_text("k")
+            with patch(f"{_RECONCILE}.subprocess.run") as run:
+                run.return_value = CompletedProcess([], 0)
+                _push_ca(key, "10.0.0.1", "-----BEGIN CERTIFICATE-----\n")
+        calls = run.call_args_list
+        self.assertEqual(3, len(calls))
+        # Each call's argv is a list; the remote command is the last element.
+        self.assertIn("mkdir", calls[0].args[0][-1])
+        self.assertIn("cat >", calls[1].args[0][-1])
+        self.assertIn("update-ca-certificates", calls[2].args[0][-1])
+
+    def test_passes_ca_pem_via_stdin(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import _push_ca
+        with tempfile.TemporaryDirectory() as td:
+            key = Path(td) / "key"
+            key.write_text("k")
+            with patch(f"{_RECONCILE}.subprocess.run") as run:
+                run.return_value = CompletedProcess([], 0)
+                _push_ca(key, "10.0.0.1", "MY-CA-PEM")
+        cat_call = run.call_args_list[1]
+        self.assertEqual("MY-CA-PEM", cat_call.kwargs.get("input"))
+
+    def test_raises_on_ssh_failure(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import _push_ca
+        with tempfile.TemporaryDirectory() as td:
+            key = Path(td) / "key"
+            key.write_text("k")
+            with patch(f"{_RECONCILE}.subprocess.run") as run:
+                run.side_effect = CalledProcessError(1, "ssh")
+                with self.assertRaises(CalledProcessError):
+                    _push_ca(key, "10.0.0.1", "pem")
+
+
+class TestReprovisionGitGate(unittest.TestCase):
+    def _make_state_dir(self, upstreams: list[dict[str, str]]) -> Path:
+        d = Path(tempfile.mkdtemp())
+        (d / "upstreams.json").write_text(json.dumps(upstreams))
+        (d / "git_gate_pre_receive.sh").write_text("#!/bin/sh")
+        (d / "git_gate_access_hook.sh").write_text("#!/bin/sh")
+        (d / "git_gate_entrypoint.sh").write_text("#!/bin/sh")
+        return d
+
+    def test_no_op_when_no_upstreams_json(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import _reprovision_git_gate
+        transport = MagicMock()
+        with tempfile.TemporaryDirectory() as td:
+            # The state dir exists but has no upstreams.json
+            with patch(f"{_RECONCILE}.git_gate_state_dir", return_value=Path(td)):
+                _reprovision_git_gate(transport, "bottle-123", "my-slug")
+        transport.exec.assert_not_called()
+
+    def test_calls_provision_git_gate_with_upstreams(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import _reprovision_git_gate
+        transport = MagicMock()
+        upstreams = [{
+            "name": "bot-bottle",
+            "upstream_url": "ssh://git@gitea.example/org/bot-bottle.git",
+            "upstream_host": "gitea.example",
+            "upstream_port": "22",
+            "identity_file": "/home/node/.ssh/id_ed25519",
+            "known_host_key": "ssh-ed25519 AAAA...",
+        }]
+        state_dir = self._make_state_dir(upstreams)
+        # Write the key file so identity_file falls back to state dir path.
+        (state_dir / "bot-bottle-key").write_bytes(b"PRIVATE")
+        try:
+            with patch(f"{_RECONCILE}.git_gate_state_dir", return_value=state_dir), \
+                    patch(f"{_RECONCILE}.provision_git_gate") as prov:
+                _reprovision_git_gate(transport, "bottle-abc", "my-slug")
+            prov.assert_called_once()
+            _, bottle_id, plan = prov.call_args.args
+            self.assertEqual("bottle-abc", bottle_id)
+            self.assertEqual(1, len(plan.upstreams))
+            self.assertEqual("bot-bottle", plan.upstreams[0].name)
+            # Key in state dir takes priority over identity_file in JSON.
+            self.assertEqual(str(state_dir / "bot-bottle-key"), plan.upstreams[0].identity_file)
+        finally:
+            import shutil; shutil.rmtree(state_dir, ignore_errors=True)
+
+    def test_falls_back_to_manifest_identity_file_when_no_key_in_state(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import _reprovision_git_gate
+        transport = MagicMock()
+        upstreams = [{
+            "name": "repo",
+            "upstream_url": "ssh://git@github.com/org/repo.git",
+            "upstream_host": "github.com",
+            "upstream_port": "22",
+            "identity_file": "/home/node/.ssh/id_ed25519",
+            "known_host_key": "",
+        }]
+        state_dir = self._make_state_dir(upstreams)
+        # No `repo-key` in state dir — static manifest path should be used.
+        try:
+            with patch(f"{_RECONCILE}.git_gate_state_dir", return_value=state_dir), \
+                    patch(f"{_RECONCILE}.provision_git_gate") as prov:
+                _reprovision_git_gate(transport, "bottle-xyz", "my-slug")
+            prov.assert_called_once()
+            _, _, plan = prov.call_args.args
+            self.assertEqual("/home/node/.ssh/id_ed25519", plan.upstreams[0].identity_file)
+        finally:
+            import shutil; shutil.rmtree(state_dir, ignore_errors=True)
+
+
+class TestAttachBottledAgentsToGateway(unittest.TestCase):
+    """Integration-level: the full reconcile loop against mocked SSH + gateway."""
+
+    def _make_run_dir(self, tmp: Path, slug: str, guest_ip: str) -> Path:
+        run_dir = tmp / slug
+        run_dir.mkdir()
+        key = run_dir / "bottle_id_ed25519"
+        key.write_text("PRIVATE")
+        cfg = {
+            "boot-source": {
+                "boot_args": f"console=ttyS0 ip={guest_ip}:{guest_ip}:255.255.255.254",
+            }
+        }
+        (run_dir / "config.json").write_text(json.dumps(cfg))
+        return run_dir
+
+    def _make_gateway(self, ca_pem: str = "-----BEGIN CERTIFICATE-----\n") -> MagicMock:
+        gw = MagicMock()
+        gw.ca_cert_pem.return_value = ca_pem
+        gw.provisioning_transport.return_value = MagicMock()
+        return gw
+
+    def test_skips_all_when_ca_fetch_fails(self) -> None:
+        from bot_bottle.backend.firecracker.gateway import FirecrackerGateway
+        from bot_bottle.backend.firecracker.reconcile import attach_bottled_agents_to_gateway
+        from bot_bottle.gateway import GatewayError
+        gw = MagicMock(spec=FirecrackerGateway)
+        gw.ca_cert_pem.side_effect = GatewayError("timeout")
+        with patch(f"{_RECONCILE}.cleanup.live_run_dirs", return_value=()):
+            attach_bottled_agents_to_gateway("http://orch:8099", gw)
+        # Nothing else is called when CA fetch fails.
+        gw.provisioning_transport.assert_not_called()
+
+    def test_ca_push_called_per_live_bottle(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import attach_bottled_agents_to_gateway
+        gw = self._make_gateway()
+        with tempfile.TemporaryDirectory() as td:
+            tmp = Path(td)
+            rd1 = self._make_run_dir(tmp, "agent-abc12", "10.243.0.5")
+            rd2 = self._make_run_dir(tmp, "agent-xyz99", "10.243.0.7")
+            with patch(f"{_RECONCILE}.cleanup.live_run_dirs", return_value=(rd1, rd2)), \
+                    patch(f"{_RECONCILE}.OrchestratorClient") as client_cls, \
+                    patch(f"{_RECONCILE}._push_ca") as push_ca, \
+                    patch(f"{_RECONCILE}._reprovision_git_gate"), \
+                    patch(f"{_RECONCILE}.subprocess.run",
+                          return_value=CompletedProcess([], 1)):
+                client_cls.return_value.list_bottles.return_value = []
+                attach_bottled_agents_to_gateway("http://orch:8099", gw)
+        # CA pushed to both bottles.
+        self.assertEqual(2, push_ca.call_count)
+
+    def test_per_bottle_ca_failure_does_not_block_others(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import attach_bottled_agents_to_gateway
+        gw = self._make_gateway()
+        with tempfile.TemporaryDirectory() as td:
+            tmp = Path(td)
+            rd1 = self._make_run_dir(tmp, "agent-fail1", "10.243.0.5")
+            rd2 = self._make_run_dir(tmp, "agent-ok99", "10.243.0.7")
+            with patch(f"{_RECONCILE}.cleanup.live_run_dirs", return_value=(rd1, rd2)), \
+                    patch(f"{_RECONCILE}.OrchestratorClient") as client_cls, \
+                    patch(f"{_RECONCILE}._push_ca",
+                          side_effect=[CalledProcessError(1, "ssh"), None]) as push_ca, \
+                    patch(f"{_RECONCILE}._reprovision_git_gate"), \
+                    patch(f"{_RECONCILE}.subprocess.run",
+                          return_value=CompletedProcess([], 1)):
+                client_cls.return_value.list_bottles.return_value = []
+                # Must not raise even though the first bottle's CA push fails.
+                attach_bottled_agents_to_gateway("http://orch:8099", gw)
+        # Both bottles were attempted.
+        self.assertEqual(2, push_ca.call_count)
+
+    def test_egress_token_restored_for_bottles_with_secret(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import attach_bottled_agents_to_gateway
+        gw = self._make_gateway()
+        with tempfile.TemporaryDirectory() as td:
+            tmp = Path(td)
+            rd = self._make_run_dir(tmp, "agent-tok11", "10.243.0.5")
+            bottles = [{"bottle_id": "bid-001", "source_ip": "10.243.0.5"}]
+            with patch(f"{_RECONCILE}.cleanup.live_run_dirs", return_value=(rd,)), \
+                    patch(f"{_RECONCILE}.OrchestratorClient") as client_cls, \
+                    patch(f"{_RECONCILE}._push_ca"), \
+                    patch(f"{_RECONCILE}._reprovision_git_gate"), \
+                    patch(f"{_RECONCILE}.reprovision_bottles") as reprov, \
+                    patch(f"{_RECONCILE}.subprocess.run",
+                          return_value=CompletedProcess([], 0, stdout="secret-key\n")):
+                client_cls.return_value.list_bottles.return_value = bottles
+                reprov.return_value = 1
+                attach_bottled_agents_to_gateway("http://orch:8099", gw)
+        reprov.assert_called_once()
+        _, secrets = reprov.call_args.args
+        self.assertIn("10.243.0.5", secrets)
+        self.assertEqual("secret-key", secrets["10.243.0.5"])
+
+    def test_git_gate_reprovision_called_with_bottle_id(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import attach_bottled_agents_to_gateway
+        gw = self._make_gateway()
+        with tempfile.TemporaryDirectory() as td:
+            tmp = Path(td)
+            rd = self._make_run_dir(tmp, "agent-git11", "10.243.0.5")
+            bottles = [{"bottle_id": "bid-git", "source_ip": "10.243.0.5"}]
+            with patch(f"{_RECONCILE}.cleanup.live_run_dirs", return_value=(rd,)), \
+                    patch(f"{_RECONCILE}.OrchestratorClient") as client_cls, \
+                    patch(f"{_RECONCILE}._push_ca"), \
+                    patch(f"{_RECONCILE}._reprovision_git_gate") as reprov_gw, \
+                    patch(f"{_RECONCILE}.subprocess.run",
+                          return_value=CompletedProcess([], 1)):
+                client_cls.return_value.list_bottles.return_value = bottles
+                attach_bottled_agents_to_gateway("http://orch:8099", gw)
+        reprov_gw.assert_called_once()
+        _, bottle_id_arg, slug_arg = reprov_gw.call_args.args
+        self.assertEqual("bid-git", bottle_id_arg)
+        self.assertEqual("agent-git11", slug_arg)
+
+    def test_run_dir_without_key_file_is_skipped(self) -> None:
+        from bot_bottle.backend.firecracker.reconcile import attach_bottled_agents_to_gateway
+        gw = self._make_gateway()
+        with tempfile.TemporaryDirectory() as td:
+            tmp = Path(td)
+            rd = self._make_run_dir(tmp, "agent-nokey", "10.243.0.5")
+            (rd / "bottle_id_ed25519").unlink()  # remove the key
+            with patch(f"{_RECONCILE}.cleanup.live_run_dirs", return_value=(rd,)), \
+                    patch(f"{_RECONCILE}.OrchestratorClient") as client_cls, \
+                    patch(f"{_RECONCILE}._push_ca") as push_ca, \
+                    patch(f"{_RECONCILE}._reprovision_git_gate"):
+                client_cls.return_value.list_bottles.return_value = []
+                attach_bottled_agents_to_gateway("http://orch:8099", gw)
+        push_ca.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/unit/test_generate_release_manifest.py

@@ -1,31 +0,0 @@
-from __future__ import annotations
-
-import json
-import tempfile
-import unittest
-from pathlib import Path
-
-from scripts.generate_release_manifest import main
-
-
-class TestGenerateReleaseManifest(unittest.TestCase):
-    def test_writes_validated_manifest(self) -> None:
-        with tempfile.TemporaryDirectory() as tmp:
-            output = Path(tmp) / "release.json"
-            rc = main([
-                "--source-commit", "1" * 40,
-                "--orchestrator-image",
-                "registry/orchestrator@sha256:" + "a" * 64,
-                "--gateway-image", "registry/gateway@sha256:" + "b" * 64,
-                "--agent-claude-image", "registry/claude@sha256:" + "e" * 64,
-                "--agent-codex-image", "registry/codex@sha256:" + "f" * 64,
-                "--agent-pi-image", "registry/pi@sha256:" + "0" * 64,
-                "--firecracker-orchestrator-version", "orch-v1",
-                "--firecracker-orchestrator-sha256", "c" * 64,
-                "--firecracker-gateway-version", "gateway-v1",
-                "--firecracker-gateway-sha256", "d" * 64,
-                "--output", str(output),
-            ])
-            self.assertEqual(0, rc)
-            data = json.loads(output.read_text(encoding="utf-8"))
-            self.assertEqual("orch-v1", data["firecracker"]["orchestrator"]["version"])

tests/unit/test_install_script.py

@@ -95,21 +95,6 @@ class TestInstallScript(unittest.TestCase):
         # Tests / local installs point BOT_BOTTLE_INSTALL_SPEC at a checkout.
         self.assertIn("BOT_BOTTLE_INSTALL_SPEC", self.text)
 
-    def test_published_install_selectors_are_mutually_exclusive(self):
-        self.assertIn("BOT_BOTTLE_CHANNEL", self.text)
-        self.assertIn("BOT_BOTTLE_VERSION", self.text)
-        self.assertIn("BOT_BOTTLE_REF", self.text)
-        self.assertIn("are mutually exclusive", self.text)
-
-    def test_commit_snapshot_warns_and_wheel_is_checksum_verified(self):
-        self.assertIn("may not have passed release qualification", self.text)
-        self.assertIn("downloaded wheel checksum mismatch", self.text)
-        self.assertIn("hashlib.sha256()", self.text)
-
-    def test_default_install_uses_production_channel(self):
-        self.assertIn('${BOT_BOTTLE_CHANNEL:-production}', self.text)
-        self.assertIn("bot-bottle-channels", self.text)
-
     def test_requires_git_for_git_specs(self):
         # A git+ / .git spec (the default) shells out to git; the script must
         # gate on it rather than failing opaquely inside pipx/pip.

tests/unit/test_invocation.py

@@ -1,98 +0,0 @@
-"""Unit: how the CLI tells users to re-run it under sudo.
-
-`sudo bot-bottle …` is wrong for the users who followed the documented
-install: sudo's secure_path excludes ~/.local/bin, where both pipx and
-install.sh put the entry point. These lock in the absolute-path form.
-"""
-
-from __future__ import annotations
-
-import contextlib
-import io
-import os
-import tempfile
-import unittest
-from pathlib import Path
-from unittest import mock
-
-from bot_bottle import invocation
-
-
-class TestSelfPath(unittest.TestCase):
-    def test_absolute_argv0_is_used_as_is(self):
-        with mock.patch.object(invocation.sys, "argv", ["/opt/venv/bin/bot-bottle"]):
-            self.assertEqual("/opt/venv/bin/bot-bottle", invocation.self_path())
-
-    def test_bare_name_is_resolved_through_path(self):
-        # The case that matters: invoked as `bot-bottle`, installed in a
-        # directory sudo would drop.
-        with tempfile.TemporaryDirectory() as d:
-            entry = Path(d, "bot-bottle")
-            entry.write_text("#!/bin/sh\n")
-            entry.chmod(0o755)
-            with mock.patch.object(invocation.sys, "argv", ["bot-bottle"]), \
-                 mock.patch.dict(os.environ, {"PATH": d}):
-                self.assertEqual(str(entry), invocation.self_path())
-
-    def test_relative_path_is_made_absolute(self):
-        with tempfile.TemporaryDirectory() as d:
-            entry = Path(d, "bot-bottle")
-            entry.write_text("#!/bin/sh\n")
-            entry.chmod(0o755)
-            cwd = os.getcwd()
-            try:
-                os.chdir(d)
-                with mock.patch.object(invocation.sys, "argv", ["./bot-bottle"]):
-                    self.assertTrue(os.path.isabs(invocation.self_path()))
-            finally:
-                os.chdir(cwd)
-
-    def test_unresolvable_entry_point_falls_back_to_the_name(self):
-        # `python -m`-style invocation, or an argv[0] that no longer exists.
-        # A slightly wrong hint beats a traceback raised while reporting some
-        # unrelated problem.
-        with mock.patch.object(invocation.sys, "argv", ["/nonexistent/gone"]), \
-             mock.patch.object(invocation.shutil, "which", return_value=None):
-            self.assertEqual("/nonexistent/gone", invocation.self_path())
-        with mock.patch.object(invocation.sys, "argv", [""]), \
-             mock.patch.object(invocation.shutil, "which", return_value=None):
-            self.assertEqual("bot-bottle", invocation.self_path())
-
-
-class TestSudoCommand(unittest.TestCase):
-    def test_names_an_absolute_path_not_the_bare_command(self):
-        with mock.patch.object(invocation, "self_path",
-                               return_value="/home/u/.local/bin/bot-bottle"):
-            cmd = invocation.sudo_command("backend", "setup", "--backend=firecracker")
-        self.assertEqual(
-            "sudo /home/u/.local/bin/bot-bottle backend setup --backend=firecracker",
-            cmd,
-        )
-        # The regression this exists to prevent.
-        self.assertNotIn("sudo bot-bottle", cmd)
-
-
-class TestFirecrackerSetupUsesIt(unittest.TestCase):
-    def test_root_reinvocation_hint_names_an_absolute_path(self):
-        # The message that prompted all this. Drive the real code path rather
-        # than scanning the source, which would also match the comment
-        # explaining why the bare form is wrong.
-        from bot_bottle.backend.firecracker import setup as fc_setup
-
-        err, out = io.StringIO(), io.StringIO()
-        with mock.patch.object(fc_setup.os, "geteuid", return_value=501), \
-             mock.patch.object(fc_setup.invocation, "self_path",
-                               return_value="/home/u/.local/bin/bot-bottle"), \
-             contextlib.redirect_stderr(err), contextlib.redirect_stdout(out):
-            fc_setup._setup_systemd()
-
-        printed = err.getvalue()
-        self.assertIn(
-            "sudo /home/u/.local/bin/bot-bottle backend setup --backend=firecracker",
-            printed,
-        )
-        self.assertNotIn("sudo bot-bottle", printed)
-
-
-if __name__ == "__main__":
-    unittest.main()

tests/unit/test_macos_infra.py

@@ -33,8 +33,8 @@ class TestInfraEnsureRunning(unittest.TestCase):
         with patch(f"{_INFRA}.ensure_networks") as nets:
             url = self.svc.ensure_running()
         nets.assert_called_once()
-        self.orch.ensure_available.assert_called_once()
+        self.orch.ensure_built.assert_called_once()
-        self.gw.ensure_available.assert_called_once()
+        self.gw.ensure_built.assert_called_once()
         self.orch.ensure_running.assert_called_once()
         # Returns the host control-plane URL (the InfraService contract).
         self.assertEqual("http://192.168.128.2:8099", url)
@@ -51,18 +51,6 @@ class TestInfraEnsureRunning(unittest.TestCase):
         self.assertTrue(self.svc.is_healthy())
         self.orch.is_healthy.assert_called_once()
 
-    def test_packaged_images_disable_local_infra_builds(self) -> None:
-        refs = [
-            ("registry/orchestrator@sha256:" + "a" * 64, False),
-            ("registry/gateway@sha256:" + "b" * 64, False),
-        ]
-        with patch(
-            f"{_INFRA}.release_manifest.oci_image", side_effect=refs,
-        ):
-            svc = MacosInfraService(repo_root=Path("/r"))
-        self.assertFalse(svc.orchestrator()._local_build)
-        self.assertFalse(svc.gateway()._local_build)
-
 
 class TestCaCertPem(unittest.TestCase):
     def test_delegates_to_the_gateway_service(self) -> None:

tests/unit/test_macos_orchestrator.py

@@ -30,16 +30,14 @@ def _spec(src: str, tgt: str, readonly: bool = False) -> str:
 
 
 class TestMacosOrchestratorRun(unittest.TestCase):
-    def _run(self, *, local_build: bool = True) -> list[str]:
+    def _run(self) -> list[str]:
         run = Mock(return_value=_ok())
         with patch(f"{_ORCH}.container_mod") as mod, \
                 patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
             mod.dns_server.return_value = "1.1.1.1"
             mod.bind_mount_spec.side_effect = _spec
             mod.run_container_argv = run
-            MacosOrchestrator(
+            MacosOrchestrator(repo_root=Path("/r"))._run_container("h1")
-                repo_root=Path("/r"), local_build=local_build,
-            )._run_container("h1")
         return run.call_args.args[0]
 
     def test_runs_on_the_control_network_only(self) -> None:
@@ -65,11 +63,6 @@ class TestMacosOrchestratorRun(unittest.TestCase):
     def test_source_hash_is_labelled_for_recreate(self) -> None:
         self.assertIn("BOT_BOTTLE_SOURCE_HASH=h1", self._run())
 
-    def test_packaged_image_does_not_overlay_host_source(self) -> None:
-        argv = self._run(local_build=False)
-        self.assertNotIn("--mount", argv)
-        self.assertFalse(any(arg.startswith("PYTHONPATH=") for arg in argv))
-
     def test_start_failure_raises(self) -> None:
         with patch(f"{_ORCH}.container_mod") as mod, \
                 patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
@@ -152,14 +145,6 @@ class TestMacosOrchestratorBuildStop(unittest.TestCase):
         _args, kwargs = mod.build_image.call_args
         self.assertEqual("Dockerfile.orchestrator", kwargs["dockerfile"])
 
-    def test_ensure_built_pulls_packaged_image(self) -> None:
-        image = "registry/orchestrator@sha256:" + "a" * 64
-        orch = MacosOrchestrator(image, local_build=False)
-        with patch(f"{_ORCH}.container_mod") as mod:
-            orch.ensure_built()
-        mod.pull_image.assert_called_once_with(image)
-        mod.build_image.assert_not_called()
-
     def test_stop_removes_the_container(self) -> None:
         orch = MacosOrchestrator()
         with patch(f"{_ORCH}.container_mod") as mod:

tests/unit/test_orchestrator_client.py

@@ -151,6 +151,31 @@ class TestReprovisionGateway(unittest.TestCase):
                 self.c.reprovision_gateway("b1", "key")
 
 
+class TestUpdateAgentSecret(unittest.TestCase):
+    def setUp(self) -> None:
+        self.c = OrchestratorClient("http://orch:8080")
+
+    def test_success_posts_single_secret(self) -> None:
+        with patch(_URLOPEN, return_value=_resp(200, {"updated": True})) as opened:
+            self.assertTrue(self.c.update_agent_secret("b1", "A", "fresh", "key"))
+        request = opened.call_args.args[0]
+        self.assertEqual("POST", request.get_method())
+        self.assertTrue(request.full_url.endswith("/bottles/b1/secret"))
+        self.assertEqual(
+            {"name": "A", "value": "fresh", "env_var_secret": "key"},
+            json.loads(request.data),
+        )
+
+    def test_unknown_bottle_is_false(self) -> None:
+        with patch(_URLOPEN, side_effect=_http_error(404)):
+            self.assertFalse(self.c.update_agent_secret("b1", "A", "v", "key"))
+
+    def test_other_status_raises(self) -> None:
+        with patch(_URLOPEN, side_effect=_http_error(400)):
+            with self.assertRaises(OrchestratorClientError):
+                self.c.update_agent_secret("b1", "A", "v", "key")
+
+
 class TestHealthAndPolicy(unittest.TestCase):
     def setUp(self) -> None:
         self.c = OrchestratorClient("http://orch:8080")

tests/unit/test_orchestrator_gateway.py

@@ -427,11 +427,11 @@ class TestDockerGatewayBuild(unittest.TestCase):
         builds = [c for c in calls if c[:2] == ["docker", "build"]]
         self.assertIn("--no-cache", builds[0])
 
-    def test_ensure_built_pulls_when_no_dockerfile(self) -> None:
+    def test_ensure_built_noop_when_no_dockerfile(self) -> None:
-        sc = DockerGateway("registry/gateway@sha256:" + "a" * 64, dockerfile=None)
+        sc = DockerGateway("busybox", dockerfile=None)
-        with patch(_RUN_DOCKER, return_value=_proc()) as m:
+        with patch(_RUN_DOCKER) as m:
             sc.ensure_built()
-        m.assert_called_once_with(["docker", "pull", sc.image_ref])
+        m.assert_not_called()
 
     def test_ensure_built_raises_on_build_failure(self) -> None:
         with patch(_RUN_DOCKER, return_value=_proc(returncode=1, stderr="build boom")):

tests/unit/test_orchestrator_registry_store.py

@@ -199,6 +199,20 @@ class TestAgentSecrets(unittest.TestCase):
         got = self.store.get_agent_secrets("bottle-1")
         self.assertEqual({"K": "new", "K2": "v2"}, got)
 
+    def test_store_agent_secret_upserts_one_key_leaving_others(self) -> None:
+        self.store.store_agent_secrets("bottle-1", {"K": "v1", "K2": "v2"})
+        self.store.store_agent_secret("bottle-1", "K", "v1-new")  # update existing
+        self.store.store_agent_secret("bottle-1", "K3", "v3")     # insert new
+        self.assertEqual(
+            {"K": "v1-new", "K2": "v2", "K3": "v3"},
+            self.store.get_agent_secrets("bottle-1"),
+        )
+
+    def test_store_agent_secret_does_not_duplicate_rows(self) -> None:
+        self.store.store_agent_secret("bottle-1", "K", "v1")
+        self.store.store_agent_secret("bottle-1", "K", "v2")
+        self.assertEqual({"K": "v2"}, self.store.get_agent_secrets("bottle-1"))
+
     def test_delete_removes_secrets(self) -> None:
         self.store.store_agent_secrets("bottle-1", {"K": "v"})
         self.store.delete_agent_secrets("bottle-1")

tests/unit/test_orchestrator_server.py

@@ -125,6 +125,47 @@ class TestDispatch(unittest.TestCase):
             {"EGRESS_TOKEN_0": "upstream-secret"}, self.orch.tokens_for(bottle_id),
         )
 
+    def test_update_agent_secret_in_place(self) -> None:
+        key = base64.urlsafe_b64encode(b"unit-test-key").rstrip(b"=").decode()
+        status, payload = dispatch(
+            self.orch, "POST", "/bottles", _body({
+                "source_ip": "10.243.0.21",
+                "tokens": {"A": "old", "B": "keep"},
+                "env_var_secret": key,
+            }),
+        )
+        self.assertEqual(201, status)
+        bottle_id = payload["bottle_id"]
+        assert isinstance(bottle_id, str)
+        status, response = dispatch(
+            self.orch, "POST", f"/bottles/{bottle_id}/secret",
+            _body({"name": "A", "value": "fresh", "env_var_secret": key}),
+        )
+        self.assertEqual((200, {"updated": True}), (status, response))
+        self.assertEqual({"A": "fresh", "B": "keep"}, self.orch.tokens_for(bottle_id))
+
+    def test_update_agent_secret_validates_and_unknown_bottle(self) -> None:
+        status, _ = dispatch(self.orch, "POST", "/bottles/b1/secret", b"not-json")
+        self.assertEqual(400, status)
+        status, _ = dispatch(
+            self.orch, "POST", "/bottles/b1/secret", _body({"name": "A"}),
+        )
+        self.assertEqual(400, status)  # value + env_var_secret missing
+        status, _ = dispatch(
+            self.orch, "POST", "/bottles/ghost/secret",
+            _body({"name": "A", "value": "v", "env_var_secret": "k"}),
+        )
+        self.assertEqual(404, status)
+
+    def test_update_agent_secret_is_cli_only(self) -> None:
+        # A data-plane (`gateway`) caller must not set a bottle's tokens.
+        status, _ = dispatch(
+            self.orch, "POST", "/bottles/b1/secret",
+            _body({"name": "A", "value": "v", "env_var_secret": "k"}),
+            role=ROLE_GATEWAY,
+        )
+        self.assertEqual(403, status)
+
     def test_reprovision_validates_request_and_missing_rows(self) -> None:
         status, _ = dispatch(
             self.orch, "POST", "/bottles/b1/reprovision_gateway", b"not-json",

tests/unit/test_orchestrator_service.py

@@ -103,6 +103,25 @@ class TestOrchestrator(unittest.TestCase):
         self.assertTrue(self.orch.reprovision_from_secret(rec.bottle_id, key))
         self.assertEqual({"EGRESS_TOKEN_0": "secret"}, self.orch.tokens_for(rec.bottle_id))
 
+    def test_update_agent_secret_refreshes_one_token_in_place(self) -> None:
+        key = new_env_var_secret()
+        rec = self.orch.launch_bottle(
+            "10.243.0.20", tokens={"A": "old-a", "B": "keep-b"}, env_var_secret=key,
+        )
+        self.assertTrue(self.orch.update_agent_secret(rec.bottle_id, "A", "new-a", key))
+        # In memory: A refreshed, B left untouched.
+        self.assertEqual({"A": "new-a", "B": "keep-b"}, self.orch.tokens_for(rec.bottle_id))
+        # At rest: the whole set stays decryptable with the SAME env_var_secret,
+        # so a later reprovision restores the refreshed value (not the launch one).
+        self.orch._tokens.clear()
+        self.assertTrue(self.orch.reprovision_from_secret(rec.bottle_id, key))
+        self.assertEqual({"A": "new-a", "B": "keep-b"}, self.orch.tokens_for(rec.bottle_id))
+
+    def test_update_agent_secret_unknown_bottle_is_false(self) -> None:
+        self.assertFalse(
+            self.orch.update_agent_secret("ghost", "A", "v", new_env_var_secret())
+        )
+
     def test_reprovision_rejects_missing_rows_and_wrong_key(self) -> None:
         self.assertFalse(self.orch.reprovision_from_secret("missing", new_env_var_secret()))
         key = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE"

tests/unit/test_release_bundle.py

@@ -1,60 +0,0 @@
-from __future__ import annotations
-
-import tempfile
-import unittest
-from pathlib import Path
-
-from bot_bottle.release_bundle import (
-    build_bundle_index,
-    canonical_bytes,
-    parse_bundle_index,
-)
-from bot_bottle.release_manifest import ReleaseManifestError
-from tests.unit.test_release_manifest import manifest
-
-
-class TestReleaseBundle(unittest.TestCase):
-    def test_builds_commit_addressed_index(self) -> None:
-        with tempfile.TemporaryDirectory() as tmp:
-            wheel = Path(tmp) / "bot_bottle-0.1.0-py3-none-any.whl"
-            wheel.write_bytes(b"wheel")
-            index = build_bundle_index(
-                manifest(),
-                wheel=wheel,
-                wheel_url=f"https://packages.example/{wheel.name}",
-                workflow_run="actions/123",
-                published_at="2026-07-27T00:00:00Z",
-            )
-        self.assertEqual("1" * 40, index["source_commit"])
-        self.assertEqual(64, len(index["wheel"]["sha256"]))
-        self.assertEqual([], index["qualifications"])
-        self.assertIs(index, parse_bundle_index(index))
-        self.assertTrue(canonical_bytes(index).endswith(b"\n"))
-
-    def test_rejects_mutable_or_mismatched_wheel_url(self) -> None:
-        with tempfile.TemporaryDirectory() as tmp:
-            wheel = Path(tmp) / "release.whl"
-            wheel.write_bytes(b"wheel")
-            with self.assertRaisesRegex(ReleaseManifestError, "HTTPS"):
-                build_bundle_index(
-                    manifest(),
-                    wheel=wheel,
-                    wheel_url="http://packages.example/other.whl",
-                    workflow_run="actions/123",
-                    published_at="now",
-                )
-
-    def test_rejects_index_with_changed_runtime_identity(self) -> None:
-        with tempfile.TemporaryDirectory() as tmp:
-            wheel = Path(tmp) / "release.whl"
-            wheel.write_bytes(b"wheel")
-            index = build_bundle_index(
-                manifest(),
-                wheel=wheel,
-                wheel_url=f"https://packages.example/{wheel.name}",
-                workflow_run="actions/123",
-                published_at="now",
-            )
-        index["oci"]["gateway"] = "registry/gateway:latest"
-        with self.assertRaisesRegex(ReleaseManifestError, "digest-pinned"):
-            parse_bundle_index(index)

tests/unit/test_release_manifest.py

@@ -1,129 +0,0 @@
-from __future__ import annotations
-
-import unittest
-from unittest.mock import patch
-
-from bot_bottle.release_manifest import (
-    ReleaseManifestError,
-    oci_image,
-    packaged_manifest,
-    parse_manifest,
-)
-
-
-def manifest() -> dict[str, object]:
-    return {
-        "schema": 1,
-        "source_commit": "1" * 40,
-        "oci": {
-            "orchestrator": "registry/orchestrator@sha256:" + "a" * 64,
-            "gateway": "registry/gateway@sha256:" + "b" * 64,
-            "agent_claude": "registry/agent-claude@sha256:" + "e" * 64,
-            "agent_codex": "registry/agent-codex@sha256:" + "f" * 64,
-            "agent_pi": "registry/agent-pi@sha256:" + "0" * 64,
-        },
-        "firecracker": {
-            "orchestrator": {"version": "orch-v1", "sha256": "c" * 64},
-            "gateway": {"version": "gateway-v1", "sha256": "d" * 64},
-        },
-    }
-
-
-class TestParseManifest(unittest.TestCase):
-    def test_parses_all_backend_artifacts(self) -> None:
-        parsed = parse_manifest(manifest())
-        self.assertTrue(parsed.orchestrator_image.endswith("a" * 64))
-        self.assertEqual("orch-v1", parsed.firecracker_orchestrator.version)
-        self.assertTrue(parsed.agent_images["codex"].endswith("f" * 64))
-
-    def test_rejects_mutable_oci_reference(self) -> None:
-        data = manifest()
-        assert isinstance(data["oci"], dict)
-        data["oci"]["orchestrator"] = "registry/orchestrator:latest"
-        with self.assertRaisesRegex(ReleaseManifestError, "digest-pinned"):
-            parse_manifest(data)
-
-    def test_rejects_malformed_firecracker_checksum(self) -> None:
-        data = manifest()
-        assert isinstance(data["firecracker"], dict)
-        artifact = data["firecracker"]["gateway"]
-        assert isinstance(artifact, dict)
-        artifact["sha256"] = "nope"
-        with self.assertRaisesRegex(ReleaseManifestError, "64 lowercase hex"):
-            parse_manifest(data)
-
-    def test_rejects_missing_manifest_sections(self) -> None:
-        cases = (
-            (None, "schema 1"),
-            ({"schema": 1}, "source_commit"),
-            (
-                {"schema": 1, "source_commit": "1" * 40},
-                "oci must be an object",
-            ),
-        )
-        for data, message in cases:
-            with self.subTest(message=message):
-                with self.assertRaisesRegex(ReleaseManifestError, message):
-                    parse_manifest(data)
-
-    def test_rejects_malformed_firecracker_artifact(self) -> None:
-        data = manifest()
-        assert isinstance(data["firecracker"], dict)
-        data["firecracker"]["orchestrator"] = None
-        with self.assertRaisesRegex(ReleaseManifestError, "must be an object"):
-            parse_manifest(data)
-
-        data["firecracker"]["orchestrator"] = {
-            "version": " ",
-            "sha256": "c" * 64,
-        }
-        with self.assertRaisesRegex(ReleaseManifestError, "version must be non-empty"):
-            parse_manifest(data)
-
-
-class TestImageSelection(unittest.TestCase):
-    def test_development_marker_selects_local_builds(self) -> None:
-        with patch(
-            "bot_bottle.release_manifest.resources.is_source_checkout",
-            return_value=False,
-        ), patch(
-            "bot_bottle.release_manifest.manifest_path",
-        ) as path:
-            path.return_value.read_text.return_value = (
-                '{"schema": 1, "development": true}')
-            self.assertIsNone(packaged_manifest())
-
-    def test_packaged_install_uses_manifest_reference_without_build(self) -> None:
-        parsed = parse_manifest(manifest())
-        with patch(
-            "bot_bottle.release_manifest.packaged_manifest", return_value=parsed,
-        ), patch(
-            "bot_bottle.release_manifest.local_build_requested", return_value=False,
-        ), patch.dict("os.environ", {}, clear=True):
-            ref, local = oci_image("orchestrator", "local:latest")
-        self.assertEqual(parsed.orchestrator_image, ref)
-        self.assertFalse(local)
-
-    def test_packaged_install_selects_agent_reference(self) -> None:
-        parsed = parse_manifest(manifest())
-        with patch(
-            "bot_bottle.release_manifest.packaged_manifest", return_value=parsed,
-        ), patch(
-            "bot_bottle.release_manifest.local_build_requested", return_value=False,
-        ), patch.dict("os.environ", {}, clear=True):
-            ref, local = oci_image("agent_codex", "local:latest")
-        self.assertEqual(parsed.agent_images["codex"], ref)
-        self.assertFalse(local)
-
-    def test_mutable_override_fails_outside_local_mode(self) -> None:
-        parsed = parse_manifest(manifest())
-        with patch(
-            "bot_bottle.release_manifest.packaged_manifest", return_value=parsed,
-        ), patch(
-            "bot_bottle.release_manifest.local_build_requested", return_value=False,
-        ), patch.dict(
-            "os.environ", {"BOT_BOTTLE_ORCHESTRATOR_IMAGE": "local:latest"},
-            clear=True,
-        ):
-            with self.assertRaisesRegex(ReleaseManifestError, "digest-pinned"):
-                oci_image("orchestrator", "fallback:latest")

tests/unit/test_release_publish.py

@@ -1,163 +0,0 @@
-from __future__ import annotations
-
-import tempfile
-import unittest
-import urllib.error
-from pathlib import Path
-from unittest.mock import MagicMock, patch
-
-from bot_bottle.release_bundle import build_bundle_index, canonical_bytes
-from bot_bottle.release_manifest import ReleaseManifestError
-from bot_bottle.release_publish import (
-    bundle_url,
-    remote_bytes,
-    remote_sha256,
-    request,
-    upload as publish_artifact,
-    publish_bundle,
-)
-from tests.unit.test_release_manifest import manifest
-
-
-class TestReleasePublish(unittest.TestCase):
-    def setUp(self) -> None:
-        env = {
-            "BOT_BOTTLE_RELEASE_BASE": "https://packages.example",
-            "BOT_BOTTLE_RELEASE_OWNER": "owner",
-            "BOT_BOTTLE_RELEASE_TOKEN": "token",
-        }
-        self.env = patch.dict("os.environ", env, clear=True)
-        self.env.start()
-        self.addCleanup(self.env.stop)
-
-    def bundle(self, root: Path) -> tuple[dict[str, object], Path]:
-        wheel = root / "bot_bottle-0.1.0-py3-none-any.whl"
-        wheel.write_bytes(b"wheel")
-        index = build_bundle_index(
-            manifest(),
-            wheel=wheel,
-            wheel_url=bundle_url("1" * 40, wheel.name),
-            workflow_run="actions/123",
-            published_at="2026-07-27T00:00:00Z",
-        )
-        return index, wheel
-
-    @staticmethod
-    def wheel_sha(index: dict[str, object]) -> str:
-        wheel = index["wheel"]
-        assert isinstance(wheel, dict)
-        digest = wheel["sha256"]
-        assert isinstance(digest, str)
-        return digest
-
-    def test_publishes_wheel_then_index(self) -> None:
-        with tempfile.TemporaryDirectory() as tmp:
-            index, wheel = self.bundle(Path(tmp))
-            with patch(
-                "bot_bottle.release_publish.remote_bytes", return_value=None,
-            ), patch(
-                "bot_bottle.release_publish.remote_sha256", return_value=None,
-            ), patch("bot_bottle.release_publish.upload") as upload:
-                result = publish_bundle(index, wheel)
-        self.assertTrue(result.endswith("/bundle-index.json"))
-        self.assertEqual(2, upload.call_count)
-        self.assertIs(wheel, upload.call_args_list[0].args[1])
-        self.assertEqual(canonical_bytes(index), upload.call_args_list[1].args[1])
-
-    def test_exact_existing_bundle_is_idempotent(self) -> None:
-        with tempfile.TemporaryDirectory() as tmp:
-            index, wheel = self.bundle(Path(tmp))
-            with patch(
-                "bot_bottle.release_publish.remote_bytes",
-                return_value=canonical_bytes(index),
-            ), patch(
-                "bot_bottle.release_publish.remote_sha256",
-                return_value=self.wheel_sha(index),
-            ), patch("bot_bottle.release_publish.upload") as upload:
-                publish_bundle(index, wheel)
-        upload.assert_not_called()
-
-    def test_rejects_partial_publication(self) -> None:
-        with tempfile.TemporaryDirectory() as tmp:
-            index, wheel = self.bundle(Path(tmp))
-            with patch(
-                "bot_bottle.release_publish.remote_bytes", return_value=None,
-            ), patch(
-                "bot_bottle.release_publish.remote_sha256",
-                return_value=self.wheel_sha(index),
-            ):
-                with self.assertRaisesRegex(
-                    ReleaseManifestError, "administrative cleanup",
-                ):
-                    publish_bundle(index, wheel)
-
-    def test_rejects_existing_bundle_with_different_index(self) -> None:
-        with tempfile.TemporaryDirectory() as tmp:
-            index, wheel = self.bundle(Path(tmp))
-            with patch(
-                "bot_bottle.release_publish.remote_bytes", return_value=b"other",
-            ), patch(
-                "bot_bottle.release_publish.remote_sha256",
-                return_value=self.wheel_sha(index),
-            ):
-                with self.assertRaisesRegex(
-                    ReleaseManifestError, "already differs",
-                ):
-                    publish_bundle(index, wheel)
-
-    def test_request_applies_auth_and_length(self) -> None:
-        result = request(
-            "https://packages.example/file",
-            method="PUT",
-            data=b"body",
-            length=4,
-        )
-        self.assertEqual(result.get_method(), "PUT")
-        self.assertEqual(result.get_header("Authorization"), "token token")
-        self.assertEqual(result.get_header("Content-length"), "4")
-
-    @patch("bot_bottle.release_publish.urllib.request.urlopen")
-    def test_remote_helpers_read_response(self, urlopen: MagicMock) -> None:
-        response = urlopen.return_value.__enter__.return_value
-        response.read.side_effect = [b"body"]
-        self.assertEqual(
-            remote_bytes("https://packages.example/file"), b"body")
-        response.read.side_effect = [b"body", b""]
-        self.assertEqual(
-            remote_sha256("https://packages.example/file"),
-            "230d8358dc8e8890b4c58deeb62912ee2"
-            "f20357ae92a5cc861b98e68fe31acb5",
-        )
-
-    @patch("bot_bottle.release_publish.urllib.request.urlopen")
-    def test_remote_helpers_handle_not_found(self, urlopen: MagicMock) -> None:
-        urlopen.side_effect = urllib.error.HTTPError(
-            "url", 404, "missing", MagicMock(), None)
-        self.assertIsNone(remote_bytes("https://packages.example/file"))
-        self.assertIsNone(remote_sha256("https://packages.example/file"))
-
-    @patch("bot_bottle.release_publish.urllib.request.urlopen")
-    def test_remote_helpers_report_registry_errors(
-        self, urlopen: MagicMock,
-    ) -> None:
-        urlopen.side_effect = urllib.error.URLError("offline")
-        with self.assertRaisesRegex(ReleaseManifestError, "offline"):
-            remote_bytes("https://packages.example/file")
-        with self.assertRaisesRegex(ReleaseManifestError, "offline"):
-            remote_sha256("https://packages.example/file")
-
-    @patch("bot_bottle.release_publish.urllib.request.urlopen")
-    def test_upload_supports_bytes_and_files(self, urlopen: MagicMock) -> None:
-        urlopen.return_value.__enter__.return_value = MagicMock()
-        publish_artifact("https://packages.example/bytes", b"body")
-        with tempfile.TemporaryDirectory() as tmp:
-            source = Path(tmp) / "artifact"
-            source.write_bytes(b"artifact")
-            publish_artifact("https://packages.example/file", source)
-        self.assertEqual(urlopen.call_count, 2)
-
-    @patch("bot_bottle.release_publish.urllib.request.urlopen")
-    def test_upload_reports_failure(self, urlopen: MagicMock) -> None:
-        urlopen.side_effect = urllib.error.URLError("offline")
-        with self.assertRaisesRegex(ReleaseManifestError, "publishing"):
-            publish_artifact("https://packages.example/file", b"body")

tests/unit/test_release_qualification.py

@@ -1,270 +0,0 @@
-"""Tests for release qualification and channel promotion."""
-
-from __future__ import annotations
-
-import json
-import unittest
-import urllib.error
-from email.message import Message
-from unittest import mock
-
-from bot_bottle.release_manifest import ReleaseManifestError
-from bot_bottle.release_qualification import (
-    build_release_pointer,
-    canonical_pointer,
-    delete,
-    parse_release_pointer,
-    publish_qualification,
-    release_channel,
-    release_version,
-)
-
-COMMIT = "a" * 40
-
-
-def bundle(source_commit: str = COMMIT) -> bytes:
-    return json.dumps({
-        "schema": 1,
-        "source_commit": source_commit,
-        "wheel": {
-            "filename": "bot_bottle.whl",
-            "url": (
-                "https://gitea.dideric.is/api/packages/didericis/generic/"
-                f"bot-bottle-builds/{source_commit}/bot_bottle.whl"
-            ),
-            "sha256": "b" * 64,
-        },
-        "oci": {
-            role: f"registry.example/{role}@sha256:{'c' * 64}"
-            for role in (
-                "orchestrator",
-                "gateway",
-                "agent_claude",
-                "agent_codex",
-                "agent_pi",
-            )
-        },
-        "firecracker": {
-            role: {"version": "v1", "sha256": "d" * 64}
-            for role in ("orchestrator", "gateway")
-        },
-        "provenance": {"workflow_run": "run", "published_at": "now"},
-        "qualifications": [],
-    }).encode()
-
-
-class ReleaseQualificationTest(unittest.TestCase):
-    def test_tag_selects_channel(self) -> None:
-        self.assertEqual(release_channel("v1.2.3"), "production")
-        self.assertEqual(release_channel("v1.2.3-rc.4"), "staging")
-        with self.assertRaises(ReleaseManifestError):
-            release_channel("latest")
-
-    def test_release_versions_are_monotonic(self) -> None:
-        self.assertLess(
-            release_version("v1.2.3-rc.4"), release_version("v1.2.3"))
-        self.assertLess(
-            release_version("v1.2.3"), release_version("v2.0.0"))
-        with self.assertRaises(ReleaseManifestError):
-            release_version("v1")
-
-    def test_builds_qualified_pointer(self) -> None:
-        pointer = build_release_pointer(
-            tag="v1.2.3",
-            source_commit=COMMIT,
-            workflow_run="https://example.test/run/1",
-            qualified_at="2026-07-27T00:00:00Z",
-        )
-        self.assertEqual(pointer["channel"], "production")
-        self.assertTrue(pointer["qualified"])
-        self.assertTrue(canonical_pointer(pointer).endswith(b"\n"))
-
-    def test_rejects_invalid_pointer_fields(self) -> None:
-        pointer = build_release_pointer(
-            tag="v1.2.3",
-            source_commit=COMMIT,
-            workflow_run="run",
-            qualified_at="now",
-        )
-        changes = (
-            ("schema", 2),
-            ("qualified", False),
-            ("channel", "staging"),
-            ("source_commit", "short"),
-            ("bundle_index_url", "https://wrong.example/index.json"),
-            ("qualification", {}),
-        )
-        for key, value in changes:
-            with self.subTest(key=key), self.assertRaises(ReleaseManifestError):
-                parse_release_pointer({**pointer, key: value})
-
-    def test_rejects_invalid_build_inputs(self) -> None:
-        with self.assertRaises(ReleaseManifestError):
-            build_release_pointer(
-                tag="v1.2.3",
-                source_commit="short",
-                workflow_run="run",
-                qualified_at="now",
-            )
-        with self.assertRaises(ReleaseManifestError):
-            build_release_pointer(
-                tag="v1.2.3",
-                source_commit=COMMIT,
-                workflow_run="",
-                qualified_at="now",
-            )
-
-    @mock.patch("bot_bottle.release_qualification.upload")
-    @mock.patch("bot_bottle.release_qualification.remote_bytes")
-    def test_publishes_bundle_release_and_channel(
-        self, remote: mock.Mock, upload: mock.Mock,
-    ) -> None:
-        remote.side_effect = [bundle(), None, None]
-        pointer = build_release_pointer(
-            tag="v1.2.3",
-            source_commit=COMMIT,
-            workflow_run="run",
-            qualified_at="now",
-        )
-        publish_qualification(pointer)
-        self.assertEqual(upload.call_count, 2)
-
-    @mock.patch("bot_bottle.release_qualification.upload")
-    @mock.patch("bot_bottle.release_qualification.remote_bytes")
-    def test_exact_existing_publication_is_noop(
-        self, remote: mock.Mock, upload: mock.Mock,
-    ) -> None:
-        pointer = build_release_pointer(
-            tag="v1.2.3",
-            source_commit=COMMIT,
-            workflow_run="run",
-            qualified_at="now",
-        )
-        wanted = canonical_pointer(pointer)
-        remote.side_effect = [bundle(), wanted, wanted]
-        publish_qualification(pointer)
-        upload.assert_not_called()
-
-    @mock.patch("bot_bottle.release_qualification.upload")
-    @mock.patch("bot_bottle.release_qualification.delete")
-    @mock.patch("bot_bottle.release_qualification.remote_bytes")
-    def test_advances_to_newer_channel(
-        self, remote: mock.Mock, remove: mock.Mock, upload: mock.Mock,
-    ) -> None:
-        old = build_release_pointer(
-            tag="v1.0.0",
-            source_commit="b" * 40,
-            workflow_run="old",
-            qualified_at="then",
-        )
-        pointer = build_release_pointer(
-            tag="v2.0.0",
-            source_commit=COMMIT,
-            workflow_run="run",
-            qualified_at="now",
-        )
-        remote.side_effect = [bundle(), None, canonical_pointer(old)]
-        publish_qualification(pointer)
-        remove.assert_called_once()
-        self.assertEqual(upload.call_count, 2)
-
-    @mock.patch("bot_bottle.release_qualification.upload")
-    @mock.patch("bot_bottle.release_qualification.remote_bytes")
-    def test_rejects_non_monotonic_channel(
-        self, remote: mock.Mock, _upload: mock.Mock,
-    ) -> None:
-        old = build_release_pointer(
-            tag="v2.0.0",
-            source_commit="b" * 40,
-            workflow_run="old-run",
-            qualified_at="old-time",
-        )
-        remote.side_effect = [bundle(), None, json.dumps(old).encode()]
-        pointer = build_release_pointer(
-            tag="v1.2.3",
-            source_commit=COMMIT,
-            workflow_run="run",
-            qualified_at="now",
-        )
-        with self.assertRaisesRegex(ReleaseManifestError, "not monotonic"):
-            publish_qualification(pointer)
-
-    @mock.patch("bot_bottle.release_qualification.upload")
-    @mock.patch("bot_bottle.release_qualification.delete")
-    @mock.patch("bot_bottle.release_qualification.remote_bytes")
-    def test_explicit_rollback_allows_older_channel(
-        self, remote: mock.Mock, remove: mock.Mock, _upload: mock.Mock,
-    ) -> None:
-        old = build_release_pointer(
-            tag="v2.0.0",
-            source_commit="b" * 40,
-            workflow_run="old",
-            qualified_at="then",
-        )
-        pointer = build_release_pointer(
-            tag="v1.0.0",
-            source_commit=COMMIT,
-            workflow_run="run",
-            qualified_at="now",
-        )
-        remote.side_effect = [bundle(), None, canonical_pointer(old)]
-        publish_qualification(pointer, allow_rollback=True)
-        remove.assert_called_once()
-
-    @mock.patch("bot_bottle.release_qualification.remote_bytes")
-    def test_rejects_missing_or_malformed_bundle(self, remote: mock.Mock) -> None:
-        pointer = build_release_pointer(
-            tag="v1.2.3",
-            source_commit=COMMIT,
-            workflow_run="run",
-            qualified_at="now",
-        )
-        remote.return_value = None
-        with self.assertRaisesRegex(ReleaseManifestError, "does not exist"):
-            publish_qualification(pointer)
-        remote.return_value = b"not-json"
-        with self.assertRaisesRegex(ReleaseManifestError, "malformed"):
-            publish_qualification(pointer)
-        remote.return_value = bundle("b" * 40)
-        with self.assertRaisesRegex(ReleaseManifestError, "does not match"):
-            publish_qualification(pointer)
-
-    @mock.patch("bot_bottle.release_qualification.upload")
-    @mock.patch("bot_bottle.release_qualification.remote_bytes")
-    def test_rejects_changed_release_or_malformed_channel(
-        self, remote: mock.Mock, _upload: mock.Mock,
-    ) -> None:
-        pointer = build_release_pointer(
-            tag="v1.2.3",
-            source_commit=COMMIT,
-            workflow_run="run",
-            qualified_at="now",
-        )
-        remote.side_effect = [bundle(), b"different"]
-        with self.assertRaisesRegex(ReleaseManifestError, "already differs"):
-            publish_qualification(pointer)
-        remote.side_effect = [bundle(), None, b"not-json"]
-        with self.assertRaisesRegex(ReleaseManifestError, "malformed"):
-            publish_qualification(pointer)
-
-    @mock.patch("bot_bottle.release_qualification.urllib.request.urlopen")
-    def test_delete_handles_success_and_not_found(self, urlopen: mock.Mock) -> None:
-        urlopen.return_value.__enter__.return_value = mock.Mock()
-        delete("https://example.test/channel")
-        urlopen.side_effect = urllib.error.HTTPError(
-            "url", 404, "missing", Message(), None)
-        delete("https://example.test/channel")
-
-    @mock.patch("bot_bottle.release_qualification.urllib.request.urlopen")
-    def test_delete_reports_registry_errors(self, urlopen: mock.Mock) -> None:
-        urlopen.side_effect = urllib.error.HTTPError(
-            "url", 500, "broken", Message(), None)
-        with self.assertRaisesRegex(ReleaseManifestError, "HTTP 500"):
-            delete("https://example.test/channel")
-        urlopen.side_effect = urllib.error.URLError("offline")
-        with self.assertRaisesRegex(ReleaseManifestError, "offline"):
-            delete("https://example.test/channel")
-
-
-if __name__ == "__main__":
-    unittest.main()

tests/unit/test_wheel_install.py

@@ -117,15 +117,6 @@ class TestWheelInstall(unittest.TestCase):
         proc = self._run(["-c", script])
         self.assertIn("SELF_CONTAINED_OK", proc.stdout, msg=proc.stderr)
 
-    def test_ad_hoc_wheel_is_marked_for_local_infrastructure_builds(self):
-        script = (
-            "from bot_bottle.release_manifest import packaged_manifest\n"
-            "assert packaged_manifest() is None\n"
-            "print('DEVELOPMENT_MANIFEST_OK')\n"
-        )
-        proc = self._run(["-c", script])
-        self.assertIn("DEVELOPMENT_MANIFEST_OK", proc.stdout, msg=proc.stderr)
-
 
 if __name__ == "__main__":
     unittest.main()