Compare commits
16 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 47cb50db4b | |||
| a750a242d1 | |||
| 4544899c79 | |||
| 89d2ba7e82 | |||
| 43a92945a3 | |||
| 7abcb2c7df | |||
| efa1c8ee4b | |||
| 249eaff53d | |||
| 9f65ab016d | |||
| 46b7e06b37 | |||
| 93190d5e0c | |||
| a2113b7f43 | |||
| d8c5532168 | |||
| a27b1a5fe9 | |||
| 1a9056c648 | |||
| 63595f123a |
@@ -19,6 +19,19 @@ name: pre-release-test
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
ref:
|
||||
description: Revision to qualify
|
||||
required: false
|
||||
default: main
|
||||
workflow_call:
|
||||
inputs:
|
||||
ref:
|
||||
required: true
|
||||
type: string
|
||||
secrets:
|
||||
BOT_BOTTLE_INFRA_ARTIFACT_TOKEN:
|
||||
required: true
|
||||
|
||||
jobs:
|
||||
unit:
|
||||
@@ -26,6 +39,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
# No actions/setup-python: the runner image already ships Python 3.12,
|
||||
# and older act_runner engines mishandle setup-python's PATH (coverage
|
||||
@@ -66,6 +81,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
# No actions/setup-python (see the note in the `unit` job); the
|
||||
# container's system Python 3.12 runs the stdlib test suite directly.
|
||||
@@ -138,10 +155,11 @@ jobs:
|
||||
# the old build-infra → integration-firecracker + coverage chain incurred.
|
||||
integration-firecracker:
|
||||
runs-on: [self-hosted, kvm]
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- name: Preflight — Firecracker host is ready
|
||||
run: |
|
||||
@@ -212,13 +230,14 @@ jobs:
|
||||
# Python >=3.11 with `coverage` importable on the launchd service PATH.
|
||||
integration-macos:
|
||||
runs-on: [self-hosted, macos]
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
concurrency:
|
||||
group: integration-macos-infra
|
||||
cancel-in-progress: false
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
# Fail loudly if the backend this job promises isn't actually usable,
|
||||
# rather than letting every test silently `unittest.skip` and the job go
|
||||
@@ -290,6 +309,7 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install coverage
|
||||
@@ -340,6 +360,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout the tested revision
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- name: Download the tested rootfs
|
||||
uses: actions/download-artifact@v3
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
name: publish-artifacts
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
ref:
|
||||
description: Branch, tag, or commit to publish
|
||||
required: true
|
||||
default: main
|
||||
workflow_call:
|
||||
inputs:
|
||||
ref:
|
||||
required: true
|
||||
type: string
|
||||
outputs:
|
||||
source_commit:
|
||||
description: Published immutable source commit
|
||||
value: ${{ jobs.resolve.outputs.sha }}
|
||||
secrets:
|
||||
BOT_BOTTLE_RELEASE_TOKEN:
|
||||
required: true
|
||||
BOT_BOTTLE_INFRA_ARTIFACT_TOKEN:
|
||||
required: true
|
||||
|
||||
concurrency:
|
||||
group: publish-artifacts-${{ inputs.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
resolve:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
sha: ${{ steps.revision.outputs.sha }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
fetch-depth: 0
|
||||
- id: revision
|
||||
name: Resolve the immutable source revision
|
||||
run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
oci:
|
||||
needs: resolve
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
orchestrator: ${{ steps.images.outputs.orchestrator }}
|
||||
gateway: ${{ steps.images.outputs.gateway }}
|
||||
agent_claude: ${{ steps.images.outputs.agent_claude }}
|
||||
agent_codex: ${{ steps.images.outputs.agent_codex }}
|
||||
agent_pi: ${{ steps.images.outputs.agent_pi }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.resolve.outputs.sha }}
|
||||
- name: Log in to the package registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: gitea.dideric.is
|
||||
username: ${{ gitea.actor }}
|
||||
password: ${{ secrets.BOT_BOTTLE_RELEASE_TOKEN }}
|
||||
- name: Set up multi-architecture builds
|
||||
uses: docker/setup-buildx-action@v3
|
||||
- id: images
|
||||
name: Build, smoke-test, and publish OCI images
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python_base=$(python3 -c \
|
||||
'import json; print(json.load(open("image-build-args.json"))["PYTHON_BASE_IMAGE"])')
|
||||
node_base=$(python3 -c \
|
||||
'import json; print(json.load(open("image-build-args.json"))["NODE_BASE_IMAGE"])')
|
||||
sha='${{ needs.resolve.outputs.sha }}'
|
||||
registry=gitea.dideric.is/didericis
|
||||
|
||||
build_image() {
|
||||
key=$1
|
||||
name=$2
|
||||
dockerfile=$3
|
||||
build_arg=$4
|
||||
smoke=$5
|
||||
local_ref="bot-bottle-${name}:candidate"
|
||||
remote_ref="${registry}/bot-bottle-${name}:commit-${sha}"
|
||||
docker build --build-arg "$build_arg" -t "$local_ref" \
|
||||
-f "$dockerfile" .
|
||||
smoke_command=$(printf "$smoke" "$local_ref")
|
||||
sh -c "docker run --rm $smoke_command"
|
||||
docker buildx build --platform linux/amd64,linux/arm64 \
|
||||
--build-arg "$build_arg" --push --iidfile "${key}.iid" \
|
||||
-t "$remote_ref" -f "$dockerfile" .
|
||||
digest=$(cat "${key}.iid")
|
||||
case "$digest" in sha256:*) ;; *) exit 1 ;; esac
|
||||
echo "${key}=${registry}/bot-bottle-${name}@${digest}" \
|
||||
>> "$GITHUB_OUTPUT"
|
||||
}
|
||||
|
||||
build_image orchestrator orchestrator Dockerfile.orchestrator \
|
||||
"PYTHON_BASE_IMAGE=$python_base" \
|
||||
"--entrypoint python3 %s -c 'import bot_bottle.orchestrator'"
|
||||
build_image gateway gateway Dockerfile.gateway \
|
||||
"PYTHON_BASE_IMAGE=$python_base" \
|
||||
"--entrypoint mitmdump %s --version"
|
||||
build_image agent_claude claude bot_bottle/contrib/claude/Dockerfile \
|
||||
"NODE_BASE_IMAGE=$node_base" "%s claude --version"
|
||||
build_image agent_codex codex bot_bottle/contrib/codex/Dockerfile \
|
||||
"NODE_BASE_IMAGE=$node_base" "%s codex --version"
|
||||
build_image agent_pi pi bot_bottle/contrib/pi/Dockerfile \
|
||||
"NODE_BASE_IMAGE=$node_base" "%s pi --version"
|
||||
|
||||
firecracker:
|
||||
needs: resolve
|
||||
runs-on: [self-hosted, kvm]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.resolve.outputs.sha }}
|
||||
- name: Build and smoke-test Firecracker artifacts
|
||||
env:
|
||||
BOT_BOTTLE_FC_DROPBEAR: /var/cache/bot-bottle-fc/dropbear
|
||||
run: |
|
||||
python3 cli.py backend status --backend=firecracker
|
||||
python3 -m bot_bottle.backend.firecracker.publish_infra \
|
||||
--output infra-candidate --reuse-published
|
||||
BOT_BOTTLE_INFRA_ARTIFACT_DIR="$PWD/infra-candidate" \
|
||||
python3 -m unittest discover -t . -s tests/integration -v
|
||||
- name: Publish tested Firecracker artifacts
|
||||
env:
|
||||
BOT_BOTTLE_INFRA_ARTIFACT_TOKEN: ${{ secrets.BOT_BOTTLE_INFRA_ARTIFACT_TOKEN }}
|
||||
BOT_BOTTLE_FC_DROPBEAR: /var/cache/bot-bottle-fc/dropbear
|
||||
run: |
|
||||
python3 -m bot_bottle.backend.firecracker.publish_infra \
|
||||
--publish-dir infra-candidate
|
||||
- name: Record Firecracker artifact identities
|
||||
run: |
|
||||
cp infra-candidate/orchestrator/version.txt fc-orchestrator-version
|
||||
cp infra-candidate/orchestrator/rootfs.ext4.gz.sha256 fc-orchestrator-sha
|
||||
cp infra-candidate/gateway/version.txt fc-gateway-version
|
||||
cp infra-candidate/gateway/rootfs.ext4.gz.sha256 fc-gateway-sha
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: firecracker-identities
|
||||
path: |
|
||||
fc-orchestrator-version
|
||||
fc-orchestrator-sha
|
||||
fc-gateway-version
|
||||
fc-gateway-sha
|
||||
|
||||
package:
|
||||
needs: [resolve, oci, firecracker]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.resolve.outputs.sha }}
|
||||
- uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: firecracker-identities
|
||||
- name: Build the wheel with the published identities
|
||||
run: |
|
||||
python3 -m pip install --break-system-packages build
|
||||
python3 scripts/generate_release_manifest.py \
|
||||
--source-commit '${{ needs.resolve.outputs.sha }}' \
|
||||
--orchestrator-image '${{ needs.oci.outputs.orchestrator }}' \
|
||||
--gateway-image '${{ needs.oci.outputs.gateway }}' \
|
||||
--agent-claude-image '${{ needs.oci.outputs.agent_claude }}' \
|
||||
--agent-codex-image '${{ needs.oci.outputs.agent_codex }}' \
|
||||
--agent-pi-image '${{ needs.oci.outputs.agent_pi }}' \
|
||||
--firecracker-orchestrator-version "$(cat fc-orchestrator-version)" \
|
||||
--firecracker-orchestrator-sha256 "$(awk '{print $1}' fc-orchestrator-sha)" \
|
||||
--firecracker-gateway-version "$(cat fc-gateway-version)" \
|
||||
--firecracker-gateway-sha256 "$(awk '{print $1}' fc-gateway-sha)" \
|
||||
--output release-manifest.json
|
||||
BOT_BOTTLE_RELEASE_MANIFEST="$PWD/release-manifest.json" \
|
||||
python3 -m build --wheel
|
||||
- name: Verify and publish the immutable commit bundle
|
||||
env:
|
||||
BOT_BOTTLE_RELEASE_TOKEN: ${{ secrets.BOT_BOTTLE_RELEASE_TOKEN }}
|
||||
run: |
|
||||
wheel=$(find dist -maxdepth 1 -name '*.whl' -type f)
|
||||
python3 -m venv verify-venv
|
||||
verify-venv/bin/pip install --no-deps "$wheel"
|
||||
verify-venv/bin/python -c \
|
||||
'from bot_bottle.release_manifest import load_manifest; load_manifest()'
|
||||
python3 scripts/generate_release_bundle.py \
|
||||
--manifest release-manifest.json \
|
||||
--wheel "$wheel" \
|
||||
--workflow-run \
|
||||
"${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
|
||||
--published-at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
||||
--output bundle-index.json --publish
|
||||
@@ -0,0 +1,93 @@
|
||||
name: release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
|
||||
concurrency:
|
||||
group: release-${{ github.ref_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
sha: ${{ steps.release.outputs.sha }}
|
||||
tag: ${{ steps.release.outputs.tag }}
|
||||
channel: ${{ steps.release.outputs.channel }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: release
|
||||
name: Validate tag shape and promotion branch
|
||||
run: |
|
||||
set -euo pipefail
|
||||
tag=${GITHUB_REF#refs/tags/}
|
||||
channel=$(python3 -c \
|
||||
'import sys; from bot_bottle.release_qualification import release_channel; print(release_channel(sys.argv[1]))' \
|
||||
"$tag")
|
||||
branch=$channel
|
||||
git fetch origin "$branch"
|
||||
sha=$(git rev-list -n 1 "$tag")
|
||||
git merge-base --is-ancestor "$sha" "origin/$branch" || {
|
||||
echo "$tag is not reachable from protected $branch" >&2
|
||||
exit 1
|
||||
}
|
||||
echo "sha=$sha" >> "$GITHUB_OUTPUT"
|
||||
echo "tag=$tag" >> "$GITHUB_OUTPUT"
|
||||
echo "channel=$channel" >> "$GITHUB_OUTPUT"
|
||||
|
||||
qualify:
|
||||
needs: validate
|
||||
uses: ./.gitea/workflows/pre-release-test.yml
|
||||
with:
|
||||
ref: ${{ needs.validate.outputs.sha }}
|
||||
secrets:
|
||||
BOT_BOTTLE_INFRA_ARTIFACT_TOKEN: ${{ secrets.BOT_BOTTLE_INFRA_ARTIFACT_TOKEN }}
|
||||
|
||||
artifacts:
|
||||
needs: [validate, qualify]
|
||||
uses: ./.gitea/workflows/publish-artifacts.yml
|
||||
with:
|
||||
ref: ${{ needs.validate.outputs.sha }}
|
||||
secrets:
|
||||
BOT_BOTTLE_RELEASE_TOKEN: ${{ secrets.BOT_BOTTLE_RELEASE_TOKEN }}
|
||||
BOT_BOTTLE_INFRA_ARTIFACT_TOKEN: ${{ secrets.BOT_BOTTLE_INFRA_ARTIFACT_TOKEN }}
|
||||
|
||||
promote:
|
||||
needs: [validate, artifacts]
|
||||
runs-on: ubuntu-latest
|
||||
concurrency:
|
||||
group: release-channel-${{ needs.validate.outputs.channel }}
|
||||
cancel-in-progress: false
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.validate.outputs.sha }}
|
||||
- name: Verify install from the published bundle
|
||||
env:
|
||||
BOT_BOTTLE_REF: ${{ needs.validate.outputs.sha }}
|
||||
run: |
|
||||
sh install.sh
|
||||
"$HOME/.local/bin/bot-bottle" --help
|
||||
- name: Publish release qualification and advance channel
|
||||
env:
|
||||
BOT_BOTTLE_RELEASE_TOKEN: ${{ secrets.BOT_BOTTLE_RELEASE_TOKEN }}
|
||||
run: |
|
||||
python3 scripts/publish_release_qualification.py \
|
||||
--tag '${{ needs.validate.outputs.tag }}' \
|
||||
--source-commit '${{ needs.validate.outputs.sha }}' \
|
||||
--workflow-run \
|
||||
"${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
|
||||
--qualified-at "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
- name: Create the Gitea release
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.BOT_BOTTLE_RELEASE_TOKEN }}
|
||||
run: |
|
||||
curl --fail-with-body --silent --show-error \
|
||||
-H "Authorization: token $GITEA_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"tag_name\":\"${{ needs.validate.outputs.tag }}\",\"name\":\"${{ needs.validate.outputs.tag }}\",\"target_commitish\":\"${{ needs.validate.outputs.sha }}\"}" \
|
||||
"${GITHUB_SERVER_URL}/api/v1/repos/${GITHUB_REPOSITORY}/releases"
|
||||
@@ -75,7 +75,15 @@ When the agent exits, `cli.py` tears down every gateway and both networks; nothi
|
||||
curl -fsSL https://gitea.dideric.is/didericis/bot-bottle/raw/branch/main/install.sh | sh
|
||||
```
|
||||
|
||||
The installer is a bootstrapper: it finds a suitable Python, installs bot-bottle with `pipx` (falling back to `pip --user`), creates `~/.bot-bottle`, and runs `bot-bottle doctor`. It is idempotent and never uses `sudo`. Python-native users can skip it entirely with `pipx install bot-bottle` or `uv tool install bot-bottle`.
|
||||
The installer is a bootstrapper: it finds a suitable Python, resolves the
|
||||
latest qualified production bundle, verifies its wheel checksum, installs with
|
||||
`pipx` (falling back to a private venv), creates `~/.bot-bottle`, and runs
|
||||
`bot-bottle doctor`. It is idempotent and never uses `sudo`.
|
||||
|
||||
Select staging with `BOT_BOTTLE_CHANNEL=staging`, an exact qualified release
|
||||
with `BOT_BOTTLE_VERSION=vX.Y.Z[-rc.N]`, or an exact published commit with
|
||||
`BOT_BOTTLE_REF=<40-character-sha>`. Commit installs print an explicit warning
|
||||
because snapshots may not have passed release qualification.
|
||||
|
||||
### Requirements
|
||||
|
||||
@@ -83,7 +91,8 @@ The installer is a bootstrapper: it finds a suitable Python, installs bot-bottle
|
||||
|
||||
**No `pipx` required.** If `pipx` is present the installer uses it and stays out of the way. If it isn't, bot-bottle installs into a private venv at `~/.bot-bottle/venv` (override with `BOT_BOTTLE_VENV`) and symlinks the entry point into `~/.local/bin`. There is deliberately no `pip install --user` path: Homebrew, python.org and Debian/Ubuntu interpreters are all externally managed (PEP 668), which blocks `--user` outright — so on a Mac it is never the fallback it appears to be. A venv is exempt from PEP 668, and `venv` is stdlib, so unlike `pipx` there is nothing to bootstrap first.
|
||||
|
||||
**`git`**, because the default install spec is a `git+` URL. Set `BOT_BOTTLE_INSTALL_SPEC` to a wheel path or index name to avoid it.
|
||||
**`git`** is needed only when `BOT_BOTTLE_INSTALL_SPEC` explicitly selects a
|
||||
`git+` URL. The normal published-wheel path does not require it.
|
||||
|
||||
**A backend**, which the installer deliberately does *not* install for you — `doctor` reports what's missing afterwards. On compatible macOS hosts, the default backend requires Apple's `container` CLI and does not require Docker. The Firecracker backend (Linux) requires Docker on the host for the gateway plus the `firecracker` binary and KVM. The legacy Docker backend requires Docker. Claude bottles also need a long-lived Claude Code OAuth token (`claude setup-token`) exported as `BOT_BOTTLE_CLAUDE_OAUTH_TOKEN`.
|
||||
|
||||
@@ -191,9 +200,19 @@ BOT_BOTTLE_BACKEND=firecracker bot-bottle start <agent>
|
||||
> **CI:** Firecracker integration runs in the manually dispatched `.gitea/workflows/pre-release-test.yml` on a self-hosted runner labelled `kvm`; privileged KVM hosts never execute unreviewed PR code automatically. Provision it like a normal Firecracker host: `firecracker` on `PATH`, `/dev/kvm`, the cached guest kernel and static dropbear, and the persistent TAP/nft pool. The required pull-request workflow runs unit plus the complete Docker integration suite on `ubuntu-latest`; see `docs/ci.md`.
|
||||
|
||||
```sh
|
||||
bot-bottle start <agent> # builds the image on first run, drops you into claude
|
||||
bot-bottle start <agent> # prepares the selected agent image, then attaches
|
||||
```
|
||||
|
||||
Packaged releases carry immutable orchestrator, gateway, and first-party agent
|
||||
image identities. Docker and Apple Container pull the package-selected OCI
|
||||
digests; Firecracker pulls the matching checksum-verified rootfs artifacts.
|
||||
Each verified wheel and its identities are published as an immutable generic
|
||||
package keyed by the full source commit; `bundle-index.json` is uploaded last
|
||||
and is the completeness marker. A source checkout uses local builds for
|
||||
development. Set `BOT_BOTTLE_INFRA_BUILD=local` to make that development
|
||||
override explicit when diagnosing a packaged release; production never falls
|
||||
back to a build after an artifact pull fails.
|
||||
|
||||
## Manifest
|
||||
|
||||
Bottles and agents are Markdown files with YAML frontmatter under `~/.bot-bottle/`. The Markdown body is the system prompt. Bottles live in `~/.bot-bottle/bottles/`; agents may also be shipped by a repo at `<repo>/.bot-bottle/agents/<name>.md`.
|
||||
|
||||
@@ -89,6 +89,10 @@ class DockerGateway(Gateway):
|
||||
when no dockerfile is configured (a pre-pulled image). BOT_BOTTLE_NO_CACHE
|
||||
forces a full rebuild (parity with `start --no-cache`)."""
|
||||
if self._dockerfile is None:
|
||||
proc = run_docker(["docker", "pull", self.image_ref])
|
||||
if proc.returncode != 0:
|
||||
raise GatewayError(
|
||||
f"gateway image pull failed: {proc.stderr.strip()}")
|
||||
return
|
||||
context = self._build_context or resources.build_root()
|
||||
argv = ["docker", "build", "-t", self.image_ref,
|
||||
|
||||
@@ -34,6 +34,7 @@ from .orchestrator import (
|
||||
ORCHESTRATOR_NETWORK,
|
||||
)
|
||||
from ... import resources
|
||||
from ... import release_manifest
|
||||
from ...gateway import (
|
||||
GATEWAY_IMAGE,
|
||||
GATEWAY_NAME,
|
||||
@@ -90,6 +91,14 @@ class DockerInfraService(InfraService):
|
||||
self._orchestrator_name = orchestrator_name
|
||||
self._orchestrator_label = orchestrator_label
|
||||
self._gateway_name = gateway_name
|
||||
resolved_orchestrator, orchestrator_local = release_manifest.oci_image(
|
||||
"orchestrator", orchestrator_image)
|
||||
resolved_gateway, gateway_local = release_manifest.oci_image(
|
||||
"gateway", gateway_image)
|
||||
self.orchestrator_image = resolved_orchestrator
|
||||
self.gateway_image = resolved_gateway
|
||||
self._orchestrator_local = orchestrator_local
|
||||
self._gateway_local = gateway_local
|
||||
|
||||
def orchestrator(self) -> DockerOrchestrator:
|
||||
"""The control-plane service. Cheap to reconstruct — `ensure_built`
|
||||
@@ -104,6 +113,8 @@ class DockerInfraService(InfraService):
|
||||
repo_root=self._repo_root,
|
||||
host_root=self._host_root,
|
||||
root_mount_source=self._root_mount_source,
|
||||
dockerfile=(
|
||||
"Dockerfile.orchestrator" if self._orchestrator_local else None),
|
||||
)
|
||||
|
||||
def gateway(self) -> DockerGateway:
|
||||
@@ -119,6 +130,7 @@ class DockerInfraService(InfraService):
|
||||
control_network=self.control_network,
|
||||
build_context=self._repo_root,
|
||||
ca_mount_source=self._gateway_ca_mount_source,
|
||||
dockerfile="Dockerfile.gateway" if self._gateway_local else None,
|
||||
)
|
||||
|
||||
def ensure_running(
|
||||
@@ -132,8 +144,8 @@ class DockerInfraService(InfraService):
|
||||
gateway = self.gateway()
|
||||
# Build both images (cache-aware; a no-op when nothing changed) before
|
||||
# bringing either plane up.
|
||||
orchestrator.ensure_built()
|
||||
gateway.ensure_built()
|
||||
orchestrator.ensure_available()
|
||||
gateway.ensure_available()
|
||||
|
||||
orchestrator.ensure_running(startup_timeout=startup_timeout)
|
||||
|
||||
|
||||
@@ -84,6 +84,15 @@ def build_or_load_images(plan: DockerBottlePlan) -> BottleImages:
|
||||
)
|
||||
info(f"using cached agent image {plan.image!r}")
|
||||
return BottleImages(agent=plan.image)
|
||||
if "@sha256:" in plan.image:
|
||||
pulled = docker_mod.run_docker(["docker", "pull", plan.image])
|
||||
if pulled.returncode != 0:
|
||||
die(f"pulling packaged agent image {plan.image!r} failed: "
|
||||
f"{pulled.stderr.strip()}")
|
||||
docker_mod.verify_agent_image(
|
||||
plan.image, runtime_for(plan.agent_provider_template).smoke_test,
|
||||
)
|
||||
return BottleImages(agent=plan.image)
|
||||
docker_mod.build_image(plan.image, str(resources.build_root()), dockerfile=plan.dockerfile_path)
|
||||
docker_mod.verify_agent_image(
|
||||
plan.image, runtime_for(plan.agent_provider_template).smoke_test,
|
||||
|
||||
@@ -125,6 +125,10 @@ class DockerOrchestrator(Orchestrator):
|
||||
no-op when nothing changed). No-op when no dockerfile is configured (a
|
||||
pre-pulled image). BOT_BOTTLE_NO_CACHE forces a full rebuild."""
|
||||
if self._dockerfile is None:
|
||||
proc = run_docker(["docker", "pull", self.image_ref])
|
||||
if proc.returncode != 0:
|
||||
raise GatewayError(
|
||||
f"orchestrator image pull failed: {proc.stderr.strip()}")
|
||||
return
|
||||
argv = ["docker", "build", "-t", self.image_ref,
|
||||
"-f", str(self._repo_root / self._dockerfile),
|
||||
|
||||
@@ -72,6 +72,46 @@ def cached_agent_rootfs_dir(dockerfile: Path) -> Path | None:
|
||||
return base if (base / ".bb-ready").is_file() else None
|
||||
|
||||
|
||||
def _image_rootfs_digest(image: str) -> str:
|
||||
h = hashlib.sha256()
|
||||
h.update(image.encode())
|
||||
h.update(b"\0")
|
||||
h.update(util._GUEST_INIT.encode())
|
||||
return h.hexdigest()[:16]
|
||||
|
||||
|
||||
def cached_agent_image_rootfs_dir(image: str) -> Path | None:
|
||||
"""Return a ready rootfs exported from an immutable OCI image."""
|
||||
base = util.cache_dir() / "rootfs" / f"agent-image-{_image_rootfs_digest(image)}"
|
||||
return base if (base / ".bb-ready").is_file() else None
|
||||
|
||||
|
||||
def acquire_agent_image_rootfs_dir(
|
||||
image: str, *, smoke_test: tuple[str, ...] = (),
|
||||
) -> Path:
|
||||
"""Pull a digest-pinned image in the infra VM and export its rootfs."""
|
||||
if "@sha256:" not in image:
|
||||
die(f"prebuilt Firecracker agent image is not digest-pinned: {image}")
|
||||
digest = _image_rootfs_digest(image)
|
||||
base = util.cache_dir() / "rootfs" / f"agent-image-{digest}"
|
||||
cached = cached_agent_image_rootfs_dir(image)
|
||||
if cached is not None:
|
||||
info(f"using cached agent rootfs {cached.name}")
|
||||
return cached
|
||||
with _build_lock():
|
||||
if (base / ".bb-ready").is_file():
|
||||
return base
|
||||
staging = util.cache_dir() / "rootfs" / f".pulling-{digest}"
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
staging.mkdir(parents=True)
|
||||
_pull_in_infra(image, staging, smoke_test, digest)
|
||||
util.inject_guest_boot(staging)
|
||||
(staging / ".bb-ready").write_text("ok\n")
|
||||
shutil.rmtree(base, ignore_errors=True)
|
||||
os.rename(staging, base)
|
||||
return base
|
||||
|
||||
|
||||
def build_agent_rootfs_dir(
|
||||
dockerfile: Path, *, image_tag: str, smoke_test: tuple[str, ...] = (),
|
||||
) -> Path:
|
||||
@@ -167,6 +207,43 @@ def _build_in_infra(
|
||||
_cleanup()
|
||||
|
||||
|
||||
def _pull_in_infra(
|
||||
image: str, base: Path, smoke_test: tuple[str, ...], digest: str,
|
||||
) -> None:
|
||||
"""Pull and export a published agent image inside the orchestrator VM."""
|
||||
service = FirecrackerInfraService()
|
||||
service.ensure_running()
|
||||
key, ip = service.orchestrator().ssh_target()
|
||||
tag = f"bot-bottle-agent-pull-{digest}"
|
||||
smoke_ctr, export_ctr = f"{tag}-smoke", f"{tag}-export"
|
||||
quoted_image = shlex.quote(image)
|
||||
|
||||
def cleanup() -> None:
|
||||
_ssh(
|
||||
key,
|
||||
ip,
|
||||
f"buildah rm {smoke_ctr} {export_ctr} >/dev/null 2>&1; "
|
||||
f"buildah rmi {_STORE_FLAG} {tag} >/dev/null 2>&1",
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
cleanup()
|
||||
try:
|
||||
result = _ssh(
|
||||
key,
|
||||
ip,
|
||||
f"buildah pull {_STORE_FLAG} {quoted_image} && "
|
||||
f"buildah tag {_STORE_FLAG} {quoted_image} {tag}",
|
||||
timeout=_BUILD_TIMEOUT_SECONDS,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
die(f"pulling pinned agent image failed: {result.stderr.strip()}")
|
||||
_smoke_test(key, ip, tag, smoke_ctr, smoke_test)
|
||||
_stream_rootfs(key, ip, tag, export_ctr, base)
|
||||
finally:
|
||||
cleanup()
|
||||
|
||||
|
||||
def _ssh(private_key: Path, guest_ip: str, script: str,
|
||||
*, timeout: float = 60.0) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
|
||||
@@ -61,7 +61,7 @@ class FirecrackerInfraService(InfraService):
|
||||
return url
|
||||
# Clear stale/hung/OUTDATED VMs holding either link before booting fresh.
|
||||
infra_vm.stop()
|
||||
infra_vm.ensure_built()
|
||||
infra_vm.ensure_available()
|
||||
# Orchestrator first — the gateway daemons reach the control plane at
|
||||
# startup. Each service owns its own boot; the orchestrator (which
|
||||
# holds the signing key) mints the role-scoped `gateway` JWT for the
|
||||
|
||||
@@ -195,7 +195,9 @@ def _sha256_file(path: Path) -> str:
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def ensure_artifact_gz(version: str, *, role: str) -> Path:
|
||||
def ensure_artifact_gz(
|
||||
version: str, *, role: str, expected_sha256: str | None = None,
|
||||
) -> Path:
|
||||
"""The verified, cached `rootfs.ext4.gz` for `role` at `version` —
|
||||
downloading it (and its `.sha256`) once, then reusing it. Fail-closed on a
|
||||
checksum mismatch: the partial is removed and we die rather than boot an
|
||||
@@ -222,6 +224,11 @@ def ensure_artifact_gz(version: str, *, role: str) -> Path:
|
||||
if not gz.is_file() or not sha.is_file():
|
||||
die(f"infra candidate bundle is incomplete: {root}")
|
||||
expected = sha.read_text().split()[0].strip().lower()
|
||||
if expected_sha256 is not None and expected != expected_sha256:
|
||||
die(
|
||||
f"infra candidate packaged checksum mismatch ({role}) for {version}:\n"
|
||||
f" packaged {expected_sha256}\n candidate {expected}"
|
||||
)
|
||||
actual = _sha256_file(gz)
|
||||
if actual != expected:
|
||||
die(
|
||||
@@ -235,7 +242,13 @@ def ensure_artifact_gz(version: str, *, role: str) -> Path:
|
||||
gz = root / _GZ_NAME
|
||||
ok = root / ".verified"
|
||||
if gz.is_file() and ok.is_file():
|
||||
return gz
|
||||
actual = _sha256_file(gz)
|
||||
recorded = ok.read_text(encoding="utf-8").strip()
|
||||
wanted = expected_sha256 or recorded
|
||||
if actual == recorded == wanted:
|
||||
return gz
|
||||
gz.unlink(missing_ok=True)
|
||||
ok.unlink(missing_ok=True)
|
||||
|
||||
info(f"pulling infra rootfs artifact {_package(role)}/{version}")
|
||||
_download(artifact_url(version, _GZ_NAME, role=role), gz)
|
||||
@@ -243,6 +256,14 @@ def ensure_artifact_gz(version: str, *, role: str) -> Path:
|
||||
_download(artifact_url(version, _SHA_NAME, role=role), sha)
|
||||
|
||||
expected = sha.read_text().split()[0].strip().lower()
|
||||
if expected_sha256 is not None and expected != expected_sha256:
|
||||
gz.unlink(missing_ok=True)
|
||||
sha.unlink(missing_ok=True)
|
||||
die(
|
||||
f"infra artifact published checksum mismatch ({role}) for {version}:\n"
|
||||
f" packaged {expected_sha256}\n"
|
||||
f" published {expected}"
|
||||
)
|
||||
actual = _sha256_file(gz)
|
||||
if actual != expected:
|
||||
gz.unlink(missing_ok=True)
|
||||
@@ -253,15 +274,22 @@ def ensure_artifact_gz(version: str, *, role: str) -> Path:
|
||||
f" actual {actual}\n"
|
||||
f" refusing to boot an unverified rootfs."
|
||||
)
|
||||
ok.write_text("ok\n")
|
||||
ok.write_text(actual + "\n")
|
||||
return gz
|
||||
|
||||
|
||||
def materialize_ext4(version: str, dest: Path, *, role: str) -> None:
|
||||
def materialize_ext4(
|
||||
version: str,
|
||||
dest: Path,
|
||||
*,
|
||||
role: str,
|
||||
expected_sha256: str | None = None,
|
||||
) -> None:
|
||||
"""Ensure the verified `role` artifact is cached, then gunzip it to `dest` —
|
||||
a fresh, writable per-boot rootfs (the VM mutates it; the cached `.gz` stays
|
||||
pristine). Atomic via a `.part` sibling."""
|
||||
gz = ensure_artifact_gz(version, role=role)
|
||||
gz = ensure_artifact_gz(
|
||||
version, role=role, expected_sha256=expected_sha256)
|
||||
tmp = dest.with_suffix(dest.suffix + ".part")
|
||||
info(f"expanding {role} infra rootfs -> {dest}")
|
||||
with gzip.open(gz, "rb") as src, open(tmp, "wb") as out:
|
||||
|
||||
@@ -43,6 +43,7 @@ from pathlib import Path
|
||||
from typing import Generator
|
||||
|
||||
from ... import resources
|
||||
from ... import release_manifest
|
||||
from ...log import die, info
|
||||
from ..docker import util as docker_mod
|
||||
from . import firecracker_vm, infra_artifact, netpool, util
|
||||
@@ -108,6 +109,18 @@ def _role_version(role: str) -> str:
|
||||
return infra_artifact.infra_artifact_version(role_init(role), role)
|
||||
|
||||
|
||||
def _role_release(role: str) -> tuple[str, str | None]:
|
||||
manifest = release_manifest.packaged_manifest()
|
||||
if manifest is None or release_manifest.local_build_requested():
|
||||
return _role_version(role), None
|
||||
artifact = (
|
||||
manifest.firecracker_orchestrator
|
||||
if role == "orchestrator"
|
||||
else manifest.firecracker_gateway
|
||||
)
|
||||
return artifact.version, artifact.sha256
|
||||
|
||||
|
||||
def ensure_built() -> None:
|
||||
"""Ensure both infra rootfs artifacts are available before boot.
|
||||
|
||||
@@ -117,11 +130,16 @@ def ensure_built() -> None:
|
||||
`BOT_BOTTLE_INFRA_BUILD=local` instead builds the images from source with
|
||||
host Docker (the orchestrator-fc image is `FROM` the orchestrator image, so
|
||||
it must exist first) — for iterating on the Dockerfiles."""
|
||||
if infra_artifact.local_build_requested():
|
||||
if release_manifest.local_build_requested():
|
||||
build_infra_images_with_docker()
|
||||
return
|
||||
for role in infra_artifact.ROLES:
|
||||
infra_artifact.ensure_artifact_gz(_role_version(role), role=role)
|
||||
version, expected = _role_release(role)
|
||||
infra_artifact.ensure_artifact_gz(
|
||||
version, role=role, expected_sha256=expected)
|
||||
|
||||
|
||||
ensure_available = ensure_built
|
||||
|
||||
|
||||
def build_infra_images_with_docker() -> None:
|
||||
@@ -214,7 +232,9 @@ def boot_vm(
|
||||
else:
|
||||
# Prebuilt artifact already carries the role's build slack; expand it to
|
||||
# a fresh writable rootfs for this boot.
|
||||
infra_artifact.materialize_ext4(_role_version(role), rootfs, role=role)
|
||||
version, expected = _role_release(role)
|
||||
infra_artifact.materialize_ext4(
|
||||
version, rootfs, role=role, expected_sha256=expected)
|
||||
private_key, pubkey = _stable_keypair()
|
||||
|
||||
info(f"booting {role} VM on {slot.iface} (guest {slot.guest_ip})")
|
||||
@@ -264,7 +284,8 @@ def _version_file() -> Path:
|
||||
def expected_version() -> str:
|
||||
"""The combined marker for the running pair: both per-plane artifact
|
||||
versions, so a change to either rootfs dislodges the adopted pair."""
|
||||
return " ".join(f"{role}={_role_version(role)}" for role in infra_artifact.ROLES)
|
||||
return " ".join(
|
||||
f"{role}={_role_release(role)[0]}" for role in infra_artifact.ROLES)
|
||||
|
||||
|
||||
def adoptable(key: Path, url: str, want: str) -> bool:
|
||||
|
||||
@@ -228,8 +228,13 @@ def build_or_load_agent_base(plan: FirecrackerBottlePlan) -> Path:
|
||||
info(f"resuming from committed rootfs {committed_tar}")
|
||||
return util.build_committed_rootfs_dir(committed_tar)
|
||||
dockerfile = Path(plan.dockerfile_path)
|
||||
prebuilt = "@sha256:" in plan.image
|
||||
if plan.spec.image_policy == "cached":
|
||||
cached = image_builder.cached_agent_rootfs_dir(dockerfile)
|
||||
cached = (
|
||||
image_builder.cached_agent_image_rootfs_dir(plan.image)
|
||||
if prebuilt
|
||||
else image_builder.cached_agent_rootfs_dir(dockerfile)
|
||||
)
|
||||
if cached is None:
|
||||
die(
|
||||
f"cached agent rootfs for {plan.image!r} not found; "
|
||||
@@ -237,6 +242,11 @@ def build_or_load_agent_base(plan: FirecrackerBottlePlan) -> Path:
|
||||
)
|
||||
info(f"using cached agent rootfs {cached.name}")
|
||||
return cached
|
||||
if prebuilt:
|
||||
return image_builder.acquire_agent_image_rootfs_dir(
|
||||
plan.image,
|
||||
smoke_test=runtime_for(plan.agent_provider_template).smoke_test,
|
||||
)
|
||||
return image_builder.build_agent_rootfs_dir(
|
||||
dockerfile,
|
||||
image_tag=plan.image,
|
||||
@@ -253,7 +263,11 @@ def stale_checks(plan: FirecrackerBottlePlan) -> None:
|
||||
if committed and committed_tar.is_file():
|
||||
check_stale_path(f"agent rootfs {committed_tar}", committed_tar)
|
||||
return
|
||||
cached = image_builder.cached_agent_rootfs_dir(Path(plan.dockerfile_path))
|
||||
cached = (
|
||||
image_builder.cached_agent_image_rootfs_dir(plan.image)
|
||||
if "@sha256:" in plan.image
|
||||
else image_builder.cached_agent_rootfs_dir(Path(plan.dockerfile_path))
|
||||
)
|
||||
if cached is not None:
|
||||
check_stale_path(f"agent rootfs {cached}", cached / ".bb-ready")
|
||||
|
||||
|
||||
@@ -20,7 +20,6 @@ import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from ... import invocation
|
||||
from ... import resources
|
||||
from . import netpool
|
||||
from . import util
|
||||
@@ -180,11 +179,9 @@ def _setup_systemd() -> None:
|
||||
f"sudo systemctl daemon-reload\n"
|
||||
f"sudo systemctl enable --now {netpool.SYSTEMD_UNIT}\n"
|
||||
)
|
||||
# Absolute path, not `sudo bot-bottle`: sudo's secure_path drops
|
||||
# ~/.local/bin, where both pipx and install.sh put the entry point.
|
||||
sys.stderr.write(
|
||||
f"\n(Or re-run this as root to install it directly:\n"
|
||||
f" {invocation.sudo_command('backend', 'setup', '--backend=firecracker')})\n"
|
||||
f"\n(Or re-run this as root to install it directly: "
|
||||
f"sudo bot-bottle backend setup --backend=firecracker)\n"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -84,6 +84,7 @@ class MacosGateway(Gateway):
|
||||
egress_network: str = GATEWAY_EGRESS_NETWORK,
|
||||
control_network: str = CONTROL_NETWORK,
|
||||
repo_root: Path | None = None,
|
||||
local_build: bool = True,
|
||||
) -> None:
|
||||
self.image_ref = image_ref
|
||||
self.name = name
|
||||
@@ -93,6 +94,7 @@ class MacosGateway(Gateway):
|
||||
# Build context: the repo root in a checkout, a staged copy from the
|
||||
# installed wheel otherwise (bot_bottle.resources).
|
||||
self._repo_root = repo_root if repo_root is not None else resources.build_root()
|
||||
self._local_build = local_build
|
||||
# Set by `connect_to_orchestrator`: the URL the daemons resolve policy
|
||||
# against + the pre-minted `gateway` token they present. The gateway
|
||||
# never mints, so it never holds the signing key (#469).
|
||||
@@ -101,8 +103,11 @@ class MacosGateway(Gateway):
|
||||
|
||||
def ensure_built(self) -> None:
|
||||
"""Build the data-plane image from `Dockerfile.gateway`."""
|
||||
container_mod.build_image(
|
||||
self.image_ref, str(self._repo_root), dockerfile="Dockerfile.gateway")
|
||||
if self._local_build:
|
||||
container_mod.build_image(
|
||||
self.image_ref, str(self._repo_root), dockerfile="Dockerfile.gateway")
|
||||
else:
|
||||
container_mod.pull_image(self.image_ref)
|
||||
|
||||
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None:
|
||||
"""Bind the gateway to this orchestrator and (re)start it, dual-homed on
|
||||
|
||||
@@ -25,6 +25,7 @@ from __future__ import annotations
|
||||
from pathlib import Path
|
||||
|
||||
from ... import resources
|
||||
from ... import release_manifest
|
||||
from ...orchestrator.lifecycle import (
|
||||
DEFAULT_PORT,
|
||||
DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||
@@ -86,6 +87,14 @@ class MacosInfraService(InfraService):
|
||||
self._orchestrator_name = orchestrator_name
|
||||
self._gateway_name = gateway_name
|
||||
self._db_volume = db_volume
|
||||
resolved_orchestrator, orchestrator_local = release_manifest.oci_image(
|
||||
"orchestrator", orchestrator_image)
|
||||
resolved_gateway, gateway_local = release_manifest.oci_image(
|
||||
"gateway", gateway_image)
|
||||
self.orchestrator_image = resolved_orchestrator
|
||||
self.gateway_image = resolved_gateway
|
||||
self._orchestrator_local = orchestrator_local
|
||||
self._gateway_local = gateway_local
|
||||
|
||||
def orchestrator(self) -> MacosOrchestrator:
|
||||
"""The control-plane service on the host-only control network. Cheap to
|
||||
@@ -98,6 +107,7 @@ class MacosInfraService(InfraService):
|
||||
control_network=self.control_network,
|
||||
repo_root=self._repo_root,
|
||||
db_volume=self._db_volume,
|
||||
local_build=self._orchestrator_local,
|
||||
)
|
||||
|
||||
def gateway(self) -> MacosGateway:
|
||||
@@ -112,6 +122,7 @@ class MacosInfraService(InfraService):
|
||||
egress_network=self.egress_network,
|
||||
control_network=self.control_network,
|
||||
repo_root=self._repo_root,
|
||||
local_build=self._gateway_local,
|
||||
)
|
||||
|
||||
def ensure_running(
|
||||
@@ -127,8 +138,8 @@ class MacosInfraService(InfraService):
|
||||
|
||||
orchestrator = self.orchestrator()
|
||||
gateway = self.gateway()
|
||||
orchestrator.ensure_built()
|
||||
gateway.ensure_built()
|
||||
orchestrator.ensure_available()
|
||||
gateway.ensure_available()
|
||||
|
||||
orchestrator.ensure_running(startup_timeout=startup_timeout)
|
||||
|
||||
|
||||
@@ -38,6 +38,7 @@ import subprocess
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from typing import Callable, Generator
|
||||
|
||||
from ...agent_provider import runtime_for
|
||||
from ...bottle_state import (
|
||||
egress_state_dir,
|
||||
git_gate_state_dir,
|
||||
@@ -93,7 +94,14 @@ def _agent_image(plan: MacosContainerBottlePlan) -> str:
|
||||
)
|
||||
info(f"using cached agent image {plan.image!r}")
|
||||
return plan.image
|
||||
container_mod.build_image(plan.image, str(resources.build_root()), dockerfile=plan.dockerfile_path)
|
||||
if "@sha256:" in plan.image:
|
||||
container_mod.pull_image(plan.image)
|
||||
container_mod.verify_agent_image(
|
||||
plan.image, runtime_for(plan.agent_provider_template).smoke_test,
|
||||
)
|
||||
return plan.image
|
||||
container_mod.build_image(
|
||||
plan.image, str(resources.build_root()), dockerfile=plan.dockerfile_path)
|
||||
return plan.image
|
||||
|
||||
|
||||
|
||||
@@ -63,6 +63,7 @@ class MacosOrchestrator(Orchestrator):
|
||||
control_network: str = CONTROL_NETWORK,
|
||||
repo_root: Path | None = None,
|
||||
db_volume: str = ORCHESTRATOR_DB_VOLUME,
|
||||
local_build: bool = True,
|
||||
) -> None:
|
||||
self.image_ref = image_ref
|
||||
self.name = name
|
||||
@@ -73,6 +74,7 @@ class MacosOrchestrator(Orchestrator):
|
||||
# staged copy from the installed wheel otherwise (bot_bottle.resources).
|
||||
self._repo_root = repo_root if repo_root is not None else resources.build_root()
|
||||
self._db_volume = db_volume
|
||||
self._local_build = local_build
|
||||
|
||||
def url(self) -> str:
|
||||
"""The orchestrator's control-network address (host CLI + registration),
|
||||
@@ -114,8 +116,12 @@ class MacosOrchestrator(Orchestrator):
|
||||
"""Build the control-plane image. The source is bind-mounted so a code
|
||||
change takes effect without a rebuild; the image still carries the
|
||||
package for its entrypoint."""
|
||||
container_mod.build_image(
|
||||
self.image_ref, str(self._repo_root), dockerfile="Dockerfile.orchestrator")
|
||||
if self._local_build:
|
||||
container_mod.build_image(
|
||||
self.image_ref, str(self._repo_root),
|
||||
dockerfile="Dockerfile.orchestrator")
|
||||
else:
|
||||
container_mod.pull_image(self.image_ref)
|
||||
|
||||
def ensure_running(
|
||||
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||
@@ -146,11 +152,6 @@ class MacosOrchestrator(Orchestrator):
|
||||
"--dns", container_mod.dns_server(),
|
||||
# Container-only DB volume: exactly one kernel writes bot-bottle.db.
|
||||
"--volume", f"{self._db_volume}:{_DB_ROOT_IN_CONTAINER}",
|
||||
# Live control-plane source (a code change takes effect on relaunch).
|
||||
"--mount",
|
||||
container_mod.bind_mount_spec(
|
||||
str(self._repo_root), _SRC_IN_CONTAINER, readonly=True),
|
||||
"--env", f"PYTHONPATH={_SRC_IN_CONTAINER}",
|
||||
"--env", f"BOT_BOTTLE_ROOT={_DB_ROOT_IN_CONTAINER}",
|
||||
# Detect a real control-plane code change and recreate.
|
||||
"--env", f"BOT_BOTTLE_SOURCE_HASH={current_hash}",
|
||||
@@ -161,6 +162,14 @@ class MacosOrchestrator(Orchestrator):
|
||||
# Dockerfile.orchestrator ENTRYPOINT is `-m bot_bottle.orchestrator`.
|
||||
"--host", "0.0.0.0", "--port", str(self.port), "--broker", "stub",
|
||||
]
|
||||
if self._local_build:
|
||||
image_index = argv.index(self.image_ref)
|
||||
argv[image_index:image_index] = [
|
||||
"--mount",
|
||||
container_mod.bind_mount_spec(
|
||||
str(self._repo_root), _SRC_IN_CONTAINER, readonly=True),
|
||||
"--env", f"PYTHONPATH={_SRC_IN_CONTAINER}",
|
||||
]
|
||||
result = container_mod.run_container_argv(
|
||||
argv, env={**os.environ, ORCHESTRATOR_TOKEN_ENV: _signing_key})
|
||||
if result.returncode != 0:
|
||||
|
||||
@@ -101,6 +101,19 @@ def build_image(
|
||||
subprocess.run(args, check=True)
|
||||
|
||||
|
||||
def pull_image(ref: str) -> None:
|
||||
"""Acquire an immutable OCI reference through Apple Container."""
|
||||
result = subprocess.run(
|
||||
[_CONTAINER, "image", "pull", ref],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
detail = (result.stderr or result.stdout or "").strip()
|
||||
raise RuntimeError(f"container image pull failed for {ref}: {detail}")
|
||||
|
||||
|
||||
def verify_agent_image(image: str, argv: tuple[str, ...]) -> None:
|
||||
"""Run `argv` inside a throwaway container of a freshly built agent
|
||||
image and die loudly if it fails, instead of shipping an image
|
||||
|
||||
@@ -9,7 +9,7 @@ backend-specific final resolution.
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import TYPE_CHECKING, Protocol
|
||||
|
||||
from ..agent_provider import AgentProvisionPlan, build_agent_provision_plan, get_provider
|
||||
@@ -17,6 +17,7 @@ from ..egress import EgressPlan
|
||||
from ..env import ResolvedEnv, resolve_env
|
||||
from ..git_gate import GitGate, GitGatePlan
|
||||
from ..manifest import Manifest
|
||||
from ..release_manifest import oci_image
|
||||
from ..supervisor.plan import SupervisePlan
|
||||
from ..workspace import workspace_plan
|
||||
from .resolve_common import (
|
||||
@@ -112,6 +113,11 @@ class BottlePreparationPlanner:
|
||||
color=spec.color,
|
||||
provider_settings=provider_config.settings,
|
||||
)
|
||||
if not provider_config.dockerfile:
|
||||
image, local = oci_image(
|
||||
f"agent_{provider_config.template}", provision.image)
|
||||
if not local:
|
||||
provision = replace(provision, image=image)
|
||||
provision = merge_provision_env_vars(provision)
|
||||
return PreparedBottle(
|
||||
manifest=manifest,
|
||||
|
||||
@@ -115,10 +115,12 @@ class Gateway(abc.ABC):
|
||||
|
||||
name: str
|
||||
|
||||
def ensure_available(self) -> None:
|
||||
"""Acquire the exact image/rootfs selected for this application."""
|
||||
self.ensure_built()
|
||||
|
||||
def ensure_built(self) -> None:
|
||||
"""Ensure the gateway's image / rootfs exists, building it if needed.
|
||||
Default: nothing to build (e.g. a stub or a pre-pulled image). Call
|
||||
before `connect_to_orchestrator`."""
|
||||
"""Local-build implementation hook retained for backend compatibility."""
|
||||
return
|
||||
|
||||
@abc.abstractmethod
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
"""How to tell a user to re-run this CLI.
|
||||
|
||||
`bot-bottle …` is the right thing to print for anything the user runs as
|
||||
themselves — it is on their PATH, since that is how they got here.
|
||||
|
||||
Under `sudo` it is not. sudo replaces PATH with sudoers' `secure_path`
|
||||
(`/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin` on Debian and
|
||||
Ubuntu, similar elsewhere), which deliberately excludes user-writable
|
||||
directories. Both supported install paths put the entry point in one of those:
|
||||
pipx uses `~/.local/bin`, and `install.sh`'s venv fallback symlinks there too.
|
||||
So `sudo bot-bottle …` fails with "command not found" for exactly the users who
|
||||
followed the documented install, while working for anyone who happened to
|
||||
install system-wide — which is why it survives review so easily.
|
||||
|
||||
Naming the absolute path sidesteps secure_path entirely.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
|
||||
def self_path() -> str:
|
||||
"""Absolute path to this CLI's entry point.
|
||||
|
||||
Falls back to the bare name when the entry point cannot be resolved (an
|
||||
unusual invocation such as `python -m`), because a slightly wrong hint is
|
||||
better than a traceback while reporting an unrelated problem.
|
||||
"""
|
||||
argv0 = sys.argv[0] or "bot-bottle"
|
||||
resolved = shutil.which(argv0) or argv0
|
||||
if not os.path.isabs(resolved):
|
||||
if os.path.exists(resolved):
|
||||
resolved = os.path.abspath(resolved)
|
||||
else:
|
||||
return "bot-bottle"
|
||||
return resolved
|
||||
|
||||
|
||||
def sudo_command(*args: str) -> str:
|
||||
"""A copy-pasteable `sudo …` invocation of this CLI.
|
||||
|
||||
>>> sudo_command("backend", "setup", "--backend=firecracker")
|
||||
'sudo /home/u/.local/bin/bot-bottle backend setup --backend=firecracker'
|
||||
"""
|
||||
return " ".join(["sudo", self_path(), *args])
|
||||
@@ -63,10 +63,12 @@ class Orchestrator(abc.ABC):
|
||||
# orchestrator never starts without its signing key.
|
||||
provisioning: ControlPlaneProvisioning = ControlPlaneProvisioning()
|
||||
|
||||
def ensure_available(self) -> None:
|
||||
"""Acquire the exact image/rootfs selected for this application."""
|
||||
self.ensure_built()
|
||||
|
||||
def ensure_built(self) -> None:
|
||||
"""Ensure the orchestrator's image / rootfs exists, building it if
|
||||
needed. Default: nothing to build (e.g. a pre-pulled image). Call before
|
||||
`ensure_running`."""
|
||||
"""Local-build implementation hook retained for backend compatibility."""
|
||||
return
|
||||
|
||||
@abc.abstractmethod
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{"schema": 1, "development": true}
|
||||
@@ -0,0 +1,127 @@
|
||||
"""External, commit-addressed release bundle metadata."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .release_manifest import ReleaseManifestError, parse_manifest
|
||||
|
||||
_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||
_SHA_RE = re.compile(r"^[0-9a-f]{64}$")
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1 << 20), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def build_bundle_index(
|
||||
manifest_data: Any,
|
||||
*,
|
||||
wheel: Path,
|
||||
wheel_url: str,
|
||||
workflow_run: str,
|
||||
published_at: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Build and validate the immutable external index for one source commit."""
|
||||
manifest = parse_manifest(manifest_data)
|
||||
if not wheel.is_file() or wheel.suffix != ".whl":
|
||||
raise ReleaseManifestError("release bundle wheel must be one .whl file")
|
||||
if not wheel_url.startswith("https://") or not wheel_url.endswith(wheel.name):
|
||||
raise ReleaseManifestError(
|
||||
"release bundle wheel URL must be HTTPS and end with the wheel filename")
|
||||
if not workflow_run.strip() or not published_at.strip():
|
||||
raise ReleaseManifestError(
|
||||
"release bundle provenance requires workflow_run and published_at")
|
||||
return {
|
||||
"schema": 1,
|
||||
"source_commit": manifest.source_commit,
|
||||
"wheel": {
|
||||
"filename": wheel.name,
|
||||
"url": wheel_url,
|
||||
"sha256": sha256_file(wheel),
|
||||
},
|
||||
"oci": {
|
||||
"orchestrator": manifest.orchestrator_image,
|
||||
"gateway": manifest.gateway_image,
|
||||
**{
|
||||
f"agent_{role}": reference
|
||||
for role, reference in manifest.agent_images.items()
|
||||
},
|
||||
},
|
||||
"firecracker": {
|
||||
"orchestrator": {
|
||||
"version": manifest.firecracker_orchestrator.version,
|
||||
"sha256": manifest.firecracker_orchestrator.sha256,
|
||||
},
|
||||
"gateway": {
|
||||
"version": manifest.firecracker_gateway.version,
|
||||
"sha256": manifest.firecracker_gateway.sha256,
|
||||
},
|
||||
},
|
||||
"provenance": {
|
||||
"workflow_run": workflow_run.strip(),
|
||||
"published_at": published_at.strip(),
|
||||
},
|
||||
"qualifications": [],
|
||||
}
|
||||
|
||||
|
||||
def parse_bundle_index(data: Any) -> dict[str, Any]:
|
||||
"""Validate an index before publication or installer consumption."""
|
||||
if not isinstance(data, dict) or data.get("schema") != 1:
|
||||
raise ReleaseManifestError("release bundle index must use schema 1")
|
||||
commit = data.get("source_commit")
|
||||
if not isinstance(commit, str) or _COMMIT_RE.fullmatch(commit) is None:
|
||||
raise ReleaseManifestError(
|
||||
"release bundle source_commit must be 40 lowercase hex")
|
||||
wheel = data.get("wheel")
|
||||
if not isinstance(wheel, dict):
|
||||
raise ReleaseManifestError("release bundle wheel must be an object")
|
||||
filename, url, digest = (
|
||||
wheel.get("filename"), wheel.get("url"), wheel.get("sha256"))
|
||||
if not isinstance(filename, str) or not filename.endswith(".whl"):
|
||||
raise ReleaseManifestError("release bundle wheel filename must end in .whl")
|
||||
if not isinstance(url, str) or not url.startswith("https://") or not url.endswith(filename):
|
||||
raise ReleaseManifestError(
|
||||
"release bundle wheel URL must be HTTPS and match its filename")
|
||||
if not isinstance(digest, str) or _SHA_RE.fullmatch(digest) is None:
|
||||
raise ReleaseManifestError(
|
||||
"release bundle wheel sha256 must be 64 lowercase hex")
|
||||
manifest_data = {
|
||||
"schema": 1,
|
||||
"source_commit": commit,
|
||||
"oci": data.get("oci"),
|
||||
"firecracker": data.get("firecracker"),
|
||||
}
|
||||
parse_manifest(manifest_data)
|
||||
provenance = data.get("provenance")
|
||||
if not isinstance(provenance, dict) or not all(
|
||||
isinstance(provenance.get(key), str) and provenance[key].strip()
|
||||
for key in ("workflow_run", "published_at")
|
||||
):
|
||||
raise ReleaseManifestError("release bundle provenance is incomplete")
|
||||
qualifications = data.get("qualifications")
|
||||
if not isinstance(qualifications, list):
|
||||
raise ReleaseManifestError("release bundle qualifications must be an array")
|
||||
return data
|
||||
|
||||
|
||||
def canonical_bytes(data: Any) -> bytes:
|
||||
parse_bundle_index(data)
|
||||
return (json.dumps(data, indent=2, sort_keys=True) + "\n").encode("utf-8")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"build_bundle_index",
|
||||
"canonical_bytes",
|
||||
"parse_bundle_index",
|
||||
"sha256_file",
|
||||
]
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Immutable infrastructure identities assigned when bot-bottle is packaged."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from . import resources
|
||||
|
||||
_MANIFEST_NAME = "release-manifest.json"
|
||||
_OCI_RE = re.compile(r"^[^@\s]+@sha256:([0-9a-f]{64})$")
|
||||
_SHA_RE = re.compile(r"^[0-9a-f]{64}$")
|
||||
_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||
|
||||
|
||||
class ReleaseManifestError(RuntimeError):
|
||||
"""The installed package has no usable infrastructure release identity."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FirecrackerArtifact:
|
||||
version: str
|
||||
sha256: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReleaseManifest:
|
||||
source_commit: str
|
||||
orchestrator_image: str
|
||||
gateway_image: str
|
||||
agent_images: dict[str, str]
|
||||
firecracker_orchestrator: FirecrackerArtifact
|
||||
firecracker_gateway: FirecrackerArtifact
|
||||
|
||||
|
||||
def manifest_path() -> Path:
|
||||
override = os.environ.get("BOT_BOTTLE_RELEASE_MANIFEST", "").strip()
|
||||
return Path(override) if override else Path(__file__).resolve().parent / _MANIFEST_NAME
|
||||
|
||||
|
||||
def _artifact(value: Any, role: str) -> FirecrackerArtifact:
|
||||
if not isinstance(value, dict):
|
||||
raise ReleaseManifestError(f"release manifest firecracker.{role} must be an object")
|
||||
version = value.get("version")
|
||||
sha256 = value.get("sha256")
|
||||
if not isinstance(version, str) or not version.strip():
|
||||
raise ReleaseManifestError(
|
||||
f"release manifest firecracker.{role}.version must be non-empty")
|
||||
if not isinstance(sha256, str) or _SHA_RE.fullmatch(sha256) is None:
|
||||
raise ReleaseManifestError(
|
||||
f"release manifest firecracker.{role}.sha256 must be 64 lowercase hex")
|
||||
return FirecrackerArtifact(version.strip(), sha256)
|
||||
|
||||
|
||||
def parse_manifest(data: Any) -> ReleaseManifest:
|
||||
if not isinstance(data, dict) or data.get("schema") != 1:
|
||||
raise ReleaseManifestError("release manifest must use schema 1")
|
||||
commit = data.get("source_commit")
|
||||
if not isinstance(commit, str) or _COMMIT_RE.fullmatch(commit) is None:
|
||||
raise ReleaseManifestError(
|
||||
"release manifest source_commit must be 40 lowercase hex")
|
||||
oci = data.get("oci")
|
||||
if not isinstance(oci, dict):
|
||||
raise ReleaseManifestError("release manifest oci must be an object")
|
||||
images: dict[str, str] = {}
|
||||
for role in ("orchestrator", "gateway", "agent_claude", "agent_codex", "agent_pi"):
|
||||
ref = oci.get(role)
|
||||
if not isinstance(ref, str) or _OCI_RE.fullmatch(ref) is None:
|
||||
raise ReleaseManifestError(
|
||||
f"release manifest oci.{role} must be digest-pinned")
|
||||
images[role] = ref
|
||||
firecracker = data.get("firecracker")
|
||||
if not isinstance(firecracker, dict):
|
||||
raise ReleaseManifestError("release manifest firecracker must be an object")
|
||||
return ReleaseManifest(
|
||||
source_commit=commit,
|
||||
orchestrator_image=images["orchestrator"],
|
||||
gateway_image=images["gateway"],
|
||||
agent_images={
|
||||
role: images[f"agent_{role}"] for role in ("claude", "codex", "pi")
|
||||
},
|
||||
firecracker_orchestrator=_artifact(
|
||||
firecracker.get("orchestrator"), "orchestrator"),
|
||||
firecracker_gateway=_artifact(firecracker.get("gateway"), "gateway"),
|
||||
)
|
||||
|
||||
|
||||
def load_manifest() -> ReleaseManifest:
|
||||
path = manifest_path()
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise ReleaseManifestError(
|
||||
f"bot-bottle was packaged without valid infrastructure pins "
|
||||
f"({path}): {exc}") from exc
|
||||
return parse_manifest(data)
|
||||
|
||||
|
||||
def packaged_manifest() -> ReleaseManifest | None:
|
||||
"""Release identity, or ``None`` for checkouts/ad-hoc development wheels."""
|
||||
if resources.is_source_checkout() and not os.environ.get(
|
||||
"BOT_BOTTLE_RELEASE_MANIFEST"
|
||||
):
|
||||
return None
|
||||
path = manifest_path()
|
||||
try:
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return load_manifest() # raise the detailed, fail-closed error
|
||||
if isinstance(raw, dict) and raw.get("development") is True:
|
||||
return None
|
||||
return load_manifest()
|
||||
|
||||
|
||||
def local_build_requested() -> bool:
|
||||
return os.environ.get("BOT_BOTTLE_INFRA_BUILD", "").strip().lower() == "local"
|
||||
|
||||
|
||||
def oci_image(role: str, local_default: str) -> tuple[str, bool]:
|
||||
"""Return ``(reference, should_build_locally)`` for a fixed OCI service."""
|
||||
override_name = f"BOT_BOTTLE_{role.upper()}_IMAGE"
|
||||
override = os.environ.get(override_name, "").strip()
|
||||
manifest = packaged_manifest()
|
||||
local = local_build_requested() or manifest is None
|
||||
if override:
|
||||
if not local and _OCI_RE.fullmatch(override) is None:
|
||||
raise ReleaseManifestError(
|
||||
f"{override_name} must be digest-pinned outside local build mode")
|
||||
return override, local
|
||||
if local:
|
||||
return local_default, True
|
||||
assert manifest is not None
|
||||
return (
|
||||
manifest.orchestrator_image if role == "orchestrator"
|
||||
else manifest.gateway_image if role == "gateway"
|
||||
else manifest.agent_images[role.removeprefix("agent_")],
|
||||
False,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"FirecrackerArtifact",
|
||||
"ReleaseManifest",
|
||||
"ReleaseManifestError",
|
||||
"load_manifest",
|
||||
"local_build_requested",
|
||||
"oci_image",
|
||||
"packaged_manifest",
|
||||
"parse_manifest",
|
||||
]
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Durable publication of immutable, commit-addressed release bundles."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from .release_bundle import canonical_bytes, parse_bundle_index, sha256_file
|
||||
from .release_manifest import ReleaseManifestError
|
||||
|
||||
HTTP_TIMEOUT_SECONDS = 60.0
|
||||
INDEX_NAME = "bundle-index.json"
|
||||
PACKAGE_NAME = "bot-bottle-builds"
|
||||
|
||||
|
||||
def config() -> tuple[str, str, str]:
|
||||
"""Return generic-package base URL, owner, and write token."""
|
||||
base = os.environ.get(
|
||||
"BOT_BOTTLE_RELEASE_BASE", "https://gitea.dideric.is").rstrip("/")
|
||||
owner = os.environ.get("BOT_BOTTLE_RELEASE_OWNER", "didericis").strip()
|
||||
token = os.environ.get("BOT_BOTTLE_RELEASE_TOKEN", "")
|
||||
return base, owner, token
|
||||
|
||||
|
||||
def bundle_url(source_commit: str, filename: str) -> str:
|
||||
base, owner, _ = config()
|
||||
return (
|
||||
f"{base}/api/packages/{owner}/generic/{PACKAGE_NAME}/"
|
||||
f"{source_commit}/{filename}"
|
||||
)
|
||||
|
||||
|
||||
def request(url: str, *, method: str = "GET", data: object | None = None,
|
||||
length: int | None = None) -> urllib.request.Request:
|
||||
result = urllib.request.Request(url, data=data, method=method) # type: ignore[arg-type]
|
||||
_, _, token = config()
|
||||
if token:
|
||||
result.add_header("Authorization", f"token {token}")
|
||||
if length is not None:
|
||||
result.add_header("Content-Length", str(length))
|
||||
result.add_header("Content-Type", "application/octet-stream")
|
||||
return result
|
||||
|
||||
|
||||
def remote_bytes(url: str) -> bytes | None:
|
||||
try:
|
||||
with urllib.request.urlopen(
|
||||
request(url), timeout=HTTP_TIMEOUT_SECONDS,
|
||||
) as response:
|
||||
return response.read()
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code == 404:
|
||||
return None
|
||||
raise ReleaseManifestError(
|
||||
f"checking release bundle failed (HTTP {exc.code}): {url}") from exc
|
||||
except urllib.error.URLError as exc:
|
||||
raise ReleaseManifestError(
|
||||
f"release registry unreachable: {url} ({exc.reason})") from exc
|
||||
|
||||
|
||||
def remote_sha256(url: str) -> str | None:
|
||||
try:
|
||||
with urllib.request.urlopen(
|
||||
request(url), timeout=HTTP_TIMEOUT_SECONDS,
|
||||
) as response:
|
||||
digest = hashlib.sha256()
|
||||
for chunk in iter(lambda: response.read(1 << 20), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code == 404:
|
||||
return None
|
||||
raise ReleaseManifestError(
|
||||
f"checking release artifact failed (HTTP {exc.code}): {url}") from exc
|
||||
except urllib.error.URLError as exc:
|
||||
raise ReleaseManifestError(
|
||||
f"release registry unreachable: {url} ({exc.reason})") from exc
|
||||
|
||||
|
||||
def upload(url: str, source: bytes | Path) -> None:
|
||||
handle = None
|
||||
if isinstance(source, Path):
|
||||
handle = source.open("rb")
|
||||
data: object = handle
|
||||
length = source.stat().st_size
|
||||
else:
|
||||
data = source
|
||||
length = len(source)
|
||||
try:
|
||||
with urllib.request.urlopen(
|
||||
request(url, method="PUT", data=data, length=length),
|
||||
timeout=HTTP_TIMEOUT_SECONDS,
|
||||
):
|
||||
pass
|
||||
except (urllib.error.HTTPError, urllib.error.URLError) as exc:
|
||||
raise ReleaseManifestError(
|
||||
f"publishing release artifact failed: {url} ({exc})") from exc
|
||||
finally:
|
||||
if handle is not None:
|
||||
handle.close()
|
||||
|
||||
|
||||
def publish_bundle(index: dict[str, object], wheel: Path) -> str:
|
||||
"""Publish wheel first and index last, or verify an existing exact bundle."""
|
||||
parse_bundle_index(index)
|
||||
source_commit = str(index["source_commit"])
|
||||
wheel_data = index["wheel"]
|
||||
assert isinstance(wheel_data, dict)
|
||||
filename = str(wheel_data["filename"])
|
||||
expected_sha = str(wheel_data["sha256"])
|
||||
if wheel.name != filename or sha256_file(wheel) != expected_sha:
|
||||
raise ReleaseManifestError(
|
||||
"release bundle wheel does not match its index")
|
||||
|
||||
wheel_url = bundle_url(source_commit, filename)
|
||||
index_url = bundle_url(source_commit, INDEX_NAME)
|
||||
if wheel_data["url"] != wheel_url:
|
||||
raise ReleaseManifestError(
|
||||
"release bundle wheel URL does not match its commit coordinate")
|
||||
|
||||
wanted_index = canonical_bytes(index)
|
||||
existing_index = remote_bytes(index_url)
|
||||
existing_wheel_sha = remote_sha256(wheel_url)
|
||||
if existing_index is not None:
|
||||
if existing_index != wanted_index or existing_wheel_sha != expected_sha:
|
||||
raise ReleaseManifestError(
|
||||
f"immutable release bundle {source_commit} already differs")
|
||||
return index_url
|
||||
if existing_wheel_sha is not None:
|
||||
raise ReleaseManifestError(
|
||||
f"partial release bundle {source_commit} exists without its index; "
|
||||
"administrative cleanup is required")
|
||||
|
||||
upload(wheel_url, wheel)
|
||||
upload(index_url, wanted_index)
|
||||
return index_url
|
||||
|
||||
|
||||
__all__ = [
|
||||
"INDEX_NAME",
|
||||
"PACKAGE_NAME",
|
||||
"bundle_url",
|
||||
"config",
|
||||
"publish_bundle",
|
||||
]
|
||||
@@ -0,0 +1,197 @@
|
||||
"""Validated release and channel pointers for qualified commit bundles."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
from .release_bundle import parse_bundle_index
|
||||
from .release_manifest import ReleaseManifestError
|
||||
from .release_publish import (
|
||||
HTTP_TIMEOUT_SECONDS,
|
||||
bundle_url,
|
||||
config,
|
||||
remote_bytes,
|
||||
request,
|
||||
upload,
|
||||
)
|
||||
|
||||
_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||
_STABLE_TAG_RE = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$")
|
||||
_STAGING_TAG_RE = re.compile(r"^v(\d+)\.(\d+)\.(\d+)-rc\.(\d+)$")
|
||||
|
||||
|
||||
def release_channel(tag: str) -> str:
|
||||
"""Return the only channel eligible for *tag*."""
|
||||
if _STABLE_TAG_RE.fullmatch(tag):
|
||||
return "production"
|
||||
if _STAGING_TAG_RE.fullmatch(tag):
|
||||
return "staging"
|
||||
raise ReleaseManifestError(
|
||||
"release tag must be vX.Y.Z or vX.Y.Z-rc.N")
|
||||
|
||||
|
||||
def release_version(tag: str) -> tuple[int, int, int, int]:
|
||||
"""Return a monotonically comparable version tuple."""
|
||||
stable = _STABLE_TAG_RE.fullmatch(tag)
|
||||
if stable:
|
||||
return (
|
||||
int(stable.group(1)),
|
||||
int(stable.group(2)),
|
||||
int(stable.group(3)),
|
||||
1 << 30,
|
||||
)
|
||||
staging = _STAGING_TAG_RE.fullmatch(tag)
|
||||
if staging:
|
||||
return (
|
||||
int(staging.group(1)),
|
||||
int(staging.group(2)),
|
||||
int(staging.group(3)),
|
||||
int(staging.group(4)),
|
||||
)
|
||||
raise ReleaseManifestError(
|
||||
"release tag must be vX.Y.Z or vX.Y.Z-rc.N")
|
||||
|
||||
|
||||
def pointer_url(package: str, version: str, filename: str) -> str:
|
||||
base, owner, _ = config()
|
||||
return f"{base}/api/packages/{owner}/generic/{package}/{version}/{filename}"
|
||||
|
||||
|
||||
def build_release_pointer(
|
||||
*,
|
||||
tag: str,
|
||||
source_commit: str,
|
||||
workflow_run: str,
|
||||
qualified_at: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Build a qualified pointer to one immutable commit bundle."""
|
||||
channel = release_channel(tag)
|
||||
if _COMMIT_RE.fullmatch(source_commit) is None:
|
||||
raise ReleaseManifestError(
|
||||
"release source_commit must be 40 lowercase hex")
|
||||
if not workflow_run.strip() or not qualified_at.strip():
|
||||
raise ReleaseManifestError(
|
||||
"release qualification provenance is incomplete")
|
||||
return {
|
||||
"schema": 1,
|
||||
"qualified": True,
|
||||
"tag": tag,
|
||||
"channel": channel,
|
||||
"source_commit": source_commit,
|
||||
"bundle_index_url": bundle_url(source_commit, "bundle-index.json"),
|
||||
"qualification": {
|
||||
"workflow_run": workflow_run.strip(),
|
||||
"qualified_at": qualified_at.strip(),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def parse_release_pointer(data: Any) -> dict[str, Any]:
|
||||
"""Validate a release or channel pointer."""
|
||||
if not isinstance(data, dict) or data.get("schema") != 1:
|
||||
raise ReleaseManifestError("release pointer must use schema 1")
|
||||
if data.get("qualified") is not True:
|
||||
raise ReleaseManifestError("release pointer must be qualified")
|
||||
tag = data.get("tag")
|
||||
channel = data.get("channel")
|
||||
commit = data.get("source_commit")
|
||||
if not isinstance(tag, str) or release_channel(tag) != channel:
|
||||
raise ReleaseManifestError("release pointer tag and channel disagree")
|
||||
if not isinstance(commit, str) or _COMMIT_RE.fullmatch(commit) is None:
|
||||
raise ReleaseManifestError("release pointer source commit is invalid")
|
||||
if data.get("bundle_index_url") != bundle_url(commit, "bundle-index.json"):
|
||||
raise ReleaseManifestError("release pointer bundle URL is invalid")
|
||||
qualification = data.get("qualification")
|
||||
if not isinstance(qualification, dict) or not all(
|
||||
isinstance(qualification.get(key), str) and qualification[key].strip()
|
||||
for key in ("workflow_run", "qualified_at")
|
||||
):
|
||||
raise ReleaseManifestError("release qualification is incomplete")
|
||||
return data
|
||||
|
||||
|
||||
def canonical_pointer(data: Any) -> bytes:
|
||||
parse_release_pointer(data)
|
||||
return (json.dumps(data, indent=2, sort_keys=True) + "\n").encode()
|
||||
|
||||
|
||||
def delete(url: str) -> None:
|
||||
try:
|
||||
with urllib.request.urlopen(
|
||||
request(url, method="DELETE"), timeout=HTTP_TIMEOUT_SECONDS,
|
||||
):
|
||||
pass
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code != 404:
|
||||
raise ReleaseManifestError(
|
||||
f"replacing release channel failed (HTTP {exc.code}): {url}"
|
||||
) from exc
|
||||
except urllib.error.URLError as exc:
|
||||
raise ReleaseManifestError(
|
||||
f"release registry unreachable: {url} ({exc.reason})") from exc
|
||||
|
||||
|
||||
def publish_qualification(
|
||||
pointer: dict[str, Any], *, allow_rollback: bool = False,
|
||||
) -> tuple[str, str]:
|
||||
"""Publish an immutable tag pointer and advance its channel."""
|
||||
parse_release_pointer(pointer)
|
||||
tag = str(pointer["tag"])
|
||||
channel = str(pointer["channel"])
|
||||
wanted = canonical_pointer(pointer)
|
||||
release_url = pointer_url(
|
||||
"bot-bottle-releases", tag, "release.json")
|
||||
channel_url = pointer_url(
|
||||
"bot-bottle-channels", channel, "channel.json")
|
||||
|
||||
bundle = remote_bytes(str(pointer["bundle_index_url"]))
|
||||
if bundle is None:
|
||||
raise ReleaseManifestError(
|
||||
f"commit bundle does not exist for {pointer['source_commit']}")
|
||||
try:
|
||||
bundle_data = parse_bundle_index(json.loads(bundle))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||
raise ReleaseManifestError("commit bundle index is malformed") from exc
|
||||
if bundle_data["source_commit"] != pointer["source_commit"]:
|
||||
raise ReleaseManifestError(
|
||||
"commit bundle index does not match the qualified source commit")
|
||||
existing_release = remote_bytes(release_url)
|
||||
if existing_release is not None and existing_release != wanted:
|
||||
raise ReleaseManifestError(
|
||||
f"immutable release pointer {tag} already differs")
|
||||
if existing_release is None:
|
||||
upload(release_url, wanted)
|
||||
|
||||
existing_channel = remote_bytes(channel_url)
|
||||
if existing_channel == wanted:
|
||||
return release_url, channel_url
|
||||
if existing_channel is not None:
|
||||
try:
|
||||
current = parse_release_pointer(json.loads(existing_channel))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||
raise ReleaseManifestError(
|
||||
f"existing {channel} channel pointer is malformed") from exc
|
||||
if (
|
||||
not allow_rollback
|
||||
and release_version(tag) <= release_version(str(current["tag"]))
|
||||
):
|
||||
raise ReleaseManifestError(
|
||||
f"{channel} channel update is not monotonic; "
|
||||
"use the explicit rollback operation")
|
||||
delete(channel_url)
|
||||
upload(channel_url, wanted)
|
||||
return release_url, channel_url
|
||||
|
||||
|
||||
__all__ = [
|
||||
"build_release_pointer",
|
||||
"canonical_pointer",
|
||||
"parse_release_pointer",
|
||||
"publish_qualification",
|
||||
"release_channel",
|
||||
"release_version",
|
||||
]
|
||||
+17
@@ -45,6 +45,23 @@ report; macOS reports advisory coverage in its own job. The macOS infra
|
||||
container is a singleton, so its job uses a concurrency group and always tears
|
||||
the service down.
|
||||
|
||||
## Artifact publication and promotion
|
||||
|
||||
[`publish-artifacts.yml`](../.gitea/workflows/publish-artifacts.yml) is the
|
||||
manual deployment-test boundary. Dispatch it with any branch, tag, or commit;
|
||||
it resolves that input once, publishes the orchestrator, gateway, Claude,
|
||||
Codex, Pi, and Firecracker artifacts, then publishes only the wheel verified
|
||||
against those identities. Install the reported commit with
|
||||
`BOT_BOTTLE_REF=<sha> sh install.sh`; the installer warns that snapshots may
|
||||
not have completed release qualification.
|
||||
|
||||
[`release.yml`](../.gitea/workflows/release.yml) handles promotion tags.
|
||||
`vX.Y.Z-rc.N` must be reachable from `staging`; `vX.Y.Z` must be reachable
|
||||
from `production`. It runs the complete pre-release workflow, ensures that the
|
||||
same commit bundle is published, verifies a clean install, creates the
|
||||
qualified release pointer, and advances the matching channel. Promotion
|
||||
reuses immutable bundle bytes and never rebuilds them.
|
||||
|
||||
## Scheduled canary
|
||||
|
||||
[`.gitea/workflows/canaries.yml`](../.gitea/workflows/canaries.yml) runs weekly
|
||||
|
||||
@@ -0,0 +1,374 @@
|
||||
# PRD 0083: Packaged infrastructure artifacts
|
||||
|
||||
- **Status:** Active
|
||||
- **Author:** Codex
|
||||
- **Created:** 2026-07-27
|
||||
- **Issue:** #536
|
||||
|
||||
## Summary
|
||||
|
||||
Publish the orchestrator and gateway infrastructure before packaging
|
||||
bot-bottle, record their immutable identities in the application package, and
|
||||
make every production backend acquire those exact artifacts instead of
|
||||
building infrastructure during bottle startup. Docker and Apple Container pull
|
||||
digest-pinned OCI images; Firecracker pulls checksum-verified rootfs artifacts
|
||||
selected by the same packaged release manifest.
|
||||
|
||||
## Problem
|
||||
|
||||
Starting a bottle currently invokes an infrastructure build on Docker and
|
||||
Apple Container even when a healthy orchestrator is already running. Both
|
||||
infra composers call `ensure_built()` before their health/currentness checks,
|
||||
and both implementations unconditionally invoke their OCI builder. Layer
|
||||
caching can make this less expensive, but startup still depends on a working
|
||||
builder, build context, base-image registry, and package installation network.
|
||||
|
||||
The Apple Container orchestrator also bind-mounts the installed source over the
|
||||
package baked into its image. A pinned image alone would therefore not pin the
|
||||
code that actually runs.
|
||||
|
||||
Firecracker already downloads fixed rootfs artifacts, but it derives their
|
||||
versions from the package contents at launch. There is no package-level record
|
||||
that says which already-published infrastructure release belongs to an
|
||||
installed bot-bottle release.
|
||||
|
||||
The result is three different delivery contracts:
|
||||
|
||||
- Docker and Apple Container build mutable `:latest` images on the launch host.
|
||||
- Firecracker computes an artifact version locally and downloads it.
|
||||
- An installed wheel contains Dockerfiles specifically so production startup
|
||||
can rebuild its own infrastructure.
|
||||
|
||||
That makes startup slower and less reliable, prevents a packaged application
|
||||
from identifying the infrastructure it was tested with, and weakens the
|
||||
supply-chain boundary between release production and runtime.
|
||||
|
||||
## Goals / Success criteria
|
||||
|
||||
- A packaged bot-bottle release contains one validated release manifest with
|
||||
immutable orchestrator and gateway identities.
|
||||
- Docker and Apple Container acquire and run the manifest's digest-pinned OCI
|
||||
images without invoking `docker build` or `container build`.
|
||||
- The shipped Claude, Codex, and Pi agent images are built, smoke-tested,
|
||||
published, and selected by digest as part of the same commit bundle.
|
||||
- Firecracker acquires the manifest's versioned, checksum-verified
|
||||
orchestrator and gateway rootfs artifacts rather than deriving versions at
|
||||
launch.
|
||||
- All three backends expose the same backend-neutral `ensure_available`
|
||||
lifecycle contract.
|
||||
- A healthy running orchestrator is retained only when its recorded release
|
||||
identity matches the installed package.
|
||||
- Apple Container runs the package baked into the orchestrator image; no host
|
||||
source overlays production code.
|
||||
- Missing, mutable, malformed, or internally inconsistent release metadata
|
||||
fails closed before infrastructure starts.
|
||||
- Source-checkout development retains an explicit local-build mode.
|
||||
- Publishing produces the infrastructure first and the installable Python
|
||||
package last, so the package cannot name artifacts that were not published.
|
||||
- Any branch, tag, or commit can be resolved once to a full source SHA and
|
||||
published as one immutable, commit-addressed artifact bundle.
|
||||
- The installer defaults to the production channel, can select staging or an
|
||||
exact commit, and warns for commit snapshots that have not been qualified as
|
||||
a release.
|
||||
- Staging and production releases promote the same tested artifact bytes;
|
||||
promotion never rebuilds them.
|
||||
- A release is published only after the complete pre-release suite and a clean
|
||||
install from the published bundle succeed.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Prebuilding user-supplied Dockerfiles. Custom agent images continue to build
|
||||
through the backend-specific agent-image paths.
|
||||
- Installing Docker, Apple Container, Firecracker, or other host prerequisites.
|
||||
- Replacing Gitea's OCI or generic-package registries.
|
||||
- Adding image signing or a transparency log. Immutable digests and existing
|
||||
Firecracker SHA-256 verification are the integrity boundary for this version.
|
||||
- Forcing a network download when an immutable, verified artifact is already
|
||||
cached locally. "Pull" means acquire the packaged identity, with safe cache
|
||||
reuse.
|
||||
- Supporting live source overlays in packaged production installs.
|
||||
- Defining the project's version-numbering policy beyond stable
|
||||
`vX.Y.Z` and staging `vX.Y.Z-rc.N` tag shapes.
|
||||
|
||||
## Design
|
||||
|
||||
### Release manifest
|
||||
|
||||
`bot_bottle/release-manifest.json` is generated during package production and
|
||||
shipped as package data:
|
||||
|
||||
```json
|
||||
{
|
||||
"schema": 1,
|
||||
"source_commit": "<40 lowercase hex>",
|
||||
"oci": {
|
||||
"orchestrator": "gitea.dideric.is/didericis/bot-bottle-orchestrator@sha256:<64 hex>",
|
||||
"gateway": "gitea.dideric.is/didericis/bot-bottle-gateway@sha256:<64 hex>",
|
||||
"agent_claude": "gitea.dideric.is/didericis/bot-bottle-claude@sha256:<64 hex>",
|
||||
"agent_codex": "gitea.dideric.is/didericis/bot-bottle-codex@sha256:<64 hex>",
|
||||
"agent_pi": "gitea.dideric.is/didericis/bot-bottle-pi@sha256:<64 hex>"
|
||||
},
|
||||
"firecracker": {
|
||||
"orchestrator": {
|
||||
"version": "<artifact version>",
|
||||
"sha256": "<rootfs.ext4.gz sha256>"
|
||||
},
|
||||
"gateway": {
|
||||
"version": "<artifact version>",
|
||||
"sha256": "<rootfs.ext4.gz sha256>"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The runtime loader accepts only schema 1, digest-qualified OCI references,
|
||||
non-empty Firecracker versions, 64-character lowercase SHA-256 values, and a
|
||||
full source commit. It returns immutable value objects rather than passing raw
|
||||
dictionaries through backend code.
|
||||
|
||||
The repository carries no pretend production pins. A source checkout or ad-hoc
|
||||
Git wheel carries a `development: true` marker and selects local infrastructure
|
||||
builds, preserving the contributor and current `install.sh` paths. The release
|
||||
packaging workflow must replace that marker with the validated schema above;
|
||||
it refuses mutable or incomplete release inputs.
|
||||
|
||||
### Packaging order
|
||||
|
||||
The release job operates on one tested commit:
|
||||
|
||||
1. Build and smoke-test the multi-architecture orchestrator, gateway, Claude,
|
||||
Codex, and Pi OCI images.
|
||||
2. Push them and resolve their registry manifest digests.
|
||||
3. Build Firecracker's orchestrator and gateway rootfs artifacts from the same
|
||||
source and pinned OCI bases.
|
||||
4. Publish both generic-package artifacts and retain their versions and
|
||||
compressed-file checksums.
|
||||
5. Generate `release-manifest.json` from those published identities.
|
||||
6. Build the wheel with that manifest. Do not produce a source distribution:
|
||||
rebuilding an sdist outside this release boundary could replace the pins
|
||||
with the development manifest.
|
||||
7. Install the wheel in a clean environment and assert that every manifest
|
||||
identity is valid and retrievable.
|
||||
8. Publish only that verified wheel.
|
||||
|
||||
The build hook receives the manifest as a file input. It must not query a
|
||||
registry or choose versions itself: package builds stay deterministic and
|
||||
network-independent once their release inputs exist.
|
||||
|
||||
### Commit-addressed bundle index
|
||||
|
||||
Every publication resolves its requested branch, tag, or commit to one
|
||||
40-character source SHA before doing any work. The publisher writes an
|
||||
immutable external bundle index under that SHA. The index contains:
|
||||
|
||||
- schema version and source SHA;
|
||||
- wheel URL and SHA-256;
|
||||
- orchestrator and gateway OCI digest references;
|
||||
- Claude, Codex, and Pi agent OCI digest references;
|
||||
- Firecracker package versions and compressed-file SHA-256 values;
|
||||
- producing workflow/run identity and publication timestamp; and
|
||||
- producing workflow provenance. Qualification is recorded separately in an
|
||||
immutable tag pointer so the commit bundle itself never changes.
|
||||
|
||||
The embedded wheel manifest contains the runtime subset of the same identities.
|
||||
Publication fails if the embedded manifest and external index differ.
|
||||
|
||||
Bundle coordinates are immutable. Re-running publication for a SHA verifies
|
||||
and reuses a byte-identical complete bundle; it never replaces one artifact or
|
||||
mixes outputs from different runs. A partial existing bundle is an error and
|
||||
requires an explicit administrative cleanup before retrying.
|
||||
|
||||
Commit bundles are durable release inputs, not expiring CI artifacts. The
|
||||
project may garbage-collect unqualified snapshots only under a documented
|
||||
retention policy; qualified staging and production bundles are retained.
|
||||
|
||||
### Branches, tags, and promotion
|
||||
|
||||
The protected promotion topology is:
|
||||
|
||||
```text
|
||||
feature -> main -> staging -> production
|
||||
| |
|
||||
vX.Y.Z-rc.N vX.Y.Z
|
||||
```
|
||||
|
||||
Changes reach `staging` and `production` through promotion pull requests; those
|
||||
branches do not accept direct development commits. A staging tag must point to
|
||||
a commit reachable from `staging` and uses `vX.Y.Z-rc.N`. A production tag must
|
||||
point to a commit reachable from `production` and uses `vX.Y.Z`.
|
||||
|
||||
Channels are small, mutable pointers to immutable qualified bundles:
|
||||
|
||||
- `production` points to the newest promoted stable tag;
|
||||
- `staging` points to the newest promoted release-candidate tag; and
|
||||
- `main` is not a release channel; its commits are installable snapshots.
|
||||
|
||||
Channel updates are serialized and monotonic. Moving a channel to an older
|
||||
release requires a distinct, explicit rollback operation that records the
|
||||
reason and target. Deleting or moving a Git tag does not mutate a published
|
||||
bundle or channel silently.
|
||||
|
||||
Promotion reuses the exact bundle selected by source SHA. It does not rebuild
|
||||
the wheel, OCI images, or Firecracker artifacts. Thus production runs the same
|
||||
bytes qualified in staging.
|
||||
|
||||
### Publication workflows
|
||||
|
||||
`publish-artifacts` is manually dispatchable for any branch, tag, or commit.
|
||||
It resolves the input to a source SHA, checks out that SHA, builds and publishes
|
||||
all infrastructure artifacts, generates the bundle index and embedded
|
||||
manifest, builds the wheel, installs the wheel in a clean environment, and
|
||||
publishes the verified wheel. It does not create a release or update staging or
|
||||
production. Its output is an unqualified snapshot and publication is
|
||||
serialized per source SHA.
|
||||
|
||||
After required CI succeeds, `main` may invoke the same publication operation
|
||||
automatically. Manual publication remains available so a branch can be tested
|
||||
before merge. Automatic and manual runs converge on the same idempotent bundle.
|
||||
|
||||
The release workflow is triggered by an eligible staging or production tag:
|
||||
|
||||
1. Resolve and validate the tag, source SHA, tag shape, and branch reachability.
|
||||
2. Run the complete pre-release suite against that SHA, including Docker,
|
||||
Firecracker, and Apple Container.
|
||||
3. After the suite passes, ensure the immutable commit bundle exists. Build it
|
||||
then if absent; otherwise verify every existing identity and checksum.
|
||||
4. Install using the published installer and bundle index on clean supported
|
||||
hosts, and run the post-install smoke test against the acquired artifacts.
|
||||
5. Publish an immutable qualified tag pointer, create the Gitea release, and
|
||||
advance the matching channel pointer.
|
||||
|
||||
No release artifact or channel update occurs before qualification succeeds.
|
||||
Jobs use per-SHA and per-channel concurrency locks so concurrent dispatches
|
||||
cannot publish twice or race a promotion.
|
||||
|
||||
### Installer selection and warnings
|
||||
|
||||
The installer installs the latest `production` channel by default. It accepts:
|
||||
|
||||
- `BOT_BOTTLE_CHANNEL=production` or `staging` to select the latest release in
|
||||
that channel;
|
||||
- `BOT_BOTTLE_REF=<40-character-sha>` to install an exact commit bundle; and
|
||||
- `BOT_BOTTLE_VERSION=<tag>` to install an exact qualified release.
|
||||
|
||||
Selectors are mutually exclusive. Branch names are accepted by the publication
|
||||
workflow but not by the installer because they are mutable; callers resolve
|
||||
and publish them first, then install the reported SHA.
|
||||
|
||||
An exact-commit install prints a prominent warning that the snapshot may not
|
||||
have passed release qualification, even if the same SHA is later associated
|
||||
with a tag. A tag or channel install suppresses that warning only when its
|
||||
validated pointer is marked qualified and selects the same immutable bundle
|
||||
SHA.
|
||||
|
||||
The installer downloads the wheel named by the external index, verifies its
|
||||
SHA-256 before invoking pipx or pip, and reports the selected channel/tag,
|
||||
source SHA, and bundle identity. It never builds a Git checkout for a packaged
|
||||
install and never trusts a mutable channel response without resolving it to an
|
||||
immutable bundle index.
|
||||
|
||||
### Backend-neutral lifecycle
|
||||
|
||||
Rename the host-side control-plane lifecycle operation from `ensure_built()` to
|
||||
`ensure_available()`. The production meaning is "make the package-selected
|
||||
artifact locally available and verified." Infrastructure composers call it
|
||||
before starting either plane.
|
||||
|
||||
The gateway receives the same contract so startup does not continue to rebuild
|
||||
the adjacent half of the fixed infrastructure pair.
|
||||
|
||||
A source checkout is itself a development boundary and selects the existing
|
||||
builders. `BOT_BOTTLE_INFRA_BUILD=local` provides the same explicit override
|
||||
for release debugging. Both paths deliberately bypass release metadata.
|
||||
Packaged production startup never silently falls back from a failed pull to a
|
||||
local build.
|
||||
|
||||
### Docker
|
||||
|
||||
The Docker backend runs `docker pull` with each digest-qualified manifest
|
||||
reference. Docker may reuse its content-addressed local store. Containers are
|
||||
created from the immutable reference, and the running container's image ID is
|
||||
compared with the locally resolved ID for currentness.
|
||||
|
||||
The orchestrator source-hash label is replaced by a release-identity label.
|
||||
The label is diagnostic; the image ID comparison remains authoritative.
|
||||
|
||||
### Apple Container
|
||||
|
||||
The Apple Container backend pulls the same multi-architecture OCI references.
|
||||
Because Apple Container's accepted syntax for pull, tag, inspect, and run is
|
||||
not identical to Docker's, one utility owns normalization from a digest
|
||||
reference to the local runnable name. Integration coverage on the macOS runner
|
||||
guards that adapter.
|
||||
|
||||
Production orchestrator launch removes the build-root bind mount, `PYTHONPATH`,
|
||||
and `BOT_BOTTLE_SOURCE_HASH`. It runs only the code baked into the pulled
|
||||
image. Currentness is based on the packaged release identity/image ID.
|
||||
|
||||
### Firecracker
|
||||
|
||||
The publishing tool may continue computing content-derived artifact versions;
|
||||
that is a producer concern. Runtime launch instead reads the version and
|
||||
expected compressed checksum from the release manifest.
|
||||
|
||||
The artifact cache validates the packaged expected checksum even when a
|
||||
previous `.verified` marker exists. The marker records the checksum it
|
||||
validated, rather than an unqualified `ok`, so changing package metadata cannot
|
||||
adopt an artifact verified against a different expectation.
|
||||
|
||||
The existing candidate-directory and explicit local-build paths remain for CI
|
||||
and development. Candidate metadata must match the packaged identity unless
|
||||
the caller selected local development mode.
|
||||
|
||||
### Overrides and failure behavior
|
||||
|
||||
`BOT_BOTTLE_ORCHESTRATOR_IMAGE` and `BOT_BOTTLE_GATEWAY_IMAGE` remain useful
|
||||
for testing but production accepts them only when they are digest-qualified.
|
||||
Mutable tag overrides require explicit local-build mode.
|
||||
|
||||
Artifact acquisition errors identify the packaged reference and backend. No
|
||||
backend catches an acquisition failure and rebuilds from local source.
|
||||
|
||||
## Testing strategy
|
||||
|
||||
- Unit-test manifest parsing, immutability validation, missing-manifest errors,
|
||||
package-data inclusion, and deterministic build-hook injection.
|
||||
- Assert Docker and Apple production preparation pulls and never builds.
|
||||
- Assert explicit local mode retains existing build behavior.
|
||||
- Assert the Apple orchestrator command has no source bind mount or
|
||||
`PYTHONPATH`.
|
||||
- Assert running-image mismatches recreate infrastructure while matching,
|
||||
healthy instances are retained.
|
||||
- Assert Firecracker uses packaged versions/checksums and rejects mismatches in
|
||||
downloaded, cached, and candidate artifacts.
|
||||
- Extend wheel-install coverage to inspect the installed manifest.
|
||||
- Run the full unit and Docker integration gates; run advisory Firecracker and
|
||||
macOS integration before publishing a release.
|
||||
- Test ref resolution, tag-shape and branch-reachability enforcement, bundle
|
||||
idempotency, partial-publication rejection, and per-SHA concurrency.
|
||||
- Test installer channel, tag, and exact-commit selection; checksum rejection;
|
||||
mutually exclusive selectors; and snapshot warning behavior.
|
||||
- Test that release qualification happens before publication and that a failed
|
||||
suite or clean-install smoke test cannot create a release or move a channel.
|
||||
- Test promotion reuses the qualified bundle byte-for-byte and that monotonic
|
||||
channel movement rejects an accidental rollback.
|
||||
|
||||
## Rollout
|
||||
|
||||
Land runtime, installer, bundle publication, and release qualification support
|
||||
together, but do not move the production installer away from its current
|
||||
source-install path until an end-to-end staging release has passed on Docker,
|
||||
Firecracker, and Apple Container.
|
||||
|
||||
Create and protect `staging` and `production`, then promote one commit from
|
||||
`main` through both branches. Publish its commit bundle, qualify an RC tag,
|
||||
perform clean installs through the staging channel, promote the same SHA, and
|
||||
qualify a stable tag. Until that first stable qualification exists, the
|
||||
installer's production default fails closed because there is no production
|
||||
pointer. The stable promotion must demonstrate that every production artifact
|
||||
identity matches the staging-qualified bundle.
|
||||
|
||||
Existing source checkouts use explicit local mode. Existing mutable local
|
||||
images are ignored by production acquisition and may be pruned independently.
|
||||
Snapshot retention and rollback procedures must be documented before automatic
|
||||
publication from every successful `main` commit is enabled.
|
||||
@@ -1,183 +0,0 @@
|
||||
# Testing a clean bot-bottle install on Linux
|
||||
|
||||
How do you exercise `install.sh` the way a brand-new user would — on a
|
||||
pristine Linux environment you can throw away afterward — *without*
|
||||
polluting your daily-driver host, and across the several package-management
|
||||
regimes Linux fragments into? This is the Linux counterpart to
|
||||
[`testing-clean-install-on-macos.md`](testing-clean-install-on-macos.md);
|
||||
the conclusion is different because Linux gives us a boundary macOS doesn't.
|
||||
|
||||
## Summary
|
||||
|
||||
On macOS the honest options were a throwaway user or a VM, and the throwaway
|
||||
user won on pragmatics (nested virtualization is gated to M3+). On Linux the
|
||||
calculus flips: a **disposable KVM virtual machine, booted from a distro
|
||||
cloud image and deleted per run, is both the cleanest boundary and the one
|
||||
that lets a single harness cover Ubuntu, Fedora, Arch, Alpine, and NixOS**.
|
||||
The host already requires KVM for the Firecracker backend, so the VM is cheap
|
||||
here.
|
||||
|
||||
The harness lives at [`scripts/linux-install-test.sh`](../../scripts/linux-install-test.sh).
|
||||
Per run it caches one read-only base image, boots a throwaway copy-on-write
|
||||
overlay (`qemu-img create -b base`), installs the distro's prerequisites,
|
||||
pipes *this checkout's* `install.sh` into the guest exactly as `curl … | sh`
|
||||
would, asserts the CLI installed, and deletes the overlay — the Linux
|
||||
equivalent of `docker run --rm`, for a whole machine.
|
||||
|
||||
## Why a VM, not a container or a throwaway user
|
||||
|
||||
| Mechanism | Why it's the wrong boundary here |
|
||||
|---|---|
|
||||
| **Container** (`docker run --rm`) | Shares the host kernel and ships a deliberately minimal userland — no systemd, a stubbed-out package manager story, and (crucially) it doesn't reproduce the *externally-managed Python* (PEP 668) that real desktop/server installs put in front of the user. It tests "does install.sh run in a container," not "does it run on a real distro." |
|
||||
| **Throwaway user** (`useradd`/`userdel`) | The macOS pick, but weaker on Linux: it reaches the real host, yet every system package it installs (python, pipx, git via `apt`/`dnf`/…) stays behind, and it can only ever test the *one* distro the host runs. The whole Linux-specific value is the cross-distro matrix. |
|
||||
| **Disposable KVM VM** (this harness) | A genuine kernel + userland + package-manager boundary that wipes to nothing on teardown, and swaps freely between distro cloud images. The one real cost — nested virtualization for the *backend* — doesn't apply, because we gate the installer, not the runtime (below). |
|
||||
|
||||
## Two variants: `test` (bare host) and `test-ready` (prepared host)
|
||||
|
||||
`install.sh` never installs a backend, and never installs its own toolchain
|
||||
prerequisites (python3, git, pipx) — it installs the `bot-bottle` package and
|
||||
runs `doctor`, which *reports* what's missing
|
||||
([`install.sh`](../../install.sh) header,
|
||||
[`bot_bottle/cli/commands/doctor.py`](../../bot_bottle/cli/commands/doctor.py)).
|
||||
That leaves two distinct things worth testing, split into two subcommands that
|
||||
mirror the macOS harness's `test` / `test-ready` convention (there the split is
|
||||
the backend service; here it is the toolchain the installer needs):
|
||||
|
||||
- **`test`** — `install.sh` runs on the **bare cloud image**, prerequisites and
|
||||
all left as the vendor ships them. This exercises `install.sh`'s own
|
||||
prerequisite-guard logic — the entire first half of the script (python
|
||||
version gate, git-for-git-specs gate, pipx/pip PEP-668 handling).
|
||||
- **`test-ready`** — the harness installs python3 + git + pipx first (the
|
||||
`prereqs` step), then runs `install.sh`. This is the *prepared-host happy
|
||||
path*: does a clean install actually land and produce a working CLI?
|
||||
|
||||
**Pass criteria differ by variant:**
|
||||
|
||||
| Variant | PASS when |
|
||||
|---|---|
|
||||
| `test` | `install.sh` **either** installs cleanly (the image already carried enough) **or** declines with one of its own recognized, actionable prerequisite errors (missing python3/git, no usable pip, PEP 668). A crash or an *unrecognized* failure is a FAIL. |
|
||||
| `test-ready` | `install.sh` actually lands: the `bot-bottle` entry point is present and runs, and `doctor` reports a usable python and config without crashing. A graceful decline is no longer good enough. |
|
||||
|
||||
Neither variant requires a green `doctor`: inside the VM there is no nested KVM
|
||||
or Docker, so **the backend is correctly reported not-ready** — install.sh does
|
||||
not install a backend and cannot regress one, and this harness does not
|
||||
provision the Docker backend. This is where Linux necessarily diverges from the
|
||||
macOS `test-ready`, which reaches the host backend; `BB_TEST_REQUIRE_BACKEND=1`
|
||||
makes readiness fatal anyway, for a nested-virt host that can satisfy it. The
|
||||
verdict instead classifies `doctor`'s output the way the macOS harness does — a
|
||||
`Traceback` is an install defect (fail), a missing `python`/`config` line is a
|
||||
fail, a not-ready backend is reported — so a genuine installer regression (a
|
||||
broken shim, an import error, a botched PATH) stays visible.
|
||||
|
||||
`test-all` runs the full matrix — every distro × both variants — each cell in
|
||||
its own throwaway VM, and prints a per-cell PASS/FAIL summary.
|
||||
|
||||
## The distro matrix is the point
|
||||
|
||||
Each distro exercises a different corner of the installer:
|
||||
|
||||
| Distro | Cloud image | What it stresses |
|
||||
|---|---|---|
|
||||
| **Ubuntu** (noble) | `cloud-images.ubuntu.com` | The common case; `apt`'s `pipx`, externally-managed Python (PEP 668) → install.sh's pipx path. |
|
||||
| **Fedora** | Fedora Cloud Base Generic | `dnf` packaging, a different default Python, BSD-style checksum file. |
|
||||
| **Arch** | `geo.mirror.pkgbuild.com/images/latest` | Rolling / newest Python; `python-pipx`. |
|
||||
| **Alpine** | Alpine `nocloud_` (cloudinit) image | musl libc + BusyBox `sh` — the harshest POSIX-`sh` host for a `#!/bin/sh` installer. |
|
||||
| **NixOS** | locally built with `nixos-generators` | No FHS `~/.local` on PATH by default; `nix profile install` prereqs; pipx laying a self-contained venv on a non-FHS host. |
|
||||
|
||||
### Validation run (2026-07-27) — full green
|
||||
|
||||
Full matrix on the delphi KVM host, QEMU 11.0.2, both variants × all five
|
||||
distros passing:
|
||||
|
||||
| Distro | `test` (bare) | `test-ready` (prepared) |
|
||||
|---|---|---|
|
||||
| Ubuntu 24.04 | ✅ declines at git gate | ✅ installs, doctor python+config green |
|
||||
| Fedora 44 | ✅ declines at git gate | ✅ installs |
|
||||
| Arch (latest) | ✅ declines at git gate | ✅ installs |
|
||||
| Alpine 3.21 | ✅ declines at git gate | ✅ installs |
|
||||
| NixOS 24.11 | ✅ declines (no python3) | ✅ installs |
|
||||
|
||||
The bare `test` sees `install.sh` decline soundly — exit 1 at the
|
||||
git-for-git-specs gate on the Debian/Fedora/Arch/Alpine images (they ship
|
||||
python3 but not git), and at the python3 gate on NixOS (no python3 on PATH) —
|
||||
and `test-ready` installs cleanly with `doctor` reporting a usable python and
|
||||
config (backends all not-ready, as expected in a plain VM).
|
||||
|
||||
Getting to green surfaced and fixed a series of real defects:
|
||||
|
||||
- **Fedora 41 was EOL/404** → bumped to 44.
|
||||
- The liveness probe used `bot-bottle --version`, which the CLI does not
|
||||
implement (unknown args die non-zero), so every *successful* install was
|
||||
misreported as failed → switched to `bot-bottle --help`.
|
||||
- **Alpine** needed three fixes: the `generic_` image ignores a NoCloud seed
|
||||
(switched to the `nocloud_` variant); OpenRC does not auto-start sshd after
|
||||
cloud-init injects the key (start it via `runcmd`); and Alpine's non-PAM
|
||||
sshd refuses pubkey auth for a cloud-init-*locked* account (give it a
|
||||
throwaway password). It also has no `sudo` by default (install it via
|
||||
cloud-init `packages:`).
|
||||
- **NixOS** publishes no downloadable cloud qcow2 (its cloud images are
|
||||
Hydra-built AMIs), so the harness builds one with `nixos-generators`
|
||||
([`linux-install-test-nixos.nix`](../../scripts/linux-install-test-nixos.nix)):
|
||||
cloud-init for the key, flakes enabled, deliberately no python/git/pipx. The
|
||||
`test-ready` prereq install pins `nixpkgs/nixos-24.11` because the guest's
|
||||
default unstable registry builds pipx from source (and its test suite
|
||||
currently fails to build).
|
||||
- Two harness-hygiene bugs also fixed: `cmd_down` left `serial.log` behind
|
||||
(orphaned run dirs), and the teardown trap was armed after `cmd_up`, leaking
|
||||
a VM when `wait_for_ssh` timed out.
|
||||
|
||||
In `test-ready` the harness installs `python3 + git + pipx` first on each
|
||||
distro (install.sh installs none of them), so all five drive the recommended
|
||||
pipx path. `test` then removes that scaffolding and lets each distro's bare
|
||||
image collide with install.sh's guards — on most cloud images python3 is
|
||||
present (cloud-init needs it) but git and pipx are not, so install.sh is
|
||||
expected to decline at the git-for-git-specs gate or the PEP-668 pip check with
|
||||
an actionable message. Both are legitimate, and the two variants together cover
|
||||
the whole first half of the installer as well as the happy path.
|
||||
|
||||
## What a clean install touches (the footprint that decides "wipeable")
|
||||
|
||||
| Artifact | Location | In the guest's `$HOME`? | Survives VM teardown? |
|
||||
|---|---|---|---|
|
||||
| Config / state / db | `~/.bot-bottle/{agents,bottles,contrib,…}` ([`install.sh`](../../install.sh)) | ✅ | ❌ overlay deleted |
|
||||
| pipx venv + shim | `~/.local/pipx/venvs/bot-bottle`, shim in `~/.local/bin` | ✅ | ❌ overlay deleted |
|
||||
| pip `--user` fallback | `~/.local/lib` + `~/.local/bin` | ✅ | ❌ overlay deleted |
|
||||
| **Distro prerequisites** (python/git/pipx, `test-ready` only) | system paths via `apt`/`dnf`/`pacman`/`apk`/`nix profile` | ❌ | ❌ **overlay deleted** |
|
||||
|
||||
Unlike the macOS throwaway user (whose Homebrew / Apple-Container / Rosetta
|
||||
footprint *survives*), **every row here dies with the overlay** — that is the
|
||||
VM's whole advantage. The cached base image is read-only backing and is the
|
||||
only thing that persists between runs, on purpose.
|
||||
|
||||
## Design notes baked into the harness
|
||||
|
||||
- **User-mode networking** (`-netdev user,hostfwd=tcp:127.0.0.1:PORT-:22`):
|
||||
no root, no bridge, no host network state touched. Only SSH is forwarded.
|
||||
- **cloud-init seed ISO** (`cloud-localds`) injects an ephemeral SSH keypair
|
||||
and a passwordless-sudo login. The keypair is generated per run and deleted
|
||||
on teardown; the guest can't be logged into after it's gone.
|
||||
- **Copy-on-write overlay**: the cached base is never mutated, so a corrupt or
|
||||
interrupted run can't poison the cache; downloads land at `*.partial` and
|
||||
are renamed only after checksum verification.
|
||||
- **Checksums**: verified against each vendor's published sums file at
|
||||
download time (GNU `hash file`, bare-hash, and Fedora's BSD
|
||||
`SHA256 (file) = hash` formats are all handled). Alpine ships `.sha512` only
|
||||
(this verifier is sha256) so it is skipped; NixOS is built locally, not
|
||||
downloaded, so there is nothing to verify.
|
||||
- **NixOS is built, not downloaded**: `ensure_base_image` runs
|
||||
`nixos-generate -f qcow` against
|
||||
[`linux-install-test-nixos.nix`](../../scripts/linux-install-test-nixos.nix)
|
||||
once and caches the result; the per-run seed/overlay flow is otherwise
|
||||
identical to the downloaded distros.
|
||||
- **`test-all`** runs every distro × both variants (`test` and `test-ready`),
|
||||
each cell in its own subshell on its own forwarded port, so one cell's
|
||||
failure (or teardown trap) can't abort the matrix; it prints a per-cell
|
||||
PASS/FAIL summary.
|
||||
|
||||
## Not wired into PR CI
|
||||
|
||||
Like the macOS harness, the runtime is host-specific (needs `/dev/kvm`,
|
||||
`qemu`, and `cloud-localds`) and is not exercised by the Linux pull-request
|
||||
runner. It is validated statically (`bash -n`, `shellcheck`) and run by hand
|
||||
on a KVM-capable host. The cloud-image URLs in the `DISTRO` table are the one
|
||||
place to bump when a distro cuts a newer build.
|
||||
+111
-4
@@ -17,10 +17,13 @@
|
||||
# Env:
|
||||
# BOT_BOTTLE_PYTHON interpreter to install with (skips the search)
|
||||
# BOT_BOTTLE_INSTALL_SPEC pip/git spec to install instead of the default
|
||||
# BOT_BOTTLE_CHANNEL production (default) or staging
|
||||
# BOT_BOTTLE_VERSION exact qualified vX.Y.Z[-rc.N] release
|
||||
# BOT_BOTTLE_REF exact 40-character commit snapshot (warns untested)
|
||||
# BOT_BOTTLE_VENV where the non-pipx install lives (~/.bot-bottle/venv)
|
||||
set -eu
|
||||
|
||||
PACKAGE_SPEC="${BOT_BOTTLE_INSTALL_SPEC:-git+https://gitea.dideric.is/didericis/bot-bottle.git}"
|
||||
PACKAGE_SPEC="${BOT_BOTTLE_INSTALL_SPEC:-}"
|
||||
VENV="${BOT_BOTTLE_VENV:-${HOME}/.bot-bottle/venv}"
|
||||
MIN_PYTHON_MAJOR=3
|
||||
MIN_PYTHON_MINOR=11
|
||||
@@ -126,9 +129,113 @@ if [ "${PYTHON}" != "${path_python}" ]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# Installing a `git+` spec (the default) shells out to git under the hood,
|
||||
# whether via pipx or pip. Fail early with a clear message rather than deep
|
||||
# inside the installer's output.
|
||||
# --- resolve an immutable published wheel -----------------------------------
|
||||
|
||||
if [ -z "${PACKAGE_SPEC}" ]; then
|
||||
selectors=0
|
||||
[ -n "${BOT_BOTTLE_REF:-}" ] && selectors=$((selectors + 1))
|
||||
[ -n "${BOT_BOTTLE_VERSION:-}" ] && selectors=$((selectors + 1))
|
||||
[ -n "${BOT_BOTTLE_CHANNEL:-}" ] && selectors=$((selectors + 1))
|
||||
[ "${selectors}" -le 1 ] || die \
|
||||
"BOT_BOTTLE_REF, BOT_BOTTLE_VERSION, and BOT_BOTTLE_CHANNEL are mutually exclusive."
|
||||
|
||||
downloads="${HOME}/.bot-bottle/downloads"
|
||||
mkdir -p "${downloads}"
|
||||
PACKAGE_SPEC="$("${PYTHON}" - \
|
||||
"${BOT_BOTTLE_REF:-}" \
|
||||
"${BOT_BOTTLE_VERSION:-}" \
|
||||
"${BOT_BOTTLE_CHANNEL:-production}" \
|
||||
"${downloads}" <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ref, version, channel, download_root = sys.argv[1:]
|
||||
base = "https://gitea.dideric.is/api/packages/didericis/generic"
|
||||
|
||||
def fetch_json(url):
|
||||
with urllib.request.urlopen(url, timeout=30) as response:
|
||||
return json.load(response)
|
||||
|
||||
if ref:
|
||||
if re.fullmatch(r"[0-9a-f]{40}", ref) is None:
|
||||
raise SystemExit("bot-bottle install: error: BOT_BOTTLE_REF must be 40 lowercase hex")
|
||||
index_url = f"{base}/bot-bottle-builds/{ref}/bundle-index.json"
|
||||
print(
|
||||
"bot-bottle install: WARNING: installing an exact commit snapshot; "
|
||||
"it may not have passed release qualification.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
selector = f"commit {ref}"
|
||||
else:
|
||||
if version:
|
||||
if re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+(?:-rc\.[0-9]+)?", version) is None:
|
||||
raise SystemExit("bot-bottle install: error: invalid BOT_BOTTLE_VERSION")
|
||||
pointer_url = f"{base}/bot-bottle-releases/{version}/release.json"
|
||||
selector = f"release {version}"
|
||||
else:
|
||||
if channel not in ("production", "staging"):
|
||||
raise SystemExit(
|
||||
"bot-bottle install: error: BOT_BOTTLE_CHANNEL must be production or staging")
|
||||
pointer_url = f"{base}/bot-bottle-channels/{channel}/channel.json"
|
||||
selector = f"{channel} channel"
|
||||
pointer = fetch_json(pointer_url)
|
||||
if pointer.get("schema") != 1 or pointer.get("qualified") is not True:
|
||||
raise SystemExit("bot-bottle install: error: release pointer is not qualified")
|
||||
index_url = pointer.get("bundle_index_url", "")
|
||||
if not isinstance(index_url, str) or not index_url.startswith("https://"):
|
||||
raise SystemExit("bot-bottle install: error: release pointer has no immutable bundle")
|
||||
|
||||
index = fetch_json(index_url)
|
||||
source_commit = index.get("source_commit", "")
|
||||
if re.fullmatch(r"[0-9a-f]{40}", source_commit) is None:
|
||||
raise SystemExit("bot-bottle install: error: bundle has an invalid source commit")
|
||||
if ref and source_commit != ref:
|
||||
raise SystemExit("bot-bottle install: error: bundle does not match BOT_BOTTLE_REF")
|
||||
wheel = index.get("wheel")
|
||||
if not isinstance(wheel, dict):
|
||||
raise SystemExit("bot-bottle install: error: bundle has no wheel")
|
||||
filename, url, expected = (
|
||||
wheel.get("filename"), wheel.get("url"), wheel.get("sha256"))
|
||||
if (
|
||||
not isinstance(filename, str)
|
||||
or re.fullmatch(r"[A-Za-z0-9_.-]+\.whl", filename) is None
|
||||
or not isinstance(url, str)
|
||||
or not url.startswith("https://")
|
||||
or not url.endswith("/" + filename)
|
||||
or not isinstance(expected, str)
|
||||
or re.fullmatch(r"[0-9a-f]{64}", expected) is None
|
||||
):
|
||||
raise SystemExit("bot-bottle install: error: bundle wheel metadata is invalid")
|
||||
|
||||
destination = Path(download_root) / source_commit / filename
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = destination.with_suffix(destination.suffix + ".part")
|
||||
digest = hashlib.sha256()
|
||||
with urllib.request.urlopen(url, timeout=60) as response, temporary.open("wb") as output:
|
||||
while chunk := response.read(1 << 20):
|
||||
digest.update(chunk)
|
||||
output.write(chunk)
|
||||
if digest.hexdigest() != expected:
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise SystemExit("bot-bottle install: error: downloaded wheel checksum mismatch")
|
||||
temporary.replace(destination)
|
||||
print(
|
||||
f"bot-bottle install: selected {selector} "
|
||||
f"(source {source_commit}, sha256 {expected[:16]}…)",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(destination)
|
||||
PY
|
||||
)"
|
||||
fi
|
||||
|
||||
# An explicit `git+` override shells out to git under the hood, whether via
|
||||
# pipx or pip. Fail early with a clear message rather than deep inside the
|
||||
# installer's output.
|
||||
case "${PACKAGE_SPEC}" in
|
||||
git+*|*.git)
|
||||
command -v git >/dev/null 2>&1 || die \
|
||||
|
||||
@@ -36,6 +36,7 @@ include = ["bot_bottle*"]
|
||||
# files shipped under bot_bottle/; test_pyproject.py asserts they exist.
|
||||
[tool.setuptools.package-data]
|
||||
bot_bottle = [
|
||||
"release-manifest.json",
|
||||
"gateway/egress/entrypoint.sh",
|
||||
"contrib/claude/Dockerfile",
|
||||
"contrib/claude/package.json",
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Generate the external commit-addressed release bundle index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from bot_bottle.release_bundle import build_bundle_index, canonical_bytes
|
||||
from bot_bottle.release_publish import bundle_url, publish_bundle
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--manifest", required=True, type=Path)
|
||||
parser.add_argument("--wheel", required=True, type=Path)
|
||||
parser.add_argument("--wheel-url")
|
||||
parser.add_argument("--workflow-run", required=True)
|
||||
parser.add_argument("--published-at", required=True)
|
||||
parser.add_argument("--output", required=True, type=Path)
|
||||
parser.add_argument("--publish", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
manifest = json.loads(args.manifest.read_text(encoding="utf-8"))
|
||||
index = build_bundle_index(
|
||||
manifest,
|
||||
wheel=args.wheel,
|
||||
wheel_url=args.wheel_url or bundle_url(
|
||||
manifest["source_commit"], args.wheel.name),
|
||||
workflow_run=args.workflow_run,
|
||||
published_at=args.published_at,
|
||||
)
|
||||
args.output.write_bytes(canonical_bytes(index))
|
||||
if args.publish:
|
||||
publish_bundle(index, args.wheel)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Generate the validated infrastructure manifest consumed by package builds."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from bot_bottle.release_manifest import parse_manifest
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
result = argparse.ArgumentParser()
|
||||
result.add_argument("--source-commit", required=True)
|
||||
result.add_argument("--orchestrator-image", required=True)
|
||||
result.add_argument("--gateway-image", required=True)
|
||||
for provider in ("claude", "codex", "pi"):
|
||||
result.add_argument(f"--agent-{provider}-image", required=True)
|
||||
for role in ("orchestrator", "gateway"):
|
||||
result.add_argument(f"--firecracker-{role}-version", required=True)
|
||||
result.add_argument(f"--firecracker-{role}-sha256", required=True)
|
||||
result.add_argument("--output", type=Path, required=True)
|
||||
return result
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = parser().parse_args(argv)
|
||||
data = {
|
||||
"schema": 1,
|
||||
"source_commit": args.source_commit,
|
||||
"oci": {
|
||||
"orchestrator": args.orchestrator_image,
|
||||
"gateway": args.gateway_image,
|
||||
**{
|
||||
f"agent_{provider}": getattr(args, f"agent_{provider}_image")
|
||||
for provider in ("claude", "codex", "pi")
|
||||
},
|
||||
},
|
||||
"firecracker": {
|
||||
role: {
|
||||
"version": getattr(args, f"firecracker_{role}_version"),
|
||||
"sha256": getattr(args, f"firecracker_{role}_sha256"),
|
||||
}
|
||||
for role in ("orchestrator", "gateway")
|
||||
},
|
||||
}
|
||||
parse_manifest(data)
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(
|
||||
json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,24 +0,0 @@
|
||||
# NixOS image for scripts/linux-install-test.sh.
|
||||
#
|
||||
# NixOS publishes no downloadable cloud qcow2 (its cloud images are Hydra-built
|
||||
# AMIs), so the harness BUILDS this one with nixos-generators (-f qcow). It is
|
||||
# deliberately minimal — no python3/git/pipx — so the bare `test` variant is
|
||||
# genuinely under-provisioned and exercises install.sh's guards; `test-ready`
|
||||
# provisions them with `nix profile install` (hence flakes below).
|
||||
{ lib, ... }:
|
||||
{
|
||||
# Consume the same NoCloud seed the other distros use: cloud-init injects the
|
||||
# per-run ephemeral SSH key for root. Leave networking to NixOS's default
|
||||
# dhcpcd (QEMU user-mode NAT) — enabling cloud-init's networkd here conflicts
|
||||
# with dhcpcd and can drop the guest's network.
|
||||
services.cloud-init.enable = true;
|
||||
services.cloud-init.network.enable = false;
|
||||
|
||||
services.openssh.enable = true;
|
||||
services.openssh.settings.PermitRootLogin = lib.mkForce "prohibit-password";
|
||||
|
||||
# Flakes so the `test-ready` prereq step can `nix profile install nixpkgs#...`.
|
||||
nix.settings.experimental-features = [ "nix-command" "flakes" ];
|
||||
|
||||
system.stateVersion = "24.11";
|
||||
}
|
||||
@@ -1,753 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Clean-install test harness for the Linux path.
|
||||
#
|
||||
# Exercises install.sh the way a brand-new user would, inside a THROWAWAY
|
||||
# QEMU/KVM virtual machine that is booted from a distro cloud image and
|
||||
# deleted afterward. install.sh's entire footprint is user-home-local (the
|
||||
# pipx venv under ~/.local, the ~/.bot-bottle config dir, and a printed PATH
|
||||
# hint), so a fresh VM's fresh $HOME is the clean surface we want — and unlike
|
||||
# a throwaway user account, tearing the VM down also wipes any OS-level
|
||||
# prerequisites installed into it, so the reset is total. Full rationale in
|
||||
# docs/research/testing-clean-install-on-linux.md.
|
||||
#
|
||||
# Why a VM and not a container or a throwaway user: a container shares the
|
||||
# host kernel and cannot exercise a genuinely pristine OS (systemd, the distro
|
||||
# package manager, PEP 668 externally-managed Python) the way a real guest
|
||||
# does, and a throwaway user leaves every system package it installs behind.
|
||||
# The host already needs KVM for the Firecracker backend, so a per-run,
|
||||
# copy-on-write VM is cheap here: one cached base image, a throwaway overlay
|
||||
# per run (`qemu-img create -b base`), deleted on teardown — the Linux
|
||||
# equivalent of `docker run --rm`, but for a whole machine.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/linux-install-test.sh test # up -> run -> verdict -> down (bare host)
|
||||
# ./scripts/linux-install-test.sh test-ready # ... with prerequisites installed first
|
||||
# ./scripts/linux-install-test.sh test-all # every distro × both variants, with a summary
|
||||
# ./scripts/linux-install-test.sh up # fetch base image, boot a fresh VM
|
||||
# ./scripts/linux-install-test.sh prereqs # install python3 + git + pipx in the VM
|
||||
# ./scripts/linux-install-test.sh run # pipe install.sh into the VM
|
||||
# ./scripts/linux-install-test.sh status # VM reachable? is the install sound?
|
||||
# ./scripts/linux-install-test.sh down # kill the VM, delete overlay + seed (the reset)
|
||||
# ./scripts/linux-install-test.sh ssh # open an interactive shell in the running VM
|
||||
#
|
||||
# TWO TEST VARIANTS, because "does install.sh handle an unprepared host" and
|
||||
# "does a prepared host get a clean install" are different questions (mirrors
|
||||
# the macOS harness's test / test-ready split):
|
||||
#
|
||||
# test A Linux system WITHOUT the prerequisites set up — the default
|
||||
# state of a stock cloud image (python3 is usually present for
|
||||
# cloud-init, but git and pipx are not). This exercises
|
||||
# install.sh's own prerequisite-guard logic. It is SOUND — a
|
||||
# PASS — when install.sh EITHER installs cleanly (the image
|
||||
# already had enough) OR declines with one of its own recognized,
|
||||
# actionable errors (missing python3/git, no usable pip, PEP 668
|
||||
# externally-managed). A crash or an unrecognized failure fails.
|
||||
#
|
||||
# test-ready A Linux system WITH the prerequisites satisfied — the harness
|
||||
# installs python3 + git + pipx first (see the DISTRO table),
|
||||
# then runs install.sh. A graceful decline is no longer good
|
||||
# enough here: the install MUST land, the entry point must run,
|
||||
# and doctor must report a usable python and config without
|
||||
# crashing.
|
||||
#
|
||||
# Neither variant requires a green backend. install.sh does not install a
|
||||
# backend and cannot regress one, and inside a plain VM there is no nested KVM
|
||||
# for Firecracker; the harness does not provision the Docker backend either. So
|
||||
# backend readiness is reported, not required (this is where Linux necessarily
|
||||
# diverges from the macOS test-ready, which reaches the host backend).
|
||||
# BB_TEST_REQUIRE_BACKEND=1 makes it fatal anyway, for a nested-virt host.
|
||||
#
|
||||
# Config via env:
|
||||
# BB_TEST_DISTRO ubuntu | fedora | arch | alpine | nixos (default: ubuntu)
|
||||
# BB_TEST_SSH_PORT host port forwarded to the guest's :22 (default: 2222)
|
||||
# BB_TEST_CACHE_DIR where base images are cached (default: ~/.cache/bot-bottle-install-test)
|
||||
# BB_TEST_RUN_DIR per-run scratch (overlay, seed, key, …) (default: a mktemp dir)
|
||||
# BB_TEST_MEM_MB guest RAM (default: 2048)
|
||||
# BB_TEST_CPUS guest vCPUs (default: 2)
|
||||
# BB_TEST_DISK overlay virtual size (default: 12G)
|
||||
# BB_TEST_BOOT_TIMEOUT seconds to wait for SSH after boot (default: 300)
|
||||
# BB_TEST_KEEP 1 = the test cycles skip teardown, to poke at a failure
|
||||
# BB_TEST_REQUIRE_BACKEND 1 = make a not-ready backend fatal (needs nested virt)
|
||||
# BB_TEST_INSTALL_URL curl this install.sh in the guest instead of piping the local checkout
|
||||
# BB_TEST_SKIP_VERIFY 1 = skip base-image checksum verification (not recommended)
|
||||
# BOT_BOTTLE_INSTALL_SPEC passed through to install.sh (pip / git spec)
|
||||
#
|
||||
# Notes:
|
||||
# * Needs /dev/kvm, qemu-system-x86_64, and cloud-localds (cloud-image-utils
|
||||
# / cloud-utils); the nixos distro additionally needs nixos-generate. On the
|
||||
# NixOS host: nix shell nixpkgs#qemu nixpkgs#cloud-utils nixpkgs#nixos-generators
|
||||
# * Networking is user-mode (`-netdev user,hostfwd`) so the harness needs no
|
||||
# root, no bridge, and touches no host network state. Only SSH is forwarded.
|
||||
# * The cloud-image URLs in the DISTRO table are the one place to bump when a
|
||||
# distro cuts a new build; each is verified against the vendor's published
|
||||
# checksum at download time (guarding against truncated/corrupt pulls).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
DISTRO="${BB_TEST_DISTRO:-ubuntu}"
|
||||
SSH_PORT="${BB_TEST_SSH_PORT:-2222}"
|
||||
CACHE_DIR="${BB_TEST_CACHE_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/bot-bottle-install-test}"
|
||||
MEM_MB="${BB_TEST_MEM_MB:-2048}"
|
||||
CPUS="${BB_TEST_CPUS:-2}"
|
||||
DISK="${BB_TEST_DISK:-12G}"
|
||||
BOOT_TIMEOUT="${BB_TEST_BOOT_TIMEOUT:-300}"
|
||||
|
||||
_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
_REPO_ROOT="$(cd "$_SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
# Per-run scratch. Persisted across sub-commands (up/prereqs/run/status/down)
|
||||
# via a marker file so `up` in one invocation and `down` in the next find the
|
||||
# same VM; the test cycles set RUN_DIR themselves and never write the marker.
|
||||
RUN_DIR="${BB_TEST_RUN_DIR:-}"
|
||||
|
||||
# Set by the test cycles, which chain the steps and suppress the per-step "next
|
||||
# command" hints. _STEPS / _PASS_CLAIM / _REQUIRE_INSTALL are the per-variant
|
||||
# knobs the shared cycle and teardown read.
|
||||
IN_TEST=0
|
||||
_STEPS=4
|
||||
_PASS_CLAIM=""
|
||||
_REQUIRE_INSTALL=0
|
||||
|
||||
# --- distro table ----------------------------------------------------
|
||||
# For each distro: cloud-image URL | checksum-file URL | default SSH user |
|
||||
# prerequisite-install command (run in the guest; uses sudo when user != root).
|
||||
#
|
||||
# The prerequisite command installs python3 + git + pipx — install.sh installs
|
||||
# none of them — so `test-ready` (and the `prereqs` sub-command) drive
|
||||
# install.sh down its recommended pipx path. Bump the URLs here when a distro
|
||||
# publishes a newer build.
|
||||
declare -A IMAGE_URL SUM_URL SSH_USER PREREQ
|
||||
|
||||
IMAGE_URL[ubuntu]="https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"
|
||||
SUM_URL[ubuntu]="https://cloud-images.ubuntu.com/noble/current/SHA256SUMS"
|
||||
SSH_USER[ubuntu]="ubuntu"
|
||||
PREREQ[ubuntu]="sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y python3 git pipx"
|
||||
|
||||
IMAGE_URL[fedora]="https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-44-1.7.x86_64.qcow2"
|
||||
SUM_URL[fedora]="https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/x86_64/images/Fedora-Cloud-44-1.7-x86_64-CHECKSUM"
|
||||
SSH_USER[fedora]="fedora"
|
||||
PREREQ[fedora]="sudo dnf install -y python3 git pipx"
|
||||
|
||||
IMAGE_URL[arch]="https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-cloudimg.qcow2"
|
||||
SUM_URL[arch]="https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-cloudimg.qcow2.SHA256"
|
||||
SSH_USER[arch]="arch"
|
||||
PREREQ[arch]="sudo pacman -Sy --noconfirm python git python-pipx"
|
||||
|
||||
# Use the *nocloud_* Alpine variant, not generic_: the generic image probes
|
||||
# network datasources and ignores the local NoCloud seed, so cloud-init never
|
||||
# runs and the SSH key is never injected. (Only published at .0 patch levels.)
|
||||
IMAGE_URL[alpine]="https://dl-cdn.alpinelinux.org/alpine/v3.21/releases/cloud/nocloud_alpine-3.21.0-x86_64-bios-cloudinit-r0.qcow2"
|
||||
SUM_URL[alpine]="" # Alpine cloud images ship .sha512 only; this verifier is sha256.
|
||||
SSH_USER[alpine]="alpine"
|
||||
PREREQ[alpine]="sudo apk add --no-cache python3 git pipx"
|
||||
|
||||
# NixOS publishes no downloadable cloud qcow2 (its cloud images are Hydra-built
|
||||
# AMIs), so the harness BUILDS one with nixos-generators — see build_nixos_image
|
||||
# and linux-install-test-nixos.nix. It's externally-managed in its own way (no
|
||||
# FHS ~/.local on PATH by default); `nix profile install` provisions the
|
||||
# prerequisites into root's profile (the image enables flakes for this).
|
||||
IMAGE_URL[nixos]="nix:build" # sentinel: ensure_base_image builds instead of downloading
|
||||
SUM_URL[nixos]=""
|
||||
SSH_USER[nixos]="root"
|
||||
# Pin to a stable release: the guest's default `nixpkgs` registry is unstable,
|
||||
# where pipx isn't in the binary cache and builds from source (its test suite
|
||||
# currently fails to build). nixos-24.11 has these cached as substitutes.
|
||||
PREREQ[nixos]="nix profile install nixpkgs/nixos-24.11#python3 nixpkgs/nixos-24.11#git nixpkgs/nixos-24.11#pipx"
|
||||
|
||||
ALL_DISTROS=(ubuntu fedora arch alpine nixos)
|
||||
|
||||
# --- guards ----------------------------------------------------------
|
||||
require_linux() {
|
||||
[ "$(uname -s)" = "Linux" ] \
|
||||
|| { echo "error: this harness is Linux-only (uname is $(uname -s))" >&2; exit 1; }
|
||||
}
|
||||
|
||||
require_kvm() {
|
||||
[ -e /dev/kvm ] && [ -r /dev/kvm ] && [ -w /dev/kvm ] \
|
||||
|| { echo "error: /dev/kvm is missing or not accessible (add yourself to the 'kvm' group)" >&2; exit 1; }
|
||||
}
|
||||
|
||||
require_tools() {
|
||||
local missing=()
|
||||
command -v qemu-system-x86_64 >/dev/null 2>&1 || missing+=(qemu-system-x86_64)
|
||||
command -v qemu-img >/dev/null 2>&1 || missing+=(qemu-img)
|
||||
command -v cloud-localds >/dev/null 2>&1 || missing+=(cloud-localds)
|
||||
command -v ssh >/dev/null 2>&1 || missing+=(ssh)
|
||||
command -v curl >/dev/null 2>&1 || missing+=(curl)
|
||||
if [ "${#missing[@]}" -ne 0 ]; then
|
||||
echo "error: missing required tools: ${missing[*]}" >&2
|
||||
echo " on NixOS: nix shell nixpkgs#qemu nixpkgs#cloud-utils nixpkgs#openssh nixpkgs#curl" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
known_distro() {
|
||||
[ -n "${IMAGE_URL[$DISTRO]:-}" ] \
|
||||
|| { echo "error: unknown distro '$DISTRO' (known: ${ALL_DISTROS[*]})" >&2; exit 1; }
|
||||
}
|
||||
|
||||
# --- run-dir bookkeeping ---------------------------------------------
|
||||
# The marker lets prereqs/run/status/down in separate invocations find the VM
|
||||
# that `up` started. The test cycles set RUN_DIR themselves and never write it.
|
||||
_marker() { echo "${TMPDIR:-/tmp}/bot-bottle-install-test.$DISTRO.run"; }
|
||||
|
||||
_ensure_run_dir() {
|
||||
if [ -z "$RUN_DIR" ]; then
|
||||
RUN_DIR="$(mktemp -d "${TMPDIR:-/tmp}/bb-install-test.$DISTRO.XXXXXX")"
|
||||
fi
|
||||
mkdir -p "$RUN_DIR"
|
||||
}
|
||||
|
||||
_load_run_dir() {
|
||||
if [ -z "$RUN_DIR" ] && [ -f "$(_marker)" ]; then
|
||||
RUN_DIR="$(cat "$(_marker)")"
|
||||
fi
|
||||
[ -n "$RUN_DIR" ] && [ -d "$RUN_DIR" ]
|
||||
}
|
||||
|
||||
_ssh_key() { echo "$RUN_DIR/id_ed25519"; }
|
||||
_overlay() { echo "$RUN_DIR/overlay.qcow2"; }
|
||||
_seed() { echo "$RUN_DIR/seed.iso"; }
|
||||
_pidfile() { echo "$RUN_DIR/qemu.pid"; }
|
||||
_serial() { echo "$RUN_DIR/serial.log"; }
|
||||
|
||||
# --- ssh helpers -----------------------------------------------------
|
||||
_ssh_opts() {
|
||||
# No host-key pinning: the guest is thrown away every run.
|
||||
printf '%s\0' \
|
||||
-i "$(_ssh_key)" \
|
||||
-p "$SSH_PORT" \
|
||||
-o StrictHostKeyChecking=no \
|
||||
-o UserKnownHostsFile=/dev/null \
|
||||
-o LogLevel=ERROR \
|
||||
-o ConnectTimeout=8 \
|
||||
-o BatchMode=yes
|
||||
}
|
||||
|
||||
guest() {
|
||||
local -a opts
|
||||
mapfile -d '' -t opts < <(_ssh_opts)
|
||||
ssh "${opts[@]}" "${SSH_USER[$DISTRO]}@127.0.0.1" "$@"
|
||||
}
|
||||
|
||||
wait_for_ssh() {
|
||||
local deadline=$(( SECONDS + BOOT_TIMEOUT ))
|
||||
echo "== waiting for SSH on 127.0.0.1:$SSH_PORT (up to ${BOOT_TIMEOUT}s) =="
|
||||
while [ "$SECONDS" -lt "$deadline" ]; do
|
||||
if guest true 2>/dev/null; then
|
||||
echo " guest is up"
|
||||
return 0
|
||||
fi
|
||||
# Bail early if QEMU has died — no point waiting out the timeout.
|
||||
if [ -f "$(_pidfile)" ] && ! kill -0 "$(cat "$(_pidfile)")" 2>/dev/null; then
|
||||
echo "error: QEMU exited before SSH came up; see $(_serial)" >&2
|
||||
return 1
|
||||
fi
|
||||
sleep 3
|
||||
done
|
||||
echo "error: timed out waiting for SSH; see $(_serial)" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# --- image cache -----------------------------------------------------
|
||||
_base_image() {
|
||||
# One cached file per distro. NixOS is built (not downloaded), so it has a
|
||||
# fixed cache name; the rest are keyed by the image's basename so a URL bump
|
||||
# lands as a new cache entry rather than a stale hit.
|
||||
if [ "$DISTRO" = nixos ]; then
|
||||
echo "$CACHE_DIR/nixos-built.qcow2"
|
||||
return
|
||||
fi
|
||||
local url="${IMAGE_URL[$DISTRO]}"
|
||||
echo "$CACHE_DIR/$DISTRO-$(basename "$url")"
|
||||
}
|
||||
|
||||
# NixOS has no upstream cloud qcow2; build one with nixos-generators and copy it
|
||||
# out of the (immutable, GC-able) store into the cache.
|
||||
build_nixos_image() {
|
||||
local out; out="$(_base_image)"
|
||||
[ -f "$out" ] && { echo "== nixos base image cached: $out =="; return 0; }
|
||||
command -v nixos-generate >/dev/null 2>&1 || {
|
||||
echo "error: 'nixos-generate' is required to build the NixOS image" >&2
|
||||
echo " run inside: nix shell nixpkgs#nixos-generators nixpkgs#qemu nixpkgs#cloud-utils" >&2
|
||||
return 1
|
||||
}
|
||||
echo "== building NixOS cloud image with nixos-generators (first run is slow) =="
|
||||
local link="$CACHE_DIR/nixos-result"
|
||||
nixos-generate -f qcow --system x86_64-linux \
|
||||
-c "$_SCRIPT_DIR/linux-install-test-nixos.nix" -o "$link"
|
||||
cp -L "$link"/*.qcow2 "$out"
|
||||
rm -f "$link"
|
||||
echo " built + cached: $out"
|
||||
}
|
||||
|
||||
verify_checksum() {
|
||||
local file="$1" sums_url="${SUM_URL[$DISTRO]}" base
|
||||
base="$(basename "${IMAGE_URL[$DISTRO]}")"
|
||||
if [ "${BB_TEST_SKIP_VERIFY:-0}" = "1" ] || [ -z "$sums_url" ]; then
|
||||
echo " checksum: SKIPPED (${sums_url:+set BB_TEST_SKIP_VERIFY=0 to enable}${sums_url:-no sums URL for $DISTRO})" >&2
|
||||
return 0
|
||||
fi
|
||||
local want
|
||||
# Vendors publish either "HASH filename" tables or a bare "HASH" (or a
|
||||
# "SHA256 (file) = HASH" BSD line, e.g. Fedora). Cover all three.
|
||||
local sums; sums="$(curl -fsSL "$sums_url")"
|
||||
want="$(printf '%s\n' "$sums" | awk -v f="$base" '
|
||||
$0 ~ f && $1 ~ /^[0-9a-fA-F]{64}$/ { print $1; exit } # GNU "hash file"
|
||||
$1=="SHA256" && $0 ~ f { gsub(/[()]/,""); print $NF; exit } # BSD "SHA256 (file) = hash"
|
||||
')"
|
||||
[ -z "$want" ] && want="$(printf '%s\n' "$sums" | awk '/^[0-9a-fA-F]{64}$/ {print $1; exit}')"
|
||||
[ -n "$want" ] || { echo "error: could not find a sha256 for $base in $sums_url" >&2; return 1; }
|
||||
local got; got="$(sha256sum "$file" | awk '{print $1}')"
|
||||
if [ "$want" != "$got" ]; then
|
||||
echo "error: checksum mismatch for $base" >&2
|
||||
echo " want $want" >&2
|
||||
echo " got $got" >&2
|
||||
return 1
|
||||
fi
|
||||
echo " checksum: OK"
|
||||
}
|
||||
|
||||
ensure_base_image() {
|
||||
mkdir -p "$CACHE_DIR"
|
||||
if [ "$DISTRO" = nixos ]; then
|
||||
build_nixos_image
|
||||
return
|
||||
fi
|
||||
local base; base="$(_base_image)"
|
||||
if [ -f "$base" ]; then
|
||||
echo "== base image cached: $base =="
|
||||
return 0
|
||||
fi
|
||||
echo "== downloading $DISTRO cloud image =="
|
||||
echo " ${IMAGE_URL[$DISTRO]}"
|
||||
# Download to a temp name and rename on success so an interrupted pull
|
||||
# never poisons the cache with a truncated image.
|
||||
local tmp="$base.partial"
|
||||
curl -fSL --retry 3 -o "$tmp" "${IMAGE_URL[$DISTRO]}"
|
||||
verify_checksum "$tmp"
|
||||
mv "$tmp" "$base"
|
||||
echo " cached: $base"
|
||||
}
|
||||
|
||||
# --- cloud-init seed -------------------------------------------------
|
||||
make_seed() {
|
||||
ssh-keygen -t ed25519 -N '' -f "$(_ssh_key)" -q
|
||||
local pub; pub="$(cat "$(_ssh_key).pub")"
|
||||
local user="${SSH_USER[$DISTRO]}"
|
||||
local user_data="$RUN_DIR/user-data"
|
||||
|
||||
if [ "$user" = "root" ]; then
|
||||
# NixOS' cloud-init lands the key straight on root; no sudo needed.
|
||||
cat > "$user_data" <<EOF
|
||||
#cloud-config
|
||||
ssh_authorized_keys:
|
||||
- $pub
|
||||
EOF
|
||||
elif [ "$DISTRO" = alpine ]; then
|
||||
# Alpine needs three things the systemd distros don't: cloud-init locks
|
||||
# the account (lock_passwd), but Alpine's non-PAM sshd then refuses
|
||||
# pubkey auth for a locked account — so give it a throwaway password;
|
||||
# and OpenRC does not auto-start sshd after the key is injected, so
|
||||
# start it via runcmd.
|
||||
cat > "$user_data" <<EOF
|
||||
#cloud-config
|
||||
users:
|
||||
- name: $user
|
||||
sudo: ALL=(ALL) NOPASSWD:ALL
|
||||
shell: /bin/sh
|
||||
lock_passwd: false
|
||||
ssh_authorized_keys:
|
||||
- $pub
|
||||
chpasswd:
|
||||
expire: false
|
||||
list: |
|
||||
$user:bbtest
|
||||
packages:
|
||||
- sudo
|
||||
runcmd:
|
||||
- [ sh, -c, "rc-service sshd start 2>/dev/null || true" ]
|
||||
EOF
|
||||
else
|
||||
cat > "$user_data" <<EOF
|
||||
#cloud-config
|
||||
users:
|
||||
- name: $user
|
||||
sudo: ALL=(ALL) NOPASSWD:ALL
|
||||
shell: /bin/sh
|
||||
lock_passwd: true
|
||||
ssh_authorized_keys:
|
||||
- $pub
|
||||
EOF
|
||||
fi
|
||||
# NoCloud wants a meta-data with an instance-id, or cloud-init may not treat
|
||||
# the seed as a new instance (the Alpine nocloud image is strict about this).
|
||||
printf 'instance-id: bbtest-%s\nlocal-hostname: bbtest-%s\n' "$DISTRO" "$DISTRO" \
|
||||
> "$RUN_DIR/meta-data"
|
||||
cloud-localds "$(_seed)" "$user_data" "$RUN_DIR/meta-data"
|
||||
}
|
||||
|
||||
# --- commands --------------------------------------------------------
|
||||
cmd_up() {
|
||||
require_linux; require_kvm; require_tools; known_distro
|
||||
_ensure_run_dir
|
||||
ensure_base_image
|
||||
|
||||
# Throwaway copy-on-write overlay: the cached base is read-only backing,
|
||||
# all guest writes land in the overlay, and `down` deletes it. Resize so
|
||||
# pipx + a git build have headroom (cloud-init grows the rootfs to fit).
|
||||
qemu-img create -q -f qcow2 -F qcow2 -b "$(_base_image)" "$(_overlay)" "$DISK"
|
||||
make_seed
|
||||
|
||||
echo "== booting $DISTRO VM (mem=${MEM_MB}M cpus=$CPUS, ssh -> :$SSH_PORT) =="
|
||||
qemu-system-x86_64 \
|
||||
-machine accel=kvm -cpu host -smp "$CPUS" -m "$MEM_MB" \
|
||||
-display none -daemonize \
|
||||
-pidfile "$(_pidfile)" \
|
||||
-serial "file:$(_serial)" \
|
||||
-drive "file=$(_overlay),if=virtio,format=qcow2" \
|
||||
-drive "file=$(_seed),if=virtio,format=raw" \
|
||||
-netdev "user,id=n0,hostfwd=tcp:127.0.0.1:$SSH_PORT-:22" \
|
||||
-device virtio-net-pci,netdev=n0
|
||||
|
||||
# Only publish the marker (so a later prereqs/run/down finds this VM) when
|
||||
# we aren't inside a test cycle, which manages its own RUN_DIR + teardown.
|
||||
[ "$IN_TEST" = 1 ] || echo "$RUN_DIR" > "$(_marker)"
|
||||
|
||||
wait_for_ssh
|
||||
if [ "$IN_TEST" != 1 ]; then
|
||||
echo "== VM is up. Prepare it with: BB_TEST_DISTRO=$DISTRO $0 prereqs (or go straight to 'run') =="
|
||||
fi
|
||||
}
|
||||
|
||||
# Install install.sh's toolchain prerequisites (python3 + git + pipx) into the
|
||||
# running VM. This is what separates `test-ready` from `test`, and it is a
|
||||
# distinct sub-command so a manual up/prereqs/run/down cycle is possible.
|
||||
cmd_prereqs() {
|
||||
require_linux
|
||||
_load_run_dir || { echo "error: no running VM for $DISTRO; run '$0 up' first" >&2; return 1; }
|
||||
echo "== installing prerequisites (python3 + git + pipx) on $DISTRO =="
|
||||
# Runs via the guest login shell; PREREQ is a client-side table value.
|
||||
guest "${PREREQ[$DISTRO]}"
|
||||
}
|
||||
|
||||
cmd_run() {
|
||||
require_linux
|
||||
_load_run_dir || { echo "error: no running VM for $DISTRO; run '$0 up' first" >&2; return 1; }
|
||||
|
||||
local spec_env=""
|
||||
[ -n "${BOT_BOTTLE_INSTALL_SPEC:-}" ] \
|
||||
&& spec_env="BOT_BOTTLE_INSTALL_SPEC='$BOT_BOTTLE_INSTALL_SPEC' "
|
||||
|
||||
echo "== installing bot-bottle as ${SSH_USER[$DISTRO]} =="
|
||||
# Capture install.sh's exit code and full output rather than aborting on
|
||||
# non-zero: on a bare host a clean prerequisite *decline* is sound, so the
|
||||
# verdict step — not set -e — decides the outcome.
|
||||
local rc
|
||||
set +e
|
||||
if [ -n "${BB_TEST_INSTALL_URL:-}" ]; then
|
||||
guest "curl -fsSL '$BB_TEST_INSTALL_URL' | ${spec_env}sh" 2>&1 | tee "$RUN_DIR/install.log"
|
||||
else
|
||||
# Feed THIS checkout's install.sh in over stdin — the same `curl … | sh`
|
||||
# shape a real user runs, and nothing is staged in the guest to leak.
|
||||
guest "${spec_env}sh -s" < "$_REPO_ROOT/install.sh" 2>&1 | tee "$RUN_DIR/install.log"
|
||||
fi
|
||||
rc="${PIPESTATUS[0]}"
|
||||
set -e
|
||||
printf '%s\n' "$rc" > "$RUN_DIR/install.rc"
|
||||
echo "== install.sh exited $rc =="
|
||||
[ "$IN_TEST" = 1 ] \
|
||||
|| echo "== verdict anytime with: BB_TEST_DISTRO=$DISTRO $0 status =="
|
||||
}
|
||||
|
||||
# Quietly report whether a runnable bot-bottle entry point exists for the
|
||||
# guest user, checking the pipx/pip locations install.sh may leave off PATH.
|
||||
entry_point_runnable() {
|
||||
# shellcheck disable=SC2016 # expand in the GUEST shell.
|
||||
guest '
|
||||
for bb in "$HOME/.local/bin/bot-bottle" "$HOME/.bot-bottle/venv/bin/bot-bottle" "$(command -v bot-bottle 2>/dev/null)"; do
|
||||
[ -n "$bb" ] && [ -x "$bb" ] || continue
|
||||
# --help exits 0 before any DB/migration/network work; it is the
|
||||
# cheapest proof the package imports and the shim runs. (bot-bottle
|
||||
# has no --version: an unknown arg would die non-zero.)
|
||||
"$bb" --help >/dev/null 2>&1 && exit 0
|
||||
done
|
||||
exit 1
|
||||
' >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# `bot-bottle doctor` in the guest, classified. doctor's own exit code
|
||||
# conflates "is the install sound" with "is a backend ready to run a bottle" —
|
||||
# and inside a plain VM no backend can be ready (no nested KVM/Docker), so the
|
||||
# raw exit code is non-zero by design. This separates the two: an unhandled
|
||||
# traceback, or a missing python/config line, is an install defect and fails;
|
||||
# a not-ready backend is reported, not fatal (unless BB_TEST_REQUIRE_BACKEND=1,
|
||||
# for a nested-virt host that can actually satisfy it).
|
||||
doctor_in_guest() {
|
||||
local out rc=0 bad=0
|
||||
out="$(mktemp "${TMPDIR:-/tmp}/bb-doctor.XXXXXX")"
|
||||
# shellcheck disable=SC2016 # $HOME/$bb must expand in the GUEST shell.
|
||||
guest '
|
||||
for bb in bot-bottle "$HOME/.local/bin/bot-bottle" "$HOME/.bot-bottle/venv/bin/bot-bottle"; do
|
||||
if command -v "$bb" >/dev/null 2>&1; then
|
||||
case "$bb" in
|
||||
bot-bottle) : ;;
|
||||
*) echo " (not on PATH — running $bb directly, as install.sh advises)" ;;
|
||||
esac
|
||||
exec "$bb" doctor
|
||||
fi
|
||||
done
|
||||
echo " no bot-bottle entry point found for this user" >&2
|
||||
exit 1
|
||||
' >"$out" 2>&1 || rc=$?
|
||||
cat "$out"
|
||||
|
||||
# An unhandled exception is always an install/product defect, never an
|
||||
# environment fact — doctor's non-zero exit alone would not distinguish it.
|
||||
if grep -q 'Traceback (most recent call last)' "$out"; then
|
||||
echo " doctor crashed (traceback above) — a defect, not a missing prerequisite" >&2
|
||||
bad=1
|
||||
fi
|
||||
grep -qE '^ok: +python:' "$out" \
|
||||
|| { echo " doctor never reported a usable python" >&2; bad=1; }
|
||||
grep -qE '^ok: +config:' "$out" \
|
||||
|| { echo " doctor never reported a usable config dir" >&2; bad=1; }
|
||||
if [ "$rc" -ne 0 ] && ! grep -qE '^(fail|warn): +backend' "$out"; then
|
||||
echo " doctor failed for something other than backend readiness" >&2
|
||||
bad=1
|
||||
fi
|
||||
|
||||
local backend_ready=1
|
||||
grep -qE '^fail: +backend' "$out" && backend_ready=0
|
||||
rm -f "$out"
|
||||
|
||||
[ "$bad" -eq 0 ] || return 1
|
||||
if [ "${BB_TEST_REQUIRE_BACKEND:-0}" = "1" ] && [ "$backend_ready" -eq 0 ]; then
|
||||
echo " backend is not ready and BB_TEST_REQUIRE_BACKEND=1 — failing" >&2
|
||||
return 1
|
||||
fi
|
||||
if [ "$backend_ready" -eq 1 ]; then
|
||||
echo " doctor: install sound; a backend is ready"
|
||||
else
|
||||
echo " doctor: install sound; no backend ready (expected in a plain VM — install gate only)"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# The prerequisite-decline messages install.sh prints via die(). On a bare host
|
||||
# ANY of these means install.sh correctly refused rather than half-installing —
|
||||
# a sound outcome for `test`.
|
||||
PREREQ_ERR_RE='is required but was not found|or newer is required|git is required to install from|neither pipx nor a usable|externally managed \(PEP 668\)|is not on PATH'
|
||||
|
||||
# The verdict: is the install sound? Reads install.sh's captured exit code and
|
||||
# output (from cmd_run) plus the guest's resulting state.
|
||||
# - installed & runnable -> the verdict is doctor's soundness classification.
|
||||
# - no entry point, but a recognized prerequisite decline, and declines are
|
||||
# allowed (bare `test`, _REQUIRE_INSTALL=0) -> sound.
|
||||
# - anything else -> not sound.
|
||||
install_verdict() {
|
||||
local rc=""
|
||||
[ -f "$RUN_DIR/install.rc" ] && rc="$(cat "$RUN_DIR/install.rc")"
|
||||
|
||||
if [ "${rc:-1}" = 0 ] && entry_point_runnable; then
|
||||
echo "doctor (in guest):"
|
||||
doctor_in_guest
|
||||
return $?
|
||||
fi
|
||||
|
||||
if [ "${_REQUIRE_INSTALL:-0}" != "1" ] \
|
||||
&& [ -n "$rc" ] && [ "$rc" != 0 ] \
|
||||
&& [ -f "$RUN_DIR/install.log" ] \
|
||||
&& grep -Eiq "$PREREQ_ERR_RE" "$RUN_DIR/install.log"; then
|
||||
echo " install.sh declined with an actionable prerequisite error (rc=$rc)"
|
||||
echo " — sound on a bare host; run 'test-ready' (or 'prereqs') to install."
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ "${_REQUIRE_INSTALL:-0}" = "1" ]; then
|
||||
echo " prerequisites were provisioned, but install.sh left no runnable entry point (rc=${rc:-?})" >&2
|
||||
else
|
||||
echo " install.sh neither installed nor gave a recognized prerequisite error (rc=${rc:-?})" >&2
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
cmd_status() {
|
||||
require_linux
|
||||
if ! _load_run_dir; then
|
||||
echo "vm: no running VM for $DISTRO"
|
||||
return 0
|
||||
fi
|
||||
if [ -f "$(_pidfile)" ] && kill -0 "$(cat "$(_pidfile)")" 2>/dev/null; then
|
||||
echo "vm: $DISTRO running (pid $(cat "$(_pidfile)"), ssh :$SSH_PORT)"
|
||||
else
|
||||
echo "vm: $DISTRO run-dir present but QEMU not alive"
|
||||
return 1
|
||||
fi
|
||||
if install_verdict; then
|
||||
echo "OK[$DISTRO]: the install is sound"
|
||||
return 0
|
||||
fi
|
||||
echo "FAIL[$DISTRO]: the install is not sound (see above)" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
cmd_down() {
|
||||
require_linux
|
||||
if ! _load_run_dir; then
|
||||
echo "$DISTRO: nothing running"
|
||||
return 0
|
||||
fi
|
||||
if [ -f "$(_pidfile)" ]; then
|
||||
local pid; pid="$(cat "$(_pidfile)")"
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
kill "$pid" 2>/dev/null || true
|
||||
for _ in 1 2 3 4 5; do kill -0 "$pid" 2>/dev/null || break; sleep 1; done
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
# Deleting the overlay + seed is the reset; the read-only base stays cached.
|
||||
rm -f "$(_overlay)" "$(_seed)" "$(_ssh_key)" "$(_ssh_key).pub" \
|
||||
"$RUN_DIR/user-data" "$RUN_DIR/meta-data" \
|
||||
"$RUN_DIR/install.rc" "$RUN_DIR/install.log" "$(_serial)"
|
||||
# Only remove a scratch dir we created (leave a user-provided one alone).
|
||||
[ -n "${BB_TEST_RUN_DIR:-}" ] || rmdir "$RUN_DIR" 2>/dev/null || true
|
||||
rm -f "$(_marker)"
|
||||
echo "removed $DISTRO VM and its overlay — install surface is clean."
|
||||
}
|
||||
|
||||
cmd_ssh() {
|
||||
require_linux
|
||||
_load_run_dir || { echo "error: no running VM for $DISTRO" >&2; return 1; }
|
||||
local -a opts
|
||||
mapfile -d '' -t opts < <(_ssh_opts)
|
||||
exec ssh -t "${opts[@]}" "${SSH_USER[$DISTRO]}@127.0.0.1"
|
||||
}
|
||||
|
||||
# Teardown half of the test cycles, armed the moment the VM exists so a failure
|
||||
# or a Ctrl-C still leaves nothing running.
|
||||
_test_teardown() {
|
||||
local rc=$?
|
||||
trap - EXIT INT TERM
|
||||
if [ "${BB_TEST_KEEP:-0}" = "1" ]; then
|
||||
echo
|
||||
echo "== [$_STEPS/$_STEPS] down: SKIPPED (BB_TEST_KEEP=1) =="
|
||||
echo " VM still up; remove with: BB_TEST_DISTRO=$DISTRO $0 down"
|
||||
echo " ssh in with: BB_TEST_RUN_DIR=$RUN_DIR BB_TEST_DISTRO=$DISTRO $0 ssh"
|
||||
exit "$rc"
|
||||
fi
|
||||
echo
|
||||
echo "== [$_STEPS/$_STEPS] down =="
|
||||
cmd_down || rc=1
|
||||
if [ "$rc" -eq 0 ]; then
|
||||
echo
|
||||
echo "PASS[$DISTRO]: $_PASS_CLAIM"
|
||||
else
|
||||
echo
|
||||
echo "FAIL[$DISTRO]: see above (the VM was torn down regardless)." >&2
|
||||
fi
|
||||
exit "$rc"
|
||||
}
|
||||
|
||||
# The two variants differ only in whether the prerequisites get installed
|
||||
# before install.sh runs, which is exactly the question each one asks:
|
||||
#
|
||||
# test a bare host — assert install.sh is SOUND (installs cleanly, or
|
||||
# declines with an actionable prerequisite error).
|
||||
# test-ready prerequisites satisfied — assert install.sh actually LANDS.
|
||||
_test_cycle() {
|
||||
local with_prereqs="$1"
|
||||
require_linux; require_kvm; require_tools; known_distro
|
||||
IN_TEST=1
|
||||
RUN_DIR="$(mktemp -d "${TMPDIR:-/tmp}/bb-install-test.$DISTRO.XXXXXX")"
|
||||
|
||||
# Arm teardown BEFORE cmd_up: its wait_for_ssh can fail after QEMU is
|
||||
# already running (e.g. a guest that never opens SSH), and without the trap
|
||||
# in place that would leak the VM.
|
||||
trap _test_teardown EXIT INT TERM
|
||||
echo "== [1/$_STEPS] up ($DISTRO) =="
|
||||
cmd_up
|
||||
|
||||
local step=2
|
||||
if [ "$with_prereqs" = 1 ]; then
|
||||
echo
|
||||
echo "== [$step/$_STEPS] prereqs =="
|
||||
cmd_prereqs || { echo "error: could not install prerequisites (see above)." >&2; return 1; }
|
||||
step=$(( step + 1 ))
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "== [$step/$_STEPS] run =="
|
||||
cmd_run
|
||||
step=$(( step + 1 ))
|
||||
|
||||
echo
|
||||
echo "== [$step/$_STEPS] verdict =="
|
||||
# install.sh exits 0 even when doctor reports unmet prerequisites, so the
|
||||
# install succeeding is not the verdict — this is.
|
||||
cmd_status || {
|
||||
echo "error: the install is not sound for $DISTRO (see above)." >&2
|
||||
echo " re-run with BB_TEST_KEEP=1 to keep the VM and dig in." >&2
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
cmd_test() {
|
||||
_STEPS=4
|
||||
_PASS_CLAIM="on a bare $DISTRO host, install.sh behaves soundly."
|
||||
_test_cycle 0
|
||||
}
|
||||
|
||||
cmd_test_ready() {
|
||||
_STEPS=5
|
||||
_PASS_CLAIM="a $DISTRO host with prerequisites satisfied installs bot-bottle cleanly."
|
||||
# Prerequisites are provisioned, so a graceful decline is no longer an
|
||||
# acceptable outcome — the install must actually land.
|
||||
_REQUIRE_INSTALL=1
|
||||
_test_cycle 1
|
||||
}
|
||||
|
||||
cmd_test_all() {
|
||||
require_linux; require_kvm; require_tools
|
||||
local -a passed=() failed=()
|
||||
local port="$SSH_PORT"
|
||||
local script
|
||||
script="$_SCRIPT_DIR/$(basename "${BASH_SOURCE[0]}")"
|
||||
# Every distro × both variants. Each cell gets its own forwarded port so a
|
||||
# leftover from a prior cell can't collide, and its own subshell so one
|
||||
# cell's failure (or teardown trap) doesn't abort the matrix.
|
||||
for sub in test test-ready; do
|
||||
for d in "${ALL_DISTROS[@]}"; do
|
||||
echo
|
||||
echo "########################################################"
|
||||
echo "# $d ($sub)"
|
||||
echo "########################################################"
|
||||
if ( DISTRO="$d" SSH_PORT="$port" \
|
||||
BB_TEST_DISTRO="$d" BB_TEST_SSH_PORT="$port" \
|
||||
bash "$script" "$sub" ); then
|
||||
passed+=("$d/$sub")
|
||||
else
|
||||
failed+=("$d/$sub")
|
||||
fi
|
||||
port=$(( port + 1 ))
|
||||
done
|
||||
done
|
||||
echo
|
||||
echo "== matrix summary =="
|
||||
echo " PASS: ${passed[*]:-(none)}"
|
||||
echo " FAIL: ${failed[*]:-(none)}"
|
||||
[ "${#failed[@]}" -eq 0 ]
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
test) cmd_test ;;
|
||||
test-ready) cmd_test_ready ;;
|
||||
test-all) cmd_test_all ;;
|
||||
up) cmd_up ;;
|
||||
prereqs) cmd_prereqs ;;
|
||||
run) cmd_run ;;
|
||||
status) cmd_status ;;
|
||||
down) cmd_down ;;
|
||||
ssh) cmd_ssh ;;
|
||||
*) echo "usage: $0 {test|test-ready|test-all|up|prereqs|run|status|down|ssh} (distro via BB_TEST_DISTRO)" >&2; exit 2 ;;
|
||||
esac
|
||||
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Publish a qualified release and advance its matching channel."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from bot_bottle.release_qualification import (
|
||||
build_release_pointer,
|
||||
canonical_pointer,
|
||||
publish_qualification,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--tag", required=True)
|
||||
parser.add_argument("--source-commit", required=True)
|
||||
parser.add_argument("--workflow-run", required=True)
|
||||
parser.add_argument("--qualified-at", required=True)
|
||||
parser.add_argument("--output", type=Path, default=Path("release.json"))
|
||||
parser.add_argument("--allow-rollback", action="store_true")
|
||||
args = parser.parse_args()
|
||||
pointer = build_release_pointer(
|
||||
tag=args.tag,
|
||||
source_commit=args.source_commit,
|
||||
workflow_run=args.workflow_run,
|
||||
qualified_at=args.qualified_at,
|
||||
)
|
||||
args.output.write_bytes(canonical_pointer(pointer))
|
||||
release_url, channel_url = publish_qualification(
|
||||
pointer, allow_rollback=args.allow_rollback)
|
||||
print(json.dumps({
|
||||
"release": release_url,
|
||||
"channel": channel_url,
|
||||
}, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -10,6 +10,8 @@ Kept in sync with ``bot_bottle.resources.BUNDLED_RESOURCES`` — the
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
@@ -44,6 +46,18 @@ class _BundleResources(build_py):
|
||||
dst = pkg_resources / rel
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(src, dst)
|
||||
manifest_dest = Path(self.build_lib) / "bot_bottle" / "release-manifest.json"
|
||||
manifest_input = os.environ.get("BOT_BOTTLE_RELEASE_MANIFEST", "").strip()
|
||||
if manifest_input:
|
||||
shutil.copy2(manifest_input, manifest_dest)
|
||||
else:
|
||||
# Wheels built ad hoc retain the contributor/install.sh local-build
|
||||
# behavior. The release workflow always replaces this marker with
|
||||
# validated immutable artifact identities.
|
||||
manifest_dest.write_text(
|
||||
json.dumps({"schema": 1, "development": True}) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
setup(cmdclass={"build_py": _BundleResources})
|
||||
|
||||
@@ -46,8 +46,8 @@ class TestDockerInfraService(unittest.TestCase):
|
||||
url = self.svc.ensure_running()
|
||||
self.assertEqual("http://127.0.0.1:8099", url)
|
||||
# Both images built; the control plane comes up before the gateway.
|
||||
self.orch.ensure_built.assert_called_once()
|
||||
self.gw.ensure_built.assert_called_once()
|
||||
self.orch.ensure_available.assert_called_once()
|
||||
self.gw.ensure_available.assert_called_once()
|
||||
self.orch.ensure_running.assert_called_once()
|
||||
|
||||
def test_ensure_running_connects_gateway_with_minted_token(self) -> None:
|
||||
@@ -76,6 +76,19 @@ class TestDockerInfraService(unittest.TestCase):
|
||||
self.assertEqual("registry-volume", svc.orchestrator()._root_mount_source)
|
||||
self.assertEqual("ca-volume", svc.gateway()._ca_mount_source)
|
||||
|
||||
def test_packaged_images_disable_local_infra_dockerfiles(self) -> None:
|
||||
refs = [
|
||||
("registry/orchestrator@sha256:" + "a" * 64, False),
|
||||
("registry/gateway@sha256:" + "b" * 64, False),
|
||||
]
|
||||
with patch(
|
||||
"bot_bottle.backend.docker.infra.release_manifest.oci_image",
|
||||
side_effect=refs,
|
||||
):
|
||||
svc = DockerInfraService()
|
||||
self.assertIsNone(svc.orchestrator()._dockerfile)
|
||||
self.assertIsNone(svc.gateway()._dockerfile)
|
||||
|
||||
def test_host_path_and_named_root_mount_are_mutually_exclusive(self) -> None:
|
||||
with self.assertRaisesRegex(ValueError, "host_root or root_mount_source"):
|
||||
DockerInfraService(
|
||||
|
||||
@@ -231,11 +231,12 @@ class TestDockerOrchestrator(unittest.TestCase):
|
||||
with self.assertRaises(GatewayError):
|
||||
self.orch.ensure_built()
|
||||
|
||||
def test_ensure_built_noop_without_dockerfile(self) -> None:
|
||||
orch = DockerOrchestrator(dockerfile=None)
|
||||
with patch(_RUN) as run:
|
||||
def test_ensure_built_pulls_without_dockerfile(self) -> None:
|
||||
orch = DockerOrchestrator("registry/orchestrator@sha256:" + "a" * 64,
|
||||
dockerfile=None)
|
||||
with patch(_RUN, return_value=_proc()) as run:
|
||||
orch.ensure_built()
|
||||
run.assert_not_called()
|
||||
run.assert_called_once_with(["docker", "pull", orch.image_ref])
|
||||
|
||||
def test_is_running_reads_docker_ps(self) -> None:
|
||||
with patch(_RUN, return_value=_proc(stdout=ORCHESTRATOR_NAME)):
|
||||
|
||||
@@ -90,6 +90,20 @@ class TestBuildAgentRootfsDir(unittest.TestCase):
|
||||
second = image_builder._rootfs_digest(self.dockerfile)
|
||||
self.assertNotEqual(first, second)
|
||||
|
||||
def test_pinned_image_is_pulled_and_cached(self):
|
||||
image = f"registry.example/agent@sha256:{'a' * 64}"
|
||||
with patch.object(image_builder.util, "cache_dir", return_value=self.cache), \
|
||||
patch.object(image_builder, "_pull_in_infra") as pull, \
|
||||
patch.object(image_builder.util, "inject_guest_boot"):
|
||||
first = image_builder.acquire_agent_image_rootfs_dir(image)
|
||||
second = image_builder.acquire_agent_image_rootfs_dir(image)
|
||||
pull.assert_called_once()
|
||||
self.assertEqual(first, second)
|
||||
|
||||
def test_mutable_image_cannot_use_prebuilt_path(self):
|
||||
with self.assertRaises(SystemExit):
|
||||
image_builder.acquire_agent_image_rootfs_dir("agent:latest")
|
||||
|
||||
|
||||
class TestSmokeTest(unittest.TestCase):
|
||||
def test_buildah_receives_centralized_image_build_args(self):
|
||||
|
||||
@@ -53,7 +53,7 @@ class TestEnsureRunning(unittest.TestCase):
|
||||
patch.object(infra_vm, "_health_ok", return_value=True), \
|
||||
patch.object(infra_vm, "_pidfile_alive", return_value=True), \
|
||||
patch.object(infra_vm, "stop") as stop, \
|
||||
patch.object(infra_vm, "ensure_built") as built:
|
||||
patch.object(infra_vm, "ensure_available") as built:
|
||||
url = FirecrackerInfraService().ensure_running()
|
||||
stop.assert_not_called()
|
||||
built.assert_not_called()
|
||||
@@ -72,7 +72,7 @@ class TestEnsureRunning(unittest.TestCase):
|
||||
patch.object(infra_vm, "_health_ok", return_value=True), \
|
||||
patch.object(infra_vm, "_pidfile_alive", return_value=True), \
|
||||
patch.object(infra_vm, "stop") as stop, \
|
||||
patch.object(infra_vm, "ensure_built"), \
|
||||
patch.object(infra_vm, "ensure_available"), \
|
||||
patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls:
|
||||
orch_cls.return_value.gateway_url.return_value = "http://10.243.255.1:8099"
|
||||
orch_cls.return_value.mint_gateway_token.return_value = "gw.jwt"
|
||||
@@ -98,7 +98,7 @@ class TestEnsureRunning(unittest.TestCase):
|
||||
patch.object(infra_vm, "_health_ok", return_value=True), \
|
||||
patch.object(infra_vm, "_pidfile_alive", return_value=False), \
|
||||
patch.object(infra_vm, "stop") as stop, \
|
||||
patch.object(infra_vm, "ensure_built"), \
|
||||
patch.object(infra_vm, "ensure_available"), \
|
||||
patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls:
|
||||
FirecrackerInfraService().ensure_running()
|
||||
stop.assert_called_once()
|
||||
@@ -111,7 +111,7 @@ class TestEnsureRunning(unittest.TestCase):
|
||||
patch.object(infra_vm, "expected_version", return_value="v-current"), \
|
||||
patch.object(infra_vm, "_health_ok", return_value=False), \
|
||||
patch.object(infra_vm, "stop") as stop, \
|
||||
patch.object(infra_vm, "ensure_built") as built, \
|
||||
patch.object(infra_vm, "ensure_available") as built, \
|
||||
patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls:
|
||||
FirecrackerInfraService().ensure_running()
|
||||
stop.assert_called_once() # clear stale VMs first
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.generate_release_manifest import main
|
||||
|
||||
|
||||
class TestGenerateReleaseManifest(unittest.TestCase):
|
||||
def test_writes_validated_manifest(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
output = Path(tmp) / "release.json"
|
||||
rc = main([
|
||||
"--source-commit", "1" * 40,
|
||||
"--orchestrator-image",
|
||||
"registry/orchestrator@sha256:" + "a" * 64,
|
||||
"--gateway-image", "registry/gateway@sha256:" + "b" * 64,
|
||||
"--agent-claude-image", "registry/claude@sha256:" + "e" * 64,
|
||||
"--agent-codex-image", "registry/codex@sha256:" + "f" * 64,
|
||||
"--agent-pi-image", "registry/pi@sha256:" + "0" * 64,
|
||||
"--firecracker-orchestrator-version", "orch-v1",
|
||||
"--firecracker-orchestrator-sha256", "c" * 64,
|
||||
"--firecracker-gateway-version", "gateway-v1",
|
||||
"--firecracker-gateway-sha256", "d" * 64,
|
||||
"--output", str(output),
|
||||
])
|
||||
self.assertEqual(0, rc)
|
||||
data = json.loads(output.read_text(encoding="utf-8"))
|
||||
self.assertEqual("orch-v1", data["firecracker"]["orchestrator"]["version"])
|
||||
@@ -95,6 +95,21 @@ class TestInstallScript(unittest.TestCase):
|
||||
# Tests / local installs point BOT_BOTTLE_INSTALL_SPEC at a checkout.
|
||||
self.assertIn("BOT_BOTTLE_INSTALL_SPEC", self.text)
|
||||
|
||||
def test_published_install_selectors_are_mutually_exclusive(self):
|
||||
self.assertIn("BOT_BOTTLE_CHANNEL", self.text)
|
||||
self.assertIn("BOT_BOTTLE_VERSION", self.text)
|
||||
self.assertIn("BOT_BOTTLE_REF", self.text)
|
||||
self.assertIn("are mutually exclusive", self.text)
|
||||
|
||||
def test_commit_snapshot_warns_and_wheel_is_checksum_verified(self):
|
||||
self.assertIn("may not have passed release qualification", self.text)
|
||||
self.assertIn("downloaded wheel checksum mismatch", self.text)
|
||||
self.assertIn("hashlib.sha256()", self.text)
|
||||
|
||||
def test_default_install_uses_production_channel(self):
|
||||
self.assertIn('${BOT_BOTTLE_CHANNEL:-production}', self.text)
|
||||
self.assertIn("bot-bottle-channels", self.text)
|
||||
|
||||
def test_requires_git_for_git_specs(self):
|
||||
# A git+ / .git spec (the default) shells out to git; the script must
|
||||
# gate on it rather than failing opaquely inside pipx/pip.
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
"""Unit: how the CLI tells users to re-run it under sudo.
|
||||
|
||||
`sudo bot-bottle …` is wrong for the users who followed the documented
|
||||
install: sudo's secure_path excludes ~/.local/bin, where both pipx and
|
||||
install.sh put the entry point. These lock in the absolute-path form.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import io
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from bot_bottle import invocation
|
||||
|
||||
|
||||
class TestSelfPath(unittest.TestCase):
|
||||
def test_absolute_argv0_is_used_as_is(self):
|
||||
with mock.patch.object(invocation.sys, "argv", ["/opt/venv/bin/bot-bottle"]):
|
||||
self.assertEqual("/opt/venv/bin/bot-bottle", invocation.self_path())
|
||||
|
||||
def test_bare_name_is_resolved_through_path(self):
|
||||
# The case that matters: invoked as `bot-bottle`, installed in a
|
||||
# directory sudo would drop.
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
entry = Path(d, "bot-bottle")
|
||||
entry.write_text("#!/bin/sh\n")
|
||||
entry.chmod(0o755)
|
||||
with mock.patch.object(invocation.sys, "argv", ["bot-bottle"]), \
|
||||
mock.patch.dict(os.environ, {"PATH": d}):
|
||||
self.assertEqual(str(entry), invocation.self_path())
|
||||
|
||||
def test_relative_path_is_made_absolute(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
entry = Path(d, "bot-bottle")
|
||||
entry.write_text("#!/bin/sh\n")
|
||||
entry.chmod(0o755)
|
||||
cwd = os.getcwd()
|
||||
try:
|
||||
os.chdir(d)
|
||||
with mock.patch.object(invocation.sys, "argv", ["./bot-bottle"]):
|
||||
self.assertTrue(os.path.isabs(invocation.self_path()))
|
||||
finally:
|
||||
os.chdir(cwd)
|
||||
|
||||
def test_unresolvable_entry_point_falls_back_to_the_name(self):
|
||||
# `python -m`-style invocation, or an argv[0] that no longer exists.
|
||||
# A slightly wrong hint beats a traceback raised while reporting some
|
||||
# unrelated problem.
|
||||
with mock.patch.object(invocation.sys, "argv", ["/nonexistent/gone"]), \
|
||||
mock.patch.object(invocation.shutil, "which", return_value=None):
|
||||
self.assertEqual("/nonexistent/gone", invocation.self_path())
|
||||
with mock.patch.object(invocation.sys, "argv", [""]), \
|
||||
mock.patch.object(invocation.shutil, "which", return_value=None):
|
||||
self.assertEqual("bot-bottle", invocation.self_path())
|
||||
|
||||
|
||||
class TestSudoCommand(unittest.TestCase):
|
||||
def test_names_an_absolute_path_not_the_bare_command(self):
|
||||
with mock.patch.object(invocation, "self_path",
|
||||
return_value="/home/u/.local/bin/bot-bottle"):
|
||||
cmd = invocation.sudo_command("backend", "setup", "--backend=firecracker")
|
||||
self.assertEqual(
|
||||
"sudo /home/u/.local/bin/bot-bottle backend setup --backend=firecracker",
|
||||
cmd,
|
||||
)
|
||||
# The regression this exists to prevent.
|
||||
self.assertNotIn("sudo bot-bottle", cmd)
|
||||
|
||||
|
||||
class TestFirecrackerSetupUsesIt(unittest.TestCase):
|
||||
def test_root_reinvocation_hint_names_an_absolute_path(self):
|
||||
# The message that prompted all this. Drive the real code path rather
|
||||
# than scanning the source, which would also match the comment
|
||||
# explaining why the bare form is wrong.
|
||||
from bot_bottle.backend.firecracker import setup as fc_setup
|
||||
|
||||
err, out = io.StringIO(), io.StringIO()
|
||||
with mock.patch.object(fc_setup.os, "geteuid", return_value=501), \
|
||||
mock.patch.object(fc_setup.invocation, "self_path",
|
||||
return_value="/home/u/.local/bin/bot-bottle"), \
|
||||
contextlib.redirect_stderr(err), contextlib.redirect_stdout(out):
|
||||
fc_setup._setup_systemd()
|
||||
|
||||
printed = err.getvalue()
|
||||
self.assertIn(
|
||||
"sudo /home/u/.local/bin/bot-bottle backend setup --backend=firecracker",
|
||||
printed,
|
||||
)
|
||||
self.assertNotIn("sudo bot-bottle", printed)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -33,8 +33,8 @@ class TestInfraEnsureRunning(unittest.TestCase):
|
||||
with patch(f"{_INFRA}.ensure_networks") as nets:
|
||||
url = self.svc.ensure_running()
|
||||
nets.assert_called_once()
|
||||
self.orch.ensure_built.assert_called_once()
|
||||
self.gw.ensure_built.assert_called_once()
|
||||
self.orch.ensure_available.assert_called_once()
|
||||
self.gw.ensure_available.assert_called_once()
|
||||
self.orch.ensure_running.assert_called_once()
|
||||
# Returns the host control-plane URL (the InfraService contract).
|
||||
self.assertEqual("http://192.168.128.2:8099", url)
|
||||
@@ -51,6 +51,18 @@ class TestInfraEnsureRunning(unittest.TestCase):
|
||||
self.assertTrue(self.svc.is_healthy())
|
||||
self.orch.is_healthy.assert_called_once()
|
||||
|
||||
def test_packaged_images_disable_local_infra_builds(self) -> None:
|
||||
refs = [
|
||||
("registry/orchestrator@sha256:" + "a" * 64, False),
|
||||
("registry/gateway@sha256:" + "b" * 64, False),
|
||||
]
|
||||
with patch(
|
||||
f"{_INFRA}.release_manifest.oci_image", side_effect=refs,
|
||||
):
|
||||
svc = MacosInfraService(repo_root=Path("/r"))
|
||||
self.assertFalse(svc.orchestrator()._local_build)
|
||||
self.assertFalse(svc.gateway()._local_build)
|
||||
|
||||
|
||||
class TestCaCertPem(unittest.TestCase):
|
||||
def test_delegates_to_the_gateway_service(self) -> None:
|
||||
|
||||
@@ -30,14 +30,16 @@ def _spec(src: str, tgt: str, readonly: bool = False) -> str:
|
||||
|
||||
|
||||
class TestMacosOrchestratorRun(unittest.TestCase):
|
||||
def _run(self) -> list[str]:
|
||||
def _run(self, *, local_build: bool = True) -> list[str]:
|
||||
run = Mock(return_value=_ok())
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
|
||||
mod.dns_server.return_value = "1.1.1.1"
|
||||
mod.bind_mount_spec.side_effect = _spec
|
||||
mod.run_container_argv = run
|
||||
MacosOrchestrator(repo_root=Path("/r"))._run_container("h1")
|
||||
MacosOrchestrator(
|
||||
repo_root=Path("/r"), local_build=local_build,
|
||||
)._run_container("h1")
|
||||
return run.call_args.args[0]
|
||||
|
||||
def test_runs_on_the_control_network_only(self) -> None:
|
||||
@@ -63,6 +65,11 @@ class TestMacosOrchestratorRun(unittest.TestCase):
|
||||
def test_source_hash_is_labelled_for_recreate(self) -> None:
|
||||
self.assertIn("BOT_BOTTLE_SOURCE_HASH=h1", self._run())
|
||||
|
||||
def test_packaged_image_does_not_overlay_host_source(self) -> None:
|
||||
argv = self._run(local_build=False)
|
||||
self.assertNotIn("--mount", argv)
|
||||
self.assertFalse(any(arg.startswith("PYTHONPATH=") for arg in argv))
|
||||
|
||||
def test_start_failure_raises(self) -> None:
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
|
||||
@@ -145,6 +152,14 @@ class TestMacosOrchestratorBuildStop(unittest.TestCase):
|
||||
_args, kwargs = mod.build_image.call_args
|
||||
self.assertEqual("Dockerfile.orchestrator", kwargs["dockerfile"])
|
||||
|
||||
def test_ensure_built_pulls_packaged_image(self) -> None:
|
||||
image = "registry/orchestrator@sha256:" + "a" * 64
|
||||
orch = MacosOrchestrator(image, local_build=False)
|
||||
with patch(f"{_ORCH}.container_mod") as mod:
|
||||
orch.ensure_built()
|
||||
mod.pull_image.assert_called_once_with(image)
|
||||
mod.build_image.assert_not_called()
|
||||
|
||||
def test_stop_removes_the_container(self) -> None:
|
||||
orch = MacosOrchestrator()
|
||||
with patch(f"{_ORCH}.container_mod") as mod:
|
||||
|
||||
@@ -427,11 +427,11 @@ class TestDockerGatewayBuild(unittest.TestCase):
|
||||
builds = [c for c in calls if c[:2] == ["docker", "build"]]
|
||||
self.assertIn("--no-cache", builds[0])
|
||||
|
||||
def test_ensure_built_noop_when_no_dockerfile(self) -> None:
|
||||
sc = DockerGateway("busybox", dockerfile=None)
|
||||
with patch(_RUN_DOCKER) as m:
|
||||
def test_ensure_built_pulls_when_no_dockerfile(self) -> None:
|
||||
sc = DockerGateway("registry/gateway@sha256:" + "a" * 64, dockerfile=None)
|
||||
with patch(_RUN_DOCKER, return_value=_proc()) as m:
|
||||
sc.ensure_built()
|
||||
m.assert_not_called()
|
||||
m.assert_called_once_with(["docker", "pull", sc.image_ref])
|
||||
|
||||
def test_ensure_built_raises_on_build_failure(self) -> None:
|
||||
with patch(_RUN_DOCKER, return_value=_proc(returncode=1, stderr="build boom")):
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from bot_bottle.release_bundle import (
|
||||
build_bundle_index,
|
||||
canonical_bytes,
|
||||
parse_bundle_index,
|
||||
)
|
||||
from bot_bottle.release_manifest import ReleaseManifestError
|
||||
from tests.unit.test_release_manifest import manifest
|
||||
|
||||
|
||||
class TestReleaseBundle(unittest.TestCase):
|
||||
def test_builds_commit_addressed_index(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
wheel = Path(tmp) / "bot_bottle-0.1.0-py3-none-any.whl"
|
||||
wheel.write_bytes(b"wheel")
|
||||
index = build_bundle_index(
|
||||
manifest(),
|
||||
wheel=wheel,
|
||||
wheel_url=f"https://packages.example/{wheel.name}",
|
||||
workflow_run="actions/123",
|
||||
published_at="2026-07-27T00:00:00Z",
|
||||
)
|
||||
self.assertEqual("1" * 40, index["source_commit"])
|
||||
self.assertEqual(64, len(index["wheel"]["sha256"]))
|
||||
self.assertEqual([], index["qualifications"])
|
||||
self.assertIs(index, parse_bundle_index(index))
|
||||
self.assertTrue(canonical_bytes(index).endswith(b"\n"))
|
||||
|
||||
def test_rejects_mutable_or_mismatched_wheel_url(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
wheel = Path(tmp) / "release.whl"
|
||||
wheel.write_bytes(b"wheel")
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "HTTPS"):
|
||||
build_bundle_index(
|
||||
manifest(),
|
||||
wheel=wheel,
|
||||
wheel_url="http://packages.example/other.whl",
|
||||
workflow_run="actions/123",
|
||||
published_at="now",
|
||||
)
|
||||
|
||||
def test_rejects_index_with_changed_runtime_identity(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
wheel = Path(tmp) / "release.whl"
|
||||
wheel.write_bytes(b"wheel")
|
||||
index = build_bundle_index(
|
||||
manifest(),
|
||||
wheel=wheel,
|
||||
wheel_url=f"https://packages.example/{wheel.name}",
|
||||
workflow_run="actions/123",
|
||||
published_at="now",
|
||||
)
|
||||
index["oci"]["gateway"] = "registry/gateway:latest"
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "digest-pinned"):
|
||||
parse_bundle_index(index)
|
||||
@@ -0,0 +1,129 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from bot_bottle.release_manifest import (
|
||||
ReleaseManifestError,
|
||||
oci_image,
|
||||
packaged_manifest,
|
||||
parse_manifest,
|
||||
)
|
||||
|
||||
|
||||
def manifest() -> dict[str, object]:
|
||||
return {
|
||||
"schema": 1,
|
||||
"source_commit": "1" * 40,
|
||||
"oci": {
|
||||
"orchestrator": "registry/orchestrator@sha256:" + "a" * 64,
|
||||
"gateway": "registry/gateway@sha256:" + "b" * 64,
|
||||
"agent_claude": "registry/agent-claude@sha256:" + "e" * 64,
|
||||
"agent_codex": "registry/agent-codex@sha256:" + "f" * 64,
|
||||
"agent_pi": "registry/agent-pi@sha256:" + "0" * 64,
|
||||
},
|
||||
"firecracker": {
|
||||
"orchestrator": {"version": "orch-v1", "sha256": "c" * 64},
|
||||
"gateway": {"version": "gateway-v1", "sha256": "d" * 64},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class TestParseManifest(unittest.TestCase):
|
||||
def test_parses_all_backend_artifacts(self) -> None:
|
||||
parsed = parse_manifest(manifest())
|
||||
self.assertTrue(parsed.orchestrator_image.endswith("a" * 64))
|
||||
self.assertEqual("orch-v1", parsed.firecracker_orchestrator.version)
|
||||
self.assertTrue(parsed.agent_images["codex"].endswith("f" * 64))
|
||||
|
||||
def test_rejects_mutable_oci_reference(self) -> None:
|
||||
data = manifest()
|
||||
assert isinstance(data["oci"], dict)
|
||||
data["oci"]["orchestrator"] = "registry/orchestrator:latest"
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "digest-pinned"):
|
||||
parse_manifest(data)
|
||||
|
||||
def test_rejects_malformed_firecracker_checksum(self) -> None:
|
||||
data = manifest()
|
||||
assert isinstance(data["firecracker"], dict)
|
||||
artifact = data["firecracker"]["gateway"]
|
||||
assert isinstance(artifact, dict)
|
||||
artifact["sha256"] = "nope"
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "64 lowercase hex"):
|
||||
parse_manifest(data)
|
||||
|
||||
def test_rejects_missing_manifest_sections(self) -> None:
|
||||
cases = (
|
||||
(None, "schema 1"),
|
||||
({"schema": 1}, "source_commit"),
|
||||
(
|
||||
{"schema": 1, "source_commit": "1" * 40},
|
||||
"oci must be an object",
|
||||
),
|
||||
)
|
||||
for data, message in cases:
|
||||
with self.subTest(message=message):
|
||||
with self.assertRaisesRegex(ReleaseManifestError, message):
|
||||
parse_manifest(data)
|
||||
|
||||
def test_rejects_malformed_firecracker_artifact(self) -> None:
|
||||
data = manifest()
|
||||
assert isinstance(data["firecracker"], dict)
|
||||
data["firecracker"]["orchestrator"] = None
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "must be an object"):
|
||||
parse_manifest(data)
|
||||
|
||||
data["firecracker"]["orchestrator"] = {
|
||||
"version": " ",
|
||||
"sha256": "c" * 64,
|
||||
}
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "version must be non-empty"):
|
||||
parse_manifest(data)
|
||||
|
||||
|
||||
class TestImageSelection(unittest.TestCase):
|
||||
def test_development_marker_selects_local_builds(self) -> None:
|
||||
with patch(
|
||||
"bot_bottle.release_manifest.resources.is_source_checkout",
|
||||
return_value=False,
|
||||
), patch(
|
||||
"bot_bottle.release_manifest.manifest_path",
|
||||
) as path:
|
||||
path.return_value.read_text.return_value = (
|
||||
'{"schema": 1, "development": true}')
|
||||
self.assertIsNone(packaged_manifest())
|
||||
|
||||
def test_packaged_install_uses_manifest_reference_without_build(self) -> None:
|
||||
parsed = parse_manifest(manifest())
|
||||
with patch(
|
||||
"bot_bottle.release_manifest.packaged_manifest", return_value=parsed,
|
||||
), patch(
|
||||
"bot_bottle.release_manifest.local_build_requested", return_value=False,
|
||||
), patch.dict("os.environ", {}, clear=True):
|
||||
ref, local = oci_image("orchestrator", "local:latest")
|
||||
self.assertEqual(parsed.orchestrator_image, ref)
|
||||
self.assertFalse(local)
|
||||
|
||||
def test_packaged_install_selects_agent_reference(self) -> None:
|
||||
parsed = parse_manifest(manifest())
|
||||
with patch(
|
||||
"bot_bottle.release_manifest.packaged_manifest", return_value=parsed,
|
||||
), patch(
|
||||
"bot_bottle.release_manifest.local_build_requested", return_value=False,
|
||||
), patch.dict("os.environ", {}, clear=True):
|
||||
ref, local = oci_image("agent_codex", "local:latest")
|
||||
self.assertEqual(parsed.agent_images["codex"], ref)
|
||||
self.assertFalse(local)
|
||||
|
||||
def test_mutable_override_fails_outside_local_mode(self) -> None:
|
||||
parsed = parse_manifest(manifest())
|
||||
with patch(
|
||||
"bot_bottle.release_manifest.packaged_manifest", return_value=parsed,
|
||||
), patch(
|
||||
"bot_bottle.release_manifest.local_build_requested", return_value=False,
|
||||
), patch.dict(
|
||||
"os.environ", {"BOT_BOTTLE_ORCHESTRATOR_IMAGE": "local:latest"},
|
||||
clear=True,
|
||||
):
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "digest-pinned"):
|
||||
oci_image("orchestrator", "fallback:latest")
|
||||
@@ -0,0 +1,163 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
import urllib.error
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from bot_bottle.release_bundle import build_bundle_index, canonical_bytes
|
||||
from bot_bottle.release_manifest import ReleaseManifestError
|
||||
from bot_bottle.release_publish import (
|
||||
bundle_url,
|
||||
remote_bytes,
|
||||
remote_sha256,
|
||||
request,
|
||||
upload as publish_artifact,
|
||||
publish_bundle,
|
||||
)
|
||||
from tests.unit.test_release_manifest import manifest
|
||||
|
||||
|
||||
class TestReleasePublish(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
env = {
|
||||
"BOT_BOTTLE_RELEASE_BASE": "https://packages.example",
|
||||
"BOT_BOTTLE_RELEASE_OWNER": "owner",
|
||||
"BOT_BOTTLE_RELEASE_TOKEN": "token",
|
||||
}
|
||||
self.env = patch.dict("os.environ", env, clear=True)
|
||||
self.env.start()
|
||||
self.addCleanup(self.env.stop)
|
||||
|
||||
def bundle(self, root: Path) -> tuple[dict[str, object], Path]:
|
||||
wheel = root / "bot_bottle-0.1.0-py3-none-any.whl"
|
||||
wheel.write_bytes(b"wheel")
|
||||
index = build_bundle_index(
|
||||
manifest(),
|
||||
wheel=wheel,
|
||||
wheel_url=bundle_url("1" * 40, wheel.name),
|
||||
workflow_run="actions/123",
|
||||
published_at="2026-07-27T00:00:00Z",
|
||||
)
|
||||
return index, wheel
|
||||
|
||||
@staticmethod
|
||||
def wheel_sha(index: dict[str, object]) -> str:
|
||||
wheel = index["wheel"]
|
||||
assert isinstance(wheel, dict)
|
||||
digest = wheel["sha256"]
|
||||
assert isinstance(digest, str)
|
||||
return digest
|
||||
|
||||
def test_publishes_wheel_then_index(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
index, wheel = self.bundle(Path(tmp))
|
||||
with patch(
|
||||
"bot_bottle.release_publish.remote_bytes", return_value=None,
|
||||
), patch(
|
||||
"bot_bottle.release_publish.remote_sha256", return_value=None,
|
||||
), patch("bot_bottle.release_publish.upload") as upload:
|
||||
result = publish_bundle(index, wheel)
|
||||
self.assertTrue(result.endswith("/bundle-index.json"))
|
||||
self.assertEqual(2, upload.call_count)
|
||||
self.assertIs(wheel, upload.call_args_list[0].args[1])
|
||||
self.assertEqual(canonical_bytes(index), upload.call_args_list[1].args[1])
|
||||
|
||||
def test_exact_existing_bundle_is_idempotent(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
index, wheel = self.bundle(Path(tmp))
|
||||
with patch(
|
||||
"bot_bottle.release_publish.remote_bytes",
|
||||
return_value=canonical_bytes(index),
|
||||
), patch(
|
||||
"bot_bottle.release_publish.remote_sha256",
|
||||
return_value=self.wheel_sha(index),
|
||||
), patch("bot_bottle.release_publish.upload") as upload:
|
||||
publish_bundle(index, wheel)
|
||||
upload.assert_not_called()
|
||||
|
||||
def test_rejects_partial_publication(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
index, wheel = self.bundle(Path(tmp))
|
||||
with patch(
|
||||
"bot_bottle.release_publish.remote_bytes", return_value=None,
|
||||
), patch(
|
||||
"bot_bottle.release_publish.remote_sha256",
|
||||
return_value=self.wheel_sha(index),
|
||||
):
|
||||
with self.assertRaisesRegex(
|
||||
ReleaseManifestError, "administrative cleanup",
|
||||
):
|
||||
publish_bundle(index, wheel)
|
||||
|
||||
def test_rejects_existing_bundle_with_different_index(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
index, wheel = self.bundle(Path(tmp))
|
||||
with patch(
|
||||
"bot_bottle.release_publish.remote_bytes", return_value=b"other",
|
||||
), patch(
|
||||
"bot_bottle.release_publish.remote_sha256",
|
||||
return_value=self.wheel_sha(index),
|
||||
):
|
||||
with self.assertRaisesRegex(
|
||||
ReleaseManifestError, "already differs",
|
||||
):
|
||||
publish_bundle(index, wheel)
|
||||
|
||||
def test_request_applies_auth_and_length(self) -> None:
|
||||
result = request(
|
||||
"https://packages.example/file",
|
||||
method="PUT",
|
||||
data=b"body",
|
||||
length=4,
|
||||
)
|
||||
self.assertEqual(result.get_method(), "PUT")
|
||||
self.assertEqual(result.get_header("Authorization"), "token token")
|
||||
self.assertEqual(result.get_header("Content-length"), "4")
|
||||
|
||||
@patch("bot_bottle.release_publish.urllib.request.urlopen")
|
||||
def test_remote_helpers_read_response(self, urlopen: MagicMock) -> None:
|
||||
response = urlopen.return_value.__enter__.return_value
|
||||
response.read.side_effect = [b"body"]
|
||||
self.assertEqual(
|
||||
remote_bytes("https://packages.example/file"), b"body")
|
||||
response.read.side_effect = [b"body", b""]
|
||||
self.assertEqual(
|
||||
remote_sha256("https://packages.example/file"),
|
||||
"230d8358dc8e8890b4c58deeb62912ee2"
|
||||
"f20357ae92a5cc861b98e68fe31acb5",
|
||||
)
|
||||
|
||||
@patch("bot_bottle.release_publish.urllib.request.urlopen")
|
||||
def test_remote_helpers_handle_not_found(self, urlopen: MagicMock) -> None:
|
||||
urlopen.side_effect = urllib.error.HTTPError(
|
||||
"url", 404, "missing", MagicMock(), None)
|
||||
self.assertIsNone(remote_bytes("https://packages.example/file"))
|
||||
self.assertIsNone(remote_sha256("https://packages.example/file"))
|
||||
|
||||
@patch("bot_bottle.release_publish.urllib.request.urlopen")
|
||||
def test_remote_helpers_report_registry_errors(
|
||||
self, urlopen: MagicMock,
|
||||
) -> None:
|
||||
urlopen.side_effect = urllib.error.URLError("offline")
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "offline"):
|
||||
remote_bytes("https://packages.example/file")
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "offline"):
|
||||
remote_sha256("https://packages.example/file")
|
||||
|
||||
@patch("bot_bottle.release_publish.urllib.request.urlopen")
|
||||
def test_upload_supports_bytes_and_files(self, urlopen: MagicMock) -> None:
|
||||
urlopen.return_value.__enter__.return_value = MagicMock()
|
||||
publish_artifact("https://packages.example/bytes", b"body")
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
source = Path(tmp) / "artifact"
|
||||
source.write_bytes(b"artifact")
|
||||
publish_artifact("https://packages.example/file", source)
|
||||
self.assertEqual(urlopen.call_count, 2)
|
||||
|
||||
@patch("bot_bottle.release_publish.urllib.request.urlopen")
|
||||
def test_upload_reports_failure(self, urlopen: MagicMock) -> None:
|
||||
urlopen.side_effect = urllib.error.URLError("offline")
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "publishing"):
|
||||
publish_artifact("https://packages.example/file", b"body")
|
||||
@@ -0,0 +1,270 @@
|
||||
"""Tests for release qualification and channel promotion."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import unittest
|
||||
import urllib.error
|
||||
from email.message import Message
|
||||
from unittest import mock
|
||||
|
||||
from bot_bottle.release_manifest import ReleaseManifestError
|
||||
from bot_bottle.release_qualification import (
|
||||
build_release_pointer,
|
||||
canonical_pointer,
|
||||
delete,
|
||||
parse_release_pointer,
|
||||
publish_qualification,
|
||||
release_channel,
|
||||
release_version,
|
||||
)
|
||||
|
||||
COMMIT = "a" * 40
|
||||
|
||||
|
||||
def bundle(source_commit: str = COMMIT) -> bytes:
|
||||
return json.dumps({
|
||||
"schema": 1,
|
||||
"source_commit": source_commit,
|
||||
"wheel": {
|
||||
"filename": "bot_bottle.whl",
|
||||
"url": (
|
||||
"https://gitea.dideric.is/api/packages/didericis/generic/"
|
||||
f"bot-bottle-builds/{source_commit}/bot_bottle.whl"
|
||||
),
|
||||
"sha256": "b" * 64,
|
||||
},
|
||||
"oci": {
|
||||
role: f"registry.example/{role}@sha256:{'c' * 64}"
|
||||
for role in (
|
||||
"orchestrator",
|
||||
"gateway",
|
||||
"agent_claude",
|
||||
"agent_codex",
|
||||
"agent_pi",
|
||||
)
|
||||
},
|
||||
"firecracker": {
|
||||
role: {"version": "v1", "sha256": "d" * 64}
|
||||
for role in ("orchestrator", "gateway")
|
||||
},
|
||||
"provenance": {"workflow_run": "run", "published_at": "now"},
|
||||
"qualifications": [],
|
||||
}).encode()
|
||||
|
||||
|
||||
class ReleaseQualificationTest(unittest.TestCase):
|
||||
def test_tag_selects_channel(self) -> None:
|
||||
self.assertEqual(release_channel("v1.2.3"), "production")
|
||||
self.assertEqual(release_channel("v1.2.3-rc.4"), "staging")
|
||||
with self.assertRaises(ReleaseManifestError):
|
||||
release_channel("latest")
|
||||
|
||||
def test_release_versions_are_monotonic(self) -> None:
|
||||
self.assertLess(
|
||||
release_version("v1.2.3-rc.4"), release_version("v1.2.3"))
|
||||
self.assertLess(
|
||||
release_version("v1.2.3"), release_version("v2.0.0"))
|
||||
with self.assertRaises(ReleaseManifestError):
|
||||
release_version("v1")
|
||||
|
||||
def test_builds_qualified_pointer(self) -> None:
|
||||
pointer = build_release_pointer(
|
||||
tag="v1.2.3",
|
||||
source_commit=COMMIT,
|
||||
workflow_run="https://example.test/run/1",
|
||||
qualified_at="2026-07-27T00:00:00Z",
|
||||
)
|
||||
self.assertEqual(pointer["channel"], "production")
|
||||
self.assertTrue(pointer["qualified"])
|
||||
self.assertTrue(canonical_pointer(pointer).endswith(b"\n"))
|
||||
|
||||
def test_rejects_invalid_pointer_fields(self) -> None:
|
||||
pointer = build_release_pointer(
|
||||
tag="v1.2.3",
|
||||
source_commit=COMMIT,
|
||||
workflow_run="run",
|
||||
qualified_at="now",
|
||||
)
|
||||
changes = (
|
||||
("schema", 2),
|
||||
("qualified", False),
|
||||
("channel", "staging"),
|
||||
("source_commit", "short"),
|
||||
("bundle_index_url", "https://wrong.example/index.json"),
|
||||
("qualification", {}),
|
||||
)
|
||||
for key, value in changes:
|
||||
with self.subTest(key=key), self.assertRaises(ReleaseManifestError):
|
||||
parse_release_pointer({**pointer, key: value})
|
||||
|
||||
def test_rejects_invalid_build_inputs(self) -> None:
|
||||
with self.assertRaises(ReleaseManifestError):
|
||||
build_release_pointer(
|
||||
tag="v1.2.3",
|
||||
source_commit="short",
|
||||
workflow_run="run",
|
||||
qualified_at="now",
|
||||
)
|
||||
with self.assertRaises(ReleaseManifestError):
|
||||
build_release_pointer(
|
||||
tag="v1.2.3",
|
||||
source_commit=COMMIT,
|
||||
workflow_run="",
|
||||
qualified_at="now",
|
||||
)
|
||||
|
||||
@mock.patch("bot_bottle.release_qualification.upload")
|
||||
@mock.patch("bot_bottle.release_qualification.remote_bytes")
|
||||
def test_publishes_bundle_release_and_channel(
|
||||
self, remote: mock.Mock, upload: mock.Mock,
|
||||
) -> None:
|
||||
remote.side_effect = [bundle(), None, None]
|
||||
pointer = build_release_pointer(
|
||||
tag="v1.2.3",
|
||||
source_commit=COMMIT,
|
||||
workflow_run="run",
|
||||
qualified_at="now",
|
||||
)
|
||||
publish_qualification(pointer)
|
||||
self.assertEqual(upload.call_count, 2)
|
||||
|
||||
@mock.patch("bot_bottle.release_qualification.upload")
|
||||
@mock.patch("bot_bottle.release_qualification.remote_bytes")
|
||||
def test_exact_existing_publication_is_noop(
|
||||
self, remote: mock.Mock, upload: mock.Mock,
|
||||
) -> None:
|
||||
pointer = build_release_pointer(
|
||||
tag="v1.2.3",
|
||||
source_commit=COMMIT,
|
||||
workflow_run="run",
|
||||
qualified_at="now",
|
||||
)
|
||||
wanted = canonical_pointer(pointer)
|
||||
remote.side_effect = [bundle(), wanted, wanted]
|
||||
publish_qualification(pointer)
|
||||
upload.assert_not_called()
|
||||
|
||||
@mock.patch("bot_bottle.release_qualification.upload")
|
||||
@mock.patch("bot_bottle.release_qualification.delete")
|
||||
@mock.patch("bot_bottle.release_qualification.remote_bytes")
|
||||
def test_advances_to_newer_channel(
|
||||
self, remote: mock.Mock, remove: mock.Mock, upload: mock.Mock,
|
||||
) -> None:
|
||||
old = build_release_pointer(
|
||||
tag="v1.0.0",
|
||||
source_commit="b" * 40,
|
||||
workflow_run="old",
|
||||
qualified_at="then",
|
||||
)
|
||||
pointer = build_release_pointer(
|
||||
tag="v2.0.0",
|
||||
source_commit=COMMIT,
|
||||
workflow_run="run",
|
||||
qualified_at="now",
|
||||
)
|
||||
remote.side_effect = [bundle(), None, canonical_pointer(old)]
|
||||
publish_qualification(pointer)
|
||||
remove.assert_called_once()
|
||||
self.assertEqual(upload.call_count, 2)
|
||||
|
||||
@mock.patch("bot_bottle.release_qualification.upload")
|
||||
@mock.patch("bot_bottle.release_qualification.remote_bytes")
|
||||
def test_rejects_non_monotonic_channel(
|
||||
self, remote: mock.Mock, _upload: mock.Mock,
|
||||
) -> None:
|
||||
old = build_release_pointer(
|
||||
tag="v2.0.0",
|
||||
source_commit="b" * 40,
|
||||
workflow_run="old-run",
|
||||
qualified_at="old-time",
|
||||
)
|
||||
remote.side_effect = [bundle(), None, json.dumps(old).encode()]
|
||||
pointer = build_release_pointer(
|
||||
tag="v1.2.3",
|
||||
source_commit=COMMIT,
|
||||
workflow_run="run",
|
||||
qualified_at="now",
|
||||
)
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "not monotonic"):
|
||||
publish_qualification(pointer)
|
||||
|
||||
@mock.patch("bot_bottle.release_qualification.upload")
|
||||
@mock.patch("bot_bottle.release_qualification.delete")
|
||||
@mock.patch("bot_bottle.release_qualification.remote_bytes")
|
||||
def test_explicit_rollback_allows_older_channel(
|
||||
self, remote: mock.Mock, remove: mock.Mock, _upload: mock.Mock,
|
||||
) -> None:
|
||||
old = build_release_pointer(
|
||||
tag="v2.0.0",
|
||||
source_commit="b" * 40,
|
||||
workflow_run="old",
|
||||
qualified_at="then",
|
||||
)
|
||||
pointer = build_release_pointer(
|
||||
tag="v1.0.0",
|
||||
source_commit=COMMIT,
|
||||
workflow_run="run",
|
||||
qualified_at="now",
|
||||
)
|
||||
remote.side_effect = [bundle(), None, canonical_pointer(old)]
|
||||
publish_qualification(pointer, allow_rollback=True)
|
||||
remove.assert_called_once()
|
||||
|
||||
@mock.patch("bot_bottle.release_qualification.remote_bytes")
|
||||
def test_rejects_missing_or_malformed_bundle(self, remote: mock.Mock) -> None:
|
||||
pointer = build_release_pointer(
|
||||
tag="v1.2.3",
|
||||
source_commit=COMMIT,
|
||||
workflow_run="run",
|
||||
qualified_at="now",
|
||||
)
|
||||
remote.return_value = None
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "does not exist"):
|
||||
publish_qualification(pointer)
|
||||
remote.return_value = b"not-json"
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "malformed"):
|
||||
publish_qualification(pointer)
|
||||
remote.return_value = bundle("b" * 40)
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "does not match"):
|
||||
publish_qualification(pointer)
|
||||
|
||||
@mock.patch("bot_bottle.release_qualification.upload")
|
||||
@mock.patch("bot_bottle.release_qualification.remote_bytes")
|
||||
def test_rejects_changed_release_or_malformed_channel(
|
||||
self, remote: mock.Mock, _upload: mock.Mock,
|
||||
) -> None:
|
||||
pointer = build_release_pointer(
|
||||
tag="v1.2.3",
|
||||
source_commit=COMMIT,
|
||||
workflow_run="run",
|
||||
qualified_at="now",
|
||||
)
|
||||
remote.side_effect = [bundle(), b"different"]
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "already differs"):
|
||||
publish_qualification(pointer)
|
||||
remote.side_effect = [bundle(), None, b"not-json"]
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "malformed"):
|
||||
publish_qualification(pointer)
|
||||
|
||||
@mock.patch("bot_bottle.release_qualification.urllib.request.urlopen")
|
||||
def test_delete_handles_success_and_not_found(self, urlopen: mock.Mock) -> None:
|
||||
urlopen.return_value.__enter__.return_value = mock.Mock()
|
||||
delete("https://example.test/channel")
|
||||
urlopen.side_effect = urllib.error.HTTPError(
|
||||
"url", 404, "missing", Message(), None)
|
||||
delete("https://example.test/channel")
|
||||
|
||||
@mock.patch("bot_bottle.release_qualification.urllib.request.urlopen")
|
||||
def test_delete_reports_registry_errors(self, urlopen: mock.Mock) -> None:
|
||||
urlopen.side_effect = urllib.error.HTTPError(
|
||||
"url", 500, "broken", Message(), None)
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "HTTP 500"):
|
||||
delete("https://example.test/channel")
|
||||
urlopen.side_effect = urllib.error.URLError("offline")
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "offline"):
|
||||
delete("https://example.test/channel")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -117,6 +117,15 @@ class TestWheelInstall(unittest.TestCase):
|
||||
proc = self._run(["-c", script])
|
||||
self.assertIn("SELF_CONTAINED_OK", proc.stdout, msg=proc.stderr)
|
||||
|
||||
def test_ad_hoc_wheel_is_marked_for_local_infrastructure_builds(self):
|
||||
script = (
|
||||
"from bot_bottle.release_manifest import packaged_manifest\n"
|
||||
"assert packaged_manifest() is None\n"
|
||||
"print('DEVELOPMENT_MANIFEST_OK')\n"
|
||||
)
|
||||
proc = self._run(["-c", script])
|
||||
self.assertIn("DEVELOPMENT_MANIFEST_OK", proc.stdout, msg=proc.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user