Compare commits
16 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 47cb50db4b | |||
| a750a242d1 | |||
| 4544899c79 | |||
| 89d2ba7e82 | |||
| 43a92945a3 | |||
| 7abcb2c7df | |||
| efa1c8ee4b | |||
| 249eaff53d | |||
| 9f65ab016d | |||
| 46b7e06b37 | |||
| 93190d5e0c | |||
| a2113b7f43 | |||
| d8c5532168 | |||
| a27b1a5fe9 | |||
| 1a9056c648 | |||
| 63595f123a |
@@ -19,6 +19,19 @@ name: pre-release-test
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
ref:
|
||||
description: Revision to qualify
|
||||
required: false
|
||||
default: main
|
||||
workflow_call:
|
||||
inputs:
|
||||
ref:
|
||||
required: true
|
||||
type: string
|
||||
secrets:
|
||||
BOT_BOTTLE_INFRA_ARTIFACT_TOKEN:
|
||||
required: true
|
||||
|
||||
jobs:
|
||||
unit:
|
||||
@@ -26,6 +39,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
# No actions/setup-python: the runner image already ships Python 3.12,
|
||||
# and older act_runner engines mishandle setup-python's PATH (coverage
|
||||
@@ -66,6 +81,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
# No actions/setup-python (see the note in the `unit` job); the
|
||||
# container's system Python 3.12 runs the stdlib test suite directly.
|
||||
@@ -138,10 +155,11 @@ jobs:
|
||||
# the old build-infra → integration-firecracker + coverage chain incurred.
|
||||
integration-firecracker:
|
||||
runs-on: [self-hosted, kvm]
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- name: Preflight — Firecracker host is ready
|
||||
run: |
|
||||
@@ -212,13 +230,14 @@ jobs:
|
||||
# Python >=3.11 with `coverage` importable on the launchd service PATH.
|
||||
integration-macos:
|
||||
runs-on: [self-hosted, macos]
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
concurrency:
|
||||
group: integration-macos-infra
|
||||
cancel-in-progress: false
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
# Fail loudly if the backend this job promises isn't actually usable,
|
||||
# rather than letting every test silently `unittest.skip` and the job go
|
||||
@@ -290,6 +309,7 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install coverage
|
||||
@@ -340,6 +360,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout the tested revision
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- name: Download the tested rootfs
|
||||
uses: actions/download-artifact@v3
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
name: publish-artifacts
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
ref:
|
||||
description: Branch, tag, or commit to publish
|
||||
required: true
|
||||
default: main
|
||||
workflow_call:
|
||||
inputs:
|
||||
ref:
|
||||
required: true
|
||||
type: string
|
||||
outputs:
|
||||
source_commit:
|
||||
description: Published immutable source commit
|
||||
value: ${{ jobs.resolve.outputs.sha }}
|
||||
secrets:
|
||||
BOT_BOTTLE_RELEASE_TOKEN:
|
||||
required: true
|
||||
BOT_BOTTLE_INFRA_ARTIFACT_TOKEN:
|
||||
required: true
|
||||
|
||||
concurrency:
|
||||
group: publish-artifacts-${{ inputs.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
resolve:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
sha: ${{ steps.revision.outputs.sha }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
fetch-depth: 0
|
||||
- id: revision
|
||||
name: Resolve the immutable source revision
|
||||
run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
oci:
|
||||
needs: resolve
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
orchestrator: ${{ steps.images.outputs.orchestrator }}
|
||||
gateway: ${{ steps.images.outputs.gateway }}
|
||||
agent_claude: ${{ steps.images.outputs.agent_claude }}
|
||||
agent_codex: ${{ steps.images.outputs.agent_codex }}
|
||||
agent_pi: ${{ steps.images.outputs.agent_pi }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.resolve.outputs.sha }}
|
||||
- name: Log in to the package registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: gitea.dideric.is
|
||||
username: ${{ gitea.actor }}
|
||||
password: ${{ secrets.BOT_BOTTLE_RELEASE_TOKEN }}
|
||||
- name: Set up multi-architecture builds
|
||||
uses: docker/setup-buildx-action@v3
|
||||
- id: images
|
||||
name: Build, smoke-test, and publish OCI images
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python_base=$(python3 -c \
|
||||
'import json; print(json.load(open("image-build-args.json"))["PYTHON_BASE_IMAGE"])')
|
||||
node_base=$(python3 -c \
|
||||
'import json; print(json.load(open("image-build-args.json"))["NODE_BASE_IMAGE"])')
|
||||
sha='${{ needs.resolve.outputs.sha }}'
|
||||
registry=gitea.dideric.is/didericis
|
||||
|
||||
build_image() {
|
||||
key=$1
|
||||
name=$2
|
||||
dockerfile=$3
|
||||
build_arg=$4
|
||||
smoke=$5
|
||||
local_ref="bot-bottle-${name}:candidate"
|
||||
remote_ref="${registry}/bot-bottle-${name}:commit-${sha}"
|
||||
docker build --build-arg "$build_arg" -t "$local_ref" \
|
||||
-f "$dockerfile" .
|
||||
smoke_command=$(printf "$smoke" "$local_ref")
|
||||
sh -c "docker run --rm $smoke_command"
|
||||
docker buildx build --platform linux/amd64,linux/arm64 \
|
||||
--build-arg "$build_arg" --push --iidfile "${key}.iid" \
|
||||
-t "$remote_ref" -f "$dockerfile" .
|
||||
digest=$(cat "${key}.iid")
|
||||
case "$digest" in sha256:*) ;; *) exit 1 ;; esac
|
||||
echo "${key}=${registry}/bot-bottle-${name}@${digest}" \
|
||||
>> "$GITHUB_OUTPUT"
|
||||
}
|
||||
|
||||
build_image orchestrator orchestrator Dockerfile.orchestrator \
|
||||
"PYTHON_BASE_IMAGE=$python_base" \
|
||||
"--entrypoint python3 %s -c 'import bot_bottle.orchestrator'"
|
||||
build_image gateway gateway Dockerfile.gateway \
|
||||
"PYTHON_BASE_IMAGE=$python_base" \
|
||||
"--entrypoint mitmdump %s --version"
|
||||
build_image agent_claude claude bot_bottle/contrib/claude/Dockerfile \
|
||||
"NODE_BASE_IMAGE=$node_base" "%s claude --version"
|
||||
build_image agent_codex codex bot_bottle/contrib/codex/Dockerfile \
|
||||
"NODE_BASE_IMAGE=$node_base" "%s codex --version"
|
||||
build_image agent_pi pi bot_bottle/contrib/pi/Dockerfile \
|
||||
"NODE_BASE_IMAGE=$node_base" "%s pi --version"
|
||||
|
||||
firecracker:
|
||||
needs: resolve
|
||||
runs-on: [self-hosted, kvm]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.resolve.outputs.sha }}
|
||||
- name: Build and smoke-test Firecracker artifacts
|
||||
env:
|
||||
BOT_BOTTLE_FC_DROPBEAR: /var/cache/bot-bottle-fc/dropbear
|
||||
run: |
|
||||
python3 cli.py backend status --backend=firecracker
|
||||
python3 -m bot_bottle.backend.firecracker.publish_infra \
|
||||
--output infra-candidate --reuse-published
|
||||
BOT_BOTTLE_INFRA_ARTIFACT_DIR="$PWD/infra-candidate" \
|
||||
python3 -m unittest discover -t . -s tests/integration -v
|
||||
- name: Publish tested Firecracker artifacts
|
||||
env:
|
||||
BOT_BOTTLE_INFRA_ARTIFACT_TOKEN: ${{ secrets.BOT_BOTTLE_INFRA_ARTIFACT_TOKEN }}
|
||||
BOT_BOTTLE_FC_DROPBEAR: /var/cache/bot-bottle-fc/dropbear
|
||||
run: |
|
||||
python3 -m bot_bottle.backend.firecracker.publish_infra \
|
||||
--publish-dir infra-candidate
|
||||
- name: Record Firecracker artifact identities
|
||||
run: |
|
||||
cp infra-candidate/orchestrator/version.txt fc-orchestrator-version
|
||||
cp infra-candidate/orchestrator/rootfs.ext4.gz.sha256 fc-orchestrator-sha
|
||||
cp infra-candidate/gateway/version.txt fc-gateway-version
|
||||
cp infra-candidate/gateway/rootfs.ext4.gz.sha256 fc-gateway-sha
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: firecracker-identities
|
||||
path: |
|
||||
fc-orchestrator-version
|
||||
fc-orchestrator-sha
|
||||
fc-gateway-version
|
||||
fc-gateway-sha
|
||||
|
||||
package:
|
||||
needs: [resolve, oci, firecracker]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.resolve.outputs.sha }}
|
||||
- uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: firecracker-identities
|
||||
- name: Build the wheel with the published identities
|
||||
run: |
|
||||
python3 -m pip install --break-system-packages build
|
||||
python3 scripts/generate_release_manifest.py \
|
||||
--source-commit '${{ needs.resolve.outputs.sha }}' \
|
||||
--orchestrator-image '${{ needs.oci.outputs.orchestrator }}' \
|
||||
--gateway-image '${{ needs.oci.outputs.gateway }}' \
|
||||
--agent-claude-image '${{ needs.oci.outputs.agent_claude }}' \
|
||||
--agent-codex-image '${{ needs.oci.outputs.agent_codex }}' \
|
||||
--agent-pi-image '${{ needs.oci.outputs.agent_pi }}' \
|
||||
--firecracker-orchestrator-version "$(cat fc-orchestrator-version)" \
|
||||
--firecracker-orchestrator-sha256 "$(awk '{print $1}' fc-orchestrator-sha)" \
|
||||
--firecracker-gateway-version "$(cat fc-gateway-version)" \
|
||||
--firecracker-gateway-sha256 "$(awk '{print $1}' fc-gateway-sha)" \
|
||||
--output release-manifest.json
|
||||
BOT_BOTTLE_RELEASE_MANIFEST="$PWD/release-manifest.json" \
|
||||
python3 -m build --wheel
|
||||
- name: Verify and publish the immutable commit bundle
|
||||
env:
|
||||
BOT_BOTTLE_RELEASE_TOKEN: ${{ secrets.BOT_BOTTLE_RELEASE_TOKEN }}
|
||||
run: |
|
||||
wheel=$(find dist -maxdepth 1 -name '*.whl' -type f)
|
||||
python3 -m venv verify-venv
|
||||
verify-venv/bin/pip install --no-deps "$wheel"
|
||||
verify-venv/bin/python -c \
|
||||
'from bot_bottle.release_manifest import load_manifest; load_manifest()'
|
||||
python3 scripts/generate_release_bundle.py \
|
||||
--manifest release-manifest.json \
|
||||
--wheel "$wheel" \
|
||||
--workflow-run \
|
||||
"${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
|
||||
--published-at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
||||
--output bundle-index.json --publish
|
||||
@@ -0,0 +1,93 @@
|
||||
name: release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
|
||||
concurrency:
|
||||
group: release-${{ github.ref_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
validate:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
sha: ${{ steps.release.outputs.sha }}
|
||||
tag: ${{ steps.release.outputs.tag }}
|
||||
channel: ${{ steps.release.outputs.channel }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: release
|
||||
name: Validate tag shape and promotion branch
|
||||
run: |
|
||||
set -euo pipefail
|
||||
tag=${GITHUB_REF#refs/tags/}
|
||||
channel=$(python3 -c \
|
||||
'import sys; from bot_bottle.release_qualification import release_channel; print(release_channel(sys.argv[1]))' \
|
||||
"$tag")
|
||||
branch=$channel
|
||||
git fetch origin "$branch"
|
||||
sha=$(git rev-list -n 1 "$tag")
|
||||
git merge-base --is-ancestor "$sha" "origin/$branch" || {
|
||||
echo "$tag is not reachable from protected $branch" >&2
|
||||
exit 1
|
||||
}
|
||||
echo "sha=$sha" >> "$GITHUB_OUTPUT"
|
||||
echo "tag=$tag" >> "$GITHUB_OUTPUT"
|
||||
echo "channel=$channel" >> "$GITHUB_OUTPUT"
|
||||
|
||||
qualify:
|
||||
needs: validate
|
||||
uses: ./.gitea/workflows/pre-release-test.yml
|
||||
with:
|
||||
ref: ${{ needs.validate.outputs.sha }}
|
||||
secrets:
|
||||
BOT_BOTTLE_INFRA_ARTIFACT_TOKEN: ${{ secrets.BOT_BOTTLE_INFRA_ARTIFACT_TOKEN }}
|
||||
|
||||
artifacts:
|
||||
needs: [validate, qualify]
|
||||
uses: ./.gitea/workflows/publish-artifacts.yml
|
||||
with:
|
||||
ref: ${{ needs.validate.outputs.sha }}
|
||||
secrets:
|
||||
BOT_BOTTLE_RELEASE_TOKEN: ${{ secrets.BOT_BOTTLE_RELEASE_TOKEN }}
|
||||
BOT_BOTTLE_INFRA_ARTIFACT_TOKEN: ${{ secrets.BOT_BOTTLE_INFRA_ARTIFACT_TOKEN }}
|
||||
|
||||
promote:
|
||||
needs: [validate, artifacts]
|
||||
runs-on: ubuntu-latest
|
||||
concurrency:
|
||||
group: release-channel-${{ needs.validate.outputs.channel }}
|
||||
cancel-in-progress: false
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.validate.outputs.sha }}
|
||||
- name: Verify install from the published bundle
|
||||
env:
|
||||
BOT_BOTTLE_REF: ${{ needs.validate.outputs.sha }}
|
||||
run: |
|
||||
sh install.sh
|
||||
"$HOME/.local/bin/bot-bottle" --help
|
||||
- name: Publish release qualification and advance channel
|
||||
env:
|
||||
BOT_BOTTLE_RELEASE_TOKEN: ${{ secrets.BOT_BOTTLE_RELEASE_TOKEN }}
|
||||
run: |
|
||||
python3 scripts/publish_release_qualification.py \
|
||||
--tag '${{ needs.validate.outputs.tag }}' \
|
||||
--source-commit '${{ needs.validate.outputs.sha }}' \
|
||||
--workflow-run \
|
||||
"${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
|
||||
--qualified-at "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
- name: Create the Gitea release
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.BOT_BOTTLE_RELEASE_TOKEN }}
|
||||
run: |
|
||||
curl --fail-with-body --silent --show-error \
|
||||
-H "Authorization: token $GITEA_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"tag_name\":\"${{ needs.validate.outputs.tag }}\",\"name\":\"${{ needs.validate.outputs.tag }}\",\"target_commitish\":\"${{ needs.validate.outputs.sha }}\"}" \
|
||||
"${GITHUB_SERVER_URL}/api/v1/repos/${GITHUB_REPOSITORY}/releases"
|
||||
@@ -75,7 +75,15 @@ When the agent exits, `cli.py` tears down every gateway and both networks; nothi
|
||||
curl -fsSL https://gitea.dideric.is/didericis/bot-bottle/raw/branch/main/install.sh | sh
|
||||
```
|
||||
|
||||
The installer is a bootstrapper: it finds a suitable Python, installs bot-bottle with `pipx` (falling back to `pip --user`), creates `~/.bot-bottle`, and runs `bot-bottle doctor`. It is idempotent and never uses `sudo`. Python-native users can skip it entirely with `pipx install bot-bottle` or `uv tool install bot-bottle`.
|
||||
The installer is a bootstrapper: it finds a suitable Python, resolves the
|
||||
latest qualified production bundle, verifies its wheel checksum, installs with
|
||||
`pipx` (falling back to a private venv), creates `~/.bot-bottle`, and runs
|
||||
`bot-bottle doctor`. It is idempotent and never uses `sudo`.
|
||||
|
||||
Select staging with `BOT_BOTTLE_CHANNEL=staging`, an exact qualified release
|
||||
with `BOT_BOTTLE_VERSION=vX.Y.Z[-rc.N]`, or an exact published commit with
|
||||
`BOT_BOTTLE_REF=<40-character-sha>`. Commit installs print an explicit warning
|
||||
because snapshots may not have passed release qualification.
|
||||
|
||||
### Requirements
|
||||
|
||||
@@ -83,7 +91,8 @@ The installer is a bootstrapper: it finds a suitable Python, installs bot-bottle
|
||||
|
||||
**No `pipx` required.** If `pipx` is present the installer uses it and stays out of the way. If it isn't, bot-bottle installs into a private venv at `~/.bot-bottle/venv` (override with `BOT_BOTTLE_VENV`) and symlinks the entry point into `~/.local/bin`. There is deliberately no `pip install --user` path: Homebrew, python.org and Debian/Ubuntu interpreters are all externally managed (PEP 668), which blocks `--user` outright — so on a Mac it is never the fallback it appears to be. A venv is exempt from PEP 668, and `venv` is stdlib, so unlike `pipx` there is nothing to bootstrap first.
|
||||
|
||||
**`git`**, because the default install spec is a `git+` URL. Set `BOT_BOTTLE_INSTALL_SPEC` to a wheel path or index name to avoid it.
|
||||
**`git`** is needed only when `BOT_BOTTLE_INSTALL_SPEC` explicitly selects a
|
||||
`git+` URL. The normal published-wheel path does not require it.
|
||||
|
||||
**A backend**, which the installer deliberately does *not* install for you — `doctor` reports what's missing afterwards. On compatible macOS hosts, the default backend requires Apple's `container` CLI and does not require Docker. The Firecracker backend (Linux) requires Docker on the host for the gateway plus the `firecracker` binary and KVM. The legacy Docker backend requires Docker. Claude bottles also need a long-lived Claude Code OAuth token (`claude setup-token`) exported as `BOT_BOTTLE_CLAUDE_OAUTH_TOKEN`.
|
||||
|
||||
@@ -191,9 +200,19 @@ BOT_BOTTLE_BACKEND=firecracker bot-bottle start <agent>
|
||||
> **CI:** Firecracker integration runs in the manually dispatched `.gitea/workflows/pre-release-test.yml` on a self-hosted runner labelled `kvm`; privileged KVM hosts never execute unreviewed PR code automatically. Provision it like a normal Firecracker host: `firecracker` on `PATH`, `/dev/kvm`, the cached guest kernel and static dropbear, and the persistent TAP/nft pool. The required pull-request workflow runs unit plus the complete Docker integration suite on `ubuntu-latest`; see `docs/ci.md`.
|
||||
|
||||
```sh
|
||||
bot-bottle start <agent> # builds the image on first run, drops you into claude
|
||||
bot-bottle start <agent> # prepares the selected agent image, then attaches
|
||||
```
|
||||
|
||||
Packaged releases carry immutable orchestrator, gateway, and first-party agent
|
||||
image identities. Docker and Apple Container pull the package-selected OCI
|
||||
digests; Firecracker pulls the matching checksum-verified rootfs artifacts.
|
||||
Each verified wheel and its identities are published as an immutable generic
|
||||
package keyed by the full source commit; `bundle-index.json` is uploaded last
|
||||
and is the completeness marker. A source checkout uses local builds for
|
||||
development. Set `BOT_BOTTLE_INFRA_BUILD=local` to make that development
|
||||
override explicit when diagnosing a packaged release; production never falls
|
||||
back to a build after an artifact pull fails.
|
||||
|
||||
## Manifest
|
||||
|
||||
Bottles and agents are Markdown files with YAML frontmatter under `~/.bot-bottle/`. The Markdown body is the system prompt. Bottles live in `~/.bot-bottle/bottles/`; agents may also be shipped by a repo at `<repo>/.bot-bottle/agents/<name>.md`.
|
||||
|
||||
@@ -89,6 +89,10 @@ class DockerGateway(Gateway):
|
||||
when no dockerfile is configured (a pre-pulled image). BOT_BOTTLE_NO_CACHE
|
||||
forces a full rebuild (parity with `start --no-cache`)."""
|
||||
if self._dockerfile is None:
|
||||
proc = run_docker(["docker", "pull", self.image_ref])
|
||||
if proc.returncode != 0:
|
||||
raise GatewayError(
|
||||
f"gateway image pull failed: {proc.stderr.strip()}")
|
||||
return
|
||||
context = self._build_context or resources.build_root()
|
||||
argv = ["docker", "build", "-t", self.image_ref,
|
||||
|
||||
@@ -34,6 +34,7 @@ from .orchestrator import (
|
||||
ORCHESTRATOR_NETWORK,
|
||||
)
|
||||
from ... import resources
|
||||
from ... import release_manifest
|
||||
from ...gateway import (
|
||||
GATEWAY_IMAGE,
|
||||
GATEWAY_NAME,
|
||||
@@ -90,6 +91,14 @@ class DockerInfraService(InfraService):
|
||||
self._orchestrator_name = orchestrator_name
|
||||
self._orchestrator_label = orchestrator_label
|
||||
self._gateway_name = gateway_name
|
||||
resolved_orchestrator, orchestrator_local = release_manifest.oci_image(
|
||||
"orchestrator", orchestrator_image)
|
||||
resolved_gateway, gateway_local = release_manifest.oci_image(
|
||||
"gateway", gateway_image)
|
||||
self.orchestrator_image = resolved_orchestrator
|
||||
self.gateway_image = resolved_gateway
|
||||
self._orchestrator_local = orchestrator_local
|
||||
self._gateway_local = gateway_local
|
||||
|
||||
def orchestrator(self) -> DockerOrchestrator:
|
||||
"""The control-plane service. Cheap to reconstruct — `ensure_built`
|
||||
@@ -104,6 +113,8 @@ class DockerInfraService(InfraService):
|
||||
repo_root=self._repo_root,
|
||||
host_root=self._host_root,
|
||||
root_mount_source=self._root_mount_source,
|
||||
dockerfile=(
|
||||
"Dockerfile.orchestrator" if self._orchestrator_local else None),
|
||||
)
|
||||
|
||||
def gateway(self) -> DockerGateway:
|
||||
@@ -119,6 +130,7 @@ class DockerInfraService(InfraService):
|
||||
control_network=self.control_network,
|
||||
build_context=self._repo_root,
|
||||
ca_mount_source=self._gateway_ca_mount_source,
|
||||
dockerfile="Dockerfile.gateway" if self._gateway_local else None,
|
||||
)
|
||||
|
||||
def ensure_running(
|
||||
@@ -132,8 +144,8 @@ class DockerInfraService(InfraService):
|
||||
gateway = self.gateway()
|
||||
# Build both images (cache-aware; a no-op when nothing changed) before
|
||||
# bringing either plane up.
|
||||
orchestrator.ensure_built()
|
||||
gateway.ensure_built()
|
||||
orchestrator.ensure_available()
|
||||
gateway.ensure_available()
|
||||
|
||||
orchestrator.ensure_running(startup_timeout=startup_timeout)
|
||||
|
||||
|
||||
@@ -84,6 +84,15 @@ def build_or_load_images(plan: DockerBottlePlan) -> BottleImages:
|
||||
)
|
||||
info(f"using cached agent image {plan.image!r}")
|
||||
return BottleImages(agent=plan.image)
|
||||
if "@sha256:" in plan.image:
|
||||
pulled = docker_mod.run_docker(["docker", "pull", plan.image])
|
||||
if pulled.returncode != 0:
|
||||
die(f"pulling packaged agent image {plan.image!r} failed: "
|
||||
f"{pulled.stderr.strip()}")
|
||||
docker_mod.verify_agent_image(
|
||||
plan.image, runtime_for(plan.agent_provider_template).smoke_test,
|
||||
)
|
||||
return BottleImages(agent=plan.image)
|
||||
docker_mod.build_image(plan.image, str(resources.build_root()), dockerfile=plan.dockerfile_path)
|
||||
docker_mod.verify_agent_image(
|
||||
plan.image, runtime_for(plan.agent_provider_template).smoke_test,
|
||||
|
||||
@@ -125,6 +125,10 @@ class DockerOrchestrator(Orchestrator):
|
||||
no-op when nothing changed). No-op when no dockerfile is configured (a
|
||||
pre-pulled image). BOT_BOTTLE_NO_CACHE forces a full rebuild."""
|
||||
if self._dockerfile is None:
|
||||
proc = run_docker(["docker", "pull", self.image_ref])
|
||||
if proc.returncode != 0:
|
||||
raise GatewayError(
|
||||
f"orchestrator image pull failed: {proc.stderr.strip()}")
|
||||
return
|
||||
argv = ["docker", "build", "-t", self.image_ref,
|
||||
"-f", str(self._repo_root / self._dockerfile),
|
||||
|
||||
@@ -72,6 +72,46 @@ def cached_agent_rootfs_dir(dockerfile: Path) -> Path | None:
|
||||
return base if (base / ".bb-ready").is_file() else None
|
||||
|
||||
|
||||
def _image_rootfs_digest(image: str) -> str:
|
||||
h = hashlib.sha256()
|
||||
h.update(image.encode())
|
||||
h.update(b"\0")
|
||||
h.update(util._GUEST_INIT.encode())
|
||||
return h.hexdigest()[:16]
|
||||
|
||||
|
||||
def cached_agent_image_rootfs_dir(image: str) -> Path | None:
|
||||
"""Return a ready rootfs exported from an immutable OCI image."""
|
||||
base = util.cache_dir() / "rootfs" / f"agent-image-{_image_rootfs_digest(image)}"
|
||||
return base if (base / ".bb-ready").is_file() else None
|
||||
|
||||
|
||||
def acquire_agent_image_rootfs_dir(
|
||||
image: str, *, smoke_test: tuple[str, ...] = (),
|
||||
) -> Path:
|
||||
"""Pull a digest-pinned image in the infra VM and export its rootfs."""
|
||||
if "@sha256:" not in image:
|
||||
die(f"prebuilt Firecracker agent image is not digest-pinned: {image}")
|
||||
digest = _image_rootfs_digest(image)
|
||||
base = util.cache_dir() / "rootfs" / f"agent-image-{digest}"
|
||||
cached = cached_agent_image_rootfs_dir(image)
|
||||
if cached is not None:
|
||||
info(f"using cached agent rootfs {cached.name}")
|
||||
return cached
|
||||
with _build_lock():
|
||||
if (base / ".bb-ready").is_file():
|
||||
return base
|
||||
staging = util.cache_dir() / "rootfs" / f".pulling-{digest}"
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
staging.mkdir(parents=True)
|
||||
_pull_in_infra(image, staging, smoke_test, digest)
|
||||
util.inject_guest_boot(staging)
|
||||
(staging / ".bb-ready").write_text("ok\n")
|
||||
shutil.rmtree(base, ignore_errors=True)
|
||||
os.rename(staging, base)
|
||||
return base
|
||||
|
||||
|
||||
def build_agent_rootfs_dir(
|
||||
dockerfile: Path, *, image_tag: str, smoke_test: tuple[str, ...] = (),
|
||||
) -> Path:
|
||||
@@ -167,6 +207,43 @@ def _build_in_infra(
|
||||
_cleanup()
|
||||
|
||||
|
||||
def _pull_in_infra(
|
||||
image: str, base: Path, smoke_test: tuple[str, ...], digest: str,
|
||||
) -> None:
|
||||
"""Pull and export a published agent image inside the orchestrator VM."""
|
||||
service = FirecrackerInfraService()
|
||||
service.ensure_running()
|
||||
key, ip = service.orchestrator().ssh_target()
|
||||
tag = f"bot-bottle-agent-pull-{digest}"
|
||||
smoke_ctr, export_ctr = f"{tag}-smoke", f"{tag}-export"
|
||||
quoted_image = shlex.quote(image)
|
||||
|
||||
def cleanup() -> None:
|
||||
_ssh(
|
||||
key,
|
||||
ip,
|
||||
f"buildah rm {smoke_ctr} {export_ctr} >/dev/null 2>&1; "
|
||||
f"buildah rmi {_STORE_FLAG} {tag} >/dev/null 2>&1",
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
cleanup()
|
||||
try:
|
||||
result = _ssh(
|
||||
key,
|
||||
ip,
|
||||
f"buildah pull {_STORE_FLAG} {quoted_image} && "
|
||||
f"buildah tag {_STORE_FLAG} {quoted_image} {tag}",
|
||||
timeout=_BUILD_TIMEOUT_SECONDS,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
die(f"pulling pinned agent image failed: {result.stderr.strip()}")
|
||||
_smoke_test(key, ip, tag, smoke_ctr, smoke_test)
|
||||
_stream_rootfs(key, ip, tag, export_ctr, base)
|
||||
finally:
|
||||
cleanup()
|
||||
|
||||
|
||||
def _ssh(private_key: Path, guest_ip: str, script: str,
|
||||
*, timeout: float = 60.0) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
|
||||
@@ -61,7 +61,7 @@ class FirecrackerInfraService(InfraService):
|
||||
return url
|
||||
# Clear stale/hung/OUTDATED VMs holding either link before booting fresh.
|
||||
infra_vm.stop()
|
||||
infra_vm.ensure_built()
|
||||
infra_vm.ensure_available()
|
||||
# Orchestrator first — the gateway daemons reach the control plane at
|
||||
# startup. Each service owns its own boot; the orchestrator (which
|
||||
# holds the signing key) mints the role-scoped `gateway` JWT for the
|
||||
|
||||
@@ -195,7 +195,9 @@ def _sha256_file(path: Path) -> str:
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def ensure_artifact_gz(version: str, *, role: str) -> Path:
|
||||
def ensure_artifact_gz(
|
||||
version: str, *, role: str, expected_sha256: str | None = None,
|
||||
) -> Path:
|
||||
"""The verified, cached `rootfs.ext4.gz` for `role` at `version` —
|
||||
downloading it (and its `.sha256`) once, then reusing it. Fail-closed on a
|
||||
checksum mismatch: the partial is removed and we die rather than boot an
|
||||
@@ -222,6 +224,11 @@ def ensure_artifact_gz(version: str, *, role: str) -> Path:
|
||||
if not gz.is_file() or not sha.is_file():
|
||||
die(f"infra candidate bundle is incomplete: {root}")
|
||||
expected = sha.read_text().split()[0].strip().lower()
|
||||
if expected_sha256 is not None and expected != expected_sha256:
|
||||
die(
|
||||
f"infra candidate packaged checksum mismatch ({role}) for {version}:\n"
|
||||
f" packaged {expected_sha256}\n candidate {expected}"
|
||||
)
|
||||
actual = _sha256_file(gz)
|
||||
if actual != expected:
|
||||
die(
|
||||
@@ -235,7 +242,13 @@ def ensure_artifact_gz(version: str, *, role: str) -> Path:
|
||||
gz = root / _GZ_NAME
|
||||
ok = root / ".verified"
|
||||
if gz.is_file() and ok.is_file():
|
||||
return gz
|
||||
actual = _sha256_file(gz)
|
||||
recorded = ok.read_text(encoding="utf-8").strip()
|
||||
wanted = expected_sha256 or recorded
|
||||
if actual == recorded == wanted:
|
||||
return gz
|
||||
gz.unlink(missing_ok=True)
|
||||
ok.unlink(missing_ok=True)
|
||||
|
||||
info(f"pulling infra rootfs artifact {_package(role)}/{version}")
|
||||
_download(artifact_url(version, _GZ_NAME, role=role), gz)
|
||||
@@ -243,6 +256,14 @@ def ensure_artifact_gz(version: str, *, role: str) -> Path:
|
||||
_download(artifact_url(version, _SHA_NAME, role=role), sha)
|
||||
|
||||
expected = sha.read_text().split()[0].strip().lower()
|
||||
if expected_sha256 is not None and expected != expected_sha256:
|
||||
gz.unlink(missing_ok=True)
|
||||
sha.unlink(missing_ok=True)
|
||||
die(
|
||||
f"infra artifact published checksum mismatch ({role}) for {version}:\n"
|
||||
f" packaged {expected_sha256}\n"
|
||||
f" published {expected}"
|
||||
)
|
||||
actual = _sha256_file(gz)
|
||||
if actual != expected:
|
||||
gz.unlink(missing_ok=True)
|
||||
@@ -253,15 +274,22 @@ def ensure_artifact_gz(version: str, *, role: str) -> Path:
|
||||
f" actual {actual}\n"
|
||||
f" refusing to boot an unverified rootfs."
|
||||
)
|
||||
ok.write_text("ok\n")
|
||||
ok.write_text(actual + "\n")
|
||||
return gz
|
||||
|
||||
|
||||
def materialize_ext4(version: str, dest: Path, *, role: str) -> None:
|
||||
def materialize_ext4(
|
||||
version: str,
|
||||
dest: Path,
|
||||
*,
|
||||
role: str,
|
||||
expected_sha256: str | None = None,
|
||||
) -> None:
|
||||
"""Ensure the verified `role` artifact is cached, then gunzip it to `dest` —
|
||||
a fresh, writable per-boot rootfs (the VM mutates it; the cached `.gz` stays
|
||||
pristine). Atomic via a `.part` sibling."""
|
||||
gz = ensure_artifact_gz(version, role=role)
|
||||
gz = ensure_artifact_gz(
|
||||
version, role=role, expected_sha256=expected_sha256)
|
||||
tmp = dest.with_suffix(dest.suffix + ".part")
|
||||
info(f"expanding {role} infra rootfs -> {dest}")
|
||||
with gzip.open(gz, "rb") as src, open(tmp, "wb") as out:
|
||||
|
||||
@@ -43,6 +43,7 @@ from pathlib import Path
|
||||
from typing import Generator
|
||||
|
||||
from ... import resources
|
||||
from ... import release_manifest
|
||||
from ...log import die, info
|
||||
from ..docker import util as docker_mod
|
||||
from . import firecracker_vm, infra_artifact, netpool, util
|
||||
@@ -108,6 +109,18 @@ def _role_version(role: str) -> str:
|
||||
return infra_artifact.infra_artifact_version(role_init(role), role)
|
||||
|
||||
|
||||
def _role_release(role: str) -> tuple[str, str | None]:
|
||||
manifest = release_manifest.packaged_manifest()
|
||||
if manifest is None or release_manifest.local_build_requested():
|
||||
return _role_version(role), None
|
||||
artifact = (
|
||||
manifest.firecracker_orchestrator
|
||||
if role == "orchestrator"
|
||||
else manifest.firecracker_gateway
|
||||
)
|
||||
return artifact.version, artifact.sha256
|
||||
|
||||
|
||||
def ensure_built() -> None:
|
||||
"""Ensure both infra rootfs artifacts are available before boot.
|
||||
|
||||
@@ -117,11 +130,16 @@ def ensure_built() -> None:
|
||||
`BOT_BOTTLE_INFRA_BUILD=local` instead builds the images from source with
|
||||
host Docker (the orchestrator-fc image is `FROM` the orchestrator image, so
|
||||
it must exist first) — for iterating on the Dockerfiles."""
|
||||
if infra_artifact.local_build_requested():
|
||||
if release_manifest.local_build_requested():
|
||||
build_infra_images_with_docker()
|
||||
return
|
||||
for role in infra_artifact.ROLES:
|
||||
infra_artifact.ensure_artifact_gz(_role_version(role), role=role)
|
||||
version, expected = _role_release(role)
|
||||
infra_artifact.ensure_artifact_gz(
|
||||
version, role=role, expected_sha256=expected)
|
||||
|
||||
|
||||
ensure_available = ensure_built
|
||||
|
||||
|
||||
def build_infra_images_with_docker() -> None:
|
||||
@@ -214,7 +232,9 @@ def boot_vm(
|
||||
else:
|
||||
# Prebuilt artifact already carries the role's build slack; expand it to
|
||||
# a fresh writable rootfs for this boot.
|
||||
infra_artifact.materialize_ext4(_role_version(role), rootfs, role=role)
|
||||
version, expected = _role_release(role)
|
||||
infra_artifact.materialize_ext4(
|
||||
version, rootfs, role=role, expected_sha256=expected)
|
||||
private_key, pubkey = _stable_keypair()
|
||||
|
||||
info(f"booting {role} VM on {slot.iface} (guest {slot.guest_ip})")
|
||||
@@ -264,7 +284,8 @@ def _version_file() -> Path:
|
||||
def expected_version() -> str:
|
||||
"""The combined marker for the running pair: both per-plane artifact
|
||||
versions, so a change to either rootfs dislodges the adopted pair."""
|
||||
return " ".join(f"{role}={_role_version(role)}" for role in infra_artifact.ROLES)
|
||||
return " ".join(
|
||||
f"{role}={_role_release(role)[0]}" for role in infra_artifact.ROLES)
|
||||
|
||||
|
||||
def adoptable(key: Path, url: str, want: str) -> bool:
|
||||
|
||||
@@ -228,8 +228,13 @@ def build_or_load_agent_base(plan: FirecrackerBottlePlan) -> Path:
|
||||
info(f"resuming from committed rootfs {committed_tar}")
|
||||
return util.build_committed_rootfs_dir(committed_tar)
|
||||
dockerfile = Path(plan.dockerfile_path)
|
||||
prebuilt = "@sha256:" in plan.image
|
||||
if plan.spec.image_policy == "cached":
|
||||
cached = image_builder.cached_agent_rootfs_dir(dockerfile)
|
||||
cached = (
|
||||
image_builder.cached_agent_image_rootfs_dir(plan.image)
|
||||
if prebuilt
|
||||
else image_builder.cached_agent_rootfs_dir(dockerfile)
|
||||
)
|
||||
if cached is None:
|
||||
die(
|
||||
f"cached agent rootfs for {plan.image!r} not found; "
|
||||
@@ -237,6 +242,11 @@ def build_or_load_agent_base(plan: FirecrackerBottlePlan) -> Path:
|
||||
)
|
||||
info(f"using cached agent rootfs {cached.name}")
|
||||
return cached
|
||||
if prebuilt:
|
||||
return image_builder.acquire_agent_image_rootfs_dir(
|
||||
plan.image,
|
||||
smoke_test=runtime_for(plan.agent_provider_template).smoke_test,
|
||||
)
|
||||
return image_builder.build_agent_rootfs_dir(
|
||||
dockerfile,
|
||||
image_tag=plan.image,
|
||||
@@ -253,7 +263,11 @@ def stale_checks(plan: FirecrackerBottlePlan) -> None:
|
||||
if committed and committed_tar.is_file():
|
||||
check_stale_path(f"agent rootfs {committed_tar}", committed_tar)
|
||||
return
|
||||
cached = image_builder.cached_agent_rootfs_dir(Path(plan.dockerfile_path))
|
||||
cached = (
|
||||
image_builder.cached_agent_image_rootfs_dir(plan.image)
|
||||
if "@sha256:" in plan.image
|
||||
else image_builder.cached_agent_rootfs_dir(Path(plan.dockerfile_path))
|
||||
)
|
||||
if cached is not None:
|
||||
check_stale_path(f"agent rootfs {cached}", cached / ".bb-ready")
|
||||
|
||||
|
||||
@@ -84,6 +84,7 @@ class MacosGateway(Gateway):
|
||||
egress_network: str = GATEWAY_EGRESS_NETWORK,
|
||||
control_network: str = CONTROL_NETWORK,
|
||||
repo_root: Path | None = None,
|
||||
local_build: bool = True,
|
||||
) -> None:
|
||||
self.image_ref = image_ref
|
||||
self.name = name
|
||||
@@ -93,6 +94,7 @@ class MacosGateway(Gateway):
|
||||
# Build context: the repo root in a checkout, a staged copy from the
|
||||
# installed wheel otherwise (bot_bottle.resources).
|
||||
self._repo_root = repo_root if repo_root is not None else resources.build_root()
|
||||
self._local_build = local_build
|
||||
# Set by `connect_to_orchestrator`: the URL the daemons resolve policy
|
||||
# against + the pre-minted `gateway` token they present. The gateway
|
||||
# never mints, so it never holds the signing key (#469).
|
||||
@@ -101,8 +103,11 @@ class MacosGateway(Gateway):
|
||||
|
||||
def ensure_built(self) -> None:
|
||||
"""Build the data-plane image from `Dockerfile.gateway`."""
|
||||
container_mod.build_image(
|
||||
self.image_ref, str(self._repo_root), dockerfile="Dockerfile.gateway")
|
||||
if self._local_build:
|
||||
container_mod.build_image(
|
||||
self.image_ref, str(self._repo_root), dockerfile="Dockerfile.gateway")
|
||||
else:
|
||||
container_mod.pull_image(self.image_ref)
|
||||
|
||||
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None:
|
||||
"""Bind the gateway to this orchestrator and (re)start it, dual-homed on
|
||||
|
||||
@@ -25,6 +25,7 @@ from __future__ import annotations
|
||||
from pathlib import Path
|
||||
|
||||
from ... import resources
|
||||
from ... import release_manifest
|
||||
from ...orchestrator.lifecycle import (
|
||||
DEFAULT_PORT,
|
||||
DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||
@@ -86,6 +87,14 @@ class MacosInfraService(InfraService):
|
||||
self._orchestrator_name = orchestrator_name
|
||||
self._gateway_name = gateway_name
|
||||
self._db_volume = db_volume
|
||||
resolved_orchestrator, orchestrator_local = release_manifest.oci_image(
|
||||
"orchestrator", orchestrator_image)
|
||||
resolved_gateway, gateway_local = release_manifest.oci_image(
|
||||
"gateway", gateway_image)
|
||||
self.orchestrator_image = resolved_orchestrator
|
||||
self.gateway_image = resolved_gateway
|
||||
self._orchestrator_local = orchestrator_local
|
||||
self._gateway_local = gateway_local
|
||||
|
||||
def orchestrator(self) -> MacosOrchestrator:
|
||||
"""The control-plane service on the host-only control network. Cheap to
|
||||
@@ -98,6 +107,7 @@ class MacosInfraService(InfraService):
|
||||
control_network=self.control_network,
|
||||
repo_root=self._repo_root,
|
||||
db_volume=self._db_volume,
|
||||
local_build=self._orchestrator_local,
|
||||
)
|
||||
|
||||
def gateway(self) -> MacosGateway:
|
||||
@@ -112,6 +122,7 @@ class MacosInfraService(InfraService):
|
||||
egress_network=self.egress_network,
|
||||
control_network=self.control_network,
|
||||
repo_root=self._repo_root,
|
||||
local_build=self._gateway_local,
|
||||
)
|
||||
|
||||
def ensure_running(
|
||||
@@ -127,8 +138,8 @@ class MacosInfraService(InfraService):
|
||||
|
||||
orchestrator = self.orchestrator()
|
||||
gateway = self.gateway()
|
||||
orchestrator.ensure_built()
|
||||
gateway.ensure_built()
|
||||
orchestrator.ensure_available()
|
||||
gateway.ensure_available()
|
||||
|
||||
orchestrator.ensure_running(startup_timeout=startup_timeout)
|
||||
|
||||
|
||||
@@ -38,6 +38,7 @@ import subprocess
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from typing import Callable, Generator
|
||||
|
||||
from ...agent_provider import runtime_for
|
||||
from ...bottle_state import (
|
||||
egress_state_dir,
|
||||
git_gate_state_dir,
|
||||
@@ -93,7 +94,14 @@ def _agent_image(plan: MacosContainerBottlePlan) -> str:
|
||||
)
|
||||
info(f"using cached agent image {plan.image!r}")
|
||||
return plan.image
|
||||
container_mod.build_image(plan.image, str(resources.build_root()), dockerfile=plan.dockerfile_path)
|
||||
if "@sha256:" in plan.image:
|
||||
container_mod.pull_image(plan.image)
|
||||
container_mod.verify_agent_image(
|
||||
plan.image, runtime_for(plan.agent_provider_template).smoke_test,
|
||||
)
|
||||
return plan.image
|
||||
container_mod.build_image(
|
||||
plan.image, str(resources.build_root()), dockerfile=plan.dockerfile_path)
|
||||
return plan.image
|
||||
|
||||
|
||||
|
||||
@@ -63,6 +63,7 @@ class MacosOrchestrator(Orchestrator):
|
||||
control_network: str = CONTROL_NETWORK,
|
||||
repo_root: Path | None = None,
|
||||
db_volume: str = ORCHESTRATOR_DB_VOLUME,
|
||||
local_build: bool = True,
|
||||
) -> None:
|
||||
self.image_ref = image_ref
|
||||
self.name = name
|
||||
@@ -73,6 +74,7 @@ class MacosOrchestrator(Orchestrator):
|
||||
# staged copy from the installed wheel otherwise (bot_bottle.resources).
|
||||
self._repo_root = repo_root if repo_root is not None else resources.build_root()
|
||||
self._db_volume = db_volume
|
||||
self._local_build = local_build
|
||||
|
||||
def url(self) -> str:
|
||||
"""The orchestrator's control-network address (host CLI + registration),
|
||||
@@ -114,8 +116,12 @@ class MacosOrchestrator(Orchestrator):
|
||||
"""Build the control-plane image. The source is bind-mounted so a code
|
||||
change takes effect without a rebuild; the image still carries the
|
||||
package for its entrypoint."""
|
||||
container_mod.build_image(
|
||||
self.image_ref, str(self._repo_root), dockerfile="Dockerfile.orchestrator")
|
||||
if self._local_build:
|
||||
container_mod.build_image(
|
||||
self.image_ref, str(self._repo_root),
|
||||
dockerfile="Dockerfile.orchestrator")
|
||||
else:
|
||||
container_mod.pull_image(self.image_ref)
|
||||
|
||||
def ensure_running(
|
||||
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||
@@ -146,11 +152,6 @@ class MacosOrchestrator(Orchestrator):
|
||||
"--dns", container_mod.dns_server(),
|
||||
# Container-only DB volume: exactly one kernel writes bot-bottle.db.
|
||||
"--volume", f"{self._db_volume}:{_DB_ROOT_IN_CONTAINER}",
|
||||
# Live control-plane source (a code change takes effect on relaunch).
|
||||
"--mount",
|
||||
container_mod.bind_mount_spec(
|
||||
str(self._repo_root), _SRC_IN_CONTAINER, readonly=True),
|
||||
"--env", f"PYTHONPATH={_SRC_IN_CONTAINER}",
|
||||
"--env", f"BOT_BOTTLE_ROOT={_DB_ROOT_IN_CONTAINER}",
|
||||
# Detect a real control-plane code change and recreate.
|
||||
"--env", f"BOT_BOTTLE_SOURCE_HASH={current_hash}",
|
||||
@@ -161,6 +162,14 @@ class MacosOrchestrator(Orchestrator):
|
||||
# Dockerfile.orchestrator ENTRYPOINT is `-m bot_bottle.orchestrator`.
|
||||
"--host", "0.0.0.0", "--port", str(self.port), "--broker", "stub",
|
||||
]
|
||||
if self._local_build:
|
||||
image_index = argv.index(self.image_ref)
|
||||
argv[image_index:image_index] = [
|
||||
"--mount",
|
||||
container_mod.bind_mount_spec(
|
||||
str(self._repo_root), _SRC_IN_CONTAINER, readonly=True),
|
||||
"--env", f"PYTHONPATH={_SRC_IN_CONTAINER}",
|
||||
]
|
||||
result = container_mod.run_container_argv(
|
||||
argv, env={**os.environ, ORCHESTRATOR_TOKEN_ENV: _signing_key})
|
||||
if result.returncode != 0:
|
||||
|
||||
@@ -101,6 +101,19 @@ def build_image(
|
||||
subprocess.run(args, check=True)
|
||||
|
||||
|
||||
def pull_image(ref: str) -> None:
|
||||
"""Acquire an immutable OCI reference through Apple Container."""
|
||||
result = subprocess.run(
|
||||
[_CONTAINER, "image", "pull", ref],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
detail = (result.stderr or result.stdout or "").strip()
|
||||
raise RuntimeError(f"container image pull failed for {ref}: {detail}")
|
||||
|
||||
|
||||
def verify_agent_image(image: str, argv: tuple[str, ...]) -> None:
|
||||
"""Run `argv` inside a throwaway container of a freshly built agent
|
||||
image and die loudly if it fails, instead of shipping an image
|
||||
|
||||
@@ -9,7 +9,7 @@ backend-specific final resolution.
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import TYPE_CHECKING, Protocol
|
||||
|
||||
from ..agent_provider import AgentProvisionPlan, build_agent_provision_plan, get_provider
|
||||
@@ -17,6 +17,7 @@ from ..egress import EgressPlan
|
||||
from ..env import ResolvedEnv, resolve_env
|
||||
from ..git_gate import GitGate, GitGatePlan
|
||||
from ..manifest import Manifest
|
||||
from ..release_manifest import oci_image
|
||||
from ..supervisor.plan import SupervisePlan
|
||||
from ..workspace import workspace_plan
|
||||
from .resolve_common import (
|
||||
@@ -112,6 +113,11 @@ class BottlePreparationPlanner:
|
||||
color=spec.color,
|
||||
provider_settings=provider_config.settings,
|
||||
)
|
||||
if not provider_config.dockerfile:
|
||||
image, local = oci_image(
|
||||
f"agent_{provider_config.template}", provision.image)
|
||||
if not local:
|
||||
provision = replace(provision, image=image)
|
||||
provision = merge_provision_env_vars(provision)
|
||||
return PreparedBottle(
|
||||
manifest=manifest,
|
||||
|
||||
@@ -115,10 +115,12 @@ class Gateway(abc.ABC):
|
||||
|
||||
name: str
|
||||
|
||||
def ensure_available(self) -> None:
|
||||
"""Acquire the exact image/rootfs selected for this application."""
|
||||
self.ensure_built()
|
||||
|
||||
def ensure_built(self) -> None:
|
||||
"""Ensure the gateway's image / rootfs exists, building it if needed.
|
||||
Default: nothing to build (e.g. a stub or a pre-pulled image). Call
|
||||
before `connect_to_orchestrator`."""
|
||||
"""Local-build implementation hook retained for backend compatibility."""
|
||||
return
|
||||
|
||||
@abc.abstractmethod
|
||||
|
||||
@@ -63,10 +63,12 @@ class Orchestrator(abc.ABC):
|
||||
# orchestrator never starts without its signing key.
|
||||
provisioning: ControlPlaneProvisioning = ControlPlaneProvisioning()
|
||||
|
||||
def ensure_available(self) -> None:
|
||||
"""Acquire the exact image/rootfs selected for this application."""
|
||||
self.ensure_built()
|
||||
|
||||
def ensure_built(self) -> None:
|
||||
"""Ensure the orchestrator's image / rootfs exists, building it if
|
||||
needed. Default: nothing to build (e.g. a pre-pulled image). Call before
|
||||
`ensure_running`."""
|
||||
"""Local-build implementation hook retained for backend compatibility."""
|
||||
return
|
||||
|
||||
@abc.abstractmethod
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{"schema": 1, "development": true}
|
||||
@@ -0,0 +1,127 @@
|
||||
"""External, commit-addressed release bundle metadata."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .release_manifest import ReleaseManifestError, parse_manifest
|
||||
|
||||
_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||
_SHA_RE = re.compile(r"^[0-9a-f]{64}$")
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1 << 20), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def build_bundle_index(
|
||||
manifest_data: Any,
|
||||
*,
|
||||
wheel: Path,
|
||||
wheel_url: str,
|
||||
workflow_run: str,
|
||||
published_at: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Build and validate the immutable external index for one source commit."""
|
||||
manifest = parse_manifest(manifest_data)
|
||||
if not wheel.is_file() or wheel.suffix != ".whl":
|
||||
raise ReleaseManifestError("release bundle wheel must be one .whl file")
|
||||
if not wheel_url.startswith("https://") or not wheel_url.endswith(wheel.name):
|
||||
raise ReleaseManifestError(
|
||||
"release bundle wheel URL must be HTTPS and end with the wheel filename")
|
||||
if not workflow_run.strip() or not published_at.strip():
|
||||
raise ReleaseManifestError(
|
||||
"release bundle provenance requires workflow_run and published_at")
|
||||
return {
|
||||
"schema": 1,
|
||||
"source_commit": manifest.source_commit,
|
||||
"wheel": {
|
||||
"filename": wheel.name,
|
||||
"url": wheel_url,
|
||||
"sha256": sha256_file(wheel),
|
||||
},
|
||||
"oci": {
|
||||
"orchestrator": manifest.orchestrator_image,
|
||||
"gateway": manifest.gateway_image,
|
||||
**{
|
||||
f"agent_{role}": reference
|
||||
for role, reference in manifest.agent_images.items()
|
||||
},
|
||||
},
|
||||
"firecracker": {
|
||||
"orchestrator": {
|
||||
"version": manifest.firecracker_orchestrator.version,
|
||||
"sha256": manifest.firecracker_orchestrator.sha256,
|
||||
},
|
||||
"gateway": {
|
||||
"version": manifest.firecracker_gateway.version,
|
||||
"sha256": manifest.firecracker_gateway.sha256,
|
||||
},
|
||||
},
|
||||
"provenance": {
|
||||
"workflow_run": workflow_run.strip(),
|
||||
"published_at": published_at.strip(),
|
||||
},
|
||||
"qualifications": [],
|
||||
}
|
||||
|
||||
|
||||
def parse_bundle_index(data: Any) -> dict[str, Any]:
|
||||
"""Validate an index before publication or installer consumption."""
|
||||
if not isinstance(data, dict) or data.get("schema") != 1:
|
||||
raise ReleaseManifestError("release bundle index must use schema 1")
|
||||
commit = data.get("source_commit")
|
||||
if not isinstance(commit, str) or _COMMIT_RE.fullmatch(commit) is None:
|
||||
raise ReleaseManifestError(
|
||||
"release bundle source_commit must be 40 lowercase hex")
|
||||
wheel = data.get("wheel")
|
||||
if not isinstance(wheel, dict):
|
||||
raise ReleaseManifestError("release bundle wheel must be an object")
|
||||
filename, url, digest = (
|
||||
wheel.get("filename"), wheel.get("url"), wheel.get("sha256"))
|
||||
if not isinstance(filename, str) or not filename.endswith(".whl"):
|
||||
raise ReleaseManifestError("release bundle wheel filename must end in .whl")
|
||||
if not isinstance(url, str) or not url.startswith("https://") or not url.endswith(filename):
|
||||
raise ReleaseManifestError(
|
||||
"release bundle wheel URL must be HTTPS and match its filename")
|
||||
if not isinstance(digest, str) or _SHA_RE.fullmatch(digest) is None:
|
||||
raise ReleaseManifestError(
|
||||
"release bundle wheel sha256 must be 64 lowercase hex")
|
||||
manifest_data = {
|
||||
"schema": 1,
|
||||
"source_commit": commit,
|
||||
"oci": data.get("oci"),
|
||||
"firecracker": data.get("firecracker"),
|
||||
}
|
||||
parse_manifest(manifest_data)
|
||||
provenance = data.get("provenance")
|
||||
if not isinstance(provenance, dict) or not all(
|
||||
isinstance(provenance.get(key), str) and provenance[key].strip()
|
||||
for key in ("workflow_run", "published_at")
|
||||
):
|
||||
raise ReleaseManifestError("release bundle provenance is incomplete")
|
||||
qualifications = data.get("qualifications")
|
||||
if not isinstance(qualifications, list):
|
||||
raise ReleaseManifestError("release bundle qualifications must be an array")
|
||||
return data
|
||||
|
||||
|
||||
def canonical_bytes(data: Any) -> bytes:
|
||||
parse_bundle_index(data)
|
||||
return (json.dumps(data, indent=2, sort_keys=True) + "\n").encode("utf-8")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"build_bundle_index",
|
||||
"canonical_bytes",
|
||||
"parse_bundle_index",
|
||||
"sha256_file",
|
||||
]
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Immutable infrastructure identities assigned when bot-bottle is packaged."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from . import resources
|
||||
|
||||
_MANIFEST_NAME = "release-manifest.json"
|
||||
_OCI_RE = re.compile(r"^[^@\s]+@sha256:([0-9a-f]{64})$")
|
||||
_SHA_RE = re.compile(r"^[0-9a-f]{64}$")
|
||||
_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||
|
||||
|
||||
class ReleaseManifestError(RuntimeError):
|
||||
"""The installed package has no usable infrastructure release identity."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FirecrackerArtifact:
|
||||
version: str
|
||||
sha256: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReleaseManifest:
|
||||
source_commit: str
|
||||
orchestrator_image: str
|
||||
gateway_image: str
|
||||
agent_images: dict[str, str]
|
||||
firecracker_orchestrator: FirecrackerArtifact
|
||||
firecracker_gateway: FirecrackerArtifact
|
||||
|
||||
|
||||
def manifest_path() -> Path:
|
||||
override = os.environ.get("BOT_BOTTLE_RELEASE_MANIFEST", "").strip()
|
||||
return Path(override) if override else Path(__file__).resolve().parent / _MANIFEST_NAME
|
||||
|
||||
|
||||
def _artifact(value: Any, role: str) -> FirecrackerArtifact:
|
||||
if not isinstance(value, dict):
|
||||
raise ReleaseManifestError(f"release manifest firecracker.{role} must be an object")
|
||||
version = value.get("version")
|
||||
sha256 = value.get("sha256")
|
||||
if not isinstance(version, str) or not version.strip():
|
||||
raise ReleaseManifestError(
|
||||
f"release manifest firecracker.{role}.version must be non-empty")
|
||||
if not isinstance(sha256, str) or _SHA_RE.fullmatch(sha256) is None:
|
||||
raise ReleaseManifestError(
|
||||
f"release manifest firecracker.{role}.sha256 must be 64 lowercase hex")
|
||||
return FirecrackerArtifact(version.strip(), sha256)
|
||||
|
||||
|
||||
def parse_manifest(data: Any) -> ReleaseManifest:
|
||||
if not isinstance(data, dict) or data.get("schema") != 1:
|
||||
raise ReleaseManifestError("release manifest must use schema 1")
|
||||
commit = data.get("source_commit")
|
||||
if not isinstance(commit, str) or _COMMIT_RE.fullmatch(commit) is None:
|
||||
raise ReleaseManifestError(
|
||||
"release manifest source_commit must be 40 lowercase hex")
|
||||
oci = data.get("oci")
|
||||
if not isinstance(oci, dict):
|
||||
raise ReleaseManifestError("release manifest oci must be an object")
|
||||
images: dict[str, str] = {}
|
||||
for role in ("orchestrator", "gateway", "agent_claude", "agent_codex", "agent_pi"):
|
||||
ref = oci.get(role)
|
||||
if not isinstance(ref, str) or _OCI_RE.fullmatch(ref) is None:
|
||||
raise ReleaseManifestError(
|
||||
f"release manifest oci.{role} must be digest-pinned")
|
||||
images[role] = ref
|
||||
firecracker = data.get("firecracker")
|
||||
if not isinstance(firecracker, dict):
|
||||
raise ReleaseManifestError("release manifest firecracker must be an object")
|
||||
return ReleaseManifest(
|
||||
source_commit=commit,
|
||||
orchestrator_image=images["orchestrator"],
|
||||
gateway_image=images["gateway"],
|
||||
agent_images={
|
||||
role: images[f"agent_{role}"] for role in ("claude", "codex", "pi")
|
||||
},
|
||||
firecracker_orchestrator=_artifact(
|
||||
firecracker.get("orchestrator"), "orchestrator"),
|
||||
firecracker_gateway=_artifact(firecracker.get("gateway"), "gateway"),
|
||||
)
|
||||
|
||||
|
||||
def load_manifest() -> ReleaseManifest:
|
||||
path = manifest_path()
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise ReleaseManifestError(
|
||||
f"bot-bottle was packaged without valid infrastructure pins "
|
||||
f"({path}): {exc}") from exc
|
||||
return parse_manifest(data)
|
||||
|
||||
|
||||
def packaged_manifest() -> ReleaseManifest | None:
|
||||
"""Release identity, or ``None`` for checkouts/ad-hoc development wheels."""
|
||||
if resources.is_source_checkout() and not os.environ.get(
|
||||
"BOT_BOTTLE_RELEASE_MANIFEST"
|
||||
):
|
||||
return None
|
||||
path = manifest_path()
|
||||
try:
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return load_manifest() # raise the detailed, fail-closed error
|
||||
if isinstance(raw, dict) and raw.get("development") is True:
|
||||
return None
|
||||
return load_manifest()
|
||||
|
||||
|
||||
def local_build_requested() -> bool:
|
||||
return os.environ.get("BOT_BOTTLE_INFRA_BUILD", "").strip().lower() == "local"
|
||||
|
||||
|
||||
def oci_image(role: str, local_default: str) -> tuple[str, bool]:
|
||||
"""Return ``(reference, should_build_locally)`` for a fixed OCI service."""
|
||||
override_name = f"BOT_BOTTLE_{role.upper()}_IMAGE"
|
||||
override = os.environ.get(override_name, "").strip()
|
||||
manifest = packaged_manifest()
|
||||
local = local_build_requested() or manifest is None
|
||||
if override:
|
||||
if not local and _OCI_RE.fullmatch(override) is None:
|
||||
raise ReleaseManifestError(
|
||||
f"{override_name} must be digest-pinned outside local build mode")
|
||||
return override, local
|
||||
if local:
|
||||
return local_default, True
|
||||
assert manifest is not None
|
||||
return (
|
||||
manifest.orchestrator_image if role == "orchestrator"
|
||||
else manifest.gateway_image if role == "gateway"
|
||||
else manifest.agent_images[role.removeprefix("agent_")],
|
||||
False,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"FirecrackerArtifact",
|
||||
"ReleaseManifest",
|
||||
"ReleaseManifestError",
|
||||
"load_manifest",
|
||||
"local_build_requested",
|
||||
"oci_image",
|
||||
"packaged_manifest",
|
||||
"parse_manifest",
|
||||
]
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Durable publication of immutable, commit-addressed release bundles."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from .release_bundle import canonical_bytes, parse_bundle_index, sha256_file
|
||||
from .release_manifest import ReleaseManifestError
|
||||
|
||||
HTTP_TIMEOUT_SECONDS = 60.0
|
||||
INDEX_NAME = "bundle-index.json"
|
||||
PACKAGE_NAME = "bot-bottle-builds"
|
||||
|
||||
|
||||
def config() -> tuple[str, str, str]:
|
||||
"""Return generic-package base URL, owner, and write token."""
|
||||
base = os.environ.get(
|
||||
"BOT_BOTTLE_RELEASE_BASE", "https://gitea.dideric.is").rstrip("/")
|
||||
owner = os.environ.get("BOT_BOTTLE_RELEASE_OWNER", "didericis").strip()
|
||||
token = os.environ.get("BOT_BOTTLE_RELEASE_TOKEN", "")
|
||||
return base, owner, token
|
||||
|
||||
|
||||
def bundle_url(source_commit: str, filename: str) -> str:
|
||||
base, owner, _ = config()
|
||||
return (
|
||||
f"{base}/api/packages/{owner}/generic/{PACKAGE_NAME}/"
|
||||
f"{source_commit}/{filename}"
|
||||
)
|
||||
|
||||
|
||||
def request(url: str, *, method: str = "GET", data: object | None = None,
|
||||
length: int | None = None) -> urllib.request.Request:
|
||||
result = urllib.request.Request(url, data=data, method=method) # type: ignore[arg-type]
|
||||
_, _, token = config()
|
||||
if token:
|
||||
result.add_header("Authorization", f"token {token}")
|
||||
if length is not None:
|
||||
result.add_header("Content-Length", str(length))
|
||||
result.add_header("Content-Type", "application/octet-stream")
|
||||
return result
|
||||
|
||||
|
||||
def remote_bytes(url: str) -> bytes | None:
|
||||
try:
|
||||
with urllib.request.urlopen(
|
||||
request(url), timeout=HTTP_TIMEOUT_SECONDS,
|
||||
) as response:
|
||||
return response.read()
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code == 404:
|
||||
return None
|
||||
raise ReleaseManifestError(
|
||||
f"checking release bundle failed (HTTP {exc.code}): {url}") from exc
|
||||
except urllib.error.URLError as exc:
|
||||
raise ReleaseManifestError(
|
||||
f"release registry unreachable: {url} ({exc.reason})") from exc
|
||||
|
||||
|
||||
def remote_sha256(url: str) -> str | None:
|
||||
try:
|
||||
with urllib.request.urlopen(
|
||||
request(url), timeout=HTTP_TIMEOUT_SECONDS,
|
||||
) as response:
|
||||
digest = hashlib.sha256()
|
||||
for chunk in iter(lambda: response.read(1 << 20), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code == 404:
|
||||
return None
|
||||
raise ReleaseManifestError(
|
||||
f"checking release artifact failed (HTTP {exc.code}): {url}") from exc
|
||||
except urllib.error.URLError as exc:
|
||||
raise ReleaseManifestError(
|
||||
f"release registry unreachable: {url} ({exc.reason})") from exc
|
||||
|
||||
|
||||
def upload(url: str, source: bytes | Path) -> None:
|
||||
handle = None
|
||||
if isinstance(source, Path):
|
||||
handle = source.open("rb")
|
||||
data: object = handle
|
||||
length = source.stat().st_size
|
||||
else:
|
||||
data = source
|
||||
length = len(source)
|
||||
try:
|
||||
with urllib.request.urlopen(
|
||||
request(url, method="PUT", data=data, length=length),
|
||||
timeout=HTTP_TIMEOUT_SECONDS,
|
||||
):
|
||||
pass
|
||||
except (urllib.error.HTTPError, urllib.error.URLError) as exc:
|
||||
raise ReleaseManifestError(
|
||||
f"publishing release artifact failed: {url} ({exc})") from exc
|
||||
finally:
|
||||
if handle is not None:
|
||||
handle.close()
|
||||
|
||||
|
||||
def publish_bundle(index: dict[str, object], wheel: Path) -> str:
|
||||
"""Publish wheel first and index last, or verify an existing exact bundle."""
|
||||
parse_bundle_index(index)
|
||||
source_commit = str(index["source_commit"])
|
||||
wheel_data = index["wheel"]
|
||||
assert isinstance(wheel_data, dict)
|
||||
filename = str(wheel_data["filename"])
|
||||
expected_sha = str(wheel_data["sha256"])
|
||||
if wheel.name != filename or sha256_file(wheel) != expected_sha:
|
||||
raise ReleaseManifestError(
|
||||
"release bundle wheel does not match its index")
|
||||
|
||||
wheel_url = bundle_url(source_commit, filename)
|
||||
index_url = bundle_url(source_commit, INDEX_NAME)
|
||||
if wheel_data["url"] != wheel_url:
|
||||
raise ReleaseManifestError(
|
||||
"release bundle wheel URL does not match its commit coordinate")
|
||||
|
||||
wanted_index = canonical_bytes(index)
|
||||
existing_index = remote_bytes(index_url)
|
||||
existing_wheel_sha = remote_sha256(wheel_url)
|
||||
if existing_index is not None:
|
||||
if existing_index != wanted_index or existing_wheel_sha != expected_sha:
|
||||
raise ReleaseManifestError(
|
||||
f"immutable release bundle {source_commit} already differs")
|
||||
return index_url
|
||||
if existing_wheel_sha is not None:
|
||||
raise ReleaseManifestError(
|
||||
f"partial release bundle {source_commit} exists without its index; "
|
||||
"administrative cleanup is required")
|
||||
|
||||
upload(wheel_url, wheel)
|
||||
upload(index_url, wanted_index)
|
||||
return index_url
|
||||
|
||||
|
||||
__all__ = [
|
||||
"INDEX_NAME",
|
||||
"PACKAGE_NAME",
|
||||
"bundle_url",
|
||||
"config",
|
||||
"publish_bundle",
|
||||
]
|
||||
@@ -0,0 +1,197 @@
|
||||
"""Validated release and channel pointers for qualified commit bundles."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
from .release_bundle import parse_bundle_index
|
||||
from .release_manifest import ReleaseManifestError
|
||||
from .release_publish import (
|
||||
HTTP_TIMEOUT_SECONDS,
|
||||
bundle_url,
|
||||
config,
|
||||
remote_bytes,
|
||||
request,
|
||||
upload,
|
||||
)
|
||||
|
||||
_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||
_STABLE_TAG_RE = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$")
|
||||
_STAGING_TAG_RE = re.compile(r"^v(\d+)\.(\d+)\.(\d+)-rc\.(\d+)$")
|
||||
|
||||
|
||||
def release_channel(tag: str) -> str:
|
||||
"""Return the only channel eligible for *tag*."""
|
||||
if _STABLE_TAG_RE.fullmatch(tag):
|
||||
return "production"
|
||||
if _STAGING_TAG_RE.fullmatch(tag):
|
||||
return "staging"
|
||||
raise ReleaseManifestError(
|
||||
"release tag must be vX.Y.Z or vX.Y.Z-rc.N")
|
||||
|
||||
|
||||
def release_version(tag: str) -> tuple[int, int, int, int]:
|
||||
"""Return a monotonically comparable version tuple."""
|
||||
stable = _STABLE_TAG_RE.fullmatch(tag)
|
||||
if stable:
|
||||
return (
|
||||
int(stable.group(1)),
|
||||
int(stable.group(2)),
|
||||
int(stable.group(3)),
|
||||
1 << 30,
|
||||
)
|
||||
staging = _STAGING_TAG_RE.fullmatch(tag)
|
||||
if staging:
|
||||
return (
|
||||
int(staging.group(1)),
|
||||
int(staging.group(2)),
|
||||
int(staging.group(3)),
|
||||
int(staging.group(4)),
|
||||
)
|
||||
raise ReleaseManifestError(
|
||||
"release tag must be vX.Y.Z or vX.Y.Z-rc.N")
|
||||
|
||||
|
||||
def pointer_url(package: str, version: str, filename: str) -> str:
|
||||
base, owner, _ = config()
|
||||
return f"{base}/api/packages/{owner}/generic/{package}/{version}/{filename}"
|
||||
|
||||
|
||||
def build_release_pointer(
|
||||
*,
|
||||
tag: str,
|
||||
source_commit: str,
|
||||
workflow_run: str,
|
||||
qualified_at: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Build a qualified pointer to one immutable commit bundle."""
|
||||
channel = release_channel(tag)
|
||||
if _COMMIT_RE.fullmatch(source_commit) is None:
|
||||
raise ReleaseManifestError(
|
||||
"release source_commit must be 40 lowercase hex")
|
||||
if not workflow_run.strip() or not qualified_at.strip():
|
||||
raise ReleaseManifestError(
|
||||
"release qualification provenance is incomplete")
|
||||
return {
|
||||
"schema": 1,
|
||||
"qualified": True,
|
||||
"tag": tag,
|
||||
"channel": channel,
|
||||
"source_commit": source_commit,
|
||||
"bundle_index_url": bundle_url(source_commit, "bundle-index.json"),
|
||||
"qualification": {
|
||||
"workflow_run": workflow_run.strip(),
|
||||
"qualified_at": qualified_at.strip(),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def parse_release_pointer(data: Any) -> dict[str, Any]:
|
||||
"""Validate a release or channel pointer."""
|
||||
if not isinstance(data, dict) or data.get("schema") != 1:
|
||||
raise ReleaseManifestError("release pointer must use schema 1")
|
||||
if data.get("qualified") is not True:
|
||||
raise ReleaseManifestError("release pointer must be qualified")
|
||||
tag = data.get("tag")
|
||||
channel = data.get("channel")
|
||||
commit = data.get("source_commit")
|
||||
if not isinstance(tag, str) or release_channel(tag) != channel:
|
||||
raise ReleaseManifestError("release pointer tag and channel disagree")
|
||||
if not isinstance(commit, str) or _COMMIT_RE.fullmatch(commit) is None:
|
||||
raise ReleaseManifestError("release pointer source commit is invalid")
|
||||
if data.get("bundle_index_url") != bundle_url(commit, "bundle-index.json"):
|
||||
raise ReleaseManifestError("release pointer bundle URL is invalid")
|
||||
qualification = data.get("qualification")
|
||||
if not isinstance(qualification, dict) or not all(
|
||||
isinstance(qualification.get(key), str) and qualification[key].strip()
|
||||
for key in ("workflow_run", "qualified_at")
|
||||
):
|
||||
raise ReleaseManifestError("release qualification is incomplete")
|
||||
return data
|
||||
|
||||
|
||||
def canonical_pointer(data: Any) -> bytes:
|
||||
parse_release_pointer(data)
|
||||
return (json.dumps(data, indent=2, sort_keys=True) + "\n").encode()
|
||||
|
||||
|
||||
def delete(url: str) -> None:
|
||||
try:
|
||||
with urllib.request.urlopen(
|
||||
request(url, method="DELETE"), timeout=HTTP_TIMEOUT_SECONDS,
|
||||
):
|
||||
pass
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code != 404:
|
||||
raise ReleaseManifestError(
|
||||
f"replacing release channel failed (HTTP {exc.code}): {url}"
|
||||
) from exc
|
||||
except urllib.error.URLError as exc:
|
||||
raise ReleaseManifestError(
|
||||
f"release registry unreachable: {url} ({exc.reason})") from exc
|
||||
|
||||
|
||||
def publish_qualification(
|
||||
pointer: dict[str, Any], *, allow_rollback: bool = False,
|
||||
) -> tuple[str, str]:
|
||||
"""Publish an immutable tag pointer and advance its channel."""
|
||||
parse_release_pointer(pointer)
|
||||
tag = str(pointer["tag"])
|
||||
channel = str(pointer["channel"])
|
||||
wanted = canonical_pointer(pointer)
|
||||
release_url = pointer_url(
|
||||
"bot-bottle-releases", tag, "release.json")
|
||||
channel_url = pointer_url(
|
||||
"bot-bottle-channels", channel, "channel.json")
|
||||
|
||||
bundle = remote_bytes(str(pointer["bundle_index_url"]))
|
||||
if bundle is None:
|
||||
raise ReleaseManifestError(
|
||||
f"commit bundle does not exist for {pointer['source_commit']}")
|
||||
try:
|
||||
bundle_data = parse_bundle_index(json.loads(bundle))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||
raise ReleaseManifestError("commit bundle index is malformed") from exc
|
||||
if bundle_data["source_commit"] != pointer["source_commit"]:
|
||||
raise ReleaseManifestError(
|
||||
"commit bundle index does not match the qualified source commit")
|
||||
existing_release = remote_bytes(release_url)
|
||||
if existing_release is not None and existing_release != wanted:
|
||||
raise ReleaseManifestError(
|
||||
f"immutable release pointer {tag} already differs")
|
||||
if existing_release is None:
|
||||
upload(release_url, wanted)
|
||||
|
||||
existing_channel = remote_bytes(channel_url)
|
||||
if existing_channel == wanted:
|
||||
return release_url, channel_url
|
||||
if existing_channel is not None:
|
||||
try:
|
||||
current = parse_release_pointer(json.loads(existing_channel))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||
raise ReleaseManifestError(
|
||||
f"existing {channel} channel pointer is malformed") from exc
|
||||
if (
|
||||
not allow_rollback
|
||||
and release_version(tag) <= release_version(str(current["tag"]))
|
||||
):
|
||||
raise ReleaseManifestError(
|
||||
f"{channel} channel update is not monotonic; "
|
||||
"use the explicit rollback operation")
|
||||
delete(channel_url)
|
||||
upload(channel_url, wanted)
|
||||
return release_url, channel_url
|
||||
|
||||
|
||||
__all__ = [
|
||||
"build_release_pointer",
|
||||
"canonical_pointer",
|
||||
"parse_release_pointer",
|
||||
"publish_qualification",
|
||||
"release_channel",
|
||||
"release_version",
|
||||
]
|
||||
+17
@@ -45,6 +45,23 @@ report; macOS reports advisory coverage in its own job. The macOS infra
|
||||
container is a singleton, so its job uses a concurrency group and always tears
|
||||
the service down.
|
||||
|
||||
## Artifact publication and promotion
|
||||
|
||||
[`publish-artifacts.yml`](../.gitea/workflows/publish-artifacts.yml) is the
|
||||
manual deployment-test boundary. Dispatch it with any branch, tag, or commit;
|
||||
it resolves that input once, publishes the orchestrator, gateway, Claude,
|
||||
Codex, Pi, and Firecracker artifacts, then publishes only the wheel verified
|
||||
against those identities. Install the reported commit with
|
||||
`BOT_BOTTLE_REF=<sha> sh install.sh`; the installer warns that snapshots may
|
||||
not have completed release qualification.
|
||||
|
||||
[`release.yml`](../.gitea/workflows/release.yml) handles promotion tags.
|
||||
`vX.Y.Z-rc.N` must be reachable from `staging`; `vX.Y.Z` must be reachable
|
||||
from `production`. It runs the complete pre-release workflow, ensures that the
|
||||
same commit bundle is published, verifies a clean install, creates the
|
||||
qualified release pointer, and advances the matching channel. Promotion
|
||||
reuses immutable bundle bytes and never rebuilds them.
|
||||
|
||||
## Scheduled canary
|
||||
|
||||
[`.gitea/workflows/canaries.yml`](../.gitea/workflows/canaries.yml) runs weekly
|
||||
|
||||
@@ -0,0 +1,374 @@
|
||||
# PRD 0083: Packaged infrastructure artifacts
|
||||
|
||||
- **Status:** Active
|
||||
- **Author:** Codex
|
||||
- **Created:** 2026-07-27
|
||||
- **Issue:** #536
|
||||
|
||||
## Summary
|
||||
|
||||
Publish the orchestrator and gateway infrastructure before packaging
|
||||
bot-bottle, record their immutable identities in the application package, and
|
||||
make every production backend acquire those exact artifacts instead of
|
||||
building infrastructure during bottle startup. Docker and Apple Container pull
|
||||
digest-pinned OCI images; Firecracker pulls checksum-verified rootfs artifacts
|
||||
selected by the same packaged release manifest.
|
||||
|
||||
## Problem
|
||||
|
||||
Starting a bottle currently invokes an infrastructure build on Docker and
|
||||
Apple Container even when a healthy orchestrator is already running. Both
|
||||
infra composers call `ensure_built()` before their health/currentness checks,
|
||||
and both implementations unconditionally invoke their OCI builder. Layer
|
||||
caching can make this less expensive, but startup still depends on a working
|
||||
builder, build context, base-image registry, and package installation network.
|
||||
|
||||
The Apple Container orchestrator also bind-mounts the installed source over the
|
||||
package baked into its image. A pinned image alone would therefore not pin the
|
||||
code that actually runs.
|
||||
|
||||
Firecracker already downloads fixed rootfs artifacts, but it derives their
|
||||
versions from the package contents at launch. There is no package-level record
|
||||
that says which already-published infrastructure release belongs to an
|
||||
installed bot-bottle release.
|
||||
|
||||
The result is three different delivery contracts:
|
||||
|
||||
- Docker and Apple Container build mutable `:latest` images on the launch host.
|
||||
- Firecracker computes an artifact version locally and downloads it.
|
||||
- An installed wheel contains Dockerfiles specifically so production startup
|
||||
can rebuild its own infrastructure.
|
||||
|
||||
That makes startup slower and less reliable, prevents a packaged application
|
||||
from identifying the infrastructure it was tested with, and weakens the
|
||||
supply-chain boundary between release production and runtime.
|
||||
|
||||
## Goals / Success criteria
|
||||
|
||||
- A packaged bot-bottle release contains one validated release manifest with
|
||||
immutable orchestrator and gateway identities.
|
||||
- Docker and Apple Container acquire and run the manifest's digest-pinned OCI
|
||||
images without invoking `docker build` or `container build`.
|
||||
- The shipped Claude, Codex, and Pi agent images are built, smoke-tested,
|
||||
published, and selected by digest as part of the same commit bundle.
|
||||
- Firecracker acquires the manifest's versioned, checksum-verified
|
||||
orchestrator and gateway rootfs artifacts rather than deriving versions at
|
||||
launch.
|
||||
- All three backends expose the same backend-neutral `ensure_available`
|
||||
lifecycle contract.
|
||||
- A healthy running orchestrator is retained only when its recorded release
|
||||
identity matches the installed package.
|
||||
- Apple Container runs the package baked into the orchestrator image; no host
|
||||
source overlays production code.
|
||||
- Missing, mutable, malformed, or internally inconsistent release metadata
|
||||
fails closed before infrastructure starts.
|
||||
- Source-checkout development retains an explicit local-build mode.
|
||||
- Publishing produces the infrastructure first and the installable Python
|
||||
package last, so the package cannot name artifacts that were not published.
|
||||
- Any branch, tag, or commit can be resolved once to a full source SHA and
|
||||
published as one immutable, commit-addressed artifact bundle.
|
||||
- The installer defaults to the production channel, can select staging or an
|
||||
exact commit, and warns for commit snapshots that have not been qualified as
|
||||
a release.
|
||||
- Staging and production releases promote the same tested artifact bytes;
|
||||
promotion never rebuilds them.
|
||||
- A release is published only after the complete pre-release suite and a clean
|
||||
install from the published bundle succeed.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Prebuilding user-supplied Dockerfiles. Custom agent images continue to build
|
||||
through the backend-specific agent-image paths.
|
||||
- Installing Docker, Apple Container, Firecracker, or other host prerequisites.
|
||||
- Replacing Gitea's OCI or generic-package registries.
|
||||
- Adding image signing or a transparency log. Immutable digests and existing
|
||||
Firecracker SHA-256 verification are the integrity boundary for this version.
|
||||
- Forcing a network download when an immutable, verified artifact is already
|
||||
cached locally. "Pull" means acquire the packaged identity, with safe cache
|
||||
reuse.
|
||||
- Supporting live source overlays in packaged production installs.
|
||||
- Defining the project's version-numbering policy beyond stable
|
||||
`vX.Y.Z` and staging `vX.Y.Z-rc.N` tag shapes.
|
||||
|
||||
## Design
|
||||
|
||||
### Release manifest
|
||||
|
||||
`bot_bottle/release-manifest.json` is generated during package production and
|
||||
shipped as package data:
|
||||
|
||||
```json
|
||||
{
|
||||
"schema": 1,
|
||||
"source_commit": "<40 lowercase hex>",
|
||||
"oci": {
|
||||
"orchestrator": "gitea.dideric.is/didericis/bot-bottle-orchestrator@sha256:<64 hex>",
|
||||
"gateway": "gitea.dideric.is/didericis/bot-bottle-gateway@sha256:<64 hex>",
|
||||
"agent_claude": "gitea.dideric.is/didericis/bot-bottle-claude@sha256:<64 hex>",
|
||||
"agent_codex": "gitea.dideric.is/didericis/bot-bottle-codex@sha256:<64 hex>",
|
||||
"agent_pi": "gitea.dideric.is/didericis/bot-bottle-pi@sha256:<64 hex>"
|
||||
},
|
||||
"firecracker": {
|
||||
"orchestrator": {
|
||||
"version": "<artifact version>",
|
||||
"sha256": "<rootfs.ext4.gz sha256>"
|
||||
},
|
||||
"gateway": {
|
||||
"version": "<artifact version>",
|
||||
"sha256": "<rootfs.ext4.gz sha256>"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The runtime loader accepts only schema 1, digest-qualified OCI references,
|
||||
non-empty Firecracker versions, 64-character lowercase SHA-256 values, and a
|
||||
full source commit. It returns immutable value objects rather than passing raw
|
||||
dictionaries through backend code.
|
||||
|
||||
The repository carries no pretend production pins. A source checkout or ad-hoc
|
||||
Git wheel carries a `development: true` marker and selects local infrastructure
|
||||
builds, preserving the contributor and current `install.sh` paths. The release
|
||||
packaging workflow must replace that marker with the validated schema above;
|
||||
it refuses mutable or incomplete release inputs.
|
||||
|
||||
### Packaging order
|
||||
|
||||
The release job operates on one tested commit:
|
||||
|
||||
1. Build and smoke-test the multi-architecture orchestrator, gateway, Claude,
|
||||
Codex, and Pi OCI images.
|
||||
2. Push them and resolve their registry manifest digests.
|
||||
3. Build Firecracker's orchestrator and gateway rootfs artifacts from the same
|
||||
source and pinned OCI bases.
|
||||
4. Publish both generic-package artifacts and retain their versions and
|
||||
compressed-file checksums.
|
||||
5. Generate `release-manifest.json` from those published identities.
|
||||
6. Build the wheel with that manifest. Do not produce a source distribution:
|
||||
rebuilding an sdist outside this release boundary could replace the pins
|
||||
with the development manifest.
|
||||
7. Install the wheel in a clean environment and assert that every manifest
|
||||
identity is valid and retrievable.
|
||||
8. Publish only that verified wheel.
|
||||
|
||||
The build hook receives the manifest as a file input. It must not query a
|
||||
registry or choose versions itself: package builds stay deterministic and
|
||||
network-independent once their release inputs exist.
|
||||
|
||||
### Commit-addressed bundle index
|
||||
|
||||
Every publication resolves its requested branch, tag, or commit to one
|
||||
40-character source SHA before doing any work. The publisher writes an
|
||||
immutable external bundle index under that SHA. The index contains:
|
||||
|
||||
- schema version and source SHA;
|
||||
- wheel URL and SHA-256;
|
||||
- orchestrator and gateway OCI digest references;
|
||||
- Claude, Codex, and Pi agent OCI digest references;
|
||||
- Firecracker package versions and compressed-file SHA-256 values;
|
||||
- producing workflow/run identity and publication timestamp; and
|
||||
- producing workflow provenance. Qualification is recorded separately in an
|
||||
immutable tag pointer so the commit bundle itself never changes.
|
||||
|
||||
The embedded wheel manifest contains the runtime subset of the same identities.
|
||||
Publication fails if the embedded manifest and external index differ.
|
||||
|
||||
Bundle coordinates are immutable. Re-running publication for a SHA verifies
|
||||
and reuses a byte-identical complete bundle; it never replaces one artifact or
|
||||
mixes outputs from different runs. A partial existing bundle is an error and
|
||||
requires an explicit administrative cleanup before retrying.
|
||||
|
||||
Commit bundles are durable release inputs, not expiring CI artifacts. The
|
||||
project may garbage-collect unqualified snapshots only under a documented
|
||||
retention policy; qualified staging and production bundles are retained.
|
||||
|
||||
### Branches, tags, and promotion
|
||||
|
||||
The protected promotion topology is:
|
||||
|
||||
```text
|
||||
feature -> main -> staging -> production
|
||||
| |
|
||||
vX.Y.Z-rc.N vX.Y.Z
|
||||
```
|
||||
|
||||
Changes reach `staging` and `production` through promotion pull requests; those
|
||||
branches do not accept direct development commits. A staging tag must point to
|
||||
a commit reachable from `staging` and uses `vX.Y.Z-rc.N`. A production tag must
|
||||
point to a commit reachable from `production` and uses `vX.Y.Z`.
|
||||
|
||||
Channels are small, mutable pointers to immutable qualified bundles:
|
||||
|
||||
- `production` points to the newest promoted stable tag;
|
||||
- `staging` points to the newest promoted release-candidate tag; and
|
||||
- `main` is not a release channel; its commits are installable snapshots.
|
||||
|
||||
Channel updates are serialized and monotonic. Moving a channel to an older
|
||||
release requires a distinct, explicit rollback operation that records the
|
||||
reason and target. Deleting or moving a Git tag does not mutate a published
|
||||
bundle or channel silently.
|
||||
|
||||
Promotion reuses the exact bundle selected by source SHA. It does not rebuild
|
||||
the wheel, OCI images, or Firecracker artifacts. Thus production runs the same
|
||||
bytes qualified in staging.
|
||||
|
||||
### Publication workflows
|
||||
|
||||
`publish-artifacts` is manually dispatchable for any branch, tag, or commit.
|
||||
It resolves the input to a source SHA, checks out that SHA, builds and publishes
|
||||
all infrastructure artifacts, generates the bundle index and embedded
|
||||
manifest, builds the wheel, installs the wheel in a clean environment, and
|
||||
publishes the verified wheel. It does not create a release or update staging or
|
||||
production. Its output is an unqualified snapshot and publication is
|
||||
serialized per source SHA.
|
||||
|
||||
After required CI succeeds, `main` may invoke the same publication operation
|
||||
automatically. Manual publication remains available so a branch can be tested
|
||||
before merge. Automatic and manual runs converge on the same idempotent bundle.
|
||||
|
||||
The release workflow is triggered by an eligible staging or production tag:
|
||||
|
||||
1. Resolve and validate the tag, source SHA, tag shape, and branch reachability.
|
||||
2. Run the complete pre-release suite against that SHA, including Docker,
|
||||
Firecracker, and Apple Container.
|
||||
3. After the suite passes, ensure the immutable commit bundle exists. Build it
|
||||
then if absent; otherwise verify every existing identity and checksum.
|
||||
4. Install using the published installer and bundle index on clean supported
|
||||
hosts, and run the post-install smoke test against the acquired artifacts.
|
||||
5. Publish an immutable qualified tag pointer, create the Gitea release, and
|
||||
advance the matching channel pointer.
|
||||
|
||||
No release artifact or channel update occurs before qualification succeeds.
|
||||
Jobs use per-SHA and per-channel concurrency locks so concurrent dispatches
|
||||
cannot publish twice or race a promotion.
|
||||
|
||||
### Installer selection and warnings
|
||||
|
||||
The installer installs the latest `production` channel by default. It accepts:
|
||||
|
||||
- `BOT_BOTTLE_CHANNEL=production` or `staging` to select the latest release in
|
||||
that channel;
|
||||
- `BOT_BOTTLE_REF=<40-character-sha>` to install an exact commit bundle; and
|
||||
- `BOT_BOTTLE_VERSION=<tag>` to install an exact qualified release.
|
||||
|
||||
Selectors are mutually exclusive. Branch names are accepted by the publication
|
||||
workflow but not by the installer because they are mutable; callers resolve
|
||||
and publish them first, then install the reported SHA.
|
||||
|
||||
An exact-commit install prints a prominent warning that the snapshot may not
|
||||
have passed release qualification, even if the same SHA is later associated
|
||||
with a tag. A tag or channel install suppresses that warning only when its
|
||||
validated pointer is marked qualified and selects the same immutable bundle
|
||||
SHA.
|
||||
|
||||
The installer downloads the wheel named by the external index, verifies its
|
||||
SHA-256 before invoking pipx or pip, and reports the selected channel/tag,
|
||||
source SHA, and bundle identity. It never builds a Git checkout for a packaged
|
||||
install and never trusts a mutable channel response without resolving it to an
|
||||
immutable bundle index.
|
||||
|
||||
### Backend-neutral lifecycle
|
||||
|
||||
Rename the host-side control-plane lifecycle operation from `ensure_built()` to
|
||||
`ensure_available()`. The production meaning is "make the package-selected
|
||||
artifact locally available and verified." Infrastructure composers call it
|
||||
before starting either plane.
|
||||
|
||||
The gateway receives the same contract so startup does not continue to rebuild
|
||||
the adjacent half of the fixed infrastructure pair.
|
||||
|
||||
A source checkout is itself a development boundary and selects the existing
|
||||
builders. `BOT_BOTTLE_INFRA_BUILD=local` provides the same explicit override
|
||||
for release debugging. Both paths deliberately bypass release metadata.
|
||||
Packaged production startup never silently falls back from a failed pull to a
|
||||
local build.
|
||||
|
||||
### Docker
|
||||
|
||||
The Docker backend runs `docker pull` with each digest-qualified manifest
|
||||
reference. Docker may reuse its content-addressed local store. Containers are
|
||||
created from the immutable reference, and the running container's image ID is
|
||||
compared with the locally resolved ID for currentness.
|
||||
|
||||
The orchestrator source-hash label is replaced by a release-identity label.
|
||||
The label is diagnostic; the image ID comparison remains authoritative.
|
||||
|
||||
### Apple Container
|
||||
|
||||
The Apple Container backend pulls the same multi-architecture OCI references.
|
||||
Because Apple Container's accepted syntax for pull, tag, inspect, and run is
|
||||
not identical to Docker's, one utility owns normalization from a digest
|
||||
reference to the local runnable name. Integration coverage on the macOS runner
|
||||
guards that adapter.
|
||||
|
||||
Production orchestrator launch removes the build-root bind mount, `PYTHONPATH`,
|
||||
and `BOT_BOTTLE_SOURCE_HASH`. It runs only the code baked into the pulled
|
||||
image. Currentness is based on the packaged release identity/image ID.
|
||||
|
||||
### Firecracker
|
||||
|
||||
The publishing tool may continue computing content-derived artifact versions;
|
||||
that is a producer concern. Runtime launch instead reads the version and
|
||||
expected compressed checksum from the release manifest.
|
||||
|
||||
The artifact cache validates the packaged expected checksum even when a
|
||||
previous `.verified` marker exists. The marker records the checksum it
|
||||
validated, rather than an unqualified `ok`, so changing package metadata cannot
|
||||
adopt an artifact verified against a different expectation.
|
||||
|
||||
The existing candidate-directory and explicit local-build paths remain for CI
|
||||
and development. Candidate metadata must match the packaged identity unless
|
||||
the caller selected local development mode.
|
||||
|
||||
### Overrides and failure behavior
|
||||
|
||||
`BOT_BOTTLE_ORCHESTRATOR_IMAGE` and `BOT_BOTTLE_GATEWAY_IMAGE` remain useful
|
||||
for testing but production accepts them only when they are digest-qualified.
|
||||
Mutable tag overrides require explicit local-build mode.
|
||||
|
||||
Artifact acquisition errors identify the packaged reference and backend. No
|
||||
backend catches an acquisition failure and rebuilds from local source.
|
||||
|
||||
## Testing strategy
|
||||
|
||||
- Unit-test manifest parsing, immutability validation, missing-manifest errors,
|
||||
package-data inclusion, and deterministic build-hook injection.
|
||||
- Assert Docker and Apple production preparation pulls and never builds.
|
||||
- Assert explicit local mode retains existing build behavior.
|
||||
- Assert the Apple orchestrator command has no source bind mount or
|
||||
`PYTHONPATH`.
|
||||
- Assert running-image mismatches recreate infrastructure while matching,
|
||||
healthy instances are retained.
|
||||
- Assert Firecracker uses packaged versions/checksums and rejects mismatches in
|
||||
downloaded, cached, and candidate artifacts.
|
||||
- Extend wheel-install coverage to inspect the installed manifest.
|
||||
- Run the full unit and Docker integration gates; run advisory Firecracker and
|
||||
macOS integration before publishing a release.
|
||||
- Test ref resolution, tag-shape and branch-reachability enforcement, bundle
|
||||
idempotency, partial-publication rejection, and per-SHA concurrency.
|
||||
- Test installer channel, tag, and exact-commit selection; checksum rejection;
|
||||
mutually exclusive selectors; and snapshot warning behavior.
|
||||
- Test that release qualification happens before publication and that a failed
|
||||
suite or clean-install smoke test cannot create a release or move a channel.
|
||||
- Test promotion reuses the qualified bundle byte-for-byte and that monotonic
|
||||
channel movement rejects an accidental rollback.
|
||||
|
||||
## Rollout
|
||||
|
||||
Land runtime, installer, bundle publication, and release qualification support
|
||||
together, but do not move the production installer away from its current
|
||||
source-install path until an end-to-end staging release has passed on Docker,
|
||||
Firecracker, and Apple Container.
|
||||
|
||||
Create and protect `staging` and `production`, then promote one commit from
|
||||
`main` through both branches. Publish its commit bundle, qualify an RC tag,
|
||||
perform clean installs through the staging channel, promote the same SHA, and
|
||||
qualify a stable tag. Until that first stable qualification exists, the
|
||||
installer's production default fails closed because there is no production
|
||||
pointer. The stable promotion must demonstrate that every production artifact
|
||||
identity matches the staging-qualified bundle.
|
||||
|
||||
Existing source checkouts use explicit local mode. Existing mutable local
|
||||
images are ignored by production acquisition and may be pruned independently.
|
||||
Snapshot retention and rollback procedures must be documented before automatic
|
||||
publication from every successful `main` commit is enabled.
|
||||
+111
-4
@@ -17,10 +17,13 @@
|
||||
# Env:
|
||||
# BOT_BOTTLE_PYTHON interpreter to install with (skips the search)
|
||||
# BOT_BOTTLE_INSTALL_SPEC pip/git spec to install instead of the default
|
||||
# BOT_BOTTLE_CHANNEL production (default) or staging
|
||||
# BOT_BOTTLE_VERSION exact qualified vX.Y.Z[-rc.N] release
|
||||
# BOT_BOTTLE_REF exact 40-character commit snapshot (warns untested)
|
||||
# BOT_BOTTLE_VENV where the non-pipx install lives (~/.bot-bottle/venv)
|
||||
set -eu
|
||||
|
||||
PACKAGE_SPEC="${BOT_BOTTLE_INSTALL_SPEC:-git+https://gitea.dideric.is/didericis/bot-bottle.git}"
|
||||
PACKAGE_SPEC="${BOT_BOTTLE_INSTALL_SPEC:-}"
|
||||
VENV="${BOT_BOTTLE_VENV:-${HOME}/.bot-bottle/venv}"
|
||||
MIN_PYTHON_MAJOR=3
|
||||
MIN_PYTHON_MINOR=11
|
||||
@@ -126,9 +129,113 @@ if [ "${PYTHON}" != "${path_python}" ]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# Installing a `git+` spec (the default) shells out to git under the hood,
|
||||
# whether via pipx or pip. Fail early with a clear message rather than deep
|
||||
# inside the installer's output.
|
||||
# --- resolve an immutable published wheel -----------------------------------
|
||||
|
||||
if [ -z "${PACKAGE_SPEC}" ]; then
|
||||
selectors=0
|
||||
[ -n "${BOT_BOTTLE_REF:-}" ] && selectors=$((selectors + 1))
|
||||
[ -n "${BOT_BOTTLE_VERSION:-}" ] && selectors=$((selectors + 1))
|
||||
[ -n "${BOT_BOTTLE_CHANNEL:-}" ] && selectors=$((selectors + 1))
|
||||
[ "${selectors}" -le 1 ] || die \
|
||||
"BOT_BOTTLE_REF, BOT_BOTTLE_VERSION, and BOT_BOTTLE_CHANNEL are mutually exclusive."
|
||||
|
||||
downloads="${HOME}/.bot-bottle/downloads"
|
||||
mkdir -p "${downloads}"
|
||||
PACKAGE_SPEC="$("${PYTHON}" - \
|
||||
"${BOT_BOTTLE_REF:-}" \
|
||||
"${BOT_BOTTLE_VERSION:-}" \
|
||||
"${BOT_BOTTLE_CHANNEL:-production}" \
|
||||
"${downloads}" <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ref, version, channel, download_root = sys.argv[1:]
|
||||
base = "https://gitea.dideric.is/api/packages/didericis/generic"
|
||||
|
||||
def fetch_json(url):
|
||||
with urllib.request.urlopen(url, timeout=30) as response:
|
||||
return json.load(response)
|
||||
|
||||
if ref:
|
||||
if re.fullmatch(r"[0-9a-f]{40}", ref) is None:
|
||||
raise SystemExit("bot-bottle install: error: BOT_BOTTLE_REF must be 40 lowercase hex")
|
||||
index_url = f"{base}/bot-bottle-builds/{ref}/bundle-index.json"
|
||||
print(
|
||||
"bot-bottle install: WARNING: installing an exact commit snapshot; "
|
||||
"it may not have passed release qualification.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
selector = f"commit {ref}"
|
||||
else:
|
||||
if version:
|
||||
if re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+(?:-rc\.[0-9]+)?", version) is None:
|
||||
raise SystemExit("bot-bottle install: error: invalid BOT_BOTTLE_VERSION")
|
||||
pointer_url = f"{base}/bot-bottle-releases/{version}/release.json"
|
||||
selector = f"release {version}"
|
||||
else:
|
||||
if channel not in ("production", "staging"):
|
||||
raise SystemExit(
|
||||
"bot-bottle install: error: BOT_BOTTLE_CHANNEL must be production or staging")
|
||||
pointer_url = f"{base}/bot-bottle-channels/{channel}/channel.json"
|
||||
selector = f"{channel} channel"
|
||||
pointer = fetch_json(pointer_url)
|
||||
if pointer.get("schema") != 1 or pointer.get("qualified") is not True:
|
||||
raise SystemExit("bot-bottle install: error: release pointer is not qualified")
|
||||
index_url = pointer.get("bundle_index_url", "")
|
||||
if not isinstance(index_url, str) or not index_url.startswith("https://"):
|
||||
raise SystemExit("bot-bottle install: error: release pointer has no immutable bundle")
|
||||
|
||||
index = fetch_json(index_url)
|
||||
source_commit = index.get("source_commit", "")
|
||||
if re.fullmatch(r"[0-9a-f]{40}", source_commit) is None:
|
||||
raise SystemExit("bot-bottle install: error: bundle has an invalid source commit")
|
||||
if ref and source_commit != ref:
|
||||
raise SystemExit("bot-bottle install: error: bundle does not match BOT_BOTTLE_REF")
|
||||
wheel = index.get("wheel")
|
||||
if not isinstance(wheel, dict):
|
||||
raise SystemExit("bot-bottle install: error: bundle has no wheel")
|
||||
filename, url, expected = (
|
||||
wheel.get("filename"), wheel.get("url"), wheel.get("sha256"))
|
||||
if (
|
||||
not isinstance(filename, str)
|
||||
or re.fullmatch(r"[A-Za-z0-9_.-]+\.whl", filename) is None
|
||||
or not isinstance(url, str)
|
||||
or not url.startswith("https://")
|
||||
or not url.endswith("/" + filename)
|
||||
or not isinstance(expected, str)
|
||||
or re.fullmatch(r"[0-9a-f]{64}", expected) is None
|
||||
):
|
||||
raise SystemExit("bot-bottle install: error: bundle wheel metadata is invalid")
|
||||
|
||||
destination = Path(download_root) / source_commit / filename
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = destination.with_suffix(destination.suffix + ".part")
|
||||
digest = hashlib.sha256()
|
||||
with urllib.request.urlopen(url, timeout=60) as response, temporary.open("wb") as output:
|
||||
while chunk := response.read(1 << 20):
|
||||
digest.update(chunk)
|
||||
output.write(chunk)
|
||||
if digest.hexdigest() != expected:
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise SystemExit("bot-bottle install: error: downloaded wheel checksum mismatch")
|
||||
temporary.replace(destination)
|
||||
print(
|
||||
f"bot-bottle install: selected {selector} "
|
||||
f"(source {source_commit}, sha256 {expected[:16]}…)",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(destination)
|
||||
PY
|
||||
)"
|
||||
fi
|
||||
|
||||
# An explicit `git+` override shells out to git under the hood, whether via
|
||||
# pipx or pip. Fail early with a clear message rather than deep inside the
|
||||
# installer's output.
|
||||
case "${PACKAGE_SPEC}" in
|
||||
git+*|*.git)
|
||||
command -v git >/dev/null 2>&1 || die \
|
||||
|
||||
@@ -36,6 +36,7 @@ include = ["bot_bottle*"]
|
||||
# files shipped under bot_bottle/; test_pyproject.py asserts they exist.
|
||||
[tool.setuptools.package-data]
|
||||
bot_bottle = [
|
||||
"release-manifest.json",
|
||||
"gateway/egress/entrypoint.sh",
|
||||
"contrib/claude/Dockerfile",
|
||||
"contrib/claude/package.json",
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Generate the external commit-addressed release bundle index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from bot_bottle.release_bundle import build_bundle_index, canonical_bytes
|
||||
from bot_bottle.release_publish import bundle_url, publish_bundle
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--manifest", required=True, type=Path)
|
||||
parser.add_argument("--wheel", required=True, type=Path)
|
||||
parser.add_argument("--wheel-url")
|
||||
parser.add_argument("--workflow-run", required=True)
|
||||
parser.add_argument("--published-at", required=True)
|
||||
parser.add_argument("--output", required=True, type=Path)
|
||||
parser.add_argument("--publish", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
manifest = json.loads(args.manifest.read_text(encoding="utf-8"))
|
||||
index = build_bundle_index(
|
||||
manifest,
|
||||
wheel=args.wheel,
|
||||
wheel_url=args.wheel_url or bundle_url(
|
||||
manifest["source_commit"], args.wheel.name),
|
||||
workflow_run=args.workflow_run,
|
||||
published_at=args.published_at,
|
||||
)
|
||||
args.output.write_bytes(canonical_bytes(index))
|
||||
if args.publish:
|
||||
publish_bundle(index, args.wheel)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Generate the validated infrastructure manifest consumed by package builds."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from bot_bottle.release_manifest import parse_manifest
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
result = argparse.ArgumentParser()
|
||||
result.add_argument("--source-commit", required=True)
|
||||
result.add_argument("--orchestrator-image", required=True)
|
||||
result.add_argument("--gateway-image", required=True)
|
||||
for provider in ("claude", "codex", "pi"):
|
||||
result.add_argument(f"--agent-{provider}-image", required=True)
|
||||
for role in ("orchestrator", "gateway"):
|
||||
result.add_argument(f"--firecracker-{role}-version", required=True)
|
||||
result.add_argument(f"--firecracker-{role}-sha256", required=True)
|
||||
result.add_argument("--output", type=Path, required=True)
|
||||
return result
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = parser().parse_args(argv)
|
||||
data = {
|
||||
"schema": 1,
|
||||
"source_commit": args.source_commit,
|
||||
"oci": {
|
||||
"orchestrator": args.orchestrator_image,
|
||||
"gateway": args.gateway_image,
|
||||
**{
|
||||
f"agent_{provider}": getattr(args, f"agent_{provider}_image")
|
||||
for provider in ("claude", "codex", "pi")
|
||||
},
|
||||
},
|
||||
"firecracker": {
|
||||
role: {
|
||||
"version": getattr(args, f"firecracker_{role}_version"),
|
||||
"sha256": getattr(args, f"firecracker_{role}_sha256"),
|
||||
}
|
||||
for role in ("orchestrator", "gateway")
|
||||
},
|
||||
}
|
||||
parse_manifest(data)
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(
|
||||
json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Publish a qualified release and advance its matching channel."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from bot_bottle.release_qualification import (
|
||||
build_release_pointer,
|
||||
canonical_pointer,
|
||||
publish_qualification,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--tag", required=True)
|
||||
parser.add_argument("--source-commit", required=True)
|
||||
parser.add_argument("--workflow-run", required=True)
|
||||
parser.add_argument("--qualified-at", required=True)
|
||||
parser.add_argument("--output", type=Path, default=Path("release.json"))
|
||||
parser.add_argument("--allow-rollback", action="store_true")
|
||||
args = parser.parse_args()
|
||||
pointer = build_release_pointer(
|
||||
tag=args.tag,
|
||||
source_commit=args.source_commit,
|
||||
workflow_run=args.workflow_run,
|
||||
qualified_at=args.qualified_at,
|
||||
)
|
||||
args.output.write_bytes(canonical_pointer(pointer))
|
||||
release_url, channel_url = publish_qualification(
|
||||
pointer, allow_rollback=args.allow_rollback)
|
||||
print(json.dumps({
|
||||
"release": release_url,
|
||||
"channel": channel_url,
|
||||
}, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -10,6 +10,8 @@ Kept in sync with ``bot_bottle.resources.BUNDLED_RESOURCES`` — the
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
@@ -44,6 +46,18 @@ class _BundleResources(build_py):
|
||||
dst = pkg_resources / rel
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(src, dst)
|
||||
manifest_dest = Path(self.build_lib) / "bot_bottle" / "release-manifest.json"
|
||||
manifest_input = os.environ.get("BOT_BOTTLE_RELEASE_MANIFEST", "").strip()
|
||||
if manifest_input:
|
||||
shutil.copy2(manifest_input, manifest_dest)
|
||||
else:
|
||||
# Wheels built ad hoc retain the contributor/install.sh local-build
|
||||
# behavior. The release workflow always replaces this marker with
|
||||
# validated immutable artifact identities.
|
||||
manifest_dest.write_text(
|
||||
json.dumps({"schema": 1, "development": True}) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
setup(cmdclass={"build_py": _BundleResources})
|
||||
|
||||
@@ -46,8 +46,8 @@ class TestDockerInfraService(unittest.TestCase):
|
||||
url = self.svc.ensure_running()
|
||||
self.assertEqual("http://127.0.0.1:8099", url)
|
||||
# Both images built; the control plane comes up before the gateway.
|
||||
self.orch.ensure_built.assert_called_once()
|
||||
self.gw.ensure_built.assert_called_once()
|
||||
self.orch.ensure_available.assert_called_once()
|
||||
self.gw.ensure_available.assert_called_once()
|
||||
self.orch.ensure_running.assert_called_once()
|
||||
|
||||
def test_ensure_running_connects_gateway_with_minted_token(self) -> None:
|
||||
@@ -76,6 +76,19 @@ class TestDockerInfraService(unittest.TestCase):
|
||||
self.assertEqual("registry-volume", svc.orchestrator()._root_mount_source)
|
||||
self.assertEqual("ca-volume", svc.gateway()._ca_mount_source)
|
||||
|
||||
def test_packaged_images_disable_local_infra_dockerfiles(self) -> None:
|
||||
refs = [
|
||||
("registry/orchestrator@sha256:" + "a" * 64, False),
|
||||
("registry/gateway@sha256:" + "b" * 64, False),
|
||||
]
|
||||
with patch(
|
||||
"bot_bottle.backend.docker.infra.release_manifest.oci_image",
|
||||
side_effect=refs,
|
||||
):
|
||||
svc = DockerInfraService()
|
||||
self.assertIsNone(svc.orchestrator()._dockerfile)
|
||||
self.assertIsNone(svc.gateway()._dockerfile)
|
||||
|
||||
def test_host_path_and_named_root_mount_are_mutually_exclusive(self) -> None:
|
||||
with self.assertRaisesRegex(ValueError, "host_root or root_mount_source"):
|
||||
DockerInfraService(
|
||||
|
||||
@@ -231,11 +231,12 @@ class TestDockerOrchestrator(unittest.TestCase):
|
||||
with self.assertRaises(GatewayError):
|
||||
self.orch.ensure_built()
|
||||
|
||||
def test_ensure_built_noop_without_dockerfile(self) -> None:
|
||||
orch = DockerOrchestrator(dockerfile=None)
|
||||
with patch(_RUN) as run:
|
||||
def test_ensure_built_pulls_without_dockerfile(self) -> None:
|
||||
orch = DockerOrchestrator("registry/orchestrator@sha256:" + "a" * 64,
|
||||
dockerfile=None)
|
||||
with patch(_RUN, return_value=_proc()) as run:
|
||||
orch.ensure_built()
|
||||
run.assert_not_called()
|
||||
run.assert_called_once_with(["docker", "pull", orch.image_ref])
|
||||
|
||||
def test_is_running_reads_docker_ps(self) -> None:
|
||||
with patch(_RUN, return_value=_proc(stdout=ORCHESTRATOR_NAME)):
|
||||
|
||||
@@ -90,6 +90,20 @@ class TestBuildAgentRootfsDir(unittest.TestCase):
|
||||
second = image_builder._rootfs_digest(self.dockerfile)
|
||||
self.assertNotEqual(first, second)
|
||||
|
||||
def test_pinned_image_is_pulled_and_cached(self):
|
||||
image = f"registry.example/agent@sha256:{'a' * 64}"
|
||||
with patch.object(image_builder.util, "cache_dir", return_value=self.cache), \
|
||||
patch.object(image_builder, "_pull_in_infra") as pull, \
|
||||
patch.object(image_builder.util, "inject_guest_boot"):
|
||||
first = image_builder.acquire_agent_image_rootfs_dir(image)
|
||||
second = image_builder.acquire_agent_image_rootfs_dir(image)
|
||||
pull.assert_called_once()
|
||||
self.assertEqual(first, second)
|
||||
|
||||
def test_mutable_image_cannot_use_prebuilt_path(self):
|
||||
with self.assertRaises(SystemExit):
|
||||
image_builder.acquire_agent_image_rootfs_dir("agent:latest")
|
||||
|
||||
|
||||
class TestSmokeTest(unittest.TestCase):
|
||||
def test_buildah_receives_centralized_image_build_args(self):
|
||||
|
||||
@@ -53,7 +53,7 @@ class TestEnsureRunning(unittest.TestCase):
|
||||
patch.object(infra_vm, "_health_ok", return_value=True), \
|
||||
patch.object(infra_vm, "_pidfile_alive", return_value=True), \
|
||||
patch.object(infra_vm, "stop") as stop, \
|
||||
patch.object(infra_vm, "ensure_built") as built:
|
||||
patch.object(infra_vm, "ensure_available") as built:
|
||||
url = FirecrackerInfraService().ensure_running()
|
||||
stop.assert_not_called()
|
||||
built.assert_not_called()
|
||||
@@ -72,7 +72,7 @@ class TestEnsureRunning(unittest.TestCase):
|
||||
patch.object(infra_vm, "_health_ok", return_value=True), \
|
||||
patch.object(infra_vm, "_pidfile_alive", return_value=True), \
|
||||
patch.object(infra_vm, "stop") as stop, \
|
||||
patch.object(infra_vm, "ensure_built"), \
|
||||
patch.object(infra_vm, "ensure_available"), \
|
||||
patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls:
|
||||
orch_cls.return_value.gateway_url.return_value = "http://10.243.255.1:8099"
|
||||
orch_cls.return_value.mint_gateway_token.return_value = "gw.jwt"
|
||||
@@ -98,7 +98,7 @@ class TestEnsureRunning(unittest.TestCase):
|
||||
patch.object(infra_vm, "_health_ok", return_value=True), \
|
||||
patch.object(infra_vm, "_pidfile_alive", return_value=False), \
|
||||
patch.object(infra_vm, "stop") as stop, \
|
||||
patch.object(infra_vm, "ensure_built"), \
|
||||
patch.object(infra_vm, "ensure_available"), \
|
||||
patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls:
|
||||
FirecrackerInfraService().ensure_running()
|
||||
stop.assert_called_once()
|
||||
@@ -111,7 +111,7 @@ class TestEnsureRunning(unittest.TestCase):
|
||||
patch.object(infra_vm, "expected_version", return_value="v-current"), \
|
||||
patch.object(infra_vm, "_health_ok", return_value=False), \
|
||||
patch.object(infra_vm, "stop") as stop, \
|
||||
patch.object(infra_vm, "ensure_built") as built, \
|
||||
patch.object(infra_vm, "ensure_available") as built, \
|
||||
patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls:
|
||||
FirecrackerInfraService().ensure_running()
|
||||
stop.assert_called_once() # clear stale VMs first
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.generate_release_manifest import main
|
||||
|
||||
|
||||
class TestGenerateReleaseManifest(unittest.TestCase):
|
||||
def test_writes_validated_manifest(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
output = Path(tmp) / "release.json"
|
||||
rc = main([
|
||||
"--source-commit", "1" * 40,
|
||||
"--orchestrator-image",
|
||||
"registry/orchestrator@sha256:" + "a" * 64,
|
||||
"--gateway-image", "registry/gateway@sha256:" + "b" * 64,
|
||||
"--agent-claude-image", "registry/claude@sha256:" + "e" * 64,
|
||||
"--agent-codex-image", "registry/codex@sha256:" + "f" * 64,
|
||||
"--agent-pi-image", "registry/pi@sha256:" + "0" * 64,
|
||||
"--firecracker-orchestrator-version", "orch-v1",
|
||||
"--firecracker-orchestrator-sha256", "c" * 64,
|
||||
"--firecracker-gateway-version", "gateway-v1",
|
||||
"--firecracker-gateway-sha256", "d" * 64,
|
||||
"--output", str(output),
|
||||
])
|
||||
self.assertEqual(0, rc)
|
||||
data = json.loads(output.read_text(encoding="utf-8"))
|
||||
self.assertEqual("orch-v1", data["firecracker"]["orchestrator"]["version"])
|
||||
@@ -95,6 +95,21 @@ class TestInstallScript(unittest.TestCase):
|
||||
# Tests / local installs point BOT_BOTTLE_INSTALL_SPEC at a checkout.
|
||||
self.assertIn("BOT_BOTTLE_INSTALL_SPEC", self.text)
|
||||
|
||||
def test_published_install_selectors_are_mutually_exclusive(self):
|
||||
self.assertIn("BOT_BOTTLE_CHANNEL", self.text)
|
||||
self.assertIn("BOT_BOTTLE_VERSION", self.text)
|
||||
self.assertIn("BOT_BOTTLE_REF", self.text)
|
||||
self.assertIn("are mutually exclusive", self.text)
|
||||
|
||||
def test_commit_snapshot_warns_and_wheel_is_checksum_verified(self):
|
||||
self.assertIn("may not have passed release qualification", self.text)
|
||||
self.assertIn("downloaded wheel checksum mismatch", self.text)
|
||||
self.assertIn("hashlib.sha256()", self.text)
|
||||
|
||||
def test_default_install_uses_production_channel(self):
|
||||
self.assertIn('${BOT_BOTTLE_CHANNEL:-production}', self.text)
|
||||
self.assertIn("bot-bottle-channels", self.text)
|
||||
|
||||
def test_requires_git_for_git_specs(self):
|
||||
# A git+ / .git spec (the default) shells out to git; the script must
|
||||
# gate on it rather than failing opaquely inside pipx/pip.
|
||||
|
||||
@@ -33,8 +33,8 @@ class TestInfraEnsureRunning(unittest.TestCase):
|
||||
with patch(f"{_INFRA}.ensure_networks") as nets:
|
||||
url = self.svc.ensure_running()
|
||||
nets.assert_called_once()
|
||||
self.orch.ensure_built.assert_called_once()
|
||||
self.gw.ensure_built.assert_called_once()
|
||||
self.orch.ensure_available.assert_called_once()
|
||||
self.gw.ensure_available.assert_called_once()
|
||||
self.orch.ensure_running.assert_called_once()
|
||||
# Returns the host control-plane URL (the InfraService contract).
|
||||
self.assertEqual("http://192.168.128.2:8099", url)
|
||||
@@ -51,6 +51,18 @@ class TestInfraEnsureRunning(unittest.TestCase):
|
||||
self.assertTrue(self.svc.is_healthy())
|
||||
self.orch.is_healthy.assert_called_once()
|
||||
|
||||
def test_packaged_images_disable_local_infra_builds(self) -> None:
|
||||
refs = [
|
||||
("registry/orchestrator@sha256:" + "a" * 64, False),
|
||||
("registry/gateway@sha256:" + "b" * 64, False),
|
||||
]
|
||||
with patch(
|
||||
f"{_INFRA}.release_manifest.oci_image", side_effect=refs,
|
||||
):
|
||||
svc = MacosInfraService(repo_root=Path("/r"))
|
||||
self.assertFalse(svc.orchestrator()._local_build)
|
||||
self.assertFalse(svc.gateway()._local_build)
|
||||
|
||||
|
||||
class TestCaCertPem(unittest.TestCase):
|
||||
def test_delegates_to_the_gateway_service(self) -> None:
|
||||
|
||||
@@ -30,14 +30,16 @@ def _spec(src: str, tgt: str, readonly: bool = False) -> str:
|
||||
|
||||
|
||||
class TestMacosOrchestratorRun(unittest.TestCase):
|
||||
def _run(self) -> list[str]:
|
||||
def _run(self, *, local_build: bool = True) -> list[str]:
|
||||
run = Mock(return_value=_ok())
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
|
||||
mod.dns_server.return_value = "1.1.1.1"
|
||||
mod.bind_mount_spec.side_effect = _spec
|
||||
mod.run_container_argv = run
|
||||
MacosOrchestrator(repo_root=Path("/r"))._run_container("h1")
|
||||
MacosOrchestrator(
|
||||
repo_root=Path("/r"), local_build=local_build,
|
||||
)._run_container("h1")
|
||||
return run.call_args.args[0]
|
||||
|
||||
def test_runs_on_the_control_network_only(self) -> None:
|
||||
@@ -63,6 +65,11 @@ class TestMacosOrchestratorRun(unittest.TestCase):
|
||||
def test_source_hash_is_labelled_for_recreate(self) -> None:
|
||||
self.assertIn("BOT_BOTTLE_SOURCE_HASH=h1", self._run())
|
||||
|
||||
def test_packaged_image_does_not_overlay_host_source(self) -> None:
|
||||
argv = self._run(local_build=False)
|
||||
self.assertNotIn("--mount", argv)
|
||||
self.assertFalse(any(arg.startswith("PYTHONPATH=") for arg in argv))
|
||||
|
||||
def test_start_failure_raises(self) -> None:
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
|
||||
@@ -145,6 +152,14 @@ class TestMacosOrchestratorBuildStop(unittest.TestCase):
|
||||
_args, kwargs = mod.build_image.call_args
|
||||
self.assertEqual("Dockerfile.orchestrator", kwargs["dockerfile"])
|
||||
|
||||
def test_ensure_built_pulls_packaged_image(self) -> None:
|
||||
image = "registry/orchestrator@sha256:" + "a" * 64
|
||||
orch = MacosOrchestrator(image, local_build=False)
|
||||
with patch(f"{_ORCH}.container_mod") as mod:
|
||||
orch.ensure_built()
|
||||
mod.pull_image.assert_called_once_with(image)
|
||||
mod.build_image.assert_not_called()
|
||||
|
||||
def test_stop_removes_the_container(self) -> None:
|
||||
orch = MacosOrchestrator()
|
||||
with patch(f"{_ORCH}.container_mod") as mod:
|
||||
|
||||
@@ -427,11 +427,11 @@ class TestDockerGatewayBuild(unittest.TestCase):
|
||||
builds = [c for c in calls if c[:2] == ["docker", "build"]]
|
||||
self.assertIn("--no-cache", builds[0])
|
||||
|
||||
def test_ensure_built_noop_when_no_dockerfile(self) -> None:
|
||||
sc = DockerGateway("busybox", dockerfile=None)
|
||||
with patch(_RUN_DOCKER) as m:
|
||||
def test_ensure_built_pulls_when_no_dockerfile(self) -> None:
|
||||
sc = DockerGateway("registry/gateway@sha256:" + "a" * 64, dockerfile=None)
|
||||
with patch(_RUN_DOCKER, return_value=_proc()) as m:
|
||||
sc.ensure_built()
|
||||
m.assert_not_called()
|
||||
m.assert_called_once_with(["docker", "pull", sc.image_ref])
|
||||
|
||||
def test_ensure_built_raises_on_build_failure(self) -> None:
|
||||
with patch(_RUN_DOCKER, return_value=_proc(returncode=1, stderr="build boom")):
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from bot_bottle.release_bundle import (
|
||||
build_bundle_index,
|
||||
canonical_bytes,
|
||||
parse_bundle_index,
|
||||
)
|
||||
from bot_bottle.release_manifest import ReleaseManifestError
|
||||
from tests.unit.test_release_manifest import manifest
|
||||
|
||||
|
||||
class TestReleaseBundle(unittest.TestCase):
|
||||
def test_builds_commit_addressed_index(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
wheel = Path(tmp) / "bot_bottle-0.1.0-py3-none-any.whl"
|
||||
wheel.write_bytes(b"wheel")
|
||||
index = build_bundle_index(
|
||||
manifest(),
|
||||
wheel=wheel,
|
||||
wheel_url=f"https://packages.example/{wheel.name}",
|
||||
workflow_run="actions/123",
|
||||
published_at="2026-07-27T00:00:00Z",
|
||||
)
|
||||
self.assertEqual("1" * 40, index["source_commit"])
|
||||
self.assertEqual(64, len(index["wheel"]["sha256"]))
|
||||
self.assertEqual([], index["qualifications"])
|
||||
self.assertIs(index, parse_bundle_index(index))
|
||||
self.assertTrue(canonical_bytes(index).endswith(b"\n"))
|
||||
|
||||
def test_rejects_mutable_or_mismatched_wheel_url(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
wheel = Path(tmp) / "release.whl"
|
||||
wheel.write_bytes(b"wheel")
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "HTTPS"):
|
||||
build_bundle_index(
|
||||
manifest(),
|
||||
wheel=wheel,
|
||||
wheel_url="http://packages.example/other.whl",
|
||||
workflow_run="actions/123",
|
||||
published_at="now",
|
||||
)
|
||||
|
||||
def test_rejects_index_with_changed_runtime_identity(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
wheel = Path(tmp) / "release.whl"
|
||||
wheel.write_bytes(b"wheel")
|
||||
index = build_bundle_index(
|
||||
manifest(),
|
||||
wheel=wheel,
|
||||
wheel_url=f"https://packages.example/{wheel.name}",
|
||||
workflow_run="actions/123",
|
||||
published_at="now",
|
||||
)
|
||||
index["oci"]["gateway"] = "registry/gateway:latest"
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "digest-pinned"):
|
||||
parse_bundle_index(index)
|
||||
@@ -0,0 +1,129 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from bot_bottle.release_manifest import (
|
||||
ReleaseManifestError,
|
||||
oci_image,
|
||||
packaged_manifest,
|
||||
parse_manifest,
|
||||
)
|
||||
|
||||
|
||||
def manifest() -> dict[str, object]:
|
||||
return {
|
||||
"schema": 1,
|
||||
"source_commit": "1" * 40,
|
||||
"oci": {
|
||||
"orchestrator": "registry/orchestrator@sha256:" + "a" * 64,
|
||||
"gateway": "registry/gateway@sha256:" + "b" * 64,
|
||||
"agent_claude": "registry/agent-claude@sha256:" + "e" * 64,
|
||||
"agent_codex": "registry/agent-codex@sha256:" + "f" * 64,
|
||||
"agent_pi": "registry/agent-pi@sha256:" + "0" * 64,
|
||||
},
|
||||
"firecracker": {
|
||||
"orchestrator": {"version": "orch-v1", "sha256": "c" * 64},
|
||||
"gateway": {"version": "gateway-v1", "sha256": "d" * 64},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class TestParseManifest(unittest.TestCase):
|
||||
def test_parses_all_backend_artifacts(self) -> None:
|
||||
parsed = parse_manifest(manifest())
|
||||
self.assertTrue(parsed.orchestrator_image.endswith("a" * 64))
|
||||
self.assertEqual("orch-v1", parsed.firecracker_orchestrator.version)
|
||||
self.assertTrue(parsed.agent_images["codex"].endswith("f" * 64))
|
||||
|
||||
def test_rejects_mutable_oci_reference(self) -> None:
|
||||
data = manifest()
|
||||
assert isinstance(data["oci"], dict)
|
||||
data["oci"]["orchestrator"] = "registry/orchestrator:latest"
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "digest-pinned"):
|
||||
parse_manifest(data)
|
||||
|
||||
def test_rejects_malformed_firecracker_checksum(self) -> None:
|
||||
data = manifest()
|
||||
assert isinstance(data["firecracker"], dict)
|
||||
artifact = data["firecracker"]["gateway"]
|
||||
assert isinstance(artifact, dict)
|
||||
artifact["sha256"] = "nope"
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "64 lowercase hex"):
|
||||
parse_manifest(data)
|
||||
|
||||
def test_rejects_missing_manifest_sections(self) -> None:
|
||||
cases = (
|
||||
(None, "schema 1"),
|
||||
({"schema": 1}, "source_commit"),
|
||||
(
|
||||
{"schema": 1, "source_commit": "1" * 40},
|
||||
"oci must be an object",
|
||||
),
|
||||
)
|
||||
for data, message in cases:
|
||||
with self.subTest(message=message):
|
||||
with self.assertRaisesRegex(ReleaseManifestError, message):
|
||||
parse_manifest(data)
|
||||
|
||||
def test_rejects_malformed_firecracker_artifact(self) -> None:
|
||||
data = manifest()
|
||||
assert isinstance(data["firecracker"], dict)
|
||||
data["firecracker"]["orchestrator"] = None
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "must be an object"):
|
||||
parse_manifest(data)
|
||||
|
||||
data["firecracker"]["orchestrator"] = {
|
||||
"version": " ",
|
||||
"sha256": "c" * 64,
|
||||
}
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "version must be non-empty"):
|
||||
parse_manifest(data)
|
||||
|
||||
|
||||
class TestImageSelection(unittest.TestCase):
|
||||
def test_development_marker_selects_local_builds(self) -> None:
|
||||
with patch(
|
||||
"bot_bottle.release_manifest.resources.is_source_checkout",
|
||||
return_value=False,
|
||||
), patch(
|
||||
"bot_bottle.release_manifest.manifest_path",
|
||||
) as path:
|
||||
path.return_value.read_text.return_value = (
|
||||
'{"schema": 1, "development": true}')
|
||||
self.assertIsNone(packaged_manifest())
|
||||
|
||||
def test_packaged_install_uses_manifest_reference_without_build(self) -> None:
|
||||
parsed = parse_manifest(manifest())
|
||||
with patch(
|
||||
"bot_bottle.release_manifest.packaged_manifest", return_value=parsed,
|
||||
), patch(
|
||||
"bot_bottle.release_manifest.local_build_requested", return_value=False,
|
||||
), patch.dict("os.environ", {}, clear=True):
|
||||
ref, local = oci_image("orchestrator", "local:latest")
|
||||
self.assertEqual(parsed.orchestrator_image, ref)
|
||||
self.assertFalse(local)
|
||||
|
||||
def test_packaged_install_selects_agent_reference(self) -> None:
|
||||
parsed = parse_manifest(manifest())
|
||||
with patch(
|
||||
"bot_bottle.release_manifest.packaged_manifest", return_value=parsed,
|
||||
), patch(
|
||||
"bot_bottle.release_manifest.local_build_requested", return_value=False,
|
||||
), patch.dict("os.environ", {}, clear=True):
|
||||
ref, local = oci_image("agent_codex", "local:latest")
|
||||
self.assertEqual(parsed.agent_images["codex"], ref)
|
||||
self.assertFalse(local)
|
||||
|
||||
def test_mutable_override_fails_outside_local_mode(self) -> None:
|
||||
parsed = parse_manifest(manifest())
|
||||
with patch(
|
||||
"bot_bottle.release_manifest.packaged_manifest", return_value=parsed,
|
||||
), patch(
|
||||
"bot_bottle.release_manifest.local_build_requested", return_value=False,
|
||||
), patch.dict(
|
||||
"os.environ", {"BOT_BOTTLE_ORCHESTRATOR_IMAGE": "local:latest"},
|
||||
clear=True,
|
||||
):
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "digest-pinned"):
|
||||
oci_image("orchestrator", "fallback:latest")
|
||||
@@ -0,0 +1,163 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
import urllib.error
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from bot_bottle.release_bundle import build_bundle_index, canonical_bytes
|
||||
from bot_bottle.release_manifest import ReleaseManifestError
|
||||
from bot_bottle.release_publish import (
|
||||
bundle_url,
|
||||
remote_bytes,
|
||||
remote_sha256,
|
||||
request,
|
||||
upload as publish_artifact,
|
||||
publish_bundle,
|
||||
)
|
||||
from tests.unit.test_release_manifest import manifest
|
||||
|
||||
|
||||
class TestReleasePublish(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
env = {
|
||||
"BOT_BOTTLE_RELEASE_BASE": "https://packages.example",
|
||||
"BOT_BOTTLE_RELEASE_OWNER": "owner",
|
||||
"BOT_BOTTLE_RELEASE_TOKEN": "token",
|
||||
}
|
||||
self.env = patch.dict("os.environ", env, clear=True)
|
||||
self.env.start()
|
||||
self.addCleanup(self.env.stop)
|
||||
|
||||
def bundle(self, root: Path) -> tuple[dict[str, object], Path]:
|
||||
wheel = root / "bot_bottle-0.1.0-py3-none-any.whl"
|
||||
wheel.write_bytes(b"wheel")
|
||||
index = build_bundle_index(
|
||||
manifest(),
|
||||
wheel=wheel,
|
||||
wheel_url=bundle_url("1" * 40, wheel.name),
|
||||
workflow_run="actions/123",
|
||||
published_at="2026-07-27T00:00:00Z",
|
||||
)
|
||||
return index, wheel
|
||||
|
||||
@staticmethod
|
||||
def wheel_sha(index: dict[str, object]) -> str:
|
||||
wheel = index["wheel"]
|
||||
assert isinstance(wheel, dict)
|
||||
digest = wheel["sha256"]
|
||||
assert isinstance(digest, str)
|
||||
return digest
|
||||
|
||||
def test_publishes_wheel_then_index(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
index, wheel = self.bundle(Path(tmp))
|
||||
with patch(
|
||||
"bot_bottle.release_publish.remote_bytes", return_value=None,
|
||||
), patch(
|
||||
"bot_bottle.release_publish.remote_sha256", return_value=None,
|
||||
), patch("bot_bottle.release_publish.upload") as upload:
|
||||
result = publish_bundle(index, wheel)
|
||||
self.assertTrue(result.endswith("/bundle-index.json"))
|
||||
self.assertEqual(2, upload.call_count)
|
||||
self.assertIs(wheel, upload.call_args_list[0].args[1])
|
||||
self.assertEqual(canonical_bytes(index), upload.call_args_list[1].args[1])
|
||||
|
||||
def test_exact_existing_bundle_is_idempotent(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
index, wheel = self.bundle(Path(tmp))
|
||||
with patch(
|
||||
"bot_bottle.release_publish.remote_bytes",
|
||||
return_value=canonical_bytes(index),
|
||||
), patch(
|
||||
"bot_bottle.release_publish.remote_sha256",
|
||||
return_value=self.wheel_sha(index),
|
||||
), patch("bot_bottle.release_publish.upload") as upload:
|
||||
publish_bundle(index, wheel)
|
||||
upload.assert_not_called()
|
||||
|
||||
def test_rejects_partial_publication(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
index, wheel = self.bundle(Path(tmp))
|
||||
with patch(
|
||||
"bot_bottle.release_publish.remote_bytes", return_value=None,
|
||||
), patch(
|
||||
"bot_bottle.release_publish.remote_sha256",
|
||||
return_value=self.wheel_sha(index),
|
||||
):
|
||||
with self.assertRaisesRegex(
|
||||
ReleaseManifestError, "administrative cleanup",
|
||||
):
|
||||
publish_bundle(index, wheel)
|
||||
|
||||
def test_rejects_existing_bundle_with_different_index(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
index, wheel = self.bundle(Path(tmp))
|
||||
with patch(
|
||||
"bot_bottle.release_publish.remote_bytes", return_value=b"other",
|
||||
), patch(
|
||||
"bot_bottle.release_publish.remote_sha256",
|
||||
return_value=self.wheel_sha(index),
|
||||
):
|
||||
with self.assertRaisesRegex(
|
||||
ReleaseManifestError, "already differs",
|
||||
):
|
||||
publish_bundle(index, wheel)
|
||||
|
||||
def test_request_applies_auth_and_length(self) -> None:
|
||||
result = request(
|
||||
"https://packages.example/file",
|
||||
method="PUT",
|
||||
data=b"body",
|
||||
length=4,
|
||||
)
|
||||
self.assertEqual(result.get_method(), "PUT")
|
||||
self.assertEqual(result.get_header("Authorization"), "token token")
|
||||
self.assertEqual(result.get_header("Content-length"), "4")
|
||||
|
||||
@patch("bot_bottle.release_publish.urllib.request.urlopen")
|
||||
def test_remote_helpers_read_response(self, urlopen: MagicMock) -> None:
|
||||
response = urlopen.return_value.__enter__.return_value
|
||||
response.read.side_effect = [b"body"]
|
||||
self.assertEqual(
|
||||
remote_bytes("https://packages.example/file"), b"body")
|
||||
response.read.side_effect = [b"body", b""]
|
||||
self.assertEqual(
|
||||
remote_sha256("https://packages.example/file"),
|
||||
"230d8358dc8e8890b4c58deeb62912ee2"
|
||||
"f20357ae92a5cc861b98e68fe31acb5",
|
||||
)
|
||||
|
||||
@patch("bot_bottle.release_publish.urllib.request.urlopen")
|
||||
def test_remote_helpers_handle_not_found(self, urlopen: MagicMock) -> None:
|
||||
urlopen.side_effect = urllib.error.HTTPError(
|
||||
"url", 404, "missing", MagicMock(), None)
|
||||
self.assertIsNone(remote_bytes("https://packages.example/file"))
|
||||
self.assertIsNone(remote_sha256("https://packages.example/file"))
|
||||
|
||||
@patch("bot_bottle.release_publish.urllib.request.urlopen")
|
||||
def test_remote_helpers_report_registry_errors(
|
||||
self, urlopen: MagicMock,
|
||||
) -> None:
|
||||
urlopen.side_effect = urllib.error.URLError("offline")
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "offline"):
|
||||
remote_bytes("https://packages.example/file")
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "offline"):
|
||||
remote_sha256("https://packages.example/file")
|
||||
|
||||
@patch("bot_bottle.release_publish.urllib.request.urlopen")
|
||||
def test_upload_supports_bytes_and_files(self, urlopen: MagicMock) -> None:
|
||||
urlopen.return_value.__enter__.return_value = MagicMock()
|
||||
publish_artifact("https://packages.example/bytes", b"body")
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
source = Path(tmp) / "artifact"
|
||||
source.write_bytes(b"artifact")
|
||||
publish_artifact("https://packages.example/file", source)
|
||||
self.assertEqual(urlopen.call_count, 2)
|
||||
|
||||
@patch("bot_bottle.release_publish.urllib.request.urlopen")
|
||||
def test_upload_reports_failure(self, urlopen: MagicMock) -> None:
|
||||
urlopen.side_effect = urllib.error.URLError("offline")
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "publishing"):
|
||||
publish_artifact("https://packages.example/file", b"body")
|
||||
@@ -0,0 +1,270 @@
|
||||
"""Tests for release qualification and channel promotion."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import unittest
|
||||
import urllib.error
|
||||
from email.message import Message
|
||||
from unittest import mock
|
||||
|
||||
from bot_bottle.release_manifest import ReleaseManifestError
|
||||
from bot_bottle.release_qualification import (
|
||||
build_release_pointer,
|
||||
canonical_pointer,
|
||||
delete,
|
||||
parse_release_pointer,
|
||||
publish_qualification,
|
||||
release_channel,
|
||||
release_version,
|
||||
)
|
||||
|
||||
COMMIT = "a" * 40
|
||||
|
||||
|
||||
def bundle(source_commit: str = COMMIT) -> bytes:
|
||||
return json.dumps({
|
||||
"schema": 1,
|
||||
"source_commit": source_commit,
|
||||
"wheel": {
|
||||
"filename": "bot_bottle.whl",
|
||||
"url": (
|
||||
"https://gitea.dideric.is/api/packages/didericis/generic/"
|
||||
f"bot-bottle-builds/{source_commit}/bot_bottle.whl"
|
||||
),
|
||||
"sha256": "b" * 64,
|
||||
},
|
||||
"oci": {
|
||||
role: f"registry.example/{role}@sha256:{'c' * 64}"
|
||||
for role in (
|
||||
"orchestrator",
|
||||
"gateway",
|
||||
"agent_claude",
|
||||
"agent_codex",
|
||||
"agent_pi",
|
||||
)
|
||||
},
|
||||
"firecracker": {
|
||||
role: {"version": "v1", "sha256": "d" * 64}
|
||||
for role in ("orchestrator", "gateway")
|
||||
},
|
||||
"provenance": {"workflow_run": "run", "published_at": "now"},
|
||||
"qualifications": [],
|
||||
}).encode()
|
||||
|
||||
|
||||
class ReleaseQualificationTest(unittest.TestCase):
|
||||
def test_tag_selects_channel(self) -> None:
|
||||
self.assertEqual(release_channel("v1.2.3"), "production")
|
||||
self.assertEqual(release_channel("v1.2.3-rc.4"), "staging")
|
||||
with self.assertRaises(ReleaseManifestError):
|
||||
release_channel("latest")
|
||||
|
||||
def test_release_versions_are_monotonic(self) -> None:
|
||||
self.assertLess(
|
||||
release_version("v1.2.3-rc.4"), release_version("v1.2.3"))
|
||||
self.assertLess(
|
||||
release_version("v1.2.3"), release_version("v2.0.0"))
|
||||
with self.assertRaises(ReleaseManifestError):
|
||||
release_version("v1")
|
||||
|
||||
def test_builds_qualified_pointer(self) -> None:
|
||||
pointer = build_release_pointer(
|
||||
tag="v1.2.3",
|
||||
source_commit=COMMIT,
|
||||
workflow_run="https://example.test/run/1",
|
||||
qualified_at="2026-07-27T00:00:00Z",
|
||||
)
|
||||
self.assertEqual(pointer["channel"], "production")
|
||||
self.assertTrue(pointer["qualified"])
|
||||
self.assertTrue(canonical_pointer(pointer).endswith(b"\n"))
|
||||
|
||||
def test_rejects_invalid_pointer_fields(self) -> None:
|
||||
pointer = build_release_pointer(
|
||||
tag="v1.2.3",
|
||||
source_commit=COMMIT,
|
||||
workflow_run="run",
|
||||
qualified_at="now",
|
||||
)
|
||||
changes = (
|
||||
("schema", 2),
|
||||
("qualified", False),
|
||||
("channel", "staging"),
|
||||
("source_commit", "short"),
|
||||
("bundle_index_url", "https://wrong.example/index.json"),
|
||||
("qualification", {}),
|
||||
)
|
||||
for key, value in changes:
|
||||
with self.subTest(key=key), self.assertRaises(ReleaseManifestError):
|
||||
parse_release_pointer({**pointer, key: value})
|
||||
|
||||
def test_rejects_invalid_build_inputs(self) -> None:
|
||||
with self.assertRaises(ReleaseManifestError):
|
||||
build_release_pointer(
|
||||
tag="v1.2.3",
|
||||
source_commit="short",
|
||||
workflow_run="run",
|
||||
qualified_at="now",
|
||||
)
|
||||
with self.assertRaises(ReleaseManifestError):
|
||||
build_release_pointer(
|
||||
tag="v1.2.3",
|
||||
source_commit=COMMIT,
|
||||
workflow_run="",
|
||||
qualified_at="now",
|
||||
)
|
||||
|
||||
@mock.patch("bot_bottle.release_qualification.upload")
|
||||
@mock.patch("bot_bottle.release_qualification.remote_bytes")
|
||||
def test_publishes_bundle_release_and_channel(
|
||||
self, remote: mock.Mock, upload: mock.Mock,
|
||||
) -> None:
|
||||
remote.side_effect = [bundle(), None, None]
|
||||
pointer = build_release_pointer(
|
||||
tag="v1.2.3",
|
||||
source_commit=COMMIT,
|
||||
workflow_run="run",
|
||||
qualified_at="now",
|
||||
)
|
||||
publish_qualification(pointer)
|
||||
self.assertEqual(upload.call_count, 2)
|
||||
|
||||
@mock.patch("bot_bottle.release_qualification.upload")
|
||||
@mock.patch("bot_bottle.release_qualification.remote_bytes")
|
||||
def test_exact_existing_publication_is_noop(
|
||||
self, remote: mock.Mock, upload: mock.Mock,
|
||||
) -> None:
|
||||
pointer = build_release_pointer(
|
||||
tag="v1.2.3",
|
||||
source_commit=COMMIT,
|
||||
workflow_run="run",
|
||||
qualified_at="now",
|
||||
)
|
||||
wanted = canonical_pointer(pointer)
|
||||
remote.side_effect = [bundle(), wanted, wanted]
|
||||
publish_qualification(pointer)
|
||||
upload.assert_not_called()
|
||||
|
||||
@mock.patch("bot_bottle.release_qualification.upload")
|
||||
@mock.patch("bot_bottle.release_qualification.delete")
|
||||
@mock.patch("bot_bottle.release_qualification.remote_bytes")
|
||||
def test_advances_to_newer_channel(
|
||||
self, remote: mock.Mock, remove: mock.Mock, upload: mock.Mock,
|
||||
) -> None:
|
||||
old = build_release_pointer(
|
||||
tag="v1.0.0",
|
||||
source_commit="b" * 40,
|
||||
workflow_run="old",
|
||||
qualified_at="then",
|
||||
)
|
||||
pointer = build_release_pointer(
|
||||
tag="v2.0.0",
|
||||
source_commit=COMMIT,
|
||||
workflow_run="run",
|
||||
qualified_at="now",
|
||||
)
|
||||
remote.side_effect = [bundle(), None, canonical_pointer(old)]
|
||||
publish_qualification(pointer)
|
||||
remove.assert_called_once()
|
||||
self.assertEqual(upload.call_count, 2)
|
||||
|
||||
@mock.patch("bot_bottle.release_qualification.upload")
|
||||
@mock.patch("bot_bottle.release_qualification.remote_bytes")
|
||||
def test_rejects_non_monotonic_channel(
|
||||
self, remote: mock.Mock, _upload: mock.Mock,
|
||||
) -> None:
|
||||
old = build_release_pointer(
|
||||
tag="v2.0.0",
|
||||
source_commit="b" * 40,
|
||||
workflow_run="old-run",
|
||||
qualified_at="old-time",
|
||||
)
|
||||
remote.side_effect = [bundle(), None, json.dumps(old).encode()]
|
||||
pointer = build_release_pointer(
|
||||
tag="v1.2.3",
|
||||
source_commit=COMMIT,
|
||||
workflow_run="run",
|
||||
qualified_at="now",
|
||||
)
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "not monotonic"):
|
||||
publish_qualification(pointer)
|
||||
|
||||
@mock.patch("bot_bottle.release_qualification.upload")
|
||||
@mock.patch("bot_bottle.release_qualification.delete")
|
||||
@mock.patch("bot_bottle.release_qualification.remote_bytes")
|
||||
def test_explicit_rollback_allows_older_channel(
|
||||
self, remote: mock.Mock, remove: mock.Mock, _upload: mock.Mock,
|
||||
) -> None:
|
||||
old = build_release_pointer(
|
||||
tag="v2.0.0",
|
||||
source_commit="b" * 40,
|
||||
workflow_run="old",
|
||||
qualified_at="then",
|
||||
)
|
||||
pointer = build_release_pointer(
|
||||
tag="v1.0.0",
|
||||
source_commit=COMMIT,
|
||||
workflow_run="run",
|
||||
qualified_at="now",
|
||||
)
|
||||
remote.side_effect = [bundle(), None, canonical_pointer(old)]
|
||||
publish_qualification(pointer, allow_rollback=True)
|
||||
remove.assert_called_once()
|
||||
|
||||
@mock.patch("bot_bottle.release_qualification.remote_bytes")
|
||||
def test_rejects_missing_or_malformed_bundle(self, remote: mock.Mock) -> None:
|
||||
pointer = build_release_pointer(
|
||||
tag="v1.2.3",
|
||||
source_commit=COMMIT,
|
||||
workflow_run="run",
|
||||
qualified_at="now",
|
||||
)
|
||||
remote.return_value = None
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "does not exist"):
|
||||
publish_qualification(pointer)
|
||||
remote.return_value = b"not-json"
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "malformed"):
|
||||
publish_qualification(pointer)
|
||||
remote.return_value = bundle("b" * 40)
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "does not match"):
|
||||
publish_qualification(pointer)
|
||||
|
||||
@mock.patch("bot_bottle.release_qualification.upload")
|
||||
@mock.patch("bot_bottle.release_qualification.remote_bytes")
|
||||
def test_rejects_changed_release_or_malformed_channel(
|
||||
self, remote: mock.Mock, _upload: mock.Mock,
|
||||
) -> None:
|
||||
pointer = build_release_pointer(
|
||||
tag="v1.2.3",
|
||||
source_commit=COMMIT,
|
||||
workflow_run="run",
|
||||
qualified_at="now",
|
||||
)
|
||||
remote.side_effect = [bundle(), b"different"]
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "already differs"):
|
||||
publish_qualification(pointer)
|
||||
remote.side_effect = [bundle(), None, b"not-json"]
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "malformed"):
|
||||
publish_qualification(pointer)
|
||||
|
||||
@mock.patch("bot_bottle.release_qualification.urllib.request.urlopen")
|
||||
def test_delete_handles_success_and_not_found(self, urlopen: mock.Mock) -> None:
|
||||
urlopen.return_value.__enter__.return_value = mock.Mock()
|
||||
delete("https://example.test/channel")
|
||||
urlopen.side_effect = urllib.error.HTTPError(
|
||||
"url", 404, "missing", Message(), None)
|
||||
delete("https://example.test/channel")
|
||||
|
||||
@mock.patch("bot_bottle.release_qualification.urllib.request.urlopen")
|
||||
def test_delete_reports_registry_errors(self, urlopen: mock.Mock) -> None:
|
||||
urlopen.side_effect = urllib.error.HTTPError(
|
||||
"url", 500, "broken", Message(), None)
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "HTTP 500"):
|
||||
delete("https://example.test/channel")
|
||||
urlopen.side_effect = urllib.error.URLError("offline")
|
||||
with self.assertRaisesRegex(ReleaseManifestError, "offline"):
|
||||
delete("https://example.test/channel")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -117,6 +117,15 @@ class TestWheelInstall(unittest.TestCase):
|
||||
proc = self._run(["-c", script])
|
||||
self.assertIn("SELF_CONTAINED_OK", proc.stdout, msg=proc.stderr)
|
||||
|
||||
def test_ad_hoc_wheel_is_marked_for_local_infrastructure_builds(self):
|
||||
script = (
|
||||
"from bot_bottle.release_manifest import packaged_manifest\n"
|
||||
"assert packaged_manifest() is None\n"
|
||||
"print('DEVELOPMENT_MANIFEST_OK')\n"
|
||||
)
|
||||
proc = self._run(["-c", script])
|
||||
self.assertIn("DEVELOPMENT_MANIFEST_OK", proc.stdout, msg=proc.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user